@evolu/common 6.0.1-preview.32 → 6.0.1-preview.34

package/src/Evolu/Protocol.ts

@@ -171,6 +171,8 @@
  *   `applyProtocolMessage` function with conditional arguments to reduce code
  *   duplication.
  * - ProtocolQuotaError should return storedBytes and actual quota.
+ * - Replace try-catch with Result + new Error (to preserve stacktraces). Measure
+ *   Result overhead, it should be super small.
  */
 
 import { Packr } from "msgpackr";
@@ -186,8 +188,8 @@ import {
   utf8ToBytes,
 } from "../Buffer.js";
 import {
+  createPadmePadding,
   EncryptionKey,
-  padmePaddingLength,
   RandomBytesDep,
   SymmetricCryptoDecryptError,
   SymmetricCryptoDep,
@@ -436,7 +438,7 @@ export interface ProtocolSyncError extends BaseOwnerError {
 export interface ProtocolTimestampMismatchError {
   readonly type: "ProtocolTimestampMismatchError";
   readonly expected: Timestamp;
-  readonly embedded: Timestamp;
+  readonly timestamp: Timestamp;
 }
 
 /**
@@ -909,8 +911,6 @@ export const applyProtocolMessageAsClient =
       | ProtocolQuotaError
     >
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1057,8 +1057,6 @@ export const applyProtocolMessageAsRelay =
   ): Promise<
     Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
   > => {
-    // try-catch instead of Result for performance and stacktraces
-    // DEV: Measure it again, I think we should use Result with new Error.
     try {
       const input = createBuffer(inputMessage);
       const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
@@ -1664,6 +1662,52 @@ export const decodeNumber = (buffer: Buffer): number => {
   return numberResult.value;
 };
 
+/**
+ * Encodes an array of boolean flags into a single byte.
+ *
+ * Each element in the array corresponds to a bit (0-7). Array can have 0-8
+ * elements.
+ *
+ * ### Example
+ *
+ * ```ts
+ * encodeFlags(buffer, [true, false, true]); // Encodes bits 0, 1, 2
+ * ```
+ */
+export const encodeFlags = (
+  buffer: Buffer,
+  flags: ReadonlyArray<boolean>,
+): void => {
+  let byte = 0;
+  for (let i = 0; i < flags.length && i < 8; i++) {
+    if (flags[i]) {
+      byte |= 1 << i;
+    }
+  }
+  buffer.extend([byte]);
+};
+
+/**
+ * Decodes a byte into an array of boolean flags.
+ *
+ * ### Example
+ *
+ * ```ts
+ * const flags = decodeFlags(buffer, 3); // Decode 3 flags
+ * ```
+ */
+export const decodeFlags = (
+  buffer: Buffer,
+  count: PositiveInt,
+): ReadonlyArray<boolean> => {
+  const byte = buffer.shift();
+  const flags: Array<boolean> = [];
+  for (let i = 0; i < count && i < 8; i++) {
+    flags.push((byte & (1 << i)) !== 0);
+  }
+  return flags;
+};
+
 /**
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
@@ -1675,36 +1719,29 @@ export const decodeNumber = (buffer: Buffer): number => {
 export const encodeAndEncryptDbChange =
   (deps: SymmetricCryptoDep) =>
   (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
-    const change = message.change;
     const buffer = createBuffer();
 
-    // Encode protocol version first for backward compatibility
     encodeNonNegativeInt(buffer, protocolVersion);
 
-    // Encode the timestamp (after version) for tamper verification
-    const timestampBytes = timestampToTimestampBytes(message.timestamp);
-    buffer.extend(timestampBytes);
+    // Encode the timestamp to prevent tampering (e.g., a malicious relay
+    // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
+    buffer.extend(timestampToTimestampBytes(message.timestamp));
 
-    encodeString(buffer, change.table);
+    encodeFlags(buffer, [message.change.isInsert]);
 
-    buffer.extend(idToIdBytes(change.id));
+    encodeString(buffer, message.change.table);
+    buffer.extend(idToIdBytes(message.change.id));
 
-    const entries = objectToEntries(change.values).map(
-      ([column, value]): [string, SqliteValue] => {
-        return [column, value];
-      },
-    );
+    const entries = objectToEntries(message.change.values);
 
     encodeLength(buffer, entries);
-
     for (const [column, value] of entries) {
       encodeString(buffer, column);
       encodeSqliteValue(buffer, value);
     }
 
-    const paddingLength = padmePaddingLength(buffer.getLength());
-    // Add zero bytes as PADMÉ padding - these will be ignored during decoding.
-    buffer.extend(new Uint8Array(paddingLength));
+    // Add PADMÉ padding (ignored during decoding)
+    buffer.extend(createPadmePadding(buffer.getLength()));
 
     const { nonce, ciphertext } = deps.symmetricCrypto.encrypt(
       buffer.unwrap(),
@@ -1735,13 +1772,11 @@ export const decryptAndDecodeDbChange =
     | ProtocolInvalidDataError
     | ProtocolTimestampMismatchError
   > => {
-    // try-catch instead of Result for performance and stacktraces
     try {
       const buffer = createBuffer(message.change);
-      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
 
-      const ciphertextLength = decodeLength(buffer);
-      const ciphertext = buffer.shiftN(ciphertextLength);
+      const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
+      const ciphertext = buffer.shiftN(decodeLength(buffer));
 
       const plaintextBytes = deps.symmetricCrypto.decrypt(
         ciphertext,
@@ -1753,24 +1788,24 @@ export const decryptAndDecodeDbChange =
       buffer.reset();
       buffer.extend(plaintextBytes.value);
 
-      // Decode version (for future compatibility, no validation needed for now)
+      // Decode version (for future compatibility, not need yet)
       decodeNonNegativeInt(buffer);
 
-      // Decode and verify the embedded timestamp
-      const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
-      const embeddedTimestamp = timestampBytesToTimestamp(
-        embeddedTimestampBytes as TimestampBytes,
+      const timestamp = timestampBytesToTimestamp(
+        buffer.shiftN(timestampBytesLength) as TimestampBytes,
       );
 
-      // Verify timestamp integrity
-      if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
+      if (!eqTimestamp(timestamp, message.timestamp)) {
         return err<ProtocolTimestampMismatchError>({
           type: "ProtocolTimestampMismatchError",
           expected: message.timestamp,
-          embedded: embeddedTimestamp,
+          timestamp,
         });
       }
 
+      const flags = decodeFlags(buffer, PositiveInt.orThrow(1));
+      const isInsert = flags[0];
+
       const table = decodeString(buffer);
       const id = decodeId(buffer);
 
@@ -1783,7 +1818,7 @@ export const decryptAndDecodeDbChange =
         values[column] = value;
       }
 
-      const dbChange = { table, id, values };
+      const dbChange = DbChange.orThrow({ table, id, values, isInsert });
 
       return ok(dbChange);
     } catch (error) {

package/src/Evolu/Query.ts

@@ -1,3 +1,4 @@
+import { Brand } from "../Brand.js";
 import { bytesToHex, hexToBytes } from "../Buffer.js";
 import { objectToEntries } from "../Object.js";
 import {
@@ -9,7 +10,6 @@ import {
 } from "../Sqlite.js";
 import { Store, StoreSubscribe } from "../Store.js";
 import { Simplify } from "../Types.js";
-import { Brand } from "../Brand.js";
 
 /**
  * A type-safe SQL query.
@@ -52,7 +52,7 @@ export const serializeQuery = <R extends Row>(query: SqliteQuery): Query<R> => {
   );
 
   const options = query.options
-    ? objectToEntries(query.options).sort(([a], [b]) => a.localeCompare(b))
+    ? objectToEntries(query.options).toSorted(([a], [b]) => a.localeCompare(b))
     : [];
 
   return JSON.stringify([query.sql, params, options]) as Query<R>;

package/src/Evolu/Schema.ts

@@ -1,4 +1,6 @@
 import * as Kysely from "kysely";
+import { assert } from "../Assert.js";
+import { createEqArrayLike, eqString } from "../Eq.js";
 import { mapObject, objectToEntries, ReadonlyRecord } from "../Object.js";
 import { ok, Result } from "../Result.js";
 import {
@@ -14,6 +16,7 @@ import {
 import {
   AnyType,
   array,
+  createIdFromString,
   DateIso,
   IdBytes,
   InferErrors,
@@ -40,7 +43,7 @@ import { Simplify } from "../Types.js";
 import { AppOwner, OwnerId } from "./Owner.js";
 import { Query, Row } from "./Query.js";
 import { CrdtMessage, DbChange } from "./Storage.js";
-import { TimestampBytes } from "./Timestamp.js";
+import { Timestamp, TimestampBytes } from "./Timestamp.js";
 
 /**
  * Defines the schema of an Evolu database.
@@ -92,7 +95,7 @@ export type EvoluSchema = ReadonlyRecord<
  *
  * 1. All tables must have an 'id' column
  * 2. The 'id' column must be a branded ID type (created with id() function)
- * 3. Tables cannot use default column names (createdAt, updatedAt, isDeleted)
+ * 3. Tables cannot use system column names (createdAt, updatedAt, isDeleted)
  * 4. All column types must be compatible with SQLite (extend SqliteValue)
  */
 export type ValidateSchema<S extends EvoluSchema> =
@@ -132,7 +135,7 @@ export type ValidateNoDefaultColumns<S extends EvoluSchema> =
       ? keyof S[TableName] extends infer ColumnName
         ? ColumnName extends keyof S[TableName]
           ? ColumnName extends "createdAt" | "updatedAt" | "isDeleted"
-            ? SchemaValidationError<`Table "${TableName & string}" uses default column name "${ColumnName & string}". Default columns (createdAt, updatedAt, isDeleted) are added automatically.`>
+            ? SchemaValidationError<`Table "${TableName & string}" uses system column name "${ColumnName & string}". System columns (createdAt, updatedAt, isDeleted) are added automatically.`>
             : never
           : never
         : never
@@ -195,7 +198,7 @@ export type CreateQuery<S extends EvoluSchema> = <R extends Row>(
               | "updatedAt"
               ? InferType<S[Table][Column]>
               : InferType<S[Table][Column]> | null;
-          } & DefaultColumns;
+          } & SystemColumns;
         } & {
           readonly evolu_history: {
             readonly timestamp: TimestampBytes;
@@ -213,21 +216,30 @@ export type CreateQuery<S extends EvoluSchema> = <R extends Row>(
 ) => Query<Simplify<R>>;
 
 /**
- * Default columns automatically added to all tables.
+ * System columns that are implicitly defined by Evolu.
  *
- * - `createdAt`: Set by Evolu when `insert` is called, or can be custom with
- *   `upsert`.
- * - `updatedAt`: Always set by Evolu, derived from {@link CrdtMessage} timestamp.
- *   If you defer sync to avoid leaking time activity, use a custom column to
- *   preserve real update time.
- * - `isDeleted`: Soft delete flag.
+ * - `createdAt`: Set by Evolu on row creation, derived from {@link Timestamp}.
+ * - `updatedAt`: Set by Evolu on every row change, derived from {@link Timestamp}.
+ * - `isDeleted`: Soft delete flag created by Evolu and used by the developer to
+ *   mark rows as deleted.
  */
-export const DefaultColumns = object({
+export const SystemColumns = object({
   createdAt: DateIso,
   updatedAt: DateIso,
   isDeleted: nullOr(SqliteBoolean),
+  // TODO: ownerId
 });
-export type DefaultColumns = typeof DefaultColumns.Type;
+export type SystemColumns = typeof SystemColumns.Type;
+
+export const systemColumns = ["createdAt", "updatedAt", "isDeleted"];
+
+assert(
+  createEqArrayLike(eqString)(
+    objectToEntries(SystemColumns.props).map(([key]) => key),
+    systemColumns,
+  ),
+  "SystemColumns keys must match systemColumnsKeys",
+);
 
 export type MutationKind = "insert" | "update" | "upsert";
 
@@ -311,7 +323,8 @@ export interface MutationChange extends DbChange {
 
 /**
  * Type Factory to create insertable {@link Type}. It makes nullable Types
- * optional, omits Id, and ensures the {@link maxMutationSize}.
+ * optional (so they are not required), omits Id, and ensures the
+ * {@link maxMutationSize}.
  *
  * ### Example
  *
@@ -341,7 +354,7 @@ export type Insertable<Props extends Record<string, AnyType>> = InferInput<
 
 /**
  * Type Factory to create updateable {@link Type}. It makes everything except for
- * the `id` column partial (i.e. optional) and ensures the
+ * the `id` column optional (so they are not required) and ensures the
  * {@link maxMutationSize}.
  *
  * ### Example
@@ -377,9 +390,15 @@ export type Updateable<Props extends Record<string, AnyType>> = InferInput<
 >;
 
 /**
- * Type Factory to create upsertable Type. It makes nullable Types optional,
- * includes optional default columns (createdAt, isDeleted), and ensures the
- * {@link maxMutationSize}.
+ * Type Factory to create an upsertable Type. It makes nullable Types optional
+ * (so they are not required) and ensures the {@link maxMutationSize}.
+ *
+ * Upsert is like insert, except it requires an ID. It's useful for inserting
+ * rows with external ID via {@link createIdFromString}.
+ *
+ * Note that it's not possible to upsert a row with `createdAt` nor `updatedAt`,
+ * because they are derived from {@link CrdtMessage} timestamp. For external
+ * createdAt, use a different column.
  *
  * ### Example
  *
@@ -389,7 +408,6 @@ export type Updateable<Props extends Record<string, AnyType>> = InferInput<
  * const todo = UpsertableTodo.from({
  *   id,
  *   title,
- *   createdAt: "2023-01-01T00:00:00.000Z",
  * });
  * if (!todo.ok) return; // handle errors
  * ```
@@ -399,7 +417,6 @@ export const upsertable = <Props extends Record<string, AnyType>>(
 ): ValidMutationSize<UpsertableProps<Props>> => {
   const propsWithDefaults = {
     ...props,
-    createdAt: optional(DateIso),
     isDeleted: optional(SqliteBoolean),
   };
   return validMutationSize(nullableToOptional(propsWithDefaults));
@@ -408,7 +425,6 @@ export const upsertable = <Props extends Record<string, AnyType>>(
 export type UpsertableProps<Props extends Record<string, AnyType>> =
   NullableToOptionalProps<
     Props & {
-      createdAt: OptionalType<typeof DateIso>;
       isDeleted: OptionalType<typeof SqliteBoolean>;
     }
   >;
@@ -535,7 +551,7 @@ export const ensureDbSchema =
       );
       if (!currentTable) {
         queries.push({
-          sql: createTableWithDefaultColumns(newTable.name, newTable.columns),
+          sql: createAppTableWithDefaultColumns(newTable),
           parameters: [],
         });
       } else {
@@ -583,16 +599,12 @@ export const ensureDbSchema =
     return ok();
   };
 
-const createTableWithDefaultColumns = (
-  tableName: string,
-  columns: ReadonlyArray<string>,
-): SafeSql =>
-  `
-    create table ${sql.identifier(tableName).sql} (
+const createAppTableWithDefaultColumns = (table: DbTable): SafeSql => {
+  return `
+    create table ${sql.identifier(table.name).sql} (
       "id" text primary key,
-      ${columns
-        // Add default columns.
-        .concat(["createdAt", "updatedAt", "isDeleted"])
+      ${table.columns
+        .concat(systemColumns)
         .filter((c) => c !== "id")
         // "A column with affinity BLOB does not prefer one storage class over another
         // and no attempt is made to coerce data from one storage class into another."
@@ -601,7 +613,7 @@ const createTableWithDefaultColumns = (
         .join(", ")}
     );
   ` as SafeSql;
-
+};
 // https://kysely.dev/docs/recipes/splitting-query-building-and-execution
 export const kysely = new Kysely.Kysely({
   dialect: {

package/src/Evolu/Storage.ts

@@ -1,5 +1,6 @@
 import { sha256 } from "@noble/hashes/sha2.js";
 import {
+  filterArray,
   firstInArray,
   isNonEmptyReadonlyArray,
   NonEmptyReadonlyArray,
@@ -9,10 +10,12 @@ import { Brand } from "../Brand.js";
 import { concatBytes } from "../Buffer.js";
 import { decrement } from "../Number.js";
 import { RandomDep } from "../Random.js";
-import { ok, Result } from "../Result.js";
+import { err, ok, Result } from "../Result.js";
 import { sql, SqliteDep, SqliteError, SqliteValue } from "../Sqlite.js";
 import { MaybeAsync } from "../Task.js";
 import {
+  Boolean,
+  brand,
   Id,
   Int64String,
   NonNegativeInt,
@@ -20,6 +23,7 @@ import {
   PositiveInt,
   record,
   String,
+  TypeError,
 } from "../Type.js";
 import {
   BaseOwnerError,
@@ -29,6 +33,7 @@ import {
   OwnerWriteKey,
 } from "./Owner.js";
 import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
+import { systemColumns } from "./Schema.js";
 
 export interface StorageConfig {
   /**
@@ -248,6 +253,38 @@ export interface CrdtMessage {
   readonly change: DbChange;
 }
 
+export const DbChangeValues = record(String, SqliteValue);
+export type DbChangeValues = typeof DbChangeValues.Type;
+
+const systemColumnsWithIdAndWithoutIsDeleted = filterArray(
+  systemColumns.concat("id"),
+  (column) => column != "isDeleted",
+);
+
+export const ValidDbChangeValues = brand(
+  "ValidDbChangeValues",
+  DbChangeValues,
+  (value) => {
+    const invalidColumns = systemColumnsWithIdAndWithoutIsDeleted.filter(
+      (key) => key in value,
+    );
+    if (invalidColumns.length > 0)
+      return err<ValidDbChangeValuesError>({
+        type: "ValidDbChangeValues",
+        value,
+        invalidColumns,
+      });
+
+    return ok(value);
+  },
+);
+export type ValidDbChangeValues = typeof ValidDbChangeValues.Type;
+
+export interface ValidDbChangeValuesError
+  extends TypeError<"ValidDbChangeValues"> {
+  readonly invalidColumns: ReadonlyArray<string>;
+}
+
 /**
  * A DbChange is a change to a table row. Together with a unique
  * {@link Timestamp}, it forms a {@link CrdtMessage}.
@@ -255,7 +292,8 @@ export interface CrdtMessage {
 export const DbChange = object({
   table: String,
   id: Id,
-  values: record(String, SqliteValue),
+  values: ValidDbChangeValues,
+  isInsert: Boolean,
 });
 export type DbChange = typeof DbChange.Type;