@evolu/common 8.11.0 → 8.13.0

package/src/local-first/Protocol.test.ts

@@ -207,6 +207,23 @@ test("encodeSqliteValue/decodeSqliteValue preserves an own __proto__ JSON object
   assertEqual(decodeSqliteValue(buffer), value);
 });
 
+test("encodeSqliteValue encodes JSON nested deeper than decoding allows as a string", () => {
+  for (const [depth, type] of [
+    [1_000, ProtocolValueType.Json],
+    [1_001, ProtocolValueType.String],
+    // Deep enough to overflow the stack of a recursive JSON.stringify.
+    [20_000, ProtocolValueType.String],
+  ] as const) {
+    const value = "[".repeat(depth) + "]".repeat(depth);
+    const buffer = createBuffer();
+
+    encodeSqliteValue(buffer, value);
+
+    assertEqual(buffer.unwrap()[0], type);
+    assertEqual(decodeSqliteValue(buffer), value);
+  }
+});
+
 test("encodeSqliteValue/decodeSqliteValue property tests", () => {
   const deps = testCreateDeps();
   // Property test: round-trip encoding/decoding should preserve the value
@@ -933,7 +950,7 @@ describe("E2E errors", () => {
     );
   });
 
-  it("rejected relay writes report quota and other causes distinctly", async () => {
+  it("reports a rejected relay write as a quota and a thrown one as a write failure", async () => {
     const deps = testCreateDeps();
     const initiatorMessage = createProtocolMessageFromCrdtMessages(deps)(
       testAppOwner,
@@ -944,50 +961,48 @@ describe("E2E errors", () => {
         },
       ],
     );
-    /** Returns the relay's response and what it logged for the rejection. */
-    const relayResponseFor = async (error: StorageWriteMessagesError) => {
+    /** Returns the relay's response and what it logged for the write. */
+    const relayResponseFor = async (
+      writeMessages: StorageDep["storage"]["writeMessages"],
+    ) => {
       await using run = testCreateRun({
         storage: {
           ...shouldNotBeCalledStorageDep.storage,
           validateWriteKey: () => true,
-          writeMessages: () => () => err(error),
+          writeMessages,
         },
       } satisfies StorageDep);
       const { message } = await run.orThrow(
         applyProtocolMessageAsRelay(initiatorMessage),
       );
-      return {
-        message,
-        logged: run.deps.console
-          .getEntriesSnapshot()
-          .map(({ method, args }) => ({ method, args })),
-      };
+      return { message, logged: run.deps.console.getEntriesSnapshot() };
     };
 
     await using run = testCreateRun(shouldNotBeCalledStorageDep);
     // A quota rejection is expected, so the relay does not log it.
-    const quota = await relayResponseFor({
-      type: "StorageQuotaError",
-      ownerId: testAppOwner.id,
-    });
+    const quota = await relayResponseFor(
+      () => () => err({ type: "StorageQuotaError", ownerId: testAppOwner.id }),
+    );
     assertEqual(
       await run(applyProtocolMessageAsClient(quota.message)),
       err({ type: "ProtocolQuotaError", ownerId: testAppOwner.id }),
     );
     assertEqual(quota.logged, []);
-    // Any other storage rejection is a relay write failure, not a quota, and
-    // the relay logs its cause.
-    const mismatch: StorageWriteMessagesError = {
-      type: "ProtocolTimestampMismatchError",
-      expected: timestampBytesToTimestamp(testTimestampsAsc[0]),
-      timestamp: timestampBytesToTimestamp(testTimestampsAsc[1]),
-    };
-    const other = await relayResponseFor(mismatch);
+    // A thrown write is a relay write failure, not a quota, and the relay
+    // logs it.
+    const failure = new Error("write failed");
+    const thrown = await relayResponseFor(() => {
+      throw failure;
+    });
     assertEqual(
-      await run(applyProtocolMessageAsClient(other.message)),
+      await run(applyProtocolMessageAsClient(thrown.message)),
       err({ type: "ProtocolWriteError", ownerId: testAppOwner.id }),
     );
-    assertEqual(other.logged, [{ method: "error", args: [mismatch] }]);
+    assertEqual(
+      thrown.logged.map(({ method }) => method),
+      ["error"],
+    );
+    assertSame(thrown.logged[0]?.args[0], failure);
   });
 });
 
@@ -1222,7 +1237,7 @@ describe("applyProtocolMessageAsClient results", () => {
     assertOk(result, { type: "Readonly" });
   });
 
-  it("preserves expected storage write rejection causes", async () => {
+  it("preserves a storage write rejection", async () => {
     const deps = testCreateDeps();
     const input = createResponse();
     input.addMessage(
@@ -1233,42 +1248,26 @@ describe("applyProtocolMessageAsClient results", () => {
     );
     const message = input.unwrap();
 
-    const errors: ReadonlyArray<StorageWriteMessagesError> = [
-      {
-        type: "DecryptWithXChaCha20Poly1305Error",
-        error: new Error("decryption failed"),
-      },
-      {
-        type: "ProtocolInvalidDataError",
-        data: Uint8Array.of(255),
-        error: new Error("decoding failed"),
-      },
-      {
-        type: "ProtocolTimestampMismatchError",
-        expected: timestampBytesToTimestamp(testTimestampsAsc[0]),
-        timestamp: timestampBytesToTimestamp(testTimestampsAsc[1]),
+    const error: StorageWriteMessagesError = {
+      type: "StorageQuotaError",
+      ownerId: testAppOwner.id,
+    };
+    await using run = testCreateRun({
+      storage: {
+        ...shouldNotBeCalledStorageDep.storage,
+        writeMessages: () => () => err(error),
       },
-      { type: "StorageQuotaError", ownerId: testAppOwner.id },
-    ];
-
-    for (const error of errors) {
-      await using run = testCreateRun({
-        storage: {
-          ...shouldNotBeCalledStorageDep.storage,
-          writeMessages: () => () => err(error),
-        },
-      } satisfies StorageDep);
-      const task = applyProtocolMessageAsClient(message, {
-        writeKey: testAppOwner.writeKey,
-      });
-      assertType<
-        InferTaskErr<typeof task>,
-        ProtocolError | StorageWriteMessagesError
-      >();
-      const result = await run(task);
-      assertErr(result);
-      assertSame(result.error, error);
-    }
+    } satisfies StorageDep);
+    const task = applyProtocolMessageAsClient(message, {
+      writeKey: testAppOwner.writeKey,
+    });
+    assertType<
+      InferTaskErr<typeof task>,
+      ProtocolError | StorageWriteMessagesError
+    >();
+    const result = await run(task);
+    assertErr(result);
+    assertSame(result.error, error);
   });
 
   it("reports a thrown write as failed", async () => {

package/src/local-first/Protocol.ts

@@ -103,9 +103,9 @@
  * fit within the limit, the protocol automatically continues synchronization in
  * subsequent rounds using range-based reconciliation.
  *
- * Database mutations are limited to 640KB, which is smaller than the protocol
- * message limit to ensure efficient sync with
- * {@link defaultProtocolMessageRangesMaxSize}.
+ * Each mutation is limited to {@link maxMutationSize}, so every change fits one
+ * message of {@link defaultProtocolMessageMaxSize} next to the largest ranges
+ * section.
  *
  * ## Why Binary?
  *
@@ -190,6 +190,7 @@ import {
 import type { Brand } from "../Brand.ts";
 import {
   type Buffer,
+  BufferError,
   createBuffer,
   createRunLengthEncoder,
   decodeFlags,
@@ -248,7 +249,7 @@ import {
   zeroNonNegativeInt,
 } from "../Type.ts";
 import type { Predicate } from "../Types.ts";
-import type { Evolu } from "./Evolu.ts";
+import type { Evolu, maxMutationSize } from "./Evolu.ts";
 import {
   type Owner,
   type OwnerError,
@@ -464,8 +465,9 @@ export interface ProtocolWriteKeyError
   extends OwnerError, Typed<"ProtocolWriteKeyError"> {}
 
 /**
- * Error indicating a serious relay-side write failure. Clients should log this
- * error and show a generic sync error to the user.
+ * Error indicating a serious relay-side write failure. Sync state shows it as
+ * the failure of that relay's route; apps show a generic sync error for the
+ * owner's `Error` status.
  */
 export interface ProtocolWriteError
   extends OwnerError, Typed<"ProtocolWriteError"> {}
@@ -488,8 +490,9 @@ export interface ProtocolQuotaError
   extends OwnerError, Typed<"ProtocolQuotaError"> {}
 
 /**
- * Error indicating a serious relay-side synchronization failure. Clients should
- * log this error and show a generic sync error to the user.
+ * Error indicating a serious relay-side synchronization failure. Sync state
+ * shows it as the failure of that relay's route; apps show a generic sync error
+ * for the owner's `Error` status.
  */
 export interface ProtocolSyncError
   extends OwnerError, Typed<"ProtocolSyncError"> {}
@@ -1285,13 +1288,9 @@ export const applyProtocolMessageAsRelay =
           );
 
           if (!result.ok) {
-            const isQuotaError = result.error.type === "StorageQuotaError";
-            if (!isQuotaError) run.deps.console.error(result.error);
             const message = createProtocolMessageBuffer(ownerId, {
               messageType: MessageType.Response,
-              errorCode: isQuotaError
-                ? ProtocolErrorCode.QuotaError
-                : ProtocolErrorCode.WriteError,
+              errorCode: ProtocolErrorCode.QuotaError,
             }).unwrap();
             return ok({ type: "Response", message });
           }
@@ -1855,30 +1854,7 @@ export const encodeAndEncryptDbChange =
   (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
     const buffer = createBuffer();
 
-    encodeNonNegativeInt(buffer, protocolVersion);
-
-    // Encode the timestamp to prevent tampering (e.g., a malicious relay
-    // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
-    buffer.extend(timestampToTimestampBytes(message.timestamp));
-
-    encodeFlags(buffer, [
-      message.change.isInsert,
-      // Encode nullable boolean as two flags: presence + value.
-      message.change.isDelete != null,
-      message.change.isDelete ?? false,
-    ]);
-
-    encodeString(buffer, message.change.table);
-    buffer.extend(idToIdBytes(message.change.id));
-
-    const entries = objectToEntries(message.change.values);
-
-    encodeLength(buffer, entries);
-    for (const [column, value] of entries) {
-      assertNotUndefined(value);
-      encodeString(buffer, column);
-      encodeSqliteValue(buffer, value);
-    }
+    encodeDbChange(buffer, message);
 
     // Add PADMÉ padding (ignored during decoding)
     buffer.extend(createPadmePadding(buffer.getLength()));
@@ -1896,6 +1872,41 @@ export const encodeAndEncryptDbChange =
     return buffer.unwrap() as EncryptedDbChange;
   };
 
+/**
+ * Encodes a {@link CrdtMessage} as {@link encodeAndEncryptDbChange} does before
+ * padding and encryption.
+ *
+ * {@link Evolu.getMutationSize} measures mutations with this encoding, so
+ * {@link maxMutationSize} limits exactly what is encoded, and every change
+ * within it fits one protocol message.
+ */
+export const encodeDbChange = (buffer: Buffer, message: CrdtMessage): void => {
+  encodeNonNegativeInt(buffer, protocolVersion);
+
+  // Encode the timestamp to prevent tampering (e.g., a malicious relay
+  // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
+  buffer.extend(timestampToTimestampBytes(message.timestamp));
+
+  encodeFlags(buffer, [
+    message.change.isInsert,
+    // Encode nullable boolean as two flags: presence + value.
+    message.change.isDelete != null,
+    message.change.isDelete ?? false,
+  ]);
+
+  encodeString(buffer, message.change.table);
+  buffer.extend(idToIdBytes(message.change.id));
+
+  const entries = objectToEntries(message.change.values);
+
+  encodeLength(buffer, entries);
+  for (const [column, value] of entries) {
+    assertNotUndefined(value);
+    encodeString(buffer, column);
+    encodeSqliteValue(buffer, value);
+  }
+};
+
 /**
  * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
  * owner's encryption key. Verifies that the embedded timestamp matches the
@@ -2069,21 +2080,30 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
       }
 
       const json = Json.from.parent(value);
-      // Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
-      // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
-      // which would cause data corruption if we don't verify round-trip safety.
-      if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
+      if (json.ok) {
+        const jsonValue = jsonToJsonValue(json.value);
         jsonBuffer.reset();
         try {
-          encodeJsonValue(jsonBuffer, jsonToJsonValue(json.value));
-          const jsonBytes = jsonBuffer.unwrap();
-          encodeNonNegativeInt(buffer, ProtocolValueType.Json);
-          encodeLength(buffer, jsonBytes);
-          buffer.extend(jsonBytes);
+          // Encoding first rejects nesting deeper than decoding allows, before
+          // the recursive JSON.stringify below could overflow the stack. Such
+          // a value is encoded as a plain string.
+          encodeJsonValue(jsonBuffer, jsonValue);
+          // Only encode as Json if it survives JSON.parse/JSON.stringify
+          // round-trip. Some valid JSON strings like "-0E0" get normalized to
+          // "0" during parsing, which would cause data corruption if we don't
+          // verify round-trip safety.
+          if (JSON.stringify(jsonValue) === value) {
+            const jsonBytes = jsonBuffer.unwrap();
+            encodeNonNegativeInt(buffer, ProtocolValueType.Json);
+            encodeLength(buffer, jsonBytes);
+            buffer.extend(jsonBytes);
+            return;
+          }
+        } catch (error) {
+          if (!(error instanceof BufferError)) throw error;
         } finally {
           jsonBuffer.reset();
         }
-        return;
       }
 
       const base64Url = Base64Url.from.parent(value);

package/src/local-first/Relay.ts

@@ -17,7 +17,7 @@ import { err, ok } from "../Result.ts";
 import type { SqliteDep } from "../Sqlite.ts";
 import { sql } from "../Sqlite.ts";
 import { createMutexByKey } from "../Task.ts";
-import { Name, PositiveInt, uint8ArrayToBase64Url } from "../Type.ts";
+import { Name, NonNegativeInt, uint8ArrayToBase64Url } from "../Type.ts";
 import { isPromiseLike, type Awaitable } from "../Types.ts";
 import {
   OwnerId,
@@ -246,7 +246,9 @@ export const createRelaySqliteStorage =
               (sum, m) => sum + m.change.length,
               0,
             );
-            const newStoredBytes = PositiveInt.orThrow(
+            // A sum of lengths can be zero. Throwing here would panic the
+            // relay's shared Run.
+            const newStoredBytes = NonNegativeInt.orThrow(
               (usage.storedBytes ?? 0) + incomingBytes,
             );
 

package/src/local-first/Schema.ts

@@ -34,6 +34,7 @@ import {
   IdBytes,
   type InferType,
   NonEmptyTrimmedString100,
+  type NonEmptyTrimmedString1000,
   Null,
   nullOr,
   object,
@@ -42,6 +43,7 @@ import {
   type withDefault,
 } from "../Type.ts";
 import type { CompileTimeError, Simplify } from "../Types.ts";
+import type { Evolu, EvoluErrorDep, maxMutationSize } from "./Evolu.ts";
 import type { AppOwner, OwnerIdBytes } from "./Owner.ts";
 import {
   OwnerEncryptionKey,
@@ -475,8 +477,17 @@ export type MutationKind = "insert" | "update" | "upsert";
  * inadvertently generate a large volume of CRDT messages. Each mutation
  * produces exactly one {@link CrdtMessage} containing all provided columns.
  *
- * Mutations never fail — values are already validated by the caller, and
- * changes are stored locally in SQLite.
+ * Each mutation must fit within {@link maxMutationSize}. Give every column a
+ * Type with a maximum length, such as {@link NonEmptyTrimmedString1000} or
+ * `maxLength(100_000)(Uint8Array)`, so that a table's largest values add up to
+ * less than the limit and input that is too large is rejected where it enters
+ * the app. A larger mutation throws before anything is saved, so the code after
+ * it does not run. Check unbounded input with {@link Evolu.getMutationSize}.
+ * Large binary data, such as images or videos, does not belong in a single
+ * mutation; a chunked API for it is planned.
+ *
+ * Binary values are copied when the mutation is made, so later changes to a
+ * `Uint8Array` do not change what is saved.
  *
  * - **insert**: all non-nullable columns required, nullable columns optional,
  *   `id` omitted (auto-generated)
@@ -503,7 +514,13 @@ export interface MutationOptions {
    * Called after the mutation's changes are stored and subscribed queries
    * reflect them. Useful for follow-up work (e.g., notifications, navigation)
    * after insert, update, or upsert. It never runs when the database is
-   * unavailable.
+   * unavailable or the mutation could not be stored, which
+   * {@link EvoluErrorDep.evoluError} reports.
+   *
+   * An Evolu instance stores its mutations in batches, each in one transaction.
+   * A batch holds the mutations made before Evolu sends it in a microtask,
+   * usually one synchronous block, and {@link Evolu.requestSync} sends it early.
+   * When one mutation cannot be stored, none of its batch is.
    *
    * Stored does not always mean visible. A change quarantined for
    * {@link QuarantineReason.TimestampDrift} is stored in