@evolu/common 8.11.0 → 8.13.0

package/src/local-first/Evolu.test.ts

@@ -13,8 +13,10 @@ import {
   assertThrowsInstanceOf,
   assertTrue,
 } from "../Assert.ts";
+import * as fc from "fast-check";
 import { describe, it } from "node:test";
 import type { Brand } from "../Brand.ts";
+import { createBuffer } from "../Bytes.ts";
 import type { ConsoleEntry, TestConsole } from "../Console.ts";
 import { testCreateConsole } from "../Console.ts";
 import { constVoid } from "../Function.ts";
@@ -25,6 +27,7 @@ import {
   createEvoluDeps,
   type Evolu,
   type EvoluConfig,
+  maxMutationSize,
   testAppName,
   type EvoluPlatformDeps,
 } from "./Evolu.ts";
@@ -34,6 +37,17 @@ import {
   createSharedOwner,
   testAppOwner,
 } from "./Owner.ts";
+import {
+  createProtocolBroadcastMessagesFromCrdtMessages,
+  createProtocolMessageBuffer,
+  createProtocolMessageFromCrdtMessages,
+  createTimestampsBuffer,
+  encodeAndEncryptDbChange,
+  encodeDbChange,
+  MessageType,
+  ProtocolErrorCode,
+  ProtocolMessageRangesMaxSize,
+} from "./Protocol.ts";
 import { createQueryBuilder } from "./Schema.ts";
 import {
   consoleEntryOrErrorBroadcastChannelName,
@@ -47,6 +61,7 @@ import {
   type SharedWorkerInput,
   type SharedWorkerOutput,
   type SyncState,
+  syncStateToOwnerSyncStatus,
 } from "./Shared.ts";
 import {
   acquireLeaderLock,
@@ -56,19 +71,25 @@ import {
 import { installPolyfills } from "../Polyfills.ts";
 import { err, ok } from "../Result.ts";
 import { SqliteBoolean } from "../Sqlite.ts";
-import { explicitAbortReason, testCreateRun } from "../Task.ts";
+import { explicitAbortReason, testCreateDeps, testCreateRun } from "../Task.ts";
 import { testCreateId } from "../Test.ts";
+import { Millis } from "../Time.ts";
 import {
   assertType,
   createIdFromString,
+  FiniteNumber,
   id,
   NonEmptyTrimmedString100,
   nullOr,
   PositiveInt,
+  String as StringType,
   testName,
   Name,
+  Uint8Array as Uint8ArrayType,
 } from "../Type.ts";
 import type { ExtractTyped } from "../Type.ts";
+import { DbChange } from "./Storage.ts";
+import { createTimestamp, type NodeId } from "./Timestamp.ts";
 import {
   testCreateBroadcastChannel,
   testCreateMessageChannel,
@@ -409,6 +430,7 @@ describe("Evolu", () => {
         tables: { todo: new Set(["title"]) },
         indexes: [],
       };
+      const sharedWorkerId = testCreateId()<"SharedWorker">();
       const sharedWorkerPort: {
         value: MessagePort<SharedWorkerOutput, SharedWorkerInput> | null;
       } = { value: null };
@@ -449,6 +471,7 @@ describe("Evolu", () => {
           sqliteSchema,
           encryptionKey: testAppOwner.encryptionKey,
           memoryOnly: true,
+          sharedWorkerId,
           port: dbWorkerChannel.port1.native,
         },
         [dbWorkerChannel.port1.native],
@@ -464,6 +487,7 @@ describe("Evolu", () => {
         sqliteSchema,
         encryptionKey: testAppOwner.encryptionKey,
         memoryOnly: true,
+        sharedWorkerId,
         port: dbWorkerChannel.port1.native,
       });
 
@@ -573,7 +597,7 @@ describe("Evolu", () => {
       );
       const createState = (label: string): SyncState => ({
         transports: [],
-        tenants: [{ name: Name.orThrow(label), refused: false, owners: [] }],
+        tenants: [{ type: "Active", name: Name.orThrow(label), owners: [] }],
       });
 
       // Nothing reaches the tab before its worker names the channel.
@@ -597,6 +621,214 @@ describe("Evolu", () => {
       assertEqual(deps.syncState.get(), createState("changed"));
     });
 
+    it("keeps the unchanged parts of each sync state snapshot", async () => {
+      using setup = await setupCreateEvoluDeps();
+      const { deps, connect } = setup;
+      const workerId = testCreateId()<"SharedWorker">();
+      using channel = testCreateBroadcastChannel<SyncState>(
+        `evolu:sync-state:${workerId}`,
+      );
+      await connect(workerId);
+      let notificationCount = 0;
+      deps.syncState.subscribe(() => {
+        notificationCount++;
+      });
+      const transportId = testCreateId()<"SyncTransport">();
+      // Every snapshot is a new structured clone. The worker wraps a caught
+      // Error inside an error in an UnknownError, whose cause can hold the
+      // bytes of a received change.
+      const createState = (
+        lastSentAt: number,
+        message = "wrong key",
+      ): SyncState => ({
+        transports: [
+          {
+            type: "WebSocket",
+            id: transportId,
+            label: "wss://relay.example",
+            connection: {
+              type: "Open",
+              openedAt: Millis.orThrow(1000),
+              error: null,
+            },
+          },
+        ],
+        tenants: [
+          {
+            type: "Active",
+            name: testName,
+            owners: [
+              {
+                type: "Writable",
+                ownerId: testAppOwner.id,
+                routes: [
+                  {
+                    type: "Settled",
+                    transportId,
+                    skippedError: {
+                      type: "DecryptWithXChaCha20Poly1305Error",
+                      error: {
+                        type: "UnknownError",
+                        error: {
+                          message,
+                          cause: { values: { photo: Uint8Array.of(1, 2, 3) } },
+                        },
+                      },
+                      at: Millis.orThrow(1500),
+                    },
+                    completeAt: null,
+                    lastSentAt: Millis.orThrow(lastSentAt),
+                    lastReceivedAt: Millis.orThrow(1500),
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      });
+      const routeOf = (state: SyncState | null) => {
+        const tenant = state?.tenants[0];
+        assert(tenant?.type === "Active", "The tenant should be active.");
+        const owner = tenant.owners[0];
+        assert(owner?.type === "Writable", "The owner should be writable.");
+        const route = owner.routes[0];
+        assert(route?.type === "Settled", "The route should be settled.");
+        return route;
+      };
+      const statusOf = (state: SyncState | null) =>
+        syncStateToOwnerSyncStatus(state, testName, testAppOwner.id);
+
+      channel.postMessage(createState(1000));
+      await testWaitForWorkerMessage();
+      const first = deps.syncState.get();
+      assertNotNull(first);
+      assertSame(notificationCount, 1);
+
+      // An equal snapshot leaves the store unchanged.
+      channel.postMessage(createState(1000));
+      await testWaitForWorkerMessage();
+      assertSame(deps.syncState.get(), first);
+      assertSame(notificationCount, 1);
+
+      // A changed route gets a new object, and every unchanged part keeps the
+      // previous one, so the owner's status is the same object.
+      channel.postMessage(createState(2000));
+      await testWaitForWorkerMessage();
+      const second = deps.syncState.get();
+      assertNotNull(second);
+      assertNotSame(second, first);
+      assertSame(notificationCount, 2);
+      assertSame(second.transports, first.transports);
+      assertNotSame(routeOf(second), routeOf(first));
+      assertSame(routeOf(second).skippedError, routeOf(first).skippedError);
+      assertSame(statusOf(second), statusOf(first));
+
+      // An error with another message is a new error and a new status.
+      channel.postMessage(createState(2000, "tampered"));
+      await testWaitForWorkerMessage();
+      const third = deps.syncState.get();
+      assertNotSame(routeOf(third).skippedError, routeOf(second).skippedError);
+      assertNotSame(statusOf(third), statusOf(second));
+    });
+
+    it("keeps each part of a sync state snapshot when one before it goes away", async () => {
+      using setup = await setupCreateEvoluDeps();
+      const { deps, connect } = setup;
+      const createId = testCreateId();
+      const workerId = createId<"SharedWorker">();
+      using channel = testCreateBroadcastChannel<SyncState>(
+        `evolu:sync-state:${workerId}`,
+      );
+      await connect(workerId);
+      const removedTransportId = createId<"SyncTransport">();
+      const transportId = createId<"SyncTransport">();
+      const at = Millis.orThrow(1000);
+      // The full snapshot lists a transport, tenant, owner, and route before
+      // each part the other snapshot keeps. Only the kept route has an error.
+      const createState = (isFull: boolean): SyncState => ({
+        transports: [removedTransportId, transportId]
+          .slice(isFull ? 0 : 1)
+          .map((id) => ({
+            type: "WebSocket",
+            id,
+            label: `wss://${id}.example`,
+            connection: { type: "Open", openedAt: at, error: null },
+          })),
+        tenants: [
+          ...(isFull
+            ? [
+                {
+                  type: "Active" as const,
+                  name: Name.orThrow("removed"),
+                  owners: [],
+                },
+              ]
+            : []),
+          {
+            type: "Active",
+            name: testName,
+            owners: [
+              ...(isFull
+                ? [
+                    {
+                      type: "Writable" as const,
+                      ownerId: createId<"OwnerId">(),
+                      routes: [],
+                    },
+                  ]
+                : []),
+              {
+                type: "Writable",
+                ownerId: testAppOwner.id,
+                routes: [
+                  ...(isFull
+                    ? [
+                        {
+                          type: "Complete" as const,
+                          transportId: removedTransportId,
+                          completeAt: at,
+                          lastSentAt: at,
+                          lastReceivedAt: at,
+                        },
+                      ]
+                    : []),
+                  {
+                    type: "Pending",
+                    transportId,
+                    failure: {
+                      type: "ProtocolQuotaError",
+                      ownerId: testAppOwner.id,
+                      at,
+                    },
+                    skippedError: null,
+                    completeAt: null,
+                    lastSentAt: at,
+                    lastReceivedAt: at,
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      });
+      const statusOf = (state: SyncState | null) =>
+        syncStateToOwnerSyncStatus(state, testName, testAppOwner.id);
+
+      channel.postMessage(createState(true));
+      await testWaitForWorkerMessage();
+      const full = deps.syncState.get();
+      assertNotNull(full);
+      assertSame(statusOf(full).type, "Error");
+
+      channel.postMessage(createState(false));
+      await testWaitForWorkerMessage();
+      const kept = deps.syncState.get();
+      assertNotNull(kept);
+      assertEqual(kept, createState(false));
+      assertSame(kept.transports[0], full.transports[1]);
+      assertSame(statusOf(kept), statusOf(full));
+    });
+
     it("listens on its worker's channel before asking for a snapshot", async () => {
       const name = "evolu:sync-state:worker";
       using worker = testCreateSharedWorker<
@@ -708,6 +940,29 @@ describe("Evolu", () => {
       assertEqual(deps.evoluError.get(), error);
     });
 
+    it("only logs a sync error that an older build broadcasts", async () => {
+      const testConsole = testCreateConsole();
+      using setup = await setupCreateEvoluDeps(testConsole);
+      const { deps, consoleEntryOrErrorBroadcastChannel } = setup;
+      const error = {
+        type: "ProtocolQuotaError",
+        ownerId: testAppOwner.id,
+      } as const;
+
+      consoleEntryOrErrorBroadcastChannel.postMessage({
+        type: "Error",
+        // @ts-expect-error A ConsoleEntryOrError Error carries only an UnknownError, but a still-running worker of an older build posts a ProtocolQuotaError.
+        error,
+      });
+
+      await testWaitForWorkerMessage();
+
+      assertSame(deps.evoluError.get(), null);
+      assertEqual(testConsole.getEntriesSnapshot(), [
+        { method: "error", path: [], args: [error] },
+      ]);
+    });
+
     for (const source of ["SharedWorker", "Error", "ConsoleEntry"] as const) {
       it(`preserves the first startup refusal while logging a later ${source} message`, async () => {
         const testConsole = testCreateConsole();
@@ -1615,6 +1870,50 @@ describe("Evolu", () => {
       assertEqual(called, 0);
     });
 
+    it("releases the onComplete callbacks of a failed mutation without running them", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const { run, evoluInputs, postEvoluOutput } = setup;
+      const evolu = await run.ok(testCreateEvolu);
+
+      let called = 0;
+      evolu.insert(
+        "todo",
+        { title: NonEmptyTrimmedString100.orThrow("First failed") },
+        {
+          onComplete: () => {
+            called += 1;
+          },
+        },
+      );
+      evolu.insert(
+        "todo",
+        { title: NonEmptyTrimmedString100.orThrow("Second failed") },
+        {
+          onComplete: () => {
+            called += 10;
+          },
+        },
+      );
+
+      await testWaitForWorkerMessage();
+
+      const mutate = evoluInputs[0] as ExtractTyped<EvoluInput, "Mutate">;
+      const { onCompleteIds } = mutate;
+      assertLength(onCompleteIds, 2);
+
+      postEvoluOutput({ type: "OnMutateFailed", onCompleteIds });
+      // Patches naming released callbacks find nothing to run.
+      postEvoluOutput({
+        type: "OnPatchesByQuery",
+        patchesByQuery: new Map(),
+        onCompleteIds,
+      });
+
+      await testWaitForWorkerMessage();
+
+      assertEqual(called, 0);
+    });
+
     it("executes mutate onComplete callback when query patches are received", async () => {
       await using setup = await setupRunWithEvoluDeps();
       const { run, evoluInputs, postEvoluOutput } = setup;
@@ -2447,6 +2746,294 @@ describe("Evolu", () => {
     });
   });
 
+  describe("mutation size", () => {
+    const NoteId = id("Note");
+    const note = {
+      id: NoteId,
+      body: nullOr(StringType),
+      data: nullOr(Uint8ArrayType),
+      count: nullOr(FiniteNumber),
+    };
+    const SizeSchema = { note, _note: note };
+
+    const createSizeEvolu = createEvolu(SizeSchema, {
+      appName: testAppName,
+      appOwner: testAppOwner,
+      transports: [],
+    });
+
+    const noteId = NoteId.orThrow(createIdFromString("note"));
+
+    const getChanges = (inputs: ReadonlyArray<EvoluInput>) =>
+      inputs.flatMap((input) => (input.type === "Mutate" ? input.changes : []));
+
+    const tooLargeMessage = (size: number) =>
+      `The mutation of table 'note' is ${size} bytes, over maxMutationSize (640000). Bound the Types of its columns or check it with evolu.getMutationSize.`;
+
+    // The length of a blob that makes a note exactly maxMutationSize. Its
+    // length varint takes three bytes, two more than an empty blob's.
+    const getDataAtLimitLength = (evolu: Evolu<typeof SizeSchema>) =>
+      maxMutationSize -
+      evolu.getMutationSize("note", { data: new Uint8Array() }) -
+      2;
+
+    it("getMutationSize measures the change as it is encoded", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const evolu = await setup.run.ok(createSizeEvolu);
+
+      const values = {
+        body: "Milk",
+        data: new Uint8Array(10),
+        count: FiniteNumber.orThrow(0.1),
+      };
+      const buffer = createBuffer();
+      encodeDbChange(buffer, {
+        timestamp: createTimestamp(),
+        change: DbChange.orThrow({
+          table: "note",
+          id: noteId,
+          values,
+          isInsert: false,
+          isDelete: true,
+        }),
+      });
+      assertEqual(evolu.getMutationSize("note", values), buffer.getLength());
+
+      // `id` and `isDeleted` are not columns.
+      assertEqual(
+        evolu.getMutationSize("note", { id: noteId, isDeleted: 1 }),
+        evolu.getMutationSize("note", {}),
+      );
+
+      // @ts-expect-error Only the schema's tables can be measured.
+      evolu.getMutationSize("unknown", {});
+      // @ts-expect-error Values must match their column's Type.
+      evolu.getMutationSize("note", { body: 1 });
+    });
+
+    it("encodes no string in more than three bytes per code unit", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const evolu = await setup.run.ok(createSizeEvolu);
+
+      const assertAtMostThreeBytesPerCodeUnit = (value: string) => {
+        // A string also takes a length varint of up to three bytes, besides
+        // the type tag that null takes too.
+        assertTrue(
+          evolu.getMutationSize("note", { body: value }) -
+            evolu.getMutationSize("note", { body: null }) <=
+            3 * value.length + 3,
+        );
+      };
+
+      for (const value of [
+        "",
+        "€".repeat(200_000),
+        "😀".repeat(1_000),
+        "\uD800".repeat(1_000),
+        // JSON takes exactly three bytes per code unit here.
+        "0.1",
+        `[${"0.1,".repeat(999)}0.1]`,
+        "[".repeat(1_001) + "]".repeat(1_001),
+        noteId,
+        "2024-01-01T00:00:00.000Z",
+      ]) {
+        assertAtMostThreeBytesPerCodeUnit(value);
+      }
+
+      fc.assert(
+        fc.property(
+          fc.oneof(fc.string(), fc.string({ unit: "binary" }), fc.json()),
+          assertAtMostThreeBytesPerCodeUnit,
+        ),
+        { numRuns: 2_000 },
+      );
+    });
+
+    it("rejects a mutation over maxMutationSize before batching it", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const { run, evoluInputs, postEvoluOutput } = setup;
+      const evolu = await run.ok(createSizeEvolu);
+
+      const dataAtLimitLength = getDataAtLimitLength(evolu);
+      const atLimit = { data: new Uint8Array(dataAtLimitLength) };
+      assertEqual(evolu.getMutationSize("note", atLimit), maxMutationSize);
+      const completed: Array<string> = [];
+
+      const { id } = evolu.insert("note", atLimit, {
+        onComplete: () => {
+          completed.push("at limit");
+        },
+      });
+      const error = assertThrowsInstanceOf(
+        () =>
+          evolu.insert(
+            "note",
+            { data: new Uint8Array(dataAtLimitLength + 1) },
+            {
+              onComplete: () => {
+                completed.push("over limit");
+              },
+            },
+          ),
+        Error,
+      );
+      assertEqual(error.message, tooLargeMessage(maxMutationSize + 1));
+
+      await testWaitForWorkerMessage();
+
+      // The mutation made before the rejected one is sent as usual.
+      assertEqual(
+        getChanges(evoluInputs).map((change) => change.id),
+        [id],
+      );
+      const mutate = evoluInputs.find((input) => input.type === "Mutate");
+      assertNotUndefined(mutate);
+      postEvoluOutput({
+        type: "OnPatchesByQuery",
+        patchesByQuery: new Map(),
+        onCompleteIds: mutate.onCompleteIds,
+      });
+      await testWaitForWorkerMessage();
+      assertEqual(completed, ["at limit"]);
+    });
+
+    it("lets plain text use the whole limit at a byte per character", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const { run, evoluInputs } = setup;
+      const evolu = await run.ok(createSizeEvolu);
+
+      // Spaces keep the text from being encoded as Base64Url, which is smaller.
+      // A long text's length varint takes two more bytes than a short one's.
+      const length =
+        maxMutationSize - evolu.getMutationSize("note", { body: " " }) - 1;
+      assertEqual(
+        evolu.getMutationSize("note", { body: " ".repeat(length) }),
+        maxMutationSize,
+      );
+      assertTrue(length > 639_900);
+
+      evolu.update("note", { id: noteId, body: " ".repeat(length) });
+      const error = assertThrowsInstanceOf(
+        () =>
+          evolu.update("note", { id: noteId, body: " ".repeat(length + 1) }),
+        Error,
+      );
+      assertEqual(error.message, tooLargeMessage(maxMutationSize + 1));
+
+      await testWaitForWorkerMessage();
+      assertLength(getChanges(evoluInputs), 1);
+    });
+
+    it("saves a copy of each binary value", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const { run, evoluInputs } = setup;
+      const evolu = await run.ok(createSizeEvolu);
+
+      const resizable = new ArrayBuffer(3, { maxByteLength: 8 });
+      const growing = new Uint8Array(resizable);
+      growing.set([1, 2, 3]);
+      const shared = new Uint8Array(new SharedArrayBuffer(3));
+      shared.set([4, 5, 6]);
+      const nodeBuffer = Buffer.from([7, 8, 9]);
+
+      evolu.insert("note", { data: growing });
+      evolu.insert("note", { data: shared });
+      evolu.insert("note", { data: nodeBuffer });
+
+      // Changes made before the batch is sent do not reach it.
+      resizable.resize(8);
+      growing.fill(0);
+      shared.fill(0);
+      nodeBuffer.fill(0);
+
+      await testWaitForWorkerMessage();
+
+      const saved = getChanges(evoluInputs).map((change) => change.values.data);
+      assertEqual(saved, [
+        new Uint8Array([1, 2, 3]),
+        new Uint8Array([4, 5, 6]),
+        new Uint8Array([7, 8, 9]),
+      ]);
+      for (const data of saved) {
+        assert(data instanceof Uint8Array, "Expected a Uint8Array.");
+        assertSame(Object.getPrototypeOf(data), Uint8Array.prototype);
+        assertFalse(data.buffer instanceof SharedArrayBuffer);
+        assertFalse((data.buffer as ArrayBuffer).resizable);
+      }
+    });
+
+    it("does not limit local-only tables but copies their binary values", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const { run, evoluInputs } = setup;
+      const evolu = await run.ok(createSizeEvolu);
+
+      const data = new Uint8Array(maxMutationSize + 1);
+      evolu.insert("_note", { data });
+      data[0] = 1;
+
+      await testWaitForWorkerMessage();
+
+      const [change] = getChanges(evoluInputs);
+      assertNotUndefined(change);
+      const saved = change.values.data;
+      assert(saved instanceof Uint8Array, "Expected a Uint8Array.");
+      assertNotSame(saved, data);
+      assertEqual(saved.byteLength, maxMutationSize + 1);
+      assertEqual(saved[0], 0);
+    });
+
+    it("fits a change at the limit into one message next to the largest ranges section", async () => {
+      await using setup = await setupRunWithEvoluDeps();
+      const evolu = await setup.run.ok(createSizeEvolu);
+      const deps = testCreateDeps();
+      const values = { data: new Uint8Array(getDataAtLimitLength(evolu)) };
+      assertEqual(evolu.getMutationSize("note", values), maxMutationSize);
+
+      const message = {
+        timestamp: createTimestamp(),
+        change: DbChange.orThrow({
+          table: "note",
+          id: noteId,
+          values,
+          isInsert: true,
+          isDelete: null,
+        }),
+      };
+
+      // A request and a broadcast assert that the change fits.
+      createProtocolMessageFromCrdtMessages(deps)(testAppOwner, [message]);
+      createProtocolBroadcastMessagesFromCrdtMessages(deps)(testAppOwner, [
+        message,
+      ]);
+
+      // A relay's response can pair it with the largest ranges section.
+      const rangesMaxSize = ProtocolMessageRangesMaxSize.orThrow(100_000);
+      const timestamps = createTimestampsBuffer();
+      for (let index = 1; timestamps.getLength() < 99_900; index++) {
+        timestamps.add(
+          createTimestamp({
+            millis: createTimestamp().millis,
+            nodeId: index.toString(16).padStart(16, "0") as NodeId,
+          }),
+        );
+      }
+      const response = createProtocolMessageBuffer(testAppOwner.id, {
+        messageType: MessageType.Response,
+        errorCode: ProtocolErrorCode.NoError,
+        rangesMaxSize,
+      });
+      assertTrue(
+        response.canAddTimestampsRangeAndMessage(timestamps, {
+          timestamp: message.timestamp,
+          change: encodeAndEncryptDbChange(deps)(
+            message,
+            testAppOwner.encryptionKey,
+          ),
+        }),
+      );
+    });
+  });
+
   describe("exportDatabase", () => {
     it("throws when OnExport arrives without pending export", async () => {
       const channels: Array<{