@evolu/common 8.12.0 → 8.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/src/Callbacks.d.ts +12 -1
  2. package/dist/src/Callbacks.d.ts.map +1 -1
  3. package/dist/src/Callbacks.js +3 -0
  4. package/dist/src/Error.d.ts +10 -4
  5. package/dist/src/Error.d.ts.map +1 -1
  6. package/dist/src/Error.js +10 -4
  7. package/dist/src/Object.d.ts +65 -0
  8. package/dist/src/Object.d.ts.map +1 -1
  9. package/dist/src/Object.js +142 -0
  10. package/dist/src/Resource.d.ts +0 -5
  11. package/dist/src/Resource.d.ts.map +1 -1
  12. package/dist/src/Resource.js +6 -13
  13. package/dist/src/Sqlite.d.ts.map +1 -1
  14. package/dist/src/Sqlite.js +7 -0
  15. package/dist/src/Task.d.ts +10 -8
  16. package/dist/src/Task.d.ts.map +1 -1
  17. package/dist/src/Task.js +41 -5
  18. package/dist/src/Worker.d.ts +3 -3
  19. package/dist/src/local-first/Db.d.ts +8 -3
  20. package/dist/src/local-first/Db.d.ts.map +1 -1
  21. package/dist/src/local-first/Db.js +56 -19
  22. package/dist/src/local-first/Evolu.d.ts +147 -39
  23. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  24. package/dist/src/local-first/Evolu.js +53 -9
  25. package/dist/src/local-first/Owner.d.ts +9 -0
  26. package/dist/src/local-first/Owner.d.ts.map +1 -1
  27. package/dist/src/local-first/Owner.js +9 -0
  28. package/dist/src/local-first/Protocol.d.ts +6 -4
  29. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  30. package/dist/src/local-first/Protocol.js +1 -6
  31. package/dist/src/local-first/Schema.d.ts +7 -1
  32. package/dist/src/local-first/Schema.d.ts.map +1 -1
  33. package/dist/src/local-first/Shared.d.ts +550 -153
  34. package/dist/src/local-first/Shared.d.ts.map +1 -1
  35. package/dist/src/local-first/Shared.js +724 -273
  36. package/dist/src/local-first/Storage.d.ts +16 -12
  37. package/dist/src/local-first/Storage.d.ts.map +1 -1
  38. package/package.json +1 -1
  39. package/src/Callbacks.test.ts +20 -0
  40. package/src/Callbacks.ts +17 -1
  41. package/src/Error.ts +10 -4
  42. package/src/Object.test.ts +296 -0
  43. package/src/Object.ts +163 -0
  44. package/src/Resource.test.ts +20 -16
  45. package/src/Resource.ts +6 -20
  46. package/src/Sqlite.ts +7 -0
  47. package/src/Task.test.ts +233 -62
  48. package/src/Task.ts +47 -13
  49. package/src/Worker.ts +3 -3
  50. package/src/local-first/Db.ts +88 -18
  51. package/src/local-first/Evolu.test.ts +381 -11
  52. package/src/local-first/Evolu.ts +219 -51
  53. package/src/local-first/Owner.ts +9 -0
  54. package/src/local-first/Protocol.test.ts +42 -60
  55. package/src/local-first/Protocol.ts +7 -9
  56. package/src/local-first/Schema.ts +8 -2
  57. package/src/local-first/Shared.test.ts +2714 -644
  58. package/src/local-first/Shared.ts +1245 -401
  59. package/src/local-first/Storage.ts +18 -19
@@ -18,6 +18,9 @@ import {
18
18
  } from "../Assert.ts";
19
19
  import { createBuffer, encodeNonNegativeInt } from "../Bytes.ts";
20
20
  import type { ConsoleEntry, ConsoleLevel } from "../Console.ts";
21
+ import type { DecryptWithXChaCha20Poly1305Error } from "../Crypto.ts";
22
+ import { createUnknownError, UnknownError } from "../Error.ts";
23
+ import { isPlainObject } from "../Object.ts";
21
24
  import type { DbWorkerInit, UnsupportedDbVersionError } from "./Db.ts";
22
25
  import {
23
26
  createAppOwner,
@@ -34,16 +37,22 @@ import {
34
37
  MessageType,
35
38
  ProtocolErrorCode,
36
39
  parseProtocolHeader,
40
+ type ProtocolInvalidDataError,
41
+ type ProtocolSyncError,
42
+ type ProtocolTimestampMismatchError,
37
43
  SubscriptionFlags,
38
44
  } from "./Protocol.ts";
45
+ import type { DevicePersistence } from "./Evolu.ts";
39
46
  import {
40
47
  createQueryBuilder,
41
48
  type EvoluSchema,
42
49
  type MutationChange,
43
50
  } from "./Schema.ts";
44
51
  import {
45
- syncStateToOwnerSyncStates,
46
52
  consoleEntryOrErrorBroadcastChannelName,
53
+ relaySyncStateToStatus,
54
+ syncStateToOwnerSyncStatus,
55
+ syncStateToRelaySyncStates,
47
56
  type EvoluInstanceId,
48
57
  initSharedWorker,
49
58
  type ConsoleEntryOrError,
@@ -55,6 +64,12 @@ import {
55
64
  type SharedWorkerId,
56
65
  type SharedWorkerInput,
57
66
  type SharedWorkerOutput,
67
+ type CompleteSyncRoute,
68
+ type OwnerSyncStatus,
69
+ type PendingSyncRoute,
70
+ type RelaySyncStatus,
71
+ type SettledSyncRoute,
72
+ type SyncConnection,
58
73
  type SyncRoute,
59
74
  type SyncState,
60
75
  type SyncTransportId,
@@ -156,11 +171,11 @@ const setupSharedWorker = async ({
156
171
  createWebSocket = testCreateWebSocket({ throwOnCreate: true }),
157
172
  time,
158
173
  seed,
159
- isPersistentStorageAvailable,
174
+ getDevicePersistence = () => Promise.resolve("Unknown"),
160
175
  }: {
161
176
  createWebSocket?: CreateWebSocket;
162
177
  time?: TestTime;
163
- isPersistentStorageAvailable?: () => Promise<boolean>;
178
+ getDevicePersistence?: () => Promise<DevicePersistence>;
164
179
  /**
165
180
  * Seeds the worker's random bytes, which are otherwise the same for every
166
181
  * worker.
@@ -168,7 +183,9 @@ const setupSharedWorker = async ({
168
183
  seed?: string;
169
184
  } = {}) => {
170
185
  await using disposer = new AsyncDisposableStack();
171
- const createTestId = testCreateId();
186
+ // Instance IDs name leader locks, as the worker's ID does, so they come from
187
+ // another seed than the worker's randomness, which would repeat one of them.
188
+ const tabDeps = testCreateDeps({ seed: "tab" });
172
189
 
173
190
  const consoleStoreOutputEntry = createStore<ConsoleEntry | null>(null);
174
191
  const worker = disposer.use(
@@ -198,7 +215,7 @@ const setupSharedWorker = async ({
198
215
  createWebSocket,
199
216
  ...(time && { time }),
200
217
  ...(seed && { randomBytes: testCreateDeps({ seed }).randomBytes }),
201
- ...(isPersistentStorageAvailable && { isPersistentStorageAvailable }),
218
+ getDevicePersistence,
202
219
  }),
203
220
  );
204
221
 
@@ -223,15 +240,17 @@ const setupSharedWorker = async ({
223
240
  releaseDbWorkerLeaderOnDispose = true,
224
241
  autoDispose = true,
225
242
  initialClock = createTimestamp(),
243
+ memoryOnly = false,
226
244
  }: {
227
245
  tenantName?: Name;
246
+ memoryOnly?: boolean;
228
247
  evoluChannel?: TestMessageChannel<EvoluOutput, EvoluInput>;
229
248
  releaseDbWorkerLeaderOnDispose?: boolean;
230
249
  autoDispose?: boolean;
231
250
  initialClock?: Timestamp;
232
251
  } = {}) => {
233
252
  const instanceDisposables = new AsyncDisposableStack();
234
- const id = createTestId<"EvoluInstance">();
253
+ const id = createId<"EvoluInstance">(tabDeps);
235
254
  const dbInputs: Array<Exclude<DbWorkerInput, { type: "Dispose" }>> = [];
236
255
  const dbDisposeInputs: Array<Extract<DbWorkerInput, { type: "Dispose" }>> =
237
256
  [];
@@ -273,7 +292,7 @@ const setupSharedWorker = async ({
273
292
  consoleLevel: "debug",
274
293
  sqliteSchema: testSqliteSchema,
275
294
  encryptionKey: testAppOwner.encryptionKey,
276
- memoryOnly: false,
295
+ memoryOnly,
277
296
  evoluPort: evoluChannel.port1.native,
278
297
  };
279
298
 
@@ -323,7 +342,25 @@ const setupSharedWorker = async ({
323
342
  ) => {
324
343
  const instance = await createEvoluBeforeDbWorkerLeader(options);
325
344
  await instance.acquireDbWorkerLeader();
326
- return instance;
345
+
346
+ // Once the DbWorker starts, the tenant adds the instance and tells it its
347
+ // device persistence first, so tests see only the outputs that follow.
348
+ const firstOutputs: Array<EvoluOutput> = [];
349
+ instance.evoluChannel.port2.onMessage = (output) => {
350
+ firstOutputs.push(output);
351
+ };
352
+ await testWaitForWorkerMessage();
353
+ instance.evoluChannel.port2.onMessage = null;
354
+ assertLength(firstOutputs, 1);
355
+ const [devicePersistenceOutput] = firstOutputs;
356
+ assert(
357
+ devicePersistenceOutput.type === "OnDevicePersistence",
358
+ "Expected the device persistence first.",
359
+ );
360
+
361
+ return Object.assign(instance, {
362
+ devicePersistence: devicePersistenceOutput.devicePersistence,
363
+ });
327
364
  };
328
365
 
329
366
  return {
@@ -419,12 +456,22 @@ const setupSyncStates = (
419
456
  return { states, latest };
420
457
  };
421
458
 
459
+ /** The owners of a database that has not refused startup. */
460
+ const ownersOf = (
461
+ state: SyncState,
462
+ tenantName: Name = testName,
463
+ ): ReadonlyArray<SyncTenantOwner> => {
464
+ const tenant = state.tenants.find(({ name }) => name === tenantName);
465
+ assert(tenant?.type === "Active", "Expected an active tenant.");
466
+ return tenant.owners;
467
+ };
468
+
422
469
  type TestEvoluInstance = Awaited<ReturnType<SharedWorkerSetup["createEvolu"]>>;
423
470
 
424
- type ApplySyncResult = Extract<
471
+ type ApplySyncResponse = Extract<
425
472
  Extract<DbWorkerQueuedResponse, { type: "ForSharedWorker" }>["message"],
426
473
  { type: "ApplySyncMessage" }
427
- >["result"];
474
+ >;
428
475
 
429
476
  /** Completes the latest queued sync round and returns the URLs it was sent to. */
430
477
  const respondToSyncRound = async (
@@ -466,7 +513,8 @@ const respondToApplySync = async (
466
513
  dbWorkerPort,
467
514
  }: Pick<TestEvoluInstance, "dbInputs" | "dbWorkerPort">,
468
515
  didWriteMessages: boolean,
469
- result: ApplySyncResult,
516
+ result: ApplySyncResponse["result"],
517
+ skippedError: ApplySyncResponse["skippedError"] = null,
470
518
  ): Promise<void> => {
471
519
  const input = dbInputs.at(-1);
472
520
  assertNotUndefined(input);
@@ -482,6 +530,7 @@ const respondToApplySync = async (
482
530
  clock: createTimestamp(),
483
531
  ownerId: input.request.message.owner.id,
484
532
  didWriteMessages,
533
+ skippedError,
485
534
  result,
486
535
  },
487
536
  },
@@ -579,17 +628,30 @@ describe("storage", () => {
579
628
  const isMemoryOnly = (output: SharedWorkerOutput | undefined): boolean =>
580
629
  getDbWorkerInit(output).memoryOnly;
581
630
 
631
+ /** Every device persistence the tenant sent to the instance so far. */
632
+ const receiveDevicePersistences = async (
633
+ evoluChannel: TestMessageChannel<EvoluOutput, EvoluInput>,
634
+ ): Promise<ReadonlyArray<DevicePersistence>> => {
635
+ const devicePersistences: Array<DevicePersistence> = [];
636
+ evoluChannel.port2.onMessage = (output) => {
637
+ if (output.type === "OnDevicePersistence") {
638
+ devicePersistences.push(output.devicePersistence);
639
+ }
640
+ };
641
+ await testWaitForWorkerMessage();
642
+ return devicePersistences;
643
+ };
644
+
582
645
  it("keeps every database in memory, replacements included, without persistent storage", async () => {
583
- const isPersistentStorageAvailable = mock.fn(() => Promise.resolve(false));
584
- await using setup = await setupSharedWorker({
585
- isPersistentStorageAvailable,
586
- });
646
+ const getDevicePersistence = mock.fn(() =>
647
+ Promise.resolve<DevicePersistence>("NotPersisted"),
648
+ );
649
+ await using setup = await setupSharedWorker({ getDevicePersistence });
587
650
  using disposer = new DisposableStack();
588
651
 
589
- // The first tab heard it right after Connected.
590
- assertEqual(setup.sharedWorkerOutputs, [{ type: "StorageUnavailable" }]);
591
652
  const first = await setup.createEvolu();
592
- assertTrue(isMemoryOnly(setup.sharedWorkerOutputs.at(1)));
653
+ assertTrue(isMemoryOnly(setup.sharedWorkerOutputs.at(0)));
654
+ assertSame(first.devicePersistence, "NotPersisted");
593
655
 
594
656
  // A replacement DbWorker, as after its tab closed, keeps it in memory too.
595
657
  await first.releaseDbWorkerLeader();
@@ -597,8 +659,7 @@ describe("storage", () => {
597
659
  tab.port.postMessage({ type: "AnnounceTabLeader", consoleLevel: "debug" });
598
660
  await testWaitForWorkerMessage();
599
661
  await testWaitForWorkerMessage();
600
- assertSame(tab.outputs.at(0)?.type, "StorageUnavailable");
601
- const replacement = getDbWorkerInit(tab.outputs.at(1));
662
+ const replacement = getDbWorkerInit(tab.outputs.at(0));
602
663
  assertTrue(replacement.memoryOnly);
603
664
  using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
604
665
  replacement.port,
@@ -610,29 +671,62 @@ describe("storage", () => {
610
671
  });
611
672
  await testWaitForWorkerMessage();
612
673
 
613
- assertSame(isPersistentStorageAvailable.mock.callCount(), 1);
674
+ assertSame(getDevicePersistence.mock.callCount(), 1);
614
675
  });
615
676
 
616
- it("keeps databases where the app configured them with persistent storage", async () => {
677
+ for (const devicePersistence of ["Persisted", "Unknown"] as const) {
678
+ it(`stores databases and tells instances ${devicePersistence} where the platform promises it`, async () => {
679
+ await using setup = await setupSharedWorker({
680
+ getDevicePersistence: () => Promise.resolve(devicePersistence),
681
+ });
682
+
683
+ const instance = await setup.createEvolu();
684
+
685
+ assertEqual(
686
+ setup.sharedWorkerOutputs.map(({ type }) => type),
687
+ ["DbWorkerInit"],
688
+ );
689
+ assertFalse(isMemoryOnly(setup.sharedWorkerOutputs.at(0)));
690
+ assertSame(instance.devicePersistence, devicePersistence);
691
+ });
692
+ }
693
+
694
+ it("tells an instance configured memoryOnly that its database is not persisted", async () => {
617
695
  await using setup = await setupSharedWorker({
618
- isPersistentStorageAvailable: () => Promise.resolve(true),
696
+ getDevicePersistence: () => Promise.resolve("Persisted"),
619
697
  });
620
698
 
621
- await setup.createEvolu();
699
+ const instance = await setup.createEvolu({ memoryOnly: true });
622
700
 
623
- assertEqual(
624
- setup.sharedWorkerOutputs.map(({ type }) => type),
625
- ["DbWorkerInit"],
626
- );
627
- assertFalse(isMemoryOnly(setup.sharedWorkerOutputs.at(0)));
701
+ assertTrue(isMemoryOnly(setup.sharedWorkerOutputs.at(0)));
702
+ assertSame(instance.devicePersistence, "NotPersisted");
628
703
  });
629
704
 
630
- it("keeps databases where the app configured them on platforms without the check", async () => {
631
- await using setup = await setupSharedWorker();
705
+ it("tells a later instance of the same name the mode the first one chose", async () => {
706
+ await using setup = await setupSharedWorker({
707
+ getDevicePersistence: () => Promise.resolve("Persisted"),
708
+ });
709
+ const { run, worker } = setup;
710
+ await setup.createEvolu({ memoryOnly: true });
632
711
 
633
- await setup.createEvolu();
712
+ const secondId: EvoluInstanceId = createId(run.deps);
713
+ await using _secondInstanceLock = await run.ok(acquireLeaderLock(secondId));
714
+ using secondChannel = testCreateMessageChannel<EvoluOutput, EvoluInput>();
715
+ worker.port.postMessage({
716
+ type: "CreateEvolu",
717
+ name: testName,
718
+ id: secondId,
719
+ consoleLevel: "debug",
720
+ sqliteSchema: testSqliteSchema,
721
+ encryptionKey: testAppOwner.encryptionKey,
722
+ memoryOnly: false,
723
+ evoluPort: secondChannel.port1.native,
724
+ });
725
+ await testWaitForWorkerMessage();
634
726
 
635
- assertFalse(isMemoryOnly(setup.sharedWorkerOutputs.at(0)));
727
+ assertEqual(await receiveDevicePersistences(secondChannel), [
728
+ "NotPersisted",
729
+ ]);
636
730
  });
637
731
  });
638
732
 
@@ -653,6 +747,7 @@ describe("builds", () => {
653
747
  lockManager,
654
748
  createMessagePort: testCreateMessagePort,
655
749
  createWebSocket: testCreateWebSocket({ throwOnCreate: true }),
750
+ getDevicePersistence: () => Promise.resolve("Unknown" as const),
656
751
  }),
657
752
  );
658
753
  const outputs: Array<SharedWorkerOutput> = [];
@@ -976,21 +1071,21 @@ describe("sync state", () => {
976
1071
  assertEqual(state.transports[0].label, "/relay");
977
1072
  assertEqual(state.tenants, [
978
1073
  {
1074
+ type: "Active",
979
1075
  name: testName,
980
- refused: false,
981
1076
  owners: [
982
1077
  {
1078
+ type: "Writable",
983
1079
  ownerId: testAppOwner.id,
984
- writable: true,
985
- transportIds: [state.transports[0].id],
986
1080
  routes: [
987
1081
  {
1082
+ type: "Pending",
988
1083
  transportId: state.transports[0].id,
989
- complete: false,
1084
+ failure: null,
1085
+ skippedError: null,
990
1086
  completeAt: null,
991
1087
  lastSentAt: null,
992
1088
  lastReceivedAt: null,
993
- error: null,
994
1089
  },
995
1090
  ],
996
1091
  },
@@ -1016,14 +1111,13 @@ describe("sync state", () => {
1016
1111
  assertLength(tab.syncStates, 1);
1017
1112
  });
1018
1113
 
1019
- it("includes transitions without an event in the next snapshot", async () => {
1020
- const createWebSocket = testCreateWebSocket({ isOpen: false });
1021
- await using setup = await setupSharedWorker({ createWebSocket });
1022
- using disposer = new DisposableStack();
1023
- const { latest } = setupSyncStates(setup, disposer);
1024
- const { evoluChannel } = await setup.createEvolu();
1114
+ /** Registers the app owner through one transport of the instance's database. */
1115
+ const setupAppOwnerTransport = async (
1116
+ { evoluChannel }: Pick<TestEvoluInstance, "evoluChannel">,
1117
+ url: string,
1118
+ ) => {
1025
1119
  const transport = createOwnerWebSocketTransport({
1026
- url: "wss://relay.example",
1120
+ url,
1027
1121
  ownerId: testAppOwner.id,
1028
1122
  });
1029
1123
  evoluChannel.port2.postMessage({
@@ -1036,21 +1130,184 @@ describe("sync state", () => {
1036
1130
  ],
1037
1131
  });
1038
1132
  await testWaitForWorkerMessage();
1133
+ return transport;
1134
+ };
1135
+
1136
+ it("publishes a transport as Connecting until its first connection opens", async () => {
1137
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1138
+ await using setup = await setupSharedWorker({ createWebSocket });
1139
+ using disposer = new DisposableStack();
1140
+ const { latest } = setupSyncStates(setup, disposer);
1141
+ const { time } = setup.run.deps;
1142
+ const instance = await setup.createEvolu();
1143
+ const transport = await setupAppOwnerTransport(
1144
+ instance,
1145
+ "wss://relay.example/sync",
1146
+ );
1147
+
1148
+ const { id } = latest().transports[0];
1149
+ assertEqual(latest().transports, [
1150
+ {
1151
+ type: "WebSocket",
1152
+ id,
1153
+ label: "wss://relay.example/sync",
1154
+ connection: { type: "Connecting" },
1155
+ },
1156
+ ]);
1157
+ // A first connection is work in progress, not an outage.
1158
+ assertEqual(
1159
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
1160
+ { type: "Syncing" },
1161
+ );
1162
+
1163
+ time.advance("1s");
1164
+ createWebSocket.open(transport.url);
1165
+ await testWaitForWorkerMessage();
1166
+ assertEqual(latest().transports[0], {
1167
+ type: "WebSocket",
1168
+ id,
1169
+ label: "wss://relay.example/sync",
1170
+ connection: { type: "Open", openedAt: time.now(), error: null },
1171
+ });
1172
+ });
1173
+
1174
+ it("keeps when a transport disconnected while its reconnects fail", async () => {
1175
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1176
+ await using setup = await setupSharedWorker({ createWebSocket });
1177
+ using disposer = new DisposableStack();
1178
+ const { latest } = setupSyncStates(setup, disposer);
1179
+ const { time } = setup.run.deps;
1180
+ const instance = await setup.createEvolu();
1181
+ const transport = await setupAppOwnerTransport(
1182
+ instance,
1183
+ "wss://relay.example",
1184
+ );
1039
1185
  createWebSocket.open(transport.url);
1040
1186
  await testWaitForWorkerMessage();
1187
+ const openedAt = time.now();
1188
+
1189
+ time.advance("1s");
1041
1190
  createWebSocket.close(transport.url);
1042
1191
  await testWaitForWorkerMessage();
1043
- assertSame(latest().transports[0].readyState, "closed");
1192
+ const disconnectedAt = time.now();
1193
+ assertEqual(latest().transports[0].connection, {
1194
+ type: "Disconnected",
1195
+ disconnectedAt,
1196
+ openedAt,
1197
+ error: null,
1198
+ });
1199
+ assertEqual(
1200
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
1201
+ { type: "Offline" },
1202
+ );
1044
1203
 
1045
- // The socket then starts reconnecting without an event, so nothing
1046
- // publishes it until the next snapshot, here one a connecting tab asks
1047
- // for, which reaches every tab.
1204
+ // A failed reconnect attempt records its error, not a new disconnect.
1205
+ time.advance("1s");
1206
+ createWebSocket.error(transport.url, {
1207
+ type: "WebSocketConnectError",
1208
+ event: new Event("error"),
1209
+ });
1210
+ await testWaitForWorkerMessage();
1211
+ const error = { type: "WebSocketConnectError", at: time.now() } as const;
1212
+ const disconnected = {
1213
+ type: "Disconnected",
1214
+ disconnectedAt,
1215
+ openedAt,
1216
+ error,
1217
+ } as const;
1218
+ assertEqual(latest().transports[0].connection, disconnected);
1219
+
1220
+ // The socket starts reconnecting without an event, and a snapshot a
1221
+ // connecting tab asks for, which reaches every tab, reads the same.
1222
+ const published = latest();
1048
1223
  setupTab(setup, disposer);
1049
1224
  await testWaitForWorkerMessage();
1050
- assertSame(latest().transports[0].readyState, "connecting");
1225
+ assertNotSame(latest(), published);
1226
+ assertEqual(latest().transports[0].connection, disconnected);
1227
+
1228
+ // The error stays after the transport opens again.
1229
+ time.advance("1s");
1230
+ createWebSocket.open(transport.url);
1231
+ await testWaitForWorkerMessage();
1232
+ const reopenedAt = time.now();
1233
+ assertEqual(latest().transports[0].connection, {
1234
+ type: "Open",
1235
+ openedAt: reopenedAt,
1236
+ error,
1237
+ });
1238
+
1239
+ // A close records no error, so the earlier one stays.
1240
+ time.advance("1s");
1241
+ createWebSocket.close(transport.url);
1242
+ await testWaitForWorkerMessage();
1243
+ assertEqual(latest().transports[0].connection, {
1244
+ type: "Disconnected",
1245
+ disconnectedAt: time.now(),
1246
+ openedAt: reopenedAt,
1247
+ error,
1248
+ });
1249
+ });
1250
+
1251
+ it("disconnects an open transport at a connection error before its close", async () => {
1252
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1253
+ await using setup = await setupSharedWorker({ createWebSocket });
1254
+ using disposer = new DisposableStack();
1255
+ const { latest } = setupSyncStates(setup, disposer);
1256
+ const { time } = setup.run.deps;
1257
+ const instance = await setup.createEvolu();
1258
+ const transport = await setupAppOwnerTransport(
1259
+ instance,
1260
+ "wss://relay.example",
1261
+ );
1262
+ createWebSocket.open(transport.url);
1263
+ await testWaitForWorkerMessage();
1264
+ const openedAt = time.now();
1265
+
1266
+ // The socket is closed when the error arrives, and its close follows.
1267
+ time.advance("1s");
1268
+ createWebSocket.error(transport.url, {
1269
+ type: "WebSocketConnectionError",
1270
+ event: new Event("error"),
1271
+ });
1272
+ await testWaitForWorkerMessage();
1273
+ const disconnected = {
1274
+ type: "Disconnected",
1275
+ disconnectedAt: time.now(),
1276
+ openedAt,
1277
+ error: { type: "WebSocketConnectionError", at: time.now() },
1278
+ } as const;
1279
+ assertEqual(latest().transports[0].connection, disconnected);
1280
+
1281
+ time.advance("1s");
1282
+ createWebSocket.close(transport.url);
1283
+ await testWaitForWorkerMessage();
1284
+ assertEqual(latest().transports[0].connection, disconnected);
1285
+
1286
+ // A reopened connection keeps the error until a later one replaces it.
1287
+ time.advance("1s");
1288
+ createWebSocket.open(transport.url);
1289
+ await testWaitForWorkerMessage();
1290
+ const reopenedAt = time.now();
1291
+ assertEqual(latest().transports[0].connection, {
1292
+ type: "Open",
1293
+ openedAt: reopenedAt,
1294
+ error: disconnected.error,
1295
+ });
1296
+ time.advance("1s");
1297
+ createWebSocket.error(transport.url, {
1298
+ type: "WebSocketConnectionError",
1299
+ event: new Event("error"),
1300
+ });
1301
+ await testWaitForWorkerMessage();
1302
+ assertEqual(latest().transports[0].connection, {
1303
+ type: "Disconnected",
1304
+ disconnectedAt: time.now(),
1305
+ openedAt: reopenedAt,
1306
+ error: { type: "WebSocketConnectionError", at: time.now() },
1307
+ });
1051
1308
  });
1052
1309
 
1053
- it("publishes transport and registration changes", async () => {
1310
+ it("publishes a readonly owner with transports and a writable owner with routes only", async () => {
1054
1311
  const createWebSocket = testCreateWebSocket({ isOpen: false });
1055
1312
  await using setup = await setupSharedWorker({ createWebSocket });
1056
1313
  using disposer = new DisposableStack();
@@ -1088,96 +1345,64 @@ describe("sync state", () => {
1088
1345
  await testWaitForWorkerMessage();
1089
1346
 
1090
1347
  const registered = latest();
1091
- assertEqual(
1092
- registered.transports.map(({ label, readyState, error }) => ({
1093
- label,
1094
- readyState,
1095
- error,
1096
- })),
1097
- [
1098
- {
1099
- label: "wss://relay.example/sync",
1100
- readyState: "connecting",
1101
- error: null,
1102
- },
1103
- {
1104
- label: "wss://relay.example/sync",
1105
- readyState: "connecting",
1106
- error: null,
1107
- },
1108
- ],
1109
- );
1110
1348
  const [writableId, readonlyId] = registered.transports.map(({ id }) => id);
1349
+ assertEqual(registered.transports, [
1350
+ {
1351
+ type: "WebSocket",
1352
+ id: writableId,
1353
+ label: "wss://relay.example/sync",
1354
+ connection: { type: "Connecting" },
1355
+ },
1356
+ {
1357
+ type: "WebSocket",
1358
+ id: readonlyId,
1359
+ label: "wss://relay.example/sync",
1360
+ connection: { type: "Connecting" },
1361
+ },
1362
+ ]);
1363
+ assertEqual(registered.tenants, [
1364
+ {
1365
+ type: "Active",
1366
+ name: testName,
1367
+ owners: [
1368
+ {
1369
+ type: "Writable",
1370
+ ownerId: testAppOwner.id,
1371
+ routes: [
1372
+ {
1373
+ type: "Pending",
1374
+ transportId: writableId,
1375
+ failure: null,
1376
+ skippedError: null,
1377
+ completeAt: null,
1378
+ lastSentAt: null,
1379
+ lastReceivedAt: null,
1380
+ },
1381
+ ],
1382
+ },
1383
+ {
1384
+ type: "Readonly",
1385
+ ownerId: readonlyOwner.id,
1386
+ transportIds: [readonlyId],
1387
+ },
1388
+ ],
1389
+ },
1390
+ ]);
1391
+ // A readonly owner is not synced here, and no relay syncs it.
1111
1392
  assertEqual(
1112
- registered.tenants.map(({ name, refused, owners }) => ({
1113
- name,
1114
- refused,
1115
- owners: owners.map(({ routes, ...owner }) => ({
1116
- ...owner,
1117
- routeCount: routes.length,
1118
- })),
1119
- })),
1120
- [
1121
- {
1122
- name: testName,
1123
- refused: false,
1124
- owners: [
1125
- {
1126
- ownerId: testAppOwner.id,
1127
- writable: true,
1128
- transportIds: [writableId],
1129
- routeCount: 1,
1130
- },
1131
- {
1132
- ownerId: readonlyOwner.id,
1133
- writable: false,
1134
- transportIds: [readonlyId],
1135
- routeCount: 0,
1136
- },
1137
- ],
1138
- },
1139
- ],
1393
+ syncStateToOwnerSyncStatus(registered, testName, readonlyOwner.id),
1394
+ { type: "NoRelays" },
1140
1395
  );
1141
1396
 
1142
- createWebSocket.open(writableTransport.url);
1143
- await testWaitForWorkerMessage();
1144
- assertSame(latest().transports[0].readyState, "open");
1145
- assertEqual(latest().transports[0].openedAt, time.now());
1146
-
1147
- time.advance("1s");
1148
- createWebSocket.close(writableTransport.url, { code: 1006 });
1149
- await testWaitForWorkerMessage();
1150
- // The close is published while the socket is still closed.
1151
- assertSame(latest().transports[0].readyState, "closed");
1152
- assertEqual(latest().transports[0].closedAt, time.now());
1153
-
1154
- const error: WebSocketError = {
1155
- type: "WebSocketConnectError",
1156
- event: new Event("error"),
1157
- };
1158
- createWebSocket.error(writableTransport.url, error);
1159
- await testWaitForWorkerMessage();
1160
- // By the next publish the socket has been dropped and is being retried.
1161
- assertSame(latest().transports[0].readyState, "connecting");
1162
- assertEqual(latest().transports[0].error, {
1163
- type: "WebSocketConnectError",
1164
- at: time.now(),
1165
- });
1166
- const lastError = latest().transports[0].error;
1167
- time.advance("1s");
1168
- createWebSocket.open(writableTransport.url);
1169
- await testWaitForWorkerMessage();
1170
- assertSame(latest().transports[0].readyState, "open");
1171
- assertEqual(latest().transports[0].openedAt, time.now());
1172
- assertEqual(latest().transports[0].error, lastError);
1173
-
1174
1397
  evoluChannel.port2.postMessage({
1175
1398
  type: "UseOwner",
1176
1399
  actions: [{ owner: readonlyRegistration, action: "remove" }],
1177
1400
  });
1178
1401
  await testWaitForWorkerMessage();
1402
+ const [tenant] = latest().tenants;
1403
+ assert(tenant?.type === "Active", "Expected an active tenant.");
1179
1404
  assertEqual(
1180
- latest().tenants[0].owners.map(({ ownerId }) => ownerId),
1405
+ tenant.owners.map(({ ownerId }) => ownerId),
1181
1406
  [testAppOwner.id],
1182
1407
  );
1183
1408
 
@@ -1193,7 +1418,7 @@ describe("sync state", () => {
1193
1418
  // follows after its own idle period.
1194
1419
  await instance[Symbol.asyncDispose]();
1195
1420
  assertEqual(latest().tenants, [
1196
- { name: testName, refused: false, owners: [] },
1421
+ { type: "Active", name: testName, owners: [] },
1197
1422
  ]);
1198
1423
  time.advance("3s");
1199
1424
  await testWaitForWorkerMessage();
@@ -1201,6 +1426,12 @@ describe("sync state", () => {
1201
1426
  });
1202
1427
 
1203
1428
  describe("routes", () => {
1429
+ /** The error of a received message the DbWorker skipped. */
1430
+ const skippedError = {
1431
+ type: "DecryptWithXChaCha20Poly1305Error",
1432
+ error: "wrong encryption key",
1433
+ } satisfies DecryptWithXChaCha20Poly1305Error;
1434
+
1204
1435
  /** A header-only relay Response: nothing to reconcile. */
1205
1436
  const relayResponse = (ownerId = testAppOwner.id): ArrayBuffer =>
1206
1437
  protocolMessageToArrayBuffer(
@@ -1216,15 +1447,23 @@ describe("sync state", () => {
1216
1447
  transportIndex = 0,
1217
1448
  ownerId: OwnerId = testAppOwner.id,
1218
1449
  ): SyncRoute => {
1219
- const tenant = state.tenants.find(({ name }) => name === tenantName);
1220
- assertNotUndefined(tenant);
1221
- const owner = tenant.owners.find((owner) => owner.ownerId === ownerId);
1222
- assertNotUndefined(owner);
1450
+ const owner = ownersOf(state, tenantName).find(
1451
+ (owner) => owner.ownerId === ownerId,
1452
+ );
1453
+ assert(owner?.type === "Writable", "Expected a writable owner.");
1223
1454
  const route = owner.routes[transportIndex];
1224
1455
  assertNotUndefined(route);
1225
1456
  return route;
1226
1457
  };
1227
1458
 
1459
+ /** The route's failure, which only a Pending route has. */
1460
+ const failureOf = (route: SyncRoute): SyncRouteError | null =>
1461
+ route.type === "Pending" ? route.failure : null;
1462
+
1463
+ /** The change the route skipped, which a Complete route never has. */
1464
+ const skippedErrorOf = (route: SyncRoute): SyncRouteError | null =>
1465
+ route.type === "Complete" ? null : route.skippedError;
1466
+
1228
1467
  /** Posts a mutation for the owner without answering it. */
1229
1468
  const queueMutation = async (
1230
1469
  instance: TestEvoluInstance,
@@ -1335,17 +1574,18 @@ describe("sync state", () => {
1335
1574
  });
1336
1575
  await testWaitForWorkerMessage();
1337
1576
  // The first claim requested a round that is not sent yet.
1338
- assertFalse(routeOf(latest()).complete);
1577
+ assertSame(routeOf(latest()).type, "Pending");
1339
1578
 
1340
1579
  await respondToSyncRound(instance, createWebSocket);
1341
1580
  const sentAt = time.now();
1342
1581
  assertEqual(routeOf(latest()), {
1582
+ type: "Pending",
1343
1583
  transportId: latest().transports[0].id,
1344
- complete: false,
1584
+ failure: null,
1585
+ skippedError: null,
1345
1586
  completeAt: null,
1346
1587
  lastSentAt: sentAt,
1347
1588
  lastReceivedAt: null,
1348
- error: null,
1349
1589
  });
1350
1590
 
1351
1591
  // The relay answers with nothing to reconcile.
@@ -1353,35 +1593,35 @@ describe("sync state", () => {
1353
1593
  createWebSocket.message(transport.url, relayResponse());
1354
1594
  await testWaitForWorkerMessage();
1355
1595
  // Received but not applied yet.
1356
- assertFalse(routeOf(latest()).complete);
1596
+ assertSame(routeOf(latest()).type, "Pending");
1357
1597
  await respondToApplySync(instance, false, {
1358
1598
  ok: true,
1359
1599
  value: { type: "Converged" },
1360
1600
  });
1361
1601
  assertEqual(routeOf(latest()), {
1602
+ type: "Complete",
1362
1603
  transportId: latest().transports[0].id,
1363
- complete: true,
1364
1604
  completeAt: time.now(),
1365
1605
  lastSentAt: sentAt,
1366
1606
  lastReceivedAt: time.now(),
1367
- error: null,
1368
1607
  });
1369
1608
  const completeAt = time.now();
1370
1609
 
1371
1610
  // A mutation needs only its upload answered.
1372
1611
  time.advance("1s");
1373
1612
  await postMutation(instance, setup.run);
1374
- assertFalse(routeOf(latest()).complete);
1613
+ assertSame(routeOf(latest()).type, "Pending");
1375
1614
  createWebSocket.message(transport.url, relayResponse());
1376
1615
  await testWaitForWorkerMessage();
1377
1616
  await respondToApplySync(instance, false, {
1378
1617
  ok: true,
1379
1618
  value: { type: "Converged" },
1380
1619
  });
1381
- assertTrue(routeOf(latest()).complete);
1620
+ assertSame(routeOf(latest()).type, "Complete");
1382
1621
  // Completing again after the upload restamps the time.
1383
- assertEqual(routeOf(latest()).completeAt, time.now());
1384
- assertNotSame(routeOf(latest()).completeAt, completeAt);
1622
+ const recompletedAt = time.now();
1623
+ assertEqual(routeOf(latest()).completeAt, recompletedAt);
1624
+ assertNotSame(recompletedAt, completeAt);
1385
1625
 
1386
1626
  // A rejected upload marks the route until a new round converges.
1387
1627
  time.advance("1s");
@@ -1392,17 +1632,21 @@ describe("sync state", () => {
1392
1632
  ok: false,
1393
1633
  error: { type: "ProtocolQuotaError", ownerId: testAppOwner.id },
1394
1634
  });
1395
- assertFalse(routeOf(latest()).complete);
1396
- assertEqual(routeOf(latest()).error, {
1635
+ const quotaFailed = routeOf(latest());
1636
+ assertSame(quotaFailed.type, "Pending");
1637
+ assertEqual(quotaFailed.failure, {
1397
1638
  type: "ProtocolQuotaError",
1639
+ ownerId: testAppOwner.id,
1398
1640
  at: time.now(),
1399
1641
  });
1642
+ // The route keeps when it last became complete.
1643
+ assertSame(quotaFailed.completeAt, recompletedAt);
1400
1644
 
1401
1645
  // The failure requested the round that clears it.
1402
1646
  time.advance("1s");
1403
1647
  await testWaitForWorkerMessage();
1404
1648
  await respondToSyncRound(instance, createWebSocket);
1405
- assertFalse(routeOf(latest()).complete);
1649
+ assertSame(routeOf(latest()).type, "Pending");
1406
1650
  createWebSocket.message(transport.url, relayResponse());
1407
1651
  await testWaitForWorkerMessage();
1408
1652
  await respondToApplySync(instance, false, {
@@ -1410,12 +1654,11 @@ describe("sync state", () => {
1410
1654
  value: { type: "Converged" },
1411
1655
  });
1412
1656
  assertEqual(routeOf(latest()), {
1657
+ type: "Complete",
1413
1658
  transportId: latest().transports[0].id,
1414
- complete: true,
1415
1659
  completeAt: time.now(),
1416
1660
  lastSentAt: time.now(),
1417
1661
  lastReceivedAt: time.now(),
1418
- error: null,
1419
1662
  });
1420
1663
  });
1421
1664
 
@@ -1463,7 +1706,7 @@ describe("sync state", () => {
1463
1706
  value: { type: "Converged" },
1464
1707
  });
1465
1708
  assertTrue(baseTime.now() > completedAt);
1466
- assertTrue(routeOf(latest()).complete);
1709
+ assertSame(routeOf(latest()).type, "Complete");
1467
1710
  assertSame(routeOf(latest()).lastReceivedAt, completedAt);
1468
1711
  assertSame(routeOf(latest()).completeAt, completedAt);
1469
1712
  });
@@ -1496,7 +1739,7 @@ describe("sync state", () => {
1496
1739
  ok: true,
1497
1740
  value: { type: "Converged" },
1498
1741
  });
1499
- assertTrue(routeOf(latest()).complete);
1742
+ assertSame(routeOf(latest()).type, "Complete");
1500
1743
 
1501
1744
  const publish = setupDeferredSyncStatePublication(
1502
1745
  t,
@@ -1541,9 +1784,8 @@ describe("sync state", () => {
1541
1784
  await testWaitForWorkerMessage();
1542
1785
  }
1543
1786
  assertTrue(states.length > publishedCount);
1544
- assertTrue(routeOf(latest()).complete);
1787
+ assertSame(routeOf(latest()).type, "Complete");
1545
1788
  assertSame(routeOf(latest()).completeAt, completedAt);
1546
- assertSame(routeOf(latest()).error, null);
1547
1789
  });
1548
1790
 
1549
1791
  it("forgets released routes before a new registration without reading a snapshot", async (t) => {
@@ -1569,7 +1811,7 @@ describe("sync state", () => {
1569
1811
  ok: true,
1570
1812
  value: { type: "Converged" },
1571
1813
  });
1572
- assertTrue(routeOf(latest()).complete);
1814
+ assertSame(routeOf(latest()).type, "Complete");
1573
1815
  assertNotSame(routeOf(latest()).completeAt, null);
1574
1816
 
1575
1817
  const publish = setupDeferredSyncStatePublication(
@@ -1598,10 +1840,11 @@ describe("sync state", () => {
1598
1840
  // No snapshot observed the registration gap. The next one must still
1599
1841
  // show a fresh route, whose new round has not been sent.
1600
1842
  const route = routeOf(latest());
1601
- assertFalse(route.complete);
1843
+ assertSame(route.type, "Pending");
1602
1844
  assertSame(route.completeAt, null);
1603
1845
  assertSame(route.lastSentAt, null);
1604
- assertSame(route.error, null);
1846
+ assertSame(route.failure, null);
1847
+ assertSame(route.skippedError, null);
1605
1848
  });
1606
1849
 
1607
1850
  for (const removed of [
@@ -1674,10 +1917,12 @@ describe("sync state", () => {
1674
1917
  queries: createSet([testQuery]),
1675
1918
  });
1676
1919
  await testWaitForWorkerMessage();
1920
+ const [ownerAfterRemoval] = ownersOf(latest());
1677
1921
  assertFalse(
1678
- latest().tenants[0].owners[0].routes.some(
1679
- ({ transportId }) => transportId === sourceTransportId,
1680
- ),
1922
+ ownerAfterRemoval?.type === "Writable" &&
1923
+ ownerAfterRemoval.routes.some(
1924
+ ({ transportId }) => transportId === sourceTransportId,
1925
+ ),
1681
1926
  );
1682
1927
  createWebSocket.sentMessages.splice(0);
1683
1928
 
@@ -1685,7 +1930,7 @@ describe("sync state", () => {
1685
1930
  type: "ProtocolInvalidDataError",
1686
1931
  data: Uint8Array.of(255),
1687
1932
  error: "late range decoding failure",
1688
- } satisfies StorageWriteMessagesError;
1933
+ } satisfies ProtocolInvalidDataError;
1689
1934
  const continuation = createProtocolMessageForUnsubscribe(
1690
1935
  testAppOwner.id,
1691
1936
  );
@@ -1701,10 +1946,9 @@ describe("sync state", () => {
1701
1946
  );
1702
1947
 
1703
1948
  // Losing the route must not discard an accepted database response.
1704
- assertEqual(
1705
- errors,
1706
- outcome === "error" ? [{ type: "Error", error }] : [],
1707
- );
1949
+ // The route is gone, so a late relay failure is dropped: it is not
1950
+ // posted as an error, and no remaining route shows it.
1951
+ assertEqual(errors, []);
1708
1952
  assertEqual(outputs, [{ type: "RefreshQueries" }]);
1709
1953
  assertEqual(
1710
1954
  createWebSocket.sentMessages.splice(0),
@@ -1712,15 +1956,17 @@ describe("sync state", () => {
1712
1956
  ? [{ url: sourceTransport.url, data: continuation }]
1713
1957
  : [],
1714
1958
  );
1715
- const owner = latest().tenants[0].owners[0];
1716
- assertSame(owner.writable, keepsWritable);
1717
- assertSame(owner.routes.length, keepsWritable ? 1 : 0);
1959
+ const [owner] = ownersOf(latest());
1960
+ assertSame(owner?.type, keepsWritable ? "Writable" : "Readonly");
1961
+ const routes = owner?.type === "Writable" ? owner.routes : [];
1962
+ assertSame(routes.length, keepsWritable ? 1 : 0);
1718
1963
  assertFalse(
1719
- owner.routes.some(
1720
- ({ transportId }) => transportId === sourceTransportId,
1721
- ),
1964
+ routes.some(({ transportId }) => transportId === sourceTransportId),
1722
1965
  );
1723
- for (const route of owner.routes) assertSame(route.error, null);
1966
+ for (const route of routes) {
1967
+ assertSame(failureOf(route), null);
1968
+ assertSame(skippedErrorOf(route), null);
1969
+ }
1724
1970
 
1725
1971
  const query = instance.dbInputs.at(-1);
1726
1972
  assertNotUndefined(query);
@@ -1786,7 +2032,7 @@ describe("sync state", () => {
1786
2032
  value: { type: "Converged" },
1787
2033
  });
1788
2034
  const otherRoute = routeOf(latest(), testName, 1);
1789
- assertTrue(otherRoute.complete);
2035
+ assertSame(otherRoute.type, "Complete");
1790
2036
  createWebSocket.message(transport.url, relayResponse());
1791
2037
  await testWaitForWorkerMessage();
1792
2038
  const inputCount = instance.dbInputs.length;
@@ -1797,12 +2043,16 @@ describe("sync state", () => {
1797
2043
 
1798
2044
  // Receiving the answer consumed the outstanding request, but the
1799
2045
  // aborted apply did not establish convergence or a domain error.
1800
- assertFalse(routeOf(latest()).complete);
2046
+ assertSame(routeOf(latest()).type, "Pending");
1801
2047
  assertSame(routeOf(latest()).completeAt, null);
1802
2048
  // Nothing was applied, so the route reports no received frame.
1803
2049
  assertSame(routeOf(latest()).lastReceivedAt, null);
1804
- assertSame(routeOf(latest()).error, null);
1805
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "syncing");
2050
+ assertSame(failureOf(routeOf(latest())), null);
2051
+ assertSame(skippedErrorOf(routeOf(latest())), null);
2052
+ assertEqual(
2053
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
2054
+ { type: "Syncing" },
2055
+ );
1806
2056
  // The frame came from one relay, so the other route stays complete.
1807
2057
  assertEqual(routeOf(latest(), testName, 1), otherRoute);
1808
2058
  await testWaitForWorkerMessage();
@@ -1815,7 +2065,7 @@ describe("sync state", () => {
1815
2065
  ok: true,
1816
2066
  value: { type: "Converged" },
1817
2067
  });
1818
- assertFalse(routeOf(latest()).complete);
2068
+ assertSame(routeOf(latest()).type, "Pending");
1819
2069
  });
1820
2070
  }
1821
2071
 
@@ -1846,7 +2096,7 @@ describe("sync state", () => {
1846
2096
  ok: true,
1847
2097
  value: { type: "Converged" },
1848
2098
  });
1849
- assertTrue(routeOf(latest()).complete);
2099
+ assertSame(routeOf(latest()).type, "Complete");
1850
2100
 
1851
2101
  // The database worker logged an exception while creating the round.
1852
2102
  instance.evoluChannel.port2.postMessage({
@@ -1877,12 +2127,28 @@ describe("sync state", () => {
1877
2127
  // Nothing is sent or retried until an explicit request or a reopen.
1878
2128
  assertEqual(createWebSocket.sentMessages, []);
1879
2129
  assertLength(instance.dbInputs, inputCount);
1880
- assertFalse(routeOf(latest()).complete);
1881
- assertEqual(routeOf(latest()).error, {
2130
+ assertSame(routeOf(latest()).type, "Pending");
2131
+ assertEqual(failureOf(routeOf(latest())), {
1882
2132
  type: "SyncFailed",
1883
2133
  at: setup.run.deps.time.now(),
1884
2134
  });
1885
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2135
+ assertSame(
2136
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2137
+ "Error",
2138
+ );
2139
+
2140
+ // SyncFailed counts as the route's failure, so a later failed apply
2141
+ // requests no round either.
2142
+ createWebSocket.message(transport.url, relayResponse());
2143
+ await testWaitForWorkerMessage();
2144
+ const error = {
2145
+ type: "ProtocolSyncError",
2146
+ ownerId: testAppOwner.id,
2147
+ } satisfies ProtocolSyncError;
2148
+ await respondToApplySync(instance, false, { ok: false, error });
2149
+ await testWaitForWorkerMessage();
2150
+ assertLength(instance.dbInputs, inputCount + 1);
2151
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
1886
2152
 
1887
2153
  instance.evoluChannel.port2.postMessage({
1888
2154
  type: "UseOwner",
@@ -1896,8 +2162,7 @@ describe("sync state", () => {
1896
2162
  ok: true,
1897
2163
  value: { type: "Converged" },
1898
2164
  });
1899
- assertTrue(routeOf(latest()).complete);
1900
- assertSame(routeOf(latest()).error, null);
2165
+ assertSame(routeOf(latest()).type, "Complete");
1901
2166
  });
1902
2167
 
1903
2168
  it("reports SyncFailed only on the routes of the failed round's target", async () => {
@@ -1930,8 +2195,8 @@ describe("sync state", () => {
1930
2195
  value: { type: "Converged" },
1931
2196
  });
1932
2197
  }
1933
- assertTrue(routeOf(latest(), testName, 0).complete);
1934
- assertTrue(routeOf(latest(), testName, 1).complete);
2198
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
2199
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
1935
2200
 
1936
2201
  // The first transport reopens, and the database worker fails to create
1937
2202
  // the round through it.
@@ -1958,13 +2223,12 @@ describe("sync state", () => {
1958
2223
  });
1959
2224
  await testWaitForWorkerMessage();
1960
2225
 
1961
- assertFalse(routeOf(latest(), testName, 0).complete);
1962
- assertEqual(routeOf(latest(), testName, 0).error, {
2226
+ assertSame(routeOf(latest(), testName, 0).type, "Pending");
2227
+ assertEqual(failureOf(routeOf(latest(), testName, 0)), {
1963
2228
  type: "SyncFailed",
1964
2229
  at: setup.run.deps.time.now(),
1965
2230
  });
1966
- assertTrue(routeOf(latest(), testName, 1).complete);
1967
- assertSame(routeOf(latest(), testName, 1).error, null);
2231
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
1968
2232
  });
1969
2233
 
1970
2234
  it("keeps SyncFailed when a round fails after an earlier round was sent", async () => {
@@ -2026,14 +2290,14 @@ describe("sync state", () => {
2026
2290
  ok: true,
2027
2291
  value: { type: "Converged" },
2028
2292
  });
2029
- assertFalse(routeOf(latest()).complete);
2030
- assertEqual(routeOf(latest()).error, {
2293
+ assertSame(routeOf(latest()).type, "Pending");
2294
+ assertEqual(failureOf(routeOf(latest())), {
2031
2295
  type: "SyncFailed",
2032
2296
  at: setup.run.deps.time.now(),
2033
2297
  });
2034
2298
  });
2035
2299
 
2036
- it("keeps completeAt while a route stays complete", async () => {
2300
+ it("keeps completeAt in a complete route and carries it into Pending", async () => {
2037
2301
  const createWebSocket = testCreateWebSocket();
2038
2302
  await using setup = await setupSharedWorker({ createWebSocket });
2039
2303
  using disposer = new DisposableStack();
@@ -2060,7 +2324,7 @@ describe("sync state", () => {
2060
2324
  ok: true,
2061
2325
  value: { type: "Converged" },
2062
2326
  });
2063
- assertTrue(routeOf(latest()).complete);
2327
+ assertSame(routeOf(latest()).type, "Complete");
2064
2328
  const { completeAt } = routeOf(latest());
2065
2329
  assertNonNullable(completeAt);
2066
2330
 
@@ -2086,32 +2350,51 @@ describe("sync state", () => {
2086
2350
  });
2087
2351
  await testWaitForWorkerMessage();
2088
2352
  await respondToSyncRound(instance, createWebSocket, testAppOwner2);
2089
- assertTrue(routeOf(latest()).complete);
2353
+ assertSame(routeOf(latest()).type, "Complete");
2090
2354
  assertSame(routeOf(latest()).completeAt, completeAt);
2355
+
2356
+ // A route that becomes Pending again keeps when it last became
2357
+ // complete.
2358
+ setup.run.deps.time.advance("5s");
2359
+ await queueMutation(instance);
2360
+ const pending = routeOf(latest());
2361
+ assertSame(pending.type, "Pending");
2362
+ assertSame(pending.completeAt, completeAt);
2363
+
2364
+ // So does a route that settles with a skipped change, and the Pending
2365
+ // route a request to check it again makes of it.
2366
+ await answerMutation(instance, setup.run);
2367
+ createWebSocket.message(transport.url, relayResponse());
2368
+ await testWaitForWorkerMessage();
2369
+ await respondToApplySync(
2370
+ instance,
2371
+ false,
2372
+ { ok: true, value: { type: "Converged" } },
2373
+ skippedError,
2374
+ );
2375
+ await testWaitForWorkerMessage();
2376
+ const settled = routeOf(latest());
2377
+ assertSame(settled.type, "Settled");
2378
+ assertSame(settled.completeAt, completeAt);
2379
+
2380
+ instance.evoluChannel.port2.postMessage({
2381
+ type: "UseOwner",
2382
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
2383
+ });
2384
+ await testWaitForWorkerMessage();
2385
+ const rechecking = routeOf(latest());
2386
+ assertSame(rechecking.type, "Pending");
2387
+ assertSame(rechecking.completeAt, completeAt);
2091
2388
  });
2092
2389
 
2093
- const writeErrors: ReadonlyArray<StorageWriteMessagesError> = [
2094
- {
2095
- type: "DecryptWithXChaCha20Poly1305Error",
2096
- error: "decryption failed",
2097
- },
2098
- {
2099
- type: "ProtocolInvalidDataError",
2100
- data: Uint8Array.of(255),
2101
- error: "decoding failed",
2102
- },
2390
+ const failedApplies = [
2103
2391
  {
2104
- type: "ProtocolTimestampMismatchError",
2105
- expected: createTimestamp(),
2106
- timestamp: createTimestamp({ millis: Millis.orThrow(1) }),
2392
+ result: {
2393
+ ok: false,
2394
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
2395
+ },
2396
+ errorType: "StorageQuotaError",
2107
2397
  },
2108
- { type: "StorageQuotaError", ownerId: testAppOwner.id },
2109
- ];
2110
- const failedApplies = [
2111
- ...writeErrors.map((error) => ({
2112
- result: { ok: false as const, error },
2113
- errorType: error.type,
2114
- })),
2115
2398
  {
2116
2399
  result: { ok: true, value: { type: "Failed", cause: "Write" } },
2117
2400
  errorType: "WriteFailed",
@@ -2121,11 +2404,11 @@ describe("sync state", () => {
2121
2404
  errorType: "SyncFailed",
2122
2405
  },
2123
2406
  ] satisfies ReadonlyArray<{
2124
- result: ApplySyncResult;
2407
+ result: ApplySyncResponse["result"];
2125
2408
  errorType: SyncRouteError["type"];
2126
2409
  }>;
2127
2410
  for (const { result, errorType } of failedApplies) {
2128
- it(`reports ${errorType} and requests only one round after repeated failure`, async () => {
2411
+ it(`shows ${errorType} on its route and requests only one round after repeated failure`, async () => {
2129
2412
  const createWebSocket = testCreateWebSocket();
2130
2413
  await using setup = await setupSharedWorker({ createWebSocket });
2131
2414
  using disposer = new DisposableStack();
@@ -2137,9 +2420,6 @@ describe("sync state", () => {
2137
2420
  errorChannel.onMessage = (output) => {
2138
2421
  errors.push(output);
2139
2422
  };
2140
- const expectedErrors: ReadonlyArray<ConsoleEntryOrError> = result.ok
2141
- ? []
2142
- : [{ type: "Error", error: result.error }];
2143
2423
  const instance = await setup.createEvolu();
2144
2424
  const transport = createOwnerWebSocketTransport({
2145
2425
  url: "wss://failed-apply.example",
@@ -2161,13 +2441,17 @@ describe("sync state", () => {
2161
2441
  await testWaitForWorkerMessage();
2162
2442
  await respondToApplySync(instance, false, result);
2163
2443
 
2164
- // The failure is reported and one round is requested through the route.
2165
- assertEqual(errors, expectedErrors);
2166
- assertEqual(routeOf(latest()).error, {
2167
- type: errorType,
2444
+ // The route shows the failure, which is not posted as an error, and
2445
+ // one round is requested through the route.
2446
+ assertEqual(errors, []);
2447
+ assertEqual(failureOf(routeOf(latest())), {
2448
+ ...(result.ok ? { type: errorType } : result.error),
2168
2449
  at: setup.run.deps.time.now(),
2169
2450
  });
2170
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2451
+ assertSame(
2452
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2453
+ "Error",
2454
+ );
2171
2455
  await testWaitForWorkerMessage();
2172
2456
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
2173
2457
  transport.url,
@@ -2179,10 +2463,13 @@ describe("sync state", () => {
2179
2463
 
2180
2464
  // A repeated failure waits for an explicit request or a reopen.
2181
2465
  await testWaitForWorkerMessage();
2182
- assertEqual(errors, [...expectedErrors, ...expectedErrors]);
2466
+ assertEqual(errors, []);
2183
2467
  assertLength(instance.dbInputs, inputCount);
2184
- assertFalse(routeOf(latest()).complete);
2185
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2468
+ assertSame(routeOf(latest()).type, "Pending");
2469
+ assertSame(
2470
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2471
+ "Error",
2472
+ );
2186
2473
 
2187
2474
  // An upload is not a round, so its answer does not clear the failure.
2188
2475
  await postMutation(instance, setup.run);
@@ -2196,8 +2483,8 @@ describe("sync state", () => {
2196
2483
  ok: true,
2197
2484
  value: { type: "Converged" },
2198
2485
  });
2199
- assertFalse(routeOf(latest()).complete);
2200
- assertSame(routeOf(latest()).error?.type, errorType);
2486
+ assertSame(routeOf(latest()).type, "Pending");
2487
+ assertSame(failureOf(routeOf(latest()))?.type, errorType);
2201
2488
 
2202
2489
  // An explicit request starts the round that clears the failure.
2203
2490
  instance.evoluChannel.port2.postMessage({
@@ -2212,23 +2499,512 @@ describe("sync state", () => {
2212
2499
  ok: true,
2213
2500
  value: { type: "Converged" },
2214
2501
  });
2215
- assertTrue(routeOf(latest()).complete);
2216
- assertSame(routeOf(latest()).error, null);
2217
- assertEqual(errors, [...expectedErrors, ...expectedErrors]);
2502
+ assertSame(routeOf(latest()).type, "Complete");
2503
+ assertEqual(errors, []);
2218
2504
  });
2219
2505
  }
2220
2506
 
2221
- it("reports a write rejection only for the active attempt after leader replacement", async () => {
2222
- const createWebSocket = testCreateWebSocket();
2223
- await using setup = await setupSharedWorker({ createWebSocket });
2224
- using disposer = new DisposableStack();
2225
- const { latest } = setupSyncStates(setup, disposer);
2226
- const errors: Array<ConsoleEntryOrError> = [];
2227
- using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
2228
- consoleEntryOrErrorBroadcastChannelName,
2229
- );
2230
- errorChannel.onMessage = (output) => {
2231
- errors.push(output);
2507
+ describe("skipped message", () => {
2508
+ /** Opens a route and queues the apply of the relay's first reply. */
2509
+ const setupRelayReply = async (
2510
+ setup: SharedWorkerSetup,
2511
+ disposer: DisposableStack,
2512
+ createWebSocket: TestCreateWebSocket,
2513
+ ) => {
2514
+ const { latest } = setupSyncStates(setup, disposer);
2515
+ const errors: Array<ConsoleEntryOrError> = [];
2516
+ const errorChannel = disposer.use(
2517
+ testCreateBroadcastChannel<ConsoleEntryOrError>(
2518
+ consoleEntryOrErrorBroadcastChannelName,
2519
+ ),
2520
+ );
2521
+ errorChannel.onMessage = (output) => {
2522
+ errors.push(output);
2523
+ };
2524
+ const instance = await setup.createEvolu();
2525
+ const transport = createOwnerWebSocketTransport({
2526
+ url: "wss://skipped-message.example",
2527
+ ownerId: testAppOwner.id,
2528
+ });
2529
+ instance.evoluChannel.port2.postMessage({
2530
+ type: "UseOwner",
2531
+ actions: [
2532
+ {
2533
+ owner: { owner: testAppOwner, transports: [transport] },
2534
+ action: "add",
2535
+ },
2536
+ ],
2537
+ });
2538
+ await testWaitForWorkerMessage();
2539
+ await respondToSyncRound(instance, createWebSocket);
2540
+ createWebSocket.message(transport.url, relayResponse());
2541
+ await testWaitForWorkerMessage();
2542
+ return { errors, instance, latest, transport };
2543
+ };
2544
+
2545
+ it("records the skip on the route while its round continues", async () => {
2546
+ const createWebSocket = testCreateWebSocket();
2547
+ await using setup = await setupSharedWorker({ createWebSocket });
2548
+ using disposer = new DisposableStack();
2549
+ const { errors, instance, latest, transport } = await setupRelayReply(
2550
+ setup,
2551
+ disposer,
2552
+ createWebSocket,
2553
+ );
2554
+ const response = createProtocolMessageForUnsubscribe(testAppOwner.id);
2555
+ const inputCount = instance.dbInputs.length;
2556
+ await respondToApplySync(
2557
+ instance,
2558
+ true,
2559
+ { ok: true, value: { type: "Response", message: response } },
2560
+ skippedError,
2561
+ );
2562
+
2563
+ // The response is sent, so the round continues, while the skipped
2564
+ // message is recorded as the route's skippedError with its details,
2565
+ // which is neither a failure nor an Evolu error. No other round is
2566
+ // requested, because this one continues.
2567
+ assertEqual(createWebSocket.sentMessages.splice(0), [
2568
+ { url: transport.url, data: response },
2569
+ ]);
2570
+ assertEqual(errors, []);
2571
+ const receivedAt = setup.run.deps.time.now();
2572
+ assertEqual(routeOf(latest()), {
2573
+ type: "Pending",
2574
+ transportId: latest().transports[0].id,
2575
+ failure: null,
2576
+ skippedError: {
2577
+ ...skippedError,
2578
+ error: createUnknownError(skippedError.error),
2579
+ at: receivedAt,
2580
+ },
2581
+ completeAt: null,
2582
+ lastSentAt: receivedAt,
2583
+ lastReceivedAt: receivedAt,
2584
+ });
2585
+ await testWaitForWorkerMessage();
2586
+ assertLength(instance.dbInputs, inputCount);
2587
+
2588
+ // The relay converges and skips a message again, whose error replaces
2589
+ // the shown one. Nothing is outstanding, yet the route does not
2590
+ // complete, and no round is requested.
2591
+ setup.run.deps.time.advance("1s");
2592
+ const mismatch = {
2593
+ type: "ProtocolTimestampMismatchError",
2594
+ expected: createTimestamp(),
2595
+ timestamp: createTimestamp({ nodeId: maxNodeId }),
2596
+ } satisfies ProtocolTimestampMismatchError;
2597
+ createWebSocket.message(transport.url, relayResponse());
2598
+ await testWaitForWorkerMessage();
2599
+ await respondToApplySync(
2600
+ instance,
2601
+ false,
2602
+ { ok: true, value: { type: "Converged" } },
2603
+ mismatch,
2604
+ );
2605
+ await testWaitForWorkerMessage();
2606
+ assertLength(instance.dbInputs, inputCount + 1);
2607
+ assertEqual(errors, []);
2608
+ assertEqual(routeOf(latest()), {
2609
+ type: "Settled",
2610
+ transportId: latest().transports[0].id,
2611
+ skippedError: { ...mismatch, at: setup.run.deps.time.now() },
2612
+ completeAt: null,
2613
+ lastSentAt: receivedAt,
2614
+ lastReceivedAt: setup.run.deps.time.now(),
2615
+ });
2616
+
2617
+ // Once the relay stops serving it, an explicit request completes the
2618
+ // route and clears the error.
2619
+ instance.evoluChannel.port2.postMessage({
2620
+ type: "UseOwner",
2621
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
2622
+ });
2623
+ await testWaitForWorkerMessage();
2624
+ await respondToSyncRound(instance, createWebSocket);
2625
+ createWebSocket.message(transport.url, relayResponse());
2626
+ await testWaitForWorkerMessage();
2627
+ await respondToApplySync(instance, false, {
2628
+ ok: true,
2629
+ value: { type: "Converged" },
2630
+ });
2631
+ assertSame(routeOf(latest()).type, "Complete");
2632
+ });
2633
+
2634
+ it("leaves its route the error that ended its round", async () => {
2635
+ const createWebSocket = testCreateWebSocket();
2636
+ await using setup = await setupSharedWorker({ createWebSocket });
2637
+ using disposer = new DisposableStack();
2638
+ const { errors, instance, latest, transport } = await setupRelayReply(
2639
+ setup,
2640
+ disposer,
2641
+ createWebSocket,
2642
+ );
2643
+ const error = {
2644
+ type: "ProtocolInvalidDataError",
2645
+ data: Uint8Array.of(255),
2646
+ error: "decoding failed",
2647
+ } satisfies ProtocolInvalidDataError;
2648
+ await respondToApplySync(
2649
+ instance,
2650
+ true,
2651
+ { ok: false, error },
2652
+ skippedError,
2653
+ );
2654
+
2655
+ // Nothing is posted as an error. The route shows the error that ended
2656
+ // the round, without its data, which requests one round, as any
2657
+ // failure does.
2658
+ assertEqual(errors, []);
2659
+ const failed = routeOf(latest());
2660
+ assertSame(failed.type, "Pending");
2661
+ assertEqual(failed.failure, {
2662
+ type: error.type,
2663
+ error: createUnknownError(error.error),
2664
+ at: setup.run.deps.time.now(),
2665
+ });
2666
+ // The skipped message stays visible beside the failure.
2667
+ assertEqual(failed.skippedError, {
2668
+ ...skippedError,
2669
+ error: createUnknownError(skippedError.error),
2670
+ at: setup.run.deps.time.now(),
2671
+ });
2672
+ await testWaitForWorkerMessage();
2673
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2674
+ transport.url,
2675
+ ]);
2676
+
2677
+ // The retry also checks the skipped message again, so once it skips
2678
+ // nothing, the route completes.
2679
+ createWebSocket.message(transport.url, relayResponse());
2680
+ await testWaitForWorkerMessage();
2681
+ await respondToApplySync(instance, false, {
2682
+ ok: true,
2683
+ value: { type: "Converged" },
2684
+ });
2685
+ assertSame(routeOf(latest()).type, "Complete");
2686
+ });
2687
+
2688
+ it("does not hide a Failed result", async () => {
2689
+ const createWebSocket = testCreateWebSocket();
2690
+ await using setup = await setupSharedWorker({ createWebSocket });
2691
+ using disposer = new DisposableStack();
2692
+ const { errors, instance, latest, transport } = await setupRelayReply(
2693
+ setup,
2694
+ disposer,
2695
+ createWebSocket,
2696
+ );
2697
+ await respondToApplySync(
2698
+ instance,
2699
+ true,
2700
+ { ok: true, value: { type: "Failed", cause: "Sync" } },
2701
+ skippedError,
2702
+ );
2703
+
2704
+ // The route shows the failure that ended the round, which requests
2705
+ // one round, as any failure does. The DbWorker already logged it.
2706
+ assertEqual(errors, []);
2707
+ assertSame(failureOf(routeOf(latest()))?.type, "SyncFailed");
2708
+ await testWaitForWorkerMessage();
2709
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2710
+ transport.url,
2711
+ ]);
2712
+ });
2713
+
2714
+ it("keeps the retry limit when failures also skip a message", async () => {
2715
+ const createWebSocket = testCreateWebSocket();
2716
+ await using setup = await setupSharedWorker({ createWebSocket });
2717
+ using disposer = new DisposableStack();
2718
+ const { instance, latest, transport } = await setupRelayReply(
2719
+ setup,
2720
+ disposer,
2721
+ createWebSocket,
2722
+ );
2723
+ const error = {
2724
+ type: "ProtocolInvalidDataError",
2725
+ data: Uint8Array.of(255),
2726
+ error: "decoding failed",
2727
+ } satisfies ProtocolInvalidDataError;
2728
+
2729
+ // The first failure requests one round, although it also skipped a
2730
+ // message.
2731
+ await respondToApplySync(
2732
+ instance,
2733
+ false,
2734
+ { ok: false, error },
2735
+ skippedError,
2736
+ );
2737
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2738
+ transport.url,
2739
+ ]);
2740
+
2741
+ // The retry's reply fails and skips again, so no further round is
2742
+ // requested.
2743
+ createWebSocket.message(transport.url, relayResponse());
2744
+ await testWaitForWorkerMessage();
2745
+ const inputCount = instance.dbInputs.length;
2746
+ await respondToApplySync(
2747
+ instance,
2748
+ false,
2749
+ { ok: false, error },
2750
+ skippedError,
2751
+ );
2752
+ await testWaitForWorkerMessage();
2753
+ assertLength(instance.dbInputs, inputCount);
2754
+
2755
+ // A converged reply that only skips does not end the wait, so a later
2756
+ // failure requests no round either; only the two applies are added.
2757
+ // The pending failure stays shown over the later skip.
2758
+ createWebSocket.message(transport.url, relayResponse());
2759
+ await testWaitForWorkerMessage();
2760
+ await respondToApplySync(
2761
+ instance,
2762
+ false,
2763
+ { ok: true, value: { type: "Converged" } },
2764
+ skippedError,
2765
+ );
2766
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
2767
+ createWebSocket.message(transport.url, relayResponse());
2768
+ await testWaitForWorkerMessage();
2769
+ await respondToApplySync(instance, false, { ok: false, error });
2770
+ await testWaitForWorkerMessage();
2771
+ assertLength(instance.dbInputs, inputCount + 2);
2772
+ assertSame(routeOf(latest()).type, "Pending");
2773
+ });
2774
+
2775
+ it("leaves a later failure its round", async () => {
2776
+ const createWebSocket = testCreateWebSocket();
2777
+ await using setup = await setupSharedWorker({ createWebSocket });
2778
+ using disposer = new DisposableStack();
2779
+ const { instance, latest, transport } = await setupRelayReply(
2780
+ setup,
2781
+ disposer,
2782
+ createWebSocket,
2783
+ );
2784
+ await respondToApplySync(
2785
+ instance,
2786
+ false,
2787
+ {
2788
+ ok: true,
2789
+ value: {
2790
+ type: "Response",
2791
+ message: createProtocolMessageForUnsubscribe(testAppOwner.id),
2792
+ },
2793
+ },
2794
+ skippedError,
2795
+ );
2796
+ createWebSocket.sentMessages.splice(0);
2797
+ const error = {
2798
+ type: "ProtocolSyncError",
2799
+ ownerId: testAppOwner.id,
2800
+ } satisfies ProtocolSyncError;
2801
+
2802
+ // The route already shows the skipped message, yet the first failure
2803
+ // still requests one round.
2804
+ createWebSocket.message(transport.url, relayResponse());
2805
+ await testWaitForWorkerMessage();
2806
+ await respondToApplySync(instance, false, { ok: false, error });
2807
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
2808
+ await testWaitForWorkerMessage();
2809
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2810
+ transport.url,
2811
+ ]);
2812
+
2813
+ // A further failure waits for an explicit request or a reopen.
2814
+ createWebSocket.message(transport.url, relayResponse());
2815
+ await testWaitForWorkerMessage();
2816
+ const inputCount = instance.dbInputs.length;
2817
+ await respondToApplySync(instance, false, { ok: false, error });
2818
+ await testWaitForWorkerMessage();
2819
+ assertLength(instance.dbInputs, inputCount);
2820
+ assertSame(routeOf(latest()).type, "Pending");
2821
+ });
2822
+
2823
+ it("lets a failure request a round after a retry skips the message again", async () => {
2824
+ const createWebSocket = testCreateWebSocket();
2825
+ await using setup = await setupSharedWorker({ createWebSocket });
2826
+ using disposer = new DisposableStack();
2827
+ const { instance, latest, transport } = await setupRelayReply(
2828
+ setup,
2829
+ disposer,
2830
+ createWebSocket,
2831
+ );
2832
+ await respondToApplySync(
2833
+ instance,
2834
+ false,
2835
+ { ok: true, value: { type: "Converged" } },
2836
+ skippedError,
2837
+ );
2838
+ const error = {
2839
+ type: "ProtocolSyncError",
2840
+ ownerId: testAppOwner.id,
2841
+ } satisfies ProtocolSyncError;
2842
+
2843
+ // A failure requests one round, whose relay converges but offers the
2844
+ // skipped message again.
2845
+ createWebSocket.message(transport.url, relayResponse());
2846
+ await testWaitForWorkerMessage();
2847
+ await respondToApplySync(instance, false, { ok: false, error });
2848
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2849
+ transport.url,
2850
+ ]);
2851
+ createWebSocket.message(transport.url, relayResponse());
2852
+ await testWaitForWorkerMessage();
2853
+ await respondToApplySync(
2854
+ instance,
2855
+ false,
2856
+ { ok: true, value: { type: "Converged" } },
2857
+ skippedError,
2858
+ );
2859
+ assertSame(routeOf(latest()).type, "Settled");
2860
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2861
+
2862
+ // The retry's reconciliation has ended, so a later failure requests a
2863
+ // round again, although the skipped message keeps the route
2864
+ // incomplete.
2865
+ createWebSocket.message(transport.url, relayResponse());
2866
+ await testWaitForWorkerMessage();
2867
+ await respondToApplySync(instance, false, { ok: false, error });
2868
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2869
+ transport.url,
2870
+ ]);
2871
+ });
2872
+
2873
+ it("publishes a caught Error inside a skipped change as an UnknownError", async () => {
2874
+ const createWebSocket = testCreateWebSocket();
2875
+ await using setup = await setupSharedWorker({ createWebSocket });
2876
+ using disposer = new DisposableStack();
2877
+ const { instance, latest } = await setupRelayReply(
2878
+ setup,
2879
+ disposer,
2880
+ createWebSocket,
2881
+ );
2882
+ await respondToApplySync(
2883
+ instance,
2884
+ false,
2885
+ { ok: true, value: { type: "Converged" } },
2886
+ {
2887
+ type: "DecryptWithXChaCha20Poly1305Error",
2888
+ error: new Error("invalid tag", { cause: new Error("tag") }),
2889
+ },
2890
+ );
2891
+
2892
+ // Sync state stays plain data, which tabs compare by content.
2893
+ const skipped = skippedErrorOf(routeOf(latest()));
2894
+ assert(
2895
+ skipped?.type === "DecryptWithXChaCha20Poly1305Error",
2896
+ "The route should show the skipped change.",
2897
+ );
2898
+ const { error } = skipped;
2899
+ assert(
2900
+ UnknownError.is(error) &&
2901
+ isPlainObject(error.error) &&
2902
+ isPlainObject(error.error.cause),
2903
+ "The Error and its cause should be plain objects in an UnknownError.",
2904
+ );
2905
+ assertSame(error.error.message, "invalid tag");
2906
+ assertSame(error.error.cause.message, "tag");
2907
+ });
2908
+
2909
+ it("publishes a caught Error inside a failure as an UnknownError", async () => {
2910
+ const createWebSocket = testCreateWebSocket();
2911
+ await using setup = await setupSharedWorker({ createWebSocket });
2912
+ using disposer = new DisposableStack();
2913
+ const { instance, latest } = await setupRelayReply(
2914
+ setup,
2915
+ disposer,
2916
+ createWebSocket,
2917
+ );
2918
+ await respondToApplySync(instance, false, {
2919
+ ok: false,
2920
+ error: {
2921
+ type: "ProtocolInvalidDataError",
2922
+ data: Uint8Array.of(255),
2923
+ error: new Error("invalid varint"),
2924
+ },
2925
+ });
2926
+
2927
+ const failure = failureOf(routeOf(latest()));
2928
+ assert(
2929
+ failure?.type === "ProtocolInvalidDataError",
2930
+ "The route should show the failure.",
2931
+ );
2932
+ const { error } = failure;
2933
+ assert(
2934
+ UnknownError.is(error) && isPlainObject(error.error),
2935
+ "The Error should be a plain object in an UnknownError.",
2936
+ );
2937
+ assertSame(error.error.message, "invalid varint");
2938
+ });
2939
+
2940
+ it("keeps its skip through an aborted apply", async () => {
2941
+ const createWebSocket = testCreateWebSocket();
2942
+ await using setup = await setupSharedWorker({ createWebSocket });
2943
+ using disposer = new DisposableStack();
2944
+ const { errors, instance, latest, transport } = await setupRelayReply(
2945
+ setup,
2946
+ disposer,
2947
+ createWebSocket,
2948
+ );
2949
+ await respondToApplySync(
2950
+ instance,
2951
+ false,
2952
+ { ok: true, value: { type: "Converged" } },
2953
+ skippedError,
2954
+ );
2955
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2956
+
2957
+ // An abort proves nothing about the skipped message, so the route
2958
+ // still shows it, and it is not a failure.
2959
+ createWebSocket.message(transport.url, relayResponse());
2960
+ await testWaitForWorkerMessage();
2961
+ await respondToApplySync(instance, false, {
2962
+ ok: false,
2963
+ error: { type: "AbortError", reason: { type: "Stop" } },
2964
+ });
2965
+ assertEqual(errors, []);
2966
+ assertSame(routeOf(latest()).type, "Pending");
2967
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2968
+ assertSame(failureOf(routeOf(latest())), null);
2969
+ });
2970
+
2971
+ it("is ignored for an aborted apply", async () => {
2972
+ const createWebSocket = testCreateWebSocket();
2973
+ await using setup = await setupSharedWorker({ createWebSocket });
2974
+ using disposer = new DisposableStack();
2975
+ const { errors, instance, latest } = await setupRelayReply(
2976
+ setup,
2977
+ disposer,
2978
+ createWebSocket,
2979
+ );
2980
+ await respondToApplySync(
2981
+ instance,
2982
+ false,
2983
+ {
2984
+ ok: false,
2985
+ error: { type: "AbortError", reason: { type: "Stop" } },
2986
+ },
2987
+ skippedError,
2988
+ );
2989
+
2990
+ assertEqual(errors, []);
2991
+ assertSame(failureOf(routeOf(latest())), null);
2992
+ assertSame(skippedErrorOf(routeOf(latest())), null);
2993
+ assertSame(routeOf(latest()).type, "Pending");
2994
+ });
2995
+ });
2996
+
2997
+ it("applies a write rejection only for the active attempt after leader replacement", async () => {
2998
+ const createWebSocket = testCreateWebSocket();
2999
+ await using setup = await setupSharedWorker({ createWebSocket });
3000
+ using disposer = new DisposableStack();
3001
+ const { latest } = setupSyncStates(setup, disposer);
3002
+ const errors: Array<ConsoleEntryOrError> = [];
3003
+ using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
3004
+ consoleEntryOrErrorBroadcastChannelName,
3005
+ );
3006
+ errorChannel.onMessage = (output) => {
3007
+ errors.push(output);
2232
3008
  };
2233
3009
  const first = await setup.createEvolu({
2234
3010
  releaseDbWorkerLeaderOnDispose: false,
@@ -2276,8 +3052,8 @@ describe("sync state", () => {
2276
3052
  assertNotSame(replay.attemptId, original.attemptId);
2277
3053
 
2278
3054
  const error = {
2279
- type: "DecryptWithXChaCha20Poly1305Error",
2280
- error: "wrong encryption key",
3055
+ type: "StorageQuotaError",
3056
+ ownerId: testAppOwner.id,
2281
3057
  } satisfies StorageWriteMessagesError;
2282
3058
  const response = {
2283
3059
  type: "OnQueuedResponse",
@@ -2289,34 +3065,36 @@ describe("sync state", () => {
2289
3065
  clock: createTimestamp(),
2290
3066
  ownerId: testAppOwner.id,
2291
3067
  didWriteMessages: false,
3068
+ skippedError: null,
2292
3069
  result: { ok: false, error },
2293
3070
  },
2294
3071
  },
2295
3072
  } satisfies DbWorkerOutput;
2296
3073
 
2297
3074
  // The retired port and an obsolete attempt on the live port must both
2298
- // leave the active replay and its error reporting untouched.
3075
+ // leave the active replay and its route untouched.
2299
3076
  first.dbWorkerPort.postMessage(response);
2300
3077
  dbWorkerPort.postMessage(response);
2301
3078
  await testWaitForWorkerMessage();
2302
3079
  assertEqual(errors, []);
2303
- assertSame(routeOf(latest()).error, null);
3080
+ assertSame(failureOf(routeOf(latest())), null);
3081
+ assertSame(skippedErrorOf(routeOf(latest())), null);
2304
3082
  assertLength(dbInputs, 1);
2305
3083
 
2306
3084
  const currentResponse = { ...response, attemptId: replay.attemptId };
2307
3085
  dbWorkerPort.postMessage(currentResponse);
2308
3086
  await testWaitForWorkerMessage();
2309
- assertEqual(errors, [{ type: "Error", error }]);
2310
- assertEqual(routeOf(latest()).error, {
2311
- type: error.type,
2312
- at: setup.run.deps.time.now(),
2313
- });
3087
+ const routeError = { ...error, at: setup.run.deps.time.now() };
3088
+ assertEqual(errors, []);
3089
+ assertEqual(failureOf(routeOf(latest())), routeError);
2314
3090
  assertLength(dbInputs, 2);
2315
3091
 
2316
- // A duplicate completion is stale as soon as its attempt is accepted.
3092
+ // A duplicate completion is stale as soon as its attempt is accepted,
3093
+ // so the route keeps the time of the accepted one.
3094
+ setup.run.deps.time.advance("1s");
2317
3095
  dbWorkerPort.postMessage(currentResponse);
2318
3096
  await testWaitForWorkerMessage();
2319
- assertEqual(errors, [{ type: "Error", error }]);
3097
+ assertEqual(failureOf(routeOf(latest())), routeError);
2320
3098
  assertLength(dbInputs, 2);
2321
3099
  const replacement = { dbInputs, dbWorkerPort };
2322
3100
  await respondToSyncRound(replacement, createWebSocket);
@@ -2326,9 +3104,8 @@ describe("sync state", () => {
2326
3104
  ok: true,
2327
3105
  value: { type: "Converged" },
2328
3106
  });
2329
- assertTrue(routeOf(latest()).complete);
2330
- assertSame(routeOf(latest()).error, null);
2331
- assertEqual(errors, [{ type: "Error", error }]);
3107
+ assertSame(routeOf(latest()).type, "Complete");
3108
+ assertEqual(errors, []);
2332
3109
  });
2333
3110
 
2334
3111
  it("keeps an aborted sibling apply incomplete without scheduling recovery", async () => {
@@ -2365,7 +3142,7 @@ describe("sync state", () => {
2365
3142
  });
2366
3143
  }
2367
3144
  }
2368
- assertTrue(routeOf(latest()).complete);
3145
+ assertSame(routeOf(latest()).type, "Complete");
2369
3146
  const completeAt = routeOf(latest()).completeAt;
2370
3147
 
2371
3148
  await postMutation(second, setup.run);
@@ -2392,10 +3169,11 @@ describe("sync state", () => {
2392
3169
  ok: true,
2393
3170
  value: { type: "Converged" },
2394
3171
  });
2395
- assertFalse(routeOf(latest()).complete);
3172
+ assertSame(routeOf(latest()).type, "Pending");
2396
3173
  assertSame(routeOf(latest()).completeAt, completeAt);
2397
- assertSame(routeOf(latest()).error, null);
2398
- assertTrue(routeOf(latest(), secondName).complete);
3174
+ assertSame(failureOf(routeOf(latest())), null);
3175
+ assertSame(skippedErrorOf(routeOf(latest())), null);
3176
+ assertSame(routeOf(latest(), secondName).type, "Complete");
2399
3177
  await testWaitForWorkerMessage();
2400
3178
  assertLength(first.dbInputs, inputCount);
2401
3179
  });
@@ -2464,6 +3242,34 @@ describe("sync state", () => {
2464
3242
  return { source, sibling, siblingName, transports };
2465
3243
  };
2466
3244
 
3245
+ it("completes a route when its sibling continuation copy is applied", async () => {
3246
+ const createWebSocket = testCreateWebSocket();
3247
+ await using setup = await setupSharedWorker({ createWebSocket });
3248
+ using disposer = new DisposableStack();
3249
+ const { latest } = setupSyncStates(setup, disposer);
3250
+ const { sibling, siblingName } = await setupSiblingContinuation(
3251
+ setup,
3252
+ createWebSocket,
3253
+ );
3254
+ const queued = routeOf(latest(), siblingName, 1);
3255
+ assertSame(queued.type, "Pending");
3256
+
3257
+ // The route through the other relay completes when the copy is applied,
3258
+ // which is later than its last frame.
3259
+ setup.run.deps.time.advance("1s");
3260
+ await respondToApplySync(sibling, false, {
3261
+ ok: true,
3262
+ value: { type: "Broadcast" },
3263
+ });
3264
+ assertEqual(routeOf(latest(), siblingName, 1), {
3265
+ type: "Complete",
3266
+ transportId: queued.transportId,
3267
+ completeAt: setup.run.deps.time.now(),
3268
+ lastSentAt: queued.lastSentAt,
3269
+ lastReceivedAt: queued.lastReceivedAt,
3270
+ });
3271
+ });
3272
+
2467
3273
  it("keeps every route incomplete after aborting a sibling continuation copy", async () => {
2468
3274
  const createWebSocket = testCreateWebSocket();
2469
3275
  await using setup = await setupSharedWorker({ createWebSocket });
@@ -2494,10 +3300,11 @@ describe("sync state", () => {
2494
3300
 
2495
3301
  for (const index of [0, 1]) {
2496
3302
  const route = routeOf(latest(), siblingName, index);
2497
- assertFalse(route.complete);
3303
+ assertSame(route.type, "Pending");
2498
3304
  assertSame(route.completeAt, completeAt[index]);
2499
- assertSame(route.error, null);
2500
- assertTrue(routeOf(latest(), testName, index).complete);
3305
+ assertSame(route.failure, null);
3306
+ assertSame(route.skippedError, null);
3307
+ assertSame(routeOf(latest(), testName, index).type, "Complete");
2501
3308
  }
2502
3309
  await testWaitForWorkerMessage();
2503
3310
  assertLength(sibling.dbInputs, inputCount);
@@ -2545,8 +3352,8 @@ describe("sync state", () => {
2545
3352
  ok: true,
2546
3353
  value: { type: "Converged" },
2547
3354
  });
2548
- assertFalse(routeOf(latest()).complete);
2549
- assertFalse(routeOf(latest(), secondName).complete);
3355
+ assertSame(routeOf(latest()).type, "Pending");
3356
+ assertSame(routeOf(latest(), secondName).type, "Pending");
2550
3357
 
2551
3358
  createWebSocket.message(transport.url, relayResponse());
2552
3359
  await testWaitForWorkerMessage();
@@ -2554,13 +3361,13 @@ describe("sync state", () => {
2554
3361
  ok: true,
2555
3362
  value: { type: "Converged" },
2556
3363
  });
2557
- assertFalse(routeOf(latest(), secondName).complete);
3364
+ assertSame(routeOf(latest(), secondName).type, "Pending");
2558
3365
  await respondToApplySync(second, false, {
2559
3366
  ok: true,
2560
3367
  value: { type: "Converged" },
2561
3368
  });
2562
- assertTrue(routeOf(latest()).complete);
2563
- assertTrue(routeOf(latest(), secondName).complete);
3369
+ assertSame(routeOf(latest()).type, "Complete");
3370
+ assertSame(routeOf(latest(), secondName).type, "Complete");
2564
3371
  });
2565
3372
 
2566
3373
  it("keeps routes incomplete until a sibling's continuation copy reconciles the other relays", async () => {
@@ -2591,12 +3398,10 @@ describe("sync state", () => {
2591
3398
  (candidate) => candidate.label === label,
2592
3399
  );
2593
3400
  assertNotUndefined(transport);
2594
- const tenant = state.tenants.find(({ name }) => name === tenantName);
2595
- assertNotUndefined(tenant);
2596
- const owner = tenant.owners.find(
3401
+ const owner = ownersOf(state, tenantName).find(
2597
3402
  ({ ownerId }) => ownerId === testAppOwner.id,
2598
3403
  );
2599
- assertNotUndefined(owner);
3404
+ assert(owner?.type === "Writable", "Expected a writable owner.");
2600
3405
  const route = owner.routes.find(
2601
3406
  ({ transportId }) => transportId === transport.id,
2602
3407
  );
@@ -2629,7 +3434,7 @@ describe("sync state", () => {
2629
3434
  }
2630
3435
  for (const label of [labelA, labelB]) {
2631
3436
  for (const name of [testName, siblingName]) {
2632
- assertTrue(routeAt(label, name).complete);
3437
+ assertSame(routeAt(label, name).type, "Complete");
2633
3438
  }
2634
3439
  }
2635
3440
  const siblingCompleteAt = routeAt(labelB, siblingName).completeAt;
@@ -2661,10 +3466,11 @@ describe("sync state", () => {
2661
3466
 
2662
3467
  // Relay B neither sent nor was sent anything, so only the queued local
2663
3468
  // copy can hold the sibling's route through B.
2664
- assertFalse(routeAt(labelB, siblingName).complete);
3469
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2665
3470
  assertSame(routeAt(labelB, siblingName).completeAt, siblingCompleteAt);
2666
- assertSame(routeAt(labelB, siblingName).error, null);
2667
- assertTrue(routeAt(labelB, testName).complete);
3471
+ assertSame(failureOf(routeAt(labelB, siblingName)), null);
3472
+ assertSame(skippedErrorOf(routeAt(labelB, siblingName)), null);
3473
+ assertSame(routeAt(labelB, testName).type, "Complete");
2668
3474
 
2669
3475
  // The copy was uploaded only through A. Storing it requires a new round
2670
3476
  // through B, and local delivery does not count as a frame received on A.
@@ -2674,13 +3480,13 @@ describe("sync state", () => {
2674
3480
  ok: true,
2675
3481
  value: { type: "Broadcast" },
2676
3482
  });
2677
- assertFalse(routeAt(labelB, siblingName).complete);
3483
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2678
3484
  assertSame(routeAt(labelB, siblingName).completeAt, siblingCompleteAt);
2679
3485
  assertSame(routeAt(labelA, siblingName).lastReceivedAt, lastReceivedAt);
2680
3486
  assertEqual(await respondToSyncRound(sibling, createWebSocket), [
2681
3487
  transports[1].url,
2682
3488
  ]);
2683
- assertFalse(routeAt(labelB, siblingName).complete);
3489
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2684
3490
  createWebSocket.message(transports[1].url, relayResponse());
2685
3491
  await testWaitForWorkerMessage();
2686
3492
  for (const instance of [source, sibling]) {
@@ -2689,7 +3495,7 @@ describe("sync state", () => {
2689
3495
  value: { type: "Converged" },
2690
3496
  });
2691
3497
  }
2692
- assertTrue(routeAt(labelB, siblingName).complete);
3498
+ assertSame(routeAt(labelB, siblingName).type, "Complete");
2693
3499
  });
2694
3500
 
2695
3501
  for (const fractional of [false, true]) {
@@ -2727,6 +3533,7 @@ describe("sync state", () => {
2727
3533
  await testWaitForWorkerMessage();
2728
3534
  createWebSocket.open(transport.url);
2729
3535
  await testWaitForWorkerMessage();
3536
+ const openedAt = time.now();
2730
3537
  await respondToSyncRound(instance, createWebSocket);
2731
3538
  await postMutation(instance, setup.run);
2732
3539
 
@@ -2745,7 +3552,7 @@ describe("sync state", () => {
2745
3552
  time.advance("5s");
2746
3553
  await testWaitForWorkerMessage();
2747
3554
  assertEqual(createWebSocket.reconnectedUrls, []);
2748
- assertSame(latest().transports[0].closedAt, null);
3555
+ assertSame(latest().transports[0].connection.type, "Open");
2749
3556
 
2750
3557
  // Nothing arrives for a whole timeout: the socket reconnects.
2751
3558
  time.advance(Millis.orThrow(84_999));
@@ -2755,9 +3562,15 @@ describe("sync state", () => {
2755
3562
  await testWaitForWorkerMessage();
2756
3563
  assertEqual(createWebSocket.createdUrls, [transport.url]);
2757
3564
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
2758
- assertSame(latest().transports[0].readyState, "connecting");
2759
- assertEqual(latest().transports[0].closedAt, time.now());
2760
-
3565
+ // The socket reports no close for a reconnect, so the transport
3566
+ // disconnects when the request times out.
3567
+ assertEqual(latest().transports[0].connection, {
3568
+ type: "Disconnected",
3569
+ disconnectedAt: time.now(),
3570
+ openedAt,
3571
+ error: null,
3572
+ });
3573
+
2761
3574
  // The reopen forgets the unanswered request and starts a round.
2762
3575
  createWebSocket.open(transport.url);
2763
3576
  await testWaitForWorkerMessage();
@@ -2768,7 +3581,7 @@ describe("sync state", () => {
2768
3581
  ok: true,
2769
3582
  value: { type: "Converged" },
2770
3583
  });
2771
- assertTrue(routeOf(latest()).complete);
3584
+ assertSame(routeOf(latest()).type, "Complete");
2772
3585
  });
2773
3586
  }
2774
3587
 
@@ -2874,7 +3687,7 @@ describe("sync state", () => {
2874
3687
  time.advance("1s");
2875
3688
  await testWaitForWorkerMessage();
2876
3689
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
2877
- assertSame(latest().transports[0].readyState, "connecting");
3690
+ assertSame(latest().transports[0].connection.type, "Disconnected");
2878
3691
  });
2879
3692
 
2880
3693
  it("stops timing a request when its socket closes", async () => {
@@ -2900,6 +3713,7 @@ describe("sync state", () => {
2900
3713
  await testWaitForWorkerMessage();
2901
3714
  createWebSocket.open(transport.url);
2902
3715
  await testWaitForWorkerMessage();
3716
+ const openedAt = time.now();
2903
3717
  await respondToSyncRound(instance, createWebSocket);
2904
3718
 
2905
3719
  // The request dies with its connection, and the WebSocket's own retry
@@ -2907,12 +3721,17 @@ describe("sync state", () => {
2907
3721
  time.advance("10s");
2908
3722
  createWebSocket.close(transport.url);
2909
3723
  await testWaitForWorkerMessage();
2910
- const { closedAt } = latest().transports[0];
2911
- assertEqual(closedAt, time.now());
3724
+ const disconnected = {
3725
+ type: "Disconnected",
3726
+ disconnectedAt: time.now(),
3727
+ openedAt,
3728
+ error: null,
3729
+ } as const;
3730
+ assertEqual(latest().transports[0].connection, disconnected);
2912
3731
  time.advance("2m");
2913
3732
  await testWaitForWorkerMessage();
2914
3733
  assertEqual(createWebSocket.reconnectedUrls, []);
2915
- assertSame(latest().transports[0].closedAt, closedAt);
3734
+ assertEqual(latest().transports[0].connection, disconnected);
2916
3735
  });
2917
3736
 
2918
3737
  it("leaves a complete route incomplete when its socket closes", async () => {
@@ -2942,13 +3761,62 @@ describe("sync state", () => {
2942
3761
  ok: true,
2943
3762
  value: { type: "Converged" },
2944
3763
  });
2945
- assertTrue(routeOf(latest()).complete);
3764
+ assertSame(routeOf(latest()).type, "Complete");
2946
3765
 
2947
3766
  createWebSocket.close(transport.url);
2948
3767
  await testWaitForWorkerMessage();
2949
- assertSame(latest().transports[0].readyState, "closed");
2950
- assertFalse(routeOf(latest()).complete);
2951
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "offline");
3768
+ assertSame(latest().transports[0].connection.type, "Disconnected");
3769
+ assertSame(routeOf(latest()).type, "Pending");
3770
+ assertEqual(
3771
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3772
+ { type: "Offline" },
3773
+ );
3774
+ });
3775
+
3776
+ it("disconnects a first connection that fails without a close", async () => {
3777
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
3778
+ await using setup = await setupSharedWorker({ createWebSocket });
3779
+ using disposer = new DisposableStack();
3780
+ const { latest } = setupSyncStates(setup, disposer);
3781
+ const { time } = setup.run.deps;
3782
+ const instance = await setup.createEvolu();
3783
+ const transport = createOwnerWebSocketTransport({
3784
+ url: "wss://first-connection.example",
3785
+ ownerId: testAppOwner.id,
3786
+ });
3787
+ instance.evoluChannel.port2.postMessage({
3788
+ type: "UseOwner",
3789
+ actions: [
3790
+ {
3791
+ owner: { owner: testAppOwner, transports: [transport] },
3792
+ action: "add",
3793
+ },
3794
+ ],
3795
+ });
3796
+ await testWaitForWorkerMessage();
3797
+ assertEqual(
3798
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3799
+ { type: "Syncing" },
3800
+ );
3801
+
3802
+ // A browser reports a failed attempt only as an error, because the
3803
+ // socket stops listening before its close arrives.
3804
+ time.advance("1s");
3805
+ createWebSocket.error(transport.url, {
3806
+ type: "WebSocketConnectError",
3807
+ event: new Event("error"),
3808
+ });
3809
+ await testWaitForWorkerMessage();
3810
+ assertEqual(latest().transports[0].connection, {
3811
+ type: "Disconnected",
3812
+ disconnectedAt: time.now(),
3813
+ openedAt: null,
3814
+ error: { type: "WebSocketConnectError", at: time.now() },
3815
+ });
3816
+ assertEqual(
3817
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3818
+ { type: "Offline" },
3819
+ );
2952
3820
  });
2953
3821
 
2954
3822
  /**
@@ -3085,6 +3953,63 @@ describe("sync state", () => {
3085
3953
  assertLength(createWebSocket.reconnectedUrls, 3);
3086
3954
  });
3087
3955
 
3956
+ it("restores the base timeout once a round settles with a skipped message", async () => {
3957
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
3958
+ await using setup = await setupSharedWorker({ createWebSocket });
3959
+ using disposer = new DisposableStack();
3960
+ const { latest } = setupSyncStates(setup, disposer);
3961
+ const instance = await setup.createEvolu();
3962
+ const { time } = setup.run.deps;
3963
+ const transport = createOwnerWebSocketTransport({
3964
+ url: "wss://skipping.example",
3965
+ ownerId: testAppOwner.id,
3966
+ });
3967
+ instance.evoluChannel.port2.postMessage({
3968
+ type: "UseOwner",
3969
+ actions: [
3970
+ {
3971
+ owner: { owner: testAppOwner, transports: [transport] },
3972
+ action: "add",
3973
+ },
3974
+ ],
3975
+ });
3976
+ await testWaitForWorkerMessage();
3977
+ await timeOutRounds(
3978
+ { setup, instance, createWebSocket, url: transport.url },
3979
+ [90],
3980
+ );
3981
+
3982
+ // The reopened round converges, but the relay offers a message the
3983
+ // database skips, so the route stays incomplete after its
3984
+ // reconciliation has ended.
3985
+ createWebSocket.open(transport.url);
3986
+ await testWaitForWorkerMessage();
3987
+ await respondToSyncRound(instance, createWebSocket);
3988
+ createWebSocket.message(transport.url, relayResponse());
3989
+ await testWaitForWorkerMessage();
3990
+ await respondToApplySync(
3991
+ instance,
3992
+ false,
3993
+ { ok: true, value: { type: "Converged" } },
3994
+ skippedError,
3995
+ );
3996
+ assertSame(routeOf(latest()).type, "Settled");
3997
+
3998
+ // The next request gets the base timeout.
3999
+ instance.evoluChannel.port2.postMessage({
4000
+ type: "UseOwner",
4001
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
4002
+ });
4003
+ await testWaitForWorkerMessage();
4004
+ await respondToSyncRound(instance, createWebSocket);
4005
+ time.advance(Millis.orThrow(89_999));
4006
+ await testWaitForWorkerMessage();
4007
+ assertLength(createWebSocket.reconnectedUrls, 1);
4008
+ time.advance("1ms");
4009
+ await testWaitForWorkerMessage();
4010
+ assertLength(createWebSocket.reconnectedUrls, 2);
4011
+ });
4012
+
3088
4013
  it("keeps a grown timeout while another database using the owner reconciles", async () => {
3089
4014
  const createWebSocket = testCreateWebSocket({ isOpen: false });
3090
4015
  await using setup = await setupSharedWorker({ createWebSocket });
@@ -3141,7 +4066,7 @@ describe("sync state", () => {
3141
4066
  ok: true,
3142
4067
  value: { type: "Converged" },
3143
4068
  });
3144
- assertTrue(routeOf(latest()).complete);
4069
+ assertSame(routeOf(latest()).type, "Complete");
3145
4070
  await respondToApplySync(second, false, {
3146
4071
  ok: true,
3147
4072
  value: {
@@ -3208,8 +4133,8 @@ describe("sync state", () => {
3208
4133
  value: { type: "Converged" },
3209
4134
  });
3210
4135
  }
3211
- assertTrue(routeOf(latest()).complete);
3212
- assertFalse(routeOf(latest(), Name.orThrow("second")).complete);
4136
+ assertSame(routeOf(latest()).type, "Complete");
4137
+ assertSame(routeOf(latest(), Name.orThrow("second")).type, "Pending");
3213
4138
 
3214
4139
  // The second database stops using the owner without completing, which
3215
4140
  // leaves no incomplete route, so the base timeout applies again.
@@ -3319,7 +4244,10 @@ describe("sync state", () => {
3319
4244
  value: { type: "Converged" },
3320
4245
  });
3321
4246
  }
3322
- assertTrue(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4247
+ assertSame(
4248
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4249
+ "Complete",
4250
+ );
3323
4251
 
3324
4252
  // The app owner's round goes unanswered, so the socket is replaced, and
3325
4253
  // the other owner's route through it is no longer reconciled.
@@ -3332,7 +4260,10 @@ describe("sync state", () => {
3332
4260
  time.advance(Millis.orThrow(90_000));
3333
4261
  await testWaitForWorkerMessage();
3334
4262
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
3335
- assertFalse(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4263
+ assertSame(
4264
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4265
+ "Pending",
4266
+ );
3336
4267
  });
3337
4268
 
3338
4269
  it("restores a relay's base timeout while another relay of the owner is down", async () => {
@@ -3373,8 +4304,8 @@ describe("sync state", () => {
3373
4304
  ok: true,
3374
4305
  value: { type: "Converged" },
3375
4306
  });
3376
- assertTrue(routeOf(latest(), testName, 0).complete);
3377
- assertFalse(routeOf(latest(), testName, 1).complete);
4307
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
4308
+ assertSame(routeOf(latest(), testName, 1).type, "Pending");
3378
4309
 
3379
4310
  instance.evoluChannel.port2.postMessage({
3380
4311
  type: "UseOwner",
@@ -3459,8 +4390,11 @@ describe("sync state", () => {
3459
4390
  ok: true,
3460
4391
  value: { type: "Converged" },
3461
4392
  });
3462
- assertTrue(routeOf(latest()).complete);
3463
- assertFalse(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4393
+ assertSame(routeOf(latest()).type, "Complete");
4394
+ assertSame(
4395
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4396
+ "Pending",
4397
+ );
3464
4398
 
3465
4399
  instance.evoluChannel.port2.postMessage({
3466
4400
  type: "UseOwner",
@@ -3515,14 +4449,15 @@ describe("sync state", () => {
3515
4449
  using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
3516
4450
  init.port,
3517
4451
  );
4452
+ const error: UnsupportedDbVersionError = {
4453
+ type: "UnsupportedDbVersionError",
4454
+ storedVersion: PositiveInt.orThrow(2),
4455
+ supportedVersion: PositiveInt.orThrow(1),
4456
+ };
3518
4457
  leaderPort.postMessage({
3519
4458
  type: "LeaderRefused",
3520
4459
  name: refusedName,
3521
- error: {
3522
- type: "UnsupportedDbVersionError",
3523
- storedVersion: PositiveInt.orThrow(2),
3524
- supportedVersion: PositiveInt.orThrow(1),
3525
- },
4460
+ error,
3526
4461
  });
3527
4462
  await testWaitForWorkerMessage();
3528
4463
  const context = { setup, instance, createWebSocket, url: transport.url };
@@ -3539,8 +4474,12 @@ describe("sync state", () => {
3539
4474
  ok: true,
3540
4475
  value: { type: "Converged" },
3541
4476
  });
3542
- assertTrue(routeOf(latest()).complete);
3543
- assertFalse(routeOf(latest(), refusedName).complete);
4477
+ assertSame(routeOf(latest()).type, "Complete");
4478
+ // The refused database publishes no route.
4479
+ assertEqual(
4480
+ latest().tenants.find(({ name }) => name === refusedName),
4481
+ { type: "Refused", name: refusedName, error },
4482
+ );
3544
4483
  instance.evoluChannel.port2.postMessage({
3545
4484
  type: "UseOwner",
3546
4485
  actions: [{ ownerId: testAppOwner.id, action: "sync" }],
@@ -3620,7 +4559,10 @@ describe("sync state", () => {
3620
4559
  ok: true,
3621
4560
  value: { type: "Converged" },
3622
4561
  });
3623
- assertTrue(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4562
+ assertSame(
4563
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4564
+ "Complete",
4565
+ );
3624
4566
  createWebSocket.message(transport.url, relayResponse());
3625
4567
  await testWaitForWorkerMessage();
3626
4568
  await respondToApplySync(instance, false, {
@@ -3682,7 +4624,8 @@ describe("sync state", () => {
3682
4624
  const completeRoutes = (): Array<boolean> =>
3683
4625
  owners.flatMap((owner) =>
3684
4626
  transports.map(
3685
- (_, index) => routeOf(latest(), testName, index, owner.id).complete,
4627
+ (_, index) =>
4628
+ routeOf(latest(), testName, index, owner.id).type === "Complete",
3686
4629
  ),
3687
4630
  );
3688
4631
  assertEqual(completeRoutes(), [true, true, true, true]);
@@ -3752,7 +4695,8 @@ describe("sync state", () => {
3752
4695
  }
3753
4696
  const completeRoutes = (): Array<boolean> =>
3754
4697
  [testAppOwner, testAppOwner2].map(
3755
- (owner) => routeOf(latest(), testName, 0, owner.id).complete,
4698
+ (owner) =>
4699
+ routeOf(latest(), testName, 0, owner.id).type === "Complete",
3756
4700
  );
3757
4701
  assertEqual(completeRoutes(), [true, true]);
3758
4702
 
@@ -3865,7 +4809,7 @@ describe("sync state", () => {
3865
4809
  ok: true,
3866
4810
  value: { type: "Converged" },
3867
4811
  });
3868
- assertTrue(routeOf(latest()).complete);
4812
+ assertSame(routeOf(latest()).type, "Complete");
3869
4813
 
3870
4814
  let changeReads = 0;
3871
4815
  const changeHandler: ProxyHandler<MutationChange> = {
@@ -3898,7 +4842,7 @@ describe("sync state", () => {
3898
4842
  });
3899
4843
  }
3900
4844
  await testWaitForWorkerMessage();
3901
- assertTrue(routeOf(latest()).complete);
4845
+ assertSame(routeOf(latest()).type, "Complete");
3902
4846
 
3903
4847
  // A transport error publishes another snapshot without enqueuing or
3904
4848
  // completing database work. The pending queue must not be rescanned.
@@ -3911,7 +4855,7 @@ describe("sync state", () => {
3911
4855
  await testWaitForWorkerMessage();
3912
4856
  assertTrue(states.length > publishedCount);
3913
4857
  assertSame(changeReads, 0);
3914
- assertTrue(routeOf(latest()).complete);
4858
+ assertSame(routeOf(latest()).type, "Complete");
3915
4859
  });
3916
4860
 
3917
4861
  it("publishes no sync state for a local-only mutation", async () => {
@@ -3945,7 +4889,7 @@ describe("sync state", () => {
3945
4889
  ok: true,
3946
4890
  value: { type: "Converged" },
3947
4891
  });
3948
- assertTrue(routeOf(latest()).complete);
4892
+ assertSame(routeOf(latest()).type, "Complete");
3949
4893
  const publishedCount = states.length;
3950
4894
 
3951
4895
  instance.evoluChannel.port2.postMessage({
@@ -3980,7 +4924,7 @@ describe("sync state", () => {
3980
4924
  // The response was handled without publishing.
3981
4925
  assertSame(evoluOutputs.at(-1)?.type, "OnPatchesByQuery");
3982
4926
  assertSame(states.length, publishedCount);
3983
- assertTrue(routeOf(latest()).complete);
4927
+ assertSame(routeOf(latest()).type, "Complete");
3984
4928
  });
3985
4929
 
3986
4930
  it("refreshes routes once per sync round, however many transports it reaches", async () => {
@@ -4067,30 +5011,30 @@ describe("sync state", () => {
4067
5011
  ok: true,
4068
5012
  value: { type: "Converged" },
4069
5013
  });
4070
- assertTrue(routeOf(latest()).complete);
5014
+ assertSame(routeOf(latest()).type, "Complete");
4071
5015
 
4072
5016
  // A write for another owner leaves the route alone.
4073
5017
  await queueMutation(instance, testAppOwner2.id);
4074
- assertTrue(routeOf(latest()).complete);
5018
+ assertSame(routeOf(latest()).type, "Complete");
4075
5019
 
4076
5020
  // The owner's write waits behind it. Its upload is sent only once the
4077
5021
  // database worker answers it, so the route is incomplete from now on.
4078
5022
  await queueMutation(instance);
4079
- assertFalse(routeOf(latest()).complete);
5023
+ assertSame(routeOf(latest()).type, "Pending");
4080
5024
  await answerMutation(instance, setup.run, testAppOwner2.id);
4081
- assertFalse(routeOf(latest()).complete);
5025
+ assertSame(routeOf(latest()).type, "Pending");
4082
5026
  assertLength(createWebSocket.sentMessages.splice(0), 0);
4083
5027
 
4084
5028
  await answerMutation(instance, setup.run);
4085
5029
  assertLength(createWebSocket.sentMessages.splice(0), 1);
4086
- assertFalse(routeOf(latest()).complete);
5030
+ assertSame(routeOf(latest()).type, "Pending");
4087
5031
  createWebSocket.message(transport.url, relayResponse());
4088
5032
  await testWaitForWorkerMessage();
4089
5033
  await respondToApplySync(instance, false, {
4090
5034
  ok: true,
4091
5035
  value: { type: "Converged" },
4092
5036
  });
4093
- assertTrue(routeOf(latest()).complete);
5037
+ assertSame(routeOf(latest()).type, "Complete");
4094
5038
  });
4095
5039
 
4096
5040
  it("keeps a route complete while a local-only mutation is queued and completed", async () => {
@@ -4122,7 +5066,7 @@ describe("sync state", () => {
4122
5066
  value: { type: "Converged" },
4123
5067
  });
4124
5068
  const completedRoute = routeOf(latest());
4125
- assertTrue(completedRoute.complete);
5069
+ assertSame(completedRoute.type, "Complete");
4126
5070
 
4127
5071
  setup.run.deps.time.advance("1s");
4128
5072
  instance.evoluChannel.port2.postMessage({
@@ -4272,13 +5216,13 @@ describe("sync state", () => {
4272
5216
  while (latest().tenants.length > 0) await testWaitForWorkerMessage();
4273
5217
  await disposingSocket.promise;
4274
5218
  // A frame the mid-disposal stalled socket still delivers publishes a
4275
- // snapshot while its transport is registered. The snapshot must not
4276
- // read that socket.
5219
+ // snapshot while its transport is registered. The transport keeps the
5220
+ // connection its last event left, without reading that socket.
4277
5221
  createWebSocket.message(transports[0].url, relayResponse());
4278
5222
  await testWaitForWorkerMessage();
4279
5223
  assertEqual(
4280
- latest().transports.map((transport) => transport.readyState),
4281
- ["connecting"],
5224
+ latest().transports.map(({ connection }) => connection.type),
5225
+ ["Disconnected"],
4282
5226
  );
4283
5227
  } finally {
4284
5228
  continueDisposing.resolve();
@@ -4505,7 +5449,7 @@ describe("sync state", () => {
4505
5449
  ok: true,
4506
5450
  value: { type: "Converged" },
4507
5451
  });
4508
- assertTrue(routeOf(latest()).complete);
5452
+ assertSame(routeOf(latest()).type, "Complete");
4509
5453
 
4510
5454
  // A sibling instance writes and is disposed before the write is
4511
5455
  // answered.
@@ -4567,14 +5511,14 @@ describe("sync state", () => {
4567
5511
  // The surviving registration uploads the write; the route waits for
4568
5512
  // the answer.
4569
5513
  assertLength(createWebSocket.sentMessages.splice(0), 1);
4570
- assertFalse(routeOf(latest()).complete);
5514
+ assertSame(routeOf(latest()).type, "Pending");
4571
5515
  createWebSocket.message(transport.url, relayResponse());
4572
5516
  await testWaitForWorkerMessage();
4573
5517
  await respondToApplySync(instance, false, {
4574
5518
  ok: true,
4575
5519
  value: { type: "Converged" },
4576
5520
  });
4577
- assertTrue(routeOf(latest()).complete);
5521
+ assertSame(routeOf(latest()).type, "Complete");
4578
5522
  });
4579
5523
 
4580
5524
  it("uploads a write from an instance without a registration", async () => {
@@ -4605,7 +5549,7 @@ describe("sync state", () => {
4605
5549
  ok: true,
4606
5550
  value: { type: "Converged" },
4607
5551
  });
4608
- assertTrue(routeOf(latest()).complete);
5552
+ assertSame(routeOf(latest()).type, "Complete");
4609
5553
 
4610
5554
  // A sibling instance of the same database has no registration.
4611
5555
  const siblingId = createId<"EvoluInstance">(setup.run.deps);
@@ -4666,14 +5610,14 @@ describe("sync state", () => {
4666
5610
  createWebSocket.sentMessages.splice(0).map(({ url }) => url),
4667
5611
  [transport.url],
4668
5612
  );
4669
- assertFalse(routeOf(latest()).complete);
5613
+ assertSame(routeOf(latest()).type, "Pending");
4670
5614
  createWebSocket.message(transport.url, relayResponse());
4671
5615
  await testWaitForWorkerMessage();
4672
5616
  await respondToApplySync(instance, false, {
4673
5617
  ok: true,
4674
5618
  value: { type: "Converged" },
4675
5619
  });
4676
- assertTrue(routeOf(latest()).complete);
5620
+ assertSame(routeOf(latest()).type, "Complete");
4677
5621
  });
4678
5622
 
4679
5623
  it("requires a round when a sibling database's messages fail to apply", async () => {
@@ -4712,8 +5656,8 @@ describe("sync state", () => {
4712
5656
  value: { type: "Converged" },
4713
5657
  });
4714
5658
  }
4715
- assertTrue(routeOf(latest()).complete);
4716
- assertTrue(routeOf(latest(), secondName).complete);
5659
+ assertSame(routeOf(latest()).type, "Complete");
5660
+ assertSame(routeOf(latest(), secondName).type, "Complete");
4717
5661
 
4718
5662
  // The second database's write reaches the first one locally but is not
4719
5663
  // stored there. The relay answers the upload before the first database
@@ -4725,20 +5669,17 @@ describe("sync state", () => {
4725
5669
  ok: true,
4726
5670
  value: { type: "Converged" },
4727
5671
  });
4728
- assertTrue(routeOf(latest(), secondName).complete);
5672
+ assertSame(routeOf(latest(), secondName).type, "Complete");
4729
5673
  await respondToApplySync(first, false, {
4730
5674
  ok: false,
4731
- error: {
4732
- type: "DecryptWithXChaCha20Poly1305Error",
4733
- error: "sibling copy rejected",
4734
- },
5675
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
4735
5676
  });
4736
5677
  await respondToApplySync(first, false, {
4737
5678
  ok: true,
4738
5679
  value: { type: "Converged" },
4739
5680
  });
4740
5681
  // Nothing is outstanding, yet the failure requires a round.
4741
- assertFalse(routeOf(latest()).complete);
5682
+ assertSame(routeOf(latest()).type, "Pending");
4742
5683
 
4743
5684
  // The round requested by the failure fetches the messages again.
4744
5685
  await respondToSyncRound(first, createWebSocket);
@@ -4752,7 +5693,7 @@ describe("sync state", () => {
4752
5693
  ok: true,
4753
5694
  value: { type: "Converged" },
4754
5695
  });
4755
- assertTrue(routeOf(latest()).complete);
5696
+ assertSame(routeOf(latest()).type, "Complete");
4756
5697
  });
4757
5698
 
4758
5699
  it("retries every route after a sibling continuation copy fails without blaming its source relay", async () => {
@@ -4777,25 +5718,24 @@ describe("sync state", () => {
4777
5718
  });
4778
5719
  await respondToApplySync(sibling, false, {
4779
5720
  ok: false,
4780
- error: {
4781
- type: "DecryptWithXChaCha20Poly1305Error",
4782
- error: "continuation copy rejected",
4783
- },
5721
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
4784
5722
  });
4785
5723
  for (const index of [0, 1]) {
4786
5724
  const route = routeOf(latest(), siblingName, index);
4787
- assertFalse(route.complete);
4788
- assertSame(route.error, null);
5725
+ assertSame(route.type, "Pending");
5726
+ assertSame(route.failure, null);
5727
+ assertSame(route.skippedError, null);
4789
5728
  assertSame(route.lastReceivedAt, lastReceivedAt[index]);
4790
5729
  }
4791
5730
  await respondToApplySync(sibling, false, {
4792
5731
  ok: true,
4793
5732
  value: { type: "Converged" },
4794
5733
  });
4795
- assertEqual(await respondToSyncRound(sibling, createWebSocket), [
4796
- transports[0].url,
4797
- transports[1].url,
4798
- ]);
5734
+ for (const transport of transports) {
5735
+ assertEqual(await respondToSyncRound(sibling, createWebSocket), [
5736
+ transport.url,
5737
+ ]);
5738
+ }
4799
5739
 
4800
5740
  for (const transport of transports) {
4801
5741
  createWebSocket.message(transport.url, relayResponse());
@@ -4808,8 +5748,181 @@ describe("sync state", () => {
4808
5748
  }
4809
5749
  }
4810
5750
  for (const index of [0, 1]) {
4811
- assertTrue(routeOf(latest(), siblingName, index).complete);
5751
+ assertSame(routeOf(latest(), siblingName, index).type, "Complete");
5752
+ }
5753
+ });
5754
+
5755
+ const siblingCopyOutcomes = [
5756
+ {
5757
+ outcome: "error",
5758
+ result: {
5759
+ ok: false,
5760
+ error: {
5761
+ type: "ProtocolInvalidDataError",
5762
+ data: Uint8Array.of(255),
5763
+ error: "decoding failed",
5764
+ },
5765
+ },
5766
+ skipped: null,
5767
+ posted: {
5768
+ type: "ProtocolInvalidDataError",
5769
+ error: createUnknownError("decoding failed"),
5770
+ },
5771
+ },
5772
+ {
5773
+ outcome: "skip",
5774
+ result: { ok: true, value: { type: "Broadcast" } },
5775
+ skipped: skippedError,
5776
+ posted: {
5777
+ ...skippedError,
5778
+ error: createUnknownError(skippedError.error),
5779
+ },
5780
+ },
5781
+ {
5782
+ outcome: "Failed result",
5783
+ result: { ok: true, value: { type: "Failed", cause: "Write" } },
5784
+ skipped: null,
5785
+ posted: null,
5786
+ },
5787
+ {
5788
+ outcome: "abort",
5789
+ result: {
5790
+ ok: false,
5791
+ error: { type: "AbortError", reason: { type: "Stop" } },
5792
+ },
5793
+ skipped: null,
5794
+ posted: null,
5795
+ },
5796
+ ] satisfies ReadonlyArray<{
5797
+ outcome: string;
5798
+ result: ApplySyncResponse["result"];
5799
+ skipped: DecryptWithXChaCha20Poly1305Error | null;
5800
+ posted:
5801
+ | Omit<ProtocolInvalidDataError, "data">
5802
+ | DecryptWithXChaCha20Poly1305Error
5803
+ | null;
5804
+ }>;
5805
+ for (const { outcome, result, skipped, posted } of siblingCopyOutcomes) {
5806
+ it(`${posted === null ? "does not post" : "posts an UnknownError for"} a sibling continuation copy's ${outcome}`, async () => {
5807
+ const createWebSocket = testCreateWebSocket();
5808
+ await using setup = await setupSharedWorker({ createWebSocket });
5809
+ const errors: Array<ConsoleEntryOrError> = [];
5810
+ using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
5811
+ consoleEntryOrErrorBroadcastChannelName,
5812
+ );
5813
+ errorChannel.onMessage = (output) => {
5814
+ if (output.type === "Error") errors.push(output);
5815
+ };
5816
+ const { sibling } = await setupSiblingContinuation(
5817
+ setup,
5818
+ createWebSocket,
5819
+ );
5820
+
5821
+ await respondToApplySync(sibling, false, result, skipped);
5822
+
5823
+ // No relay sent the copy, so no route can show its problem. An error
5824
+ // or a skip means a bug and is posted as an unexpected failure,
5825
+ // without the frame, as a route shows it. The protocol already logged
5826
+ // a Failed result, and the tab turns that log into an unexpected
5827
+ // failure, so it is not posted again. An abort is not a failure.
5828
+ assertEqual(
5829
+ errors,
5830
+ posted === null
5831
+ ? []
5832
+ : [
5833
+ {
5834
+ type: "Error",
5835
+ error: createUnknownError({
5836
+ ...posted,
5837
+ at: setup.run.deps.time.now(),
5838
+ }),
5839
+ },
5840
+ ],
5841
+ );
5842
+ });
5843
+ }
5844
+
5845
+ it("retries every route after a sibling continuation copy skips a message", async () => {
5846
+ const createWebSocket = testCreateWebSocket();
5847
+ await using setup = await setupSharedWorker({ createWebSocket });
5848
+ using disposer = new DisposableStack();
5849
+ const { latest } = setupSyncStates(setup, disposer);
5850
+ const { source, sibling, siblingName, transports } =
5851
+ await setupSiblingContinuation(setup, createWebSocket);
5852
+
5853
+ createWebSocket.message(transports[0].url, relayResponse());
5854
+ await testWaitForWorkerMessage();
5855
+ await respondToApplySync(source, false, {
5856
+ ok: true,
5857
+ value: { type: "Converged" },
5858
+ });
5859
+ await respondToApplySync(
5860
+ sibling,
5861
+ false,
5862
+ { ok: true, value: { type: "Broadcast" } },
5863
+ skippedError,
5864
+ );
5865
+ await respondToApplySync(sibling, false, {
5866
+ ok: true,
5867
+ value: { type: "Converged" },
5868
+ });
5869
+ for (const transport of transports) {
5870
+ assertEqual(await respondToSyncRound(sibling, createWebSocket), [
5871
+ transport.url,
5872
+ ]);
5873
+ }
5874
+
5875
+ // The relays offer the skipped message, which fails their routes
5876
+ // without requesting another round, so each relay frame adds only its
5877
+ // apply.
5878
+ const inputCount = sibling.dbInputs.length;
5879
+ for (const transport of transports) {
5880
+ createWebSocket.message(transport.url, relayResponse());
5881
+ await testWaitForWorkerMessage();
5882
+ await respondToApplySync(source, false, {
5883
+ ok: true,
5884
+ value: { type: "Converged" },
5885
+ });
5886
+ await respondToApplySync(
5887
+ sibling,
5888
+ false,
5889
+ { ok: true, value: { type: "Converged" } },
5890
+ skippedError,
5891
+ );
5892
+ }
5893
+ assertLength(sibling.dbInputs, inputCount + transports.length);
5894
+ for (const index of [0, 1]) {
5895
+ const route = routeOf(latest(), siblingName, index);
5896
+ assertSame(route.type, "Settled");
5897
+ assertSame(route.skippedError.type, skippedError.type);
4812
5898
  }
5899
+
5900
+ // Another copy that skips a message requests no round, because each
5901
+ // round would download the skipped messages again.
5902
+ createWebSocket.message(transports[0].url, relayResponse());
5903
+ await testWaitForWorkerMessage();
5904
+ await respondToApplySync(source, false, {
5905
+ ok: true,
5906
+ value: {
5907
+ type: "Response",
5908
+ message: createProtocolMessageForUnsubscribe(testAppOwner.id),
5909
+ broadcast: createProtocolMessageBuffer(testAppOwner.id, {
5910
+ messageType: MessageType.Broadcast,
5911
+ }).unwrap(),
5912
+ },
5913
+ });
5914
+ await respondToApplySync(sibling, false, {
5915
+ ok: true,
5916
+ value: { type: "Converged" },
5917
+ });
5918
+ const copyInputCount = sibling.dbInputs.length;
5919
+ await respondToApplySync(
5920
+ sibling,
5921
+ false,
5922
+ { ok: true, value: { type: "Broadcast" } },
5923
+ skippedError,
5924
+ );
5925
+ assertLength(sibling.dbInputs, copyInputCount);
4813
5926
  });
4814
5927
 
4815
5928
  it("requires a new round through the other transports when a relay stores messages", async () => {
@@ -4846,8 +5959,8 @@ describe("sync state", () => {
4846
5959
  value: { type: "Converged" },
4847
5960
  });
4848
5961
  }
4849
- assertTrue(routeOf(latest(), testName, 0).complete);
4850
- assertTrue(routeOf(latest(), testName, 1).complete);
5962
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5963
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
4851
5964
 
4852
5965
  // Messages stored from the first relay invalidate the second route.
4853
5966
  createWebSocket.message(transports[0].url, relayResponse());
@@ -4856,8 +5969,8 @@ describe("sync state", () => {
4856
5969
  ok: true,
4857
5970
  value: { type: "Converged" },
4858
5971
  });
4859
- assertTrue(routeOf(latest(), testName, 0).complete);
4860
- assertFalse(routeOf(latest(), testName, 1).complete);
5972
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5973
+ assertSame(routeOf(latest(), testName, 1).type, "Pending");
4861
5974
 
4862
5975
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
4863
5976
  transports[1].url,
@@ -4868,267 +5981,888 @@ describe("sync state", () => {
4868
5981
  ok: true,
4869
5982
  value: { type: "Converged" },
4870
5983
  });
4871
- assertTrue(routeOf(latest(), testName, 1).complete);
5984
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
4872
5985
  });
4873
5986
 
4874
- it("never completes a route after a replacement leader refused startup", async () => {
4875
- const createWebSocket = testCreateWebSocket();
4876
- await using setup = await setupSharedWorker({ createWebSocket });
4877
- using disposer = new DisposableStack();
5987
+ /**
5988
+ * Opens routes through two relays and settles the second with a skipped
5989
+ * message.
5990
+ */
5991
+ const setupSkippingSecondRoute = async (
5992
+ setup: SharedWorkerSetup,
5993
+ disposer: DisposableStack,
5994
+ createWebSocket: TestCreateWebSocket,
5995
+ ) => {
4878
5996
  const { latest } = setupSyncStates(setup, disposer);
4879
- const { time } = setup.run.deps;
4880
- const instance = await setup.createEvolu({
4881
- releaseDbWorkerLeaderOnDispose: false,
4882
- });
4883
- const transport = createOwnerWebSocketTransport({
4884
- url: "wss://refused-route.example",
4885
- ownerId: testAppOwner.id,
4886
- });
5997
+ const instance = await setup.createEvolu();
5998
+ const transports = [
5999
+ createOwnerWebSocketTransport({
6000
+ url: "wss://first.example",
6001
+ ownerId: testAppOwner.id,
6002
+ }),
6003
+ createOwnerWebSocketTransport({
6004
+ url: "wss://skipping.example",
6005
+ ownerId: testAppOwner.id,
6006
+ }),
6007
+ ] as const;
6008
+
4887
6009
  instance.evoluChannel.port2.postMessage({
4888
6010
  type: "UseOwner",
4889
6011
  actions: [
4890
- {
4891
- owner: { owner: testAppOwner, transports: [transport] },
4892
- action: "add",
4893
- },
6012
+ { owner: { owner: testAppOwner, transports }, action: "add" },
4894
6013
  ],
4895
6014
  });
4896
6015
  await testWaitForWorkerMessage();
4897
6016
  await respondToSyncRound(instance, createWebSocket);
4898
- createWebSocket.message(transport.url, relayResponse());
6017
+ await respondToSyncRound(instance, createWebSocket);
6018
+ createWebSocket.message(transports[0].url, relayResponse());
4899
6019
  await testWaitForWorkerMessage();
4900
6020
  await respondToApplySync(instance, false, {
4901
6021
  ok: true,
4902
6022
  value: { type: "Converged" },
4903
6023
  });
4904
- assertTrue(routeOf(latest()).complete);
4905
- const completeAt = routeOf(latest()).completeAt;
6024
+ createWebSocket.message(transports[1].url, relayResponse());
6025
+ await testWaitForWorkerMessage();
6026
+ await respondToApplySync(
6027
+ instance,
6028
+ false,
6029
+ { ok: true, value: { type: "Converged" } },
6030
+ skippedError,
6031
+ );
6032
+ return { instance, latest, transports };
6033
+ };
4906
6034
 
4907
- // The queued write alone keeps the route incomplete.
4908
- time.advance("1s");
4909
- await queueMutation(instance);
4910
- assertFalse(routeOf(latest()).complete);
4911
- await instance.releaseDbWorkerLeader();
6035
+ it("requests no round through a route that skipped a message when a relay stores messages", async () => {
6036
+ const createWebSocket = testCreateWebSocket();
6037
+ await using setup = await setupSharedWorker({ createWebSocket });
6038
+ using disposer = new DisposableStack();
6039
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
6040
+ setup,
6041
+ disposer,
6042
+ createWebSocket,
6043
+ );
6044
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
6045
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
4912
6046
 
4913
- const init = await setupTabLeader(setup, disposer);
4914
- using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
4915
- init.port,
6047
+ // Messages stored from the first relay request no round through the
6048
+ // second, whose relay would offer the skipped message again.
6049
+ createWebSocket.message(transports[0].url, relayResponse());
6050
+ await testWaitForWorkerMessage();
6051
+ const inputCount = instance.dbInputs.length;
6052
+ await respondToApplySync(instance, true, {
6053
+ ok: true,
6054
+ value: { type: "Converged" },
6055
+ });
6056
+ assertLength(instance.dbInputs, inputCount);
6057
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
6058
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
6059
+
6060
+ // Messages stored from the second relay still reconcile through the
6061
+ // first.
6062
+ createWebSocket.message(transports[1].url, relayResponse());
6063
+ await testWaitForWorkerMessage();
6064
+ await respondToApplySync(
6065
+ instance,
6066
+ true,
6067
+ { ok: true, value: { type: "Converged" } },
6068
+ skippedError,
4916
6069
  );
4917
- leaderPort.postMessage({
4918
- type: "LeaderRefused",
4919
- name: testName,
4920
- error: {
4921
- type: "UnsupportedDbVersionError",
4922
- storedVersion: PositiveInt.orThrow(2),
4923
- supportedVersion: PositiveInt.orThrow(1),
6070
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6071
+ transports[0].url,
6072
+ ]);
6073
+
6074
+ // An explicit request checks the second relay again. Messages stored
6075
+ // from the first relay meanwhile request no round through it either,
6076
+ // and the check may not include them, so it no longer completes the
6077
+ // route even when it skips nothing.
6078
+ const sync = () => {
6079
+ instance.evoluChannel.port2.postMessage({
6080
+ type: "UseOwner",
6081
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
6082
+ });
6083
+ };
6084
+ sync();
6085
+ await testWaitForWorkerMessage();
6086
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6087
+ transports[0].url,
6088
+ transports[1].url,
6089
+ ]);
6090
+ createWebSocket.message(transports[0].url, relayResponse());
6091
+ await testWaitForWorkerMessage();
6092
+ const checkInputCount = instance.dbInputs.length;
6093
+ await respondToApplySync(instance, true, {
6094
+ ok: true,
6095
+ value: { type: "Converged" },
6096
+ });
6097
+ assertLength(instance.dbInputs, checkInputCount);
6098
+ createWebSocket.message(transports[1].url, relayResponse());
6099
+ await testWaitForWorkerMessage();
6100
+ await respondToApplySync(instance, false, {
6101
+ ok: true,
6102
+ value: { type: "Converged" },
6103
+ });
6104
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
6105
+
6106
+ // The next request checks it again with those messages and completes
6107
+ // the route once the relay no longer offers the skipped message.
6108
+ sync();
6109
+ await testWaitForWorkerMessage();
6110
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6111
+ transports[0].url,
6112
+ transports[1].url,
6113
+ ]);
6114
+ createWebSocket.message(transports[1].url, relayResponse());
6115
+ await testWaitForWorkerMessage();
6116
+ await respondToApplySync(instance, false, {
6117
+ ok: true,
6118
+ value: { type: "Converged" },
6119
+ });
6120
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
6121
+ });
6122
+
6123
+ it("keeps a route's skip when the database fails to create a requested round", async () => {
6124
+ const createWebSocket = testCreateWebSocket();
6125
+ await using setup = await setupSharedWorker({ createWebSocket });
6126
+ using disposer = new DisposableStack();
6127
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
6128
+ setup,
6129
+ disposer,
6130
+ createWebSocket,
6131
+ );
6132
+
6133
+ // The database worker fails to create the requested round, so the
6134
+ // second route shows the failure.
6135
+ instance.evoluChannel.port2.postMessage({
6136
+ type: "UseOwner",
6137
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
6138
+ });
6139
+ await testWaitForWorkerMessage();
6140
+ const round = instance.dbInputs.at(-1);
6141
+ assertNotUndefined(round);
6142
+ assertSame(round.request.type, "ForSharedWorker");
6143
+ assertSame(round.request.message.type, "CreateSyncMessages");
6144
+ instance.dbWorkerPort.postMessage({
6145
+ type: "OnQueuedResponse",
6146
+ attemptId: round.attemptId,
6147
+ response: {
6148
+ type: "ForSharedWorker",
6149
+ message: {
6150
+ type: "CreateSyncMessages",
6151
+ protocolMessagesByOwnerId: new Map(),
6152
+ failedOwnerIds: new Set([testAppOwner.id]),
6153
+ },
4924
6154
  },
4925
6155
  });
4926
6156
  await testWaitForWorkerMessage();
6157
+ assertSame(failureOf(routeOf(latest(), testName, 1))?.type, "SyncFailed");
4927
6158
 
4928
- // Refusal discards the write, which is never uploaded.
4929
- assertTrue(latest().tenants[0].refused);
4930
- assertFalse(routeOf(latest()).complete);
4931
- assertSame(routeOf(latest()).completeAt, completeAt);
6159
+ // The failed route keeps its skip, so messages stored from the first
6160
+ // relay still request no round through the second.
6161
+ createWebSocket.message(transports[0].url, relayResponse());
6162
+ await testWaitForWorkerMessage();
6163
+ const inputCount = instance.dbInputs.length;
6164
+ await respondToApplySync(instance, true, {
6165
+ ok: true,
6166
+ value: { type: "Converged" },
6167
+ });
6168
+ assertLength(instance.dbInputs, inputCount);
6169
+ });
4932
6170
 
4933
- // Later refreshes, such as a relay frame, keep it incomplete.
4934
- createWebSocket.message(transport.url, relayResponse());
6171
+ it("shows the skip, not a recovered failure, once a route settles", async () => {
6172
+ const createWebSocket = testCreateWebSocket();
6173
+ await using setup = await setupSharedWorker({ createWebSocket });
6174
+ using disposer = new DisposableStack();
6175
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
6176
+ setup,
6177
+ disposer,
6178
+ createWebSocket,
6179
+ );
6180
+ assertSame(
6181
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6182
+ skippedError.type,
6183
+ );
6184
+
6185
+ // A failure on the second route requests a round that checks the relay
6186
+ // again, but messages stored from the first relay meanwhile keep it from
6187
+ // completing the route.
6188
+ createWebSocket.message(transports[1].url, relayResponse());
6189
+ await testWaitForWorkerMessage();
6190
+ await respondToApplySync(instance, false, {
6191
+ ok: false,
6192
+ error: { type: "ProtocolQuotaError", ownerId: testAppOwner.id },
6193
+ });
6194
+ assertSame(
6195
+ failureOf(routeOf(latest(), testName, 1))?.type,
6196
+ "ProtocolQuotaError",
6197
+ );
6198
+ // The skip stays visible beside the failure.
6199
+ assertSame(
6200
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6201
+ skippedError.type,
6202
+ );
6203
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6204
+ transports[1].url,
6205
+ ]);
6206
+ createWebSocket.message(transports[0].url, relayResponse());
6207
+ await testWaitForWorkerMessage();
6208
+ await respondToApplySync(instance, true, {
6209
+ ok: true,
6210
+ value: { type: "Converged" },
6211
+ });
6212
+
6213
+ // The round recovers from the failure, and the route settles showing the
6214
+ // skip that keeps it incomplete.
6215
+ createWebSocket.message(transports[1].url, relayResponse());
4935
6216
  await testWaitForWorkerMessage();
4936
- assertFalse(routeOf(latest()).complete);
6217
+ await respondToApplySync(instance, false, {
6218
+ ok: true,
6219
+ value: { type: "Converged" },
6220
+ });
6221
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
6222
+ assertSame(
6223
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6224
+ skippedError.type,
6225
+ );
6226
+ });
6227
+
6228
+ it("publishes a refused database with its error and no owners", async () => {
6229
+ const createWebSocket = testCreateWebSocket();
6230
+ await using setup = await setupSharedWorker({ createWebSocket });
6231
+ using disposer = new DisposableStack();
6232
+ const { latest } = setupSyncStates(setup, disposer);
6233
+ const instance = await setup.createEvolu({
6234
+ releaseDbWorkerLeaderOnDispose: false,
6235
+ });
6236
+ instance.evoluChannel.port2.postMessage({
6237
+ type: "UseOwner",
6238
+ actions: [
6239
+ {
6240
+ owner: {
6241
+ owner: testAppOwner,
6242
+ transports: [
6243
+ createOwnerWebSocketTransport({
6244
+ url: "wss://refused-route.example",
6245
+ ownerId: testAppOwner.id,
6246
+ }),
6247
+ ],
6248
+ },
6249
+ action: "add",
6250
+ },
6251
+ ],
6252
+ });
6253
+ await testWaitForWorkerMessage();
6254
+ assertSame(routeOf(latest()).type, "Pending");
6255
+ await instance.releaseDbWorkerLeader();
6256
+
6257
+ const init = await setupTabLeader(setup, disposer);
6258
+ using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
6259
+ init.port,
6260
+ );
6261
+ const error: UnsupportedDbVersionError = {
6262
+ type: "UnsupportedDbVersionError",
6263
+ storedVersion: PositiveInt.orThrow(2),
6264
+ supportedVersion: PositiveInt.orThrow(1),
6265
+ };
6266
+ leaderPort.postMessage({ type: "LeaderRefused", name: testName, error });
6267
+ await testWaitForWorkerMessage();
6268
+
6269
+ // Nothing the refused database holds syncs, so it publishes its error
6270
+ // instead of its registrations, and no relay syncs its owner.
6271
+ const refused: ReadonlyArray<SyncTenant> = [
6272
+ { type: "Refused", name: testName, error },
6273
+ ];
6274
+ assertEqual(latest().tenants, refused);
6275
+ assertEqual(
6276
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
6277
+ { type: "NoRelays" },
6278
+ );
6279
+ assertEqual(
6280
+ syncStateToRelaySyncStates(latest(), testName, testAppOwner.id),
6281
+ [],
6282
+ );
6283
+ });
6284
+ });
6285
+ });
6286
+
6287
+ const derivationDeps = testCreateDeps();
6288
+ const connectingId = createId<"SyncTransport">(derivationDeps);
6289
+ const openId = createId<"SyncTransport">(derivationDeps);
6290
+ const otherOpenId = createId<"SyncTransport">(derivationDeps);
6291
+ const disconnectedId = createId<"SyncTransport">(derivationDeps);
6292
+
6293
+ const connectingConnection: SyncConnection = { type: "Connecting" };
6294
+ const openConnection: SyncConnection = {
6295
+ type: "Open",
6296
+ openedAt: Millis.orThrow(1000),
6297
+ error: null,
6298
+ };
6299
+ const disconnectedConnection: SyncConnection = {
6300
+ type: "Disconnected",
6301
+ disconnectedAt: Millis.orThrow(2000),
6302
+ openedAt: Millis.orThrow(1000),
6303
+ error: null,
6304
+ };
6305
+
6306
+ const createSyncTransport = (
6307
+ id: SyncTransportId,
6308
+ connection: SyncConnection,
6309
+ ): SyncTransport => ({
6310
+ type: "WebSocket",
6311
+ id,
6312
+ label: "wss://relay.example",
6313
+ connection,
6314
+ });
6315
+
6316
+ const syncTransports = [
6317
+ createSyncTransport(connectingId, connectingConnection),
6318
+ createSyncTransport(openId, openConnection),
6319
+ createSyncTransport(otherOpenId, openConnection),
6320
+ createSyncTransport(disconnectedId, disconnectedConnection),
6321
+ ] as const;
6322
+
6323
+ const createPendingRoute = (
6324
+ transportId: SyncTransportId,
6325
+ route: Partial<Omit<PendingSyncRoute, "type" | "transportId">> = {},
6326
+ ): PendingSyncRoute => ({
6327
+ type: "Pending",
6328
+ transportId,
6329
+ failure: null,
6330
+ skippedError: null,
6331
+ completeAt: null,
6332
+ lastSentAt: null,
6333
+ lastReceivedAt: null,
6334
+ ...route,
6335
+ });
6336
+
6337
+ const createSettledRoute = (
6338
+ transportId: SyncTransportId,
6339
+ skippedError: SyncRouteError,
6340
+ ): SettledSyncRoute => ({
6341
+ type: "Settled",
6342
+ transportId,
6343
+ skippedError,
6344
+ completeAt: null,
6345
+ lastSentAt: Millis.orThrow(1500),
6346
+ lastReceivedAt: Millis.orThrow(1600),
6347
+ });
6348
+
6349
+ const createCompleteRoute = (
6350
+ transportId: SyncTransportId,
6351
+ ): CompleteSyncRoute => ({
6352
+ type: "Complete",
6353
+ transportId,
6354
+ completeAt: Millis.orThrow(3000),
6355
+ lastSentAt: Millis.orThrow(2500),
6356
+ lastReceivedAt: Millis.orThrow(3000),
6357
+ });
6358
+
6359
+ const quotaFailure = (at: number): SyncRouteError => ({
6360
+ type: "ProtocolQuotaError",
6361
+ ownerId: testAppOwner.id,
6362
+ at: Millis.orThrow(at),
6363
+ });
6364
+
6365
+ const skippedChange = (at: number): SyncRouteError => ({
6366
+ type: "DecryptWithXChaCha20Poly1305Error",
6367
+ error: "wrong encryption key",
6368
+ at: Millis.orThrow(at),
6369
+ });
6370
+
6371
+ const derivationRefusal: UnsupportedDbVersionError = {
6372
+ type: "UnsupportedDbVersionError",
6373
+ storedVersion: PositiveInt.orThrow(2),
6374
+ supportedVersion: PositiveInt.orThrow(1),
6375
+ };
6376
+
6377
+ /** A snapshot whose database syncs the app owner through `routes`. */
6378
+ const createRoutesState = (routes: ReadonlyArray<SyncRoute>): SyncState => ({
6379
+ transports: syncTransports,
6380
+ tenants: [
6381
+ {
6382
+ type: "Active",
6383
+ name: testName,
6384
+ owners: [{ type: "Writable", ownerId: testAppOwner.id, routes }],
6385
+ },
6386
+ ],
6387
+ });
6388
+
6389
+ const ownerStatusOf = (routes: ReadonlyArray<SyncRoute>): OwnerSyncStatus =>
6390
+ syncStateToOwnerSyncStatus(
6391
+ createRoutesState(routes),
6392
+ testName,
6393
+ testAppOwner.id,
6394
+ );
6395
+
6396
+ describe("SyncState", () => {
6397
+ it("accepts only states the shared worker can be in", () => {
6398
+ const transportId = openId;
6399
+ const ownerId = testAppOwner.id;
6400
+
6401
+ const connections: ReadonlyArray<SyncConnection> = [
6402
+ { type: "Connecting" },
6403
+ {
6404
+ type: "Connecting",
6405
+ // @ts-expect-error A first connection has no error; one that failed is Disconnected.
6406
+ error: null,
6407
+ },
6408
+ { type: "Open", openedAt: Millis.orThrow(1000), error: null },
6409
+ // @ts-expect-error An open connection has openedAt, when it opened.
6410
+ { type: "Open", error: null },
6411
+ ];
6412
+
6413
+ const routes: ReadonlyArray<SyncRoute> = [
6414
+ {
6415
+ type: "Complete",
6416
+ transportId,
6417
+ completeAt: Millis.orThrow(1000),
6418
+ lastSentAt: Millis.orThrow(900),
6419
+ lastReceivedAt: null,
6420
+ },
6421
+ // @ts-expect-error A complete route has when it became complete.
6422
+ {
6423
+ type: "Complete",
6424
+ transportId,
6425
+ completeAt: null,
6426
+ lastSentAt: Millis.orThrow(900),
6427
+ lastReceivedAt: null,
6428
+ },
6429
+ // @ts-expect-error A complete route has sent a round, so it has when it last sent.
6430
+ {
6431
+ type: "Complete",
6432
+ transportId,
6433
+ completeAt: Millis.orThrow(1000),
6434
+ lastSentAt: null,
6435
+ lastReceivedAt: null,
6436
+ },
6437
+ {
6438
+ type: "Complete",
6439
+ transportId,
6440
+ completeAt: Millis.orThrow(1000),
6441
+ lastSentAt: Millis.orThrow(900),
6442
+ lastReceivedAt: null,
6443
+ // @ts-expect-error A complete route has no failure.
6444
+ failure: null,
6445
+ },
6446
+ {
6447
+ type: "Settled",
6448
+ transportId,
6449
+ skippedError: skippedChange(1000),
6450
+ completeAt: null,
6451
+ lastSentAt: Millis.orThrow(900),
6452
+ lastReceivedAt: null,
6453
+ },
6454
+ // @ts-expect-error A settled route has sent a round, so it has when it last sent.
6455
+ {
6456
+ type: "Settled",
6457
+ transportId,
6458
+ skippedError: skippedChange(1000),
6459
+ completeAt: null,
6460
+ lastSentAt: null,
6461
+ lastReceivedAt: null,
6462
+ },
6463
+ {
6464
+ type: "Settled",
6465
+ transportId,
6466
+ // @ts-expect-error A settled route has the skipped change that keeps it incomplete.
6467
+ skippedError: null,
6468
+ completeAt: null,
6469
+ lastSentAt: Millis.orThrow(900),
6470
+ lastReceivedAt: null,
6471
+ },
6472
+ ];
6473
+
6474
+ const owners: ReadonlyArray<SyncTenantOwner> = [
6475
+ { type: "Readonly", ownerId, transportIds: [transportId] },
6476
+ {
6477
+ type: "Readonly",
6478
+ ownerId,
6479
+ transportIds: [transportId],
6480
+ // @ts-expect-error A readonly owner has no routes; its database does not sync it.
6481
+ routes: [],
6482
+ },
6483
+ ];
6484
+
6485
+ const tenants: ReadonlyArray<SyncTenant> = [
6486
+ { type: "Refused", name: testName, error: derivationRefusal },
6487
+ {
6488
+ type: "Refused",
6489
+ name: testName,
6490
+ error: derivationRefusal,
6491
+ // @ts-expect-error A refused database has no owners; nothing it holds syncs.
6492
+ owners: [],
6493
+ },
6494
+ ];
6495
+
6496
+ assertLength(connections, 4);
6497
+ assertLength(routes, 7);
6498
+ assertLength(owners, 2);
6499
+ assertLength(tenants, 2);
6500
+
6501
+ const transport = createSyncTransport(openId, openConnection);
6502
+ // @ts-expect-error A transport has no readyState; its connection tells whether it is open.
6503
+ assertSame(transport.readyState, undefined);
6504
+ });
6505
+ });
6506
+
6507
+ describe("syncStateToRelaySyncStates", () => {
6508
+ it("pairs each route with its transport in route order", () => {
6509
+ const routes = [
6510
+ createPendingRoute(disconnectedId),
6511
+ createCompleteRoute(openId),
6512
+ createPendingRoute(connectingId),
6513
+ ];
6514
+ assertEqual(
6515
+ syncStateToRelaySyncStates(
6516
+ createRoutesState(routes),
6517
+ testName,
6518
+ testAppOwner.id,
6519
+ ),
6520
+ [
6521
+ { transport: syncTransports[3], route: routes[0] },
6522
+ { transport: syncTransports[1], route: routes[1] },
6523
+ { transport: syncTransports[0], route: routes[2] },
6524
+ ],
6525
+ );
6526
+ });
6527
+
6528
+ it("returns none without an active database or a writable owner", () => {
6529
+ const state = createRoutesState([createCompleteRoute(openId)]);
6530
+ const secondName = Name.orThrow("second");
6531
+ const withTenant = (tenant: SyncTenant): SyncState => ({
6532
+ transports: syncTransports,
6533
+ tenants: [tenant],
4937
6534
  });
6535
+
6536
+ assertEqual(
6537
+ syncStateToRelaySyncStates(null, testName, testAppOwner.id),
6538
+ [],
6539
+ );
6540
+ assertEqual(
6541
+ syncStateToRelaySyncStates(state, secondName, testAppOwner.id),
6542
+ [],
6543
+ );
6544
+ assertEqual(
6545
+ syncStateToRelaySyncStates(state, testName, testAppOwner2.id),
6546
+ [],
6547
+ );
6548
+ assertEqual(
6549
+ syncStateToRelaySyncStates(
6550
+ withTenant({
6551
+ type: "Refused",
6552
+ name: testName,
6553
+ error: derivationRefusal,
6554
+ }),
6555
+ testName,
6556
+ testAppOwner.id,
6557
+ ),
6558
+ [],
6559
+ );
6560
+ assertEqual(
6561
+ syncStateToRelaySyncStates(
6562
+ withTenant({
6563
+ type: "Active",
6564
+ name: testName,
6565
+ owners: [
6566
+ {
6567
+ type: "Readonly",
6568
+ ownerId: testAppOwner.id,
6569
+ transportIds: [openId],
6570
+ },
6571
+ ],
6572
+ }),
6573
+ testName,
6574
+ testAppOwner.id,
6575
+ ),
6576
+ [],
6577
+ );
4938
6578
  });
4939
6579
  });
4940
6580
 
4941
- describe("syncStateToOwnerSyncStates", () => {
4942
- const deps = testCreateDeps();
4943
- const openA = createId<"SyncTransport">(deps);
4944
- const openB = createId<"SyncTransport">(deps);
4945
- const closedId = createId<"SyncTransport">(deps);
4946
- const connectingId = createId<"SyncTransport">(deps);
4947
- const createTransport = (
4948
- id: SyncTransportId,
4949
- readyState: SyncTransport["readyState"],
4950
- ): SyncTransport => ({
4951
- id,
4952
- label: "wss://relay.example",
4953
- readyState,
4954
- openedAt: null,
4955
- closedAt: null,
4956
- error: null,
4957
- });
4958
- const transports = [
4959
- createTransport(openA, "open"),
4960
- createTransport(openB, "open"),
4961
- createTransport(closedId, "closed"),
4962
- createTransport(connectingId, "connecting"),
4963
- ];
4964
- const createRoute = (
4965
- transportId: SyncTransportId,
4966
- route: Partial<Omit<SyncRoute, "transportId">> = {},
4967
- ): SyncRoute => ({
4968
- transportId,
4969
- complete: false,
4970
- completeAt: null,
4971
- lastSentAt: null,
4972
- lastReceivedAt: null,
4973
- error: null,
4974
- ...route,
4975
- });
4976
- const createOwner = (
4977
- ownerId: OwnerId,
4978
- routes: ReadonlyArray<SyncRoute>,
4979
- writable = true,
4980
- ): SyncTenantOwner => ({
4981
- ownerId,
4982
- writable,
4983
- transportIds: routes.map(({ transportId }) => transportId),
4984
- routes,
4985
- });
4986
- const createState = (tenants: ReadonlyArray<SyncTenant>): SyncState => ({
4987
- transports,
4988
- tenants,
6581
+ describe("relaySyncStateToStatus", () => {
6582
+ it("tells the status of every route on every connection", () => {
6583
+ for (const transport of [
6584
+ syncTransports[0],
6585
+ syncTransports[1],
6586
+ syncTransports[3],
6587
+ ]) {
6588
+ const statusOf = (route: SyncRoute): RelaySyncStatus =>
6589
+ relaySyncStateToStatus({ transport, route });
6590
+ const { id } = transport;
6591
+
6592
+ assertEqual(statusOf(createCompleteRoute(id)), { type: "Synced" });
6593
+ assertEqual(statusOf(createSettledRoute(id, skippedChange(2000))), {
6594
+ type: "Error",
6595
+ error: skippedChange(2000),
6596
+ });
6597
+ assertEqual(
6598
+ statusOf(createPendingRoute(id)),
6599
+ transport.connection.type === "Disconnected"
6600
+ ? { type: "Offline" }
6601
+ : { type: "Syncing" },
6602
+ );
6603
+ assertEqual(
6604
+ statusOf(createPendingRoute(id, { skippedError: skippedChange(2000) })),
6605
+ { type: "Error", error: skippedChange(2000) },
6606
+ );
6607
+ // A failure shows before a skipped change, even a newer one.
6608
+ assertEqual(
6609
+ statusOf(
6610
+ createPendingRoute(id, {
6611
+ failure: quotaFailure(1000),
6612
+ skippedError: skippedChange(2000),
6613
+ }),
6614
+ ),
6615
+ { type: "Error", error: quotaFailure(1000) },
6616
+ );
6617
+ }
4989
6618
  });
4990
- const statusOf = (routes: ReadonlyArray<SyncRoute>): string => {
4991
- const [owner] = syncStateToOwnerSyncStates(
4992
- createState([
4993
- {
6619
+ });
6620
+
6621
+ describe("syncStateToOwnerSyncStatus", () => {
6622
+ it("reports NoRelays without a route that syncs the owner", () => {
6623
+ const noRelays = { type: "NoRelays" } as const;
6624
+ const withTenant = (tenant: SyncTenant): SyncState => ({
6625
+ transports: syncTransports,
6626
+ tenants: [tenant],
6627
+ });
6628
+
6629
+ assertEqual(
6630
+ syncStateToOwnerSyncStatus(null, testName, testAppOwner.id),
6631
+ noRelays,
6632
+ );
6633
+ assertEqual(
6634
+ syncStateToOwnerSyncStatus(
6635
+ createRoutesState([createCompleteRoute(openId)]),
6636
+ Name.orThrow("second"),
6637
+ testAppOwner.id,
6638
+ ),
6639
+ noRelays,
6640
+ );
6641
+ assertEqual(
6642
+ syncStateToOwnerSyncStatus(
6643
+ withTenant({
6644
+ type: "Refused",
4994
6645
  name: testName,
4995
- refused: false,
4996
- owners: [createOwner(testAppOwner.id, routes)],
4997
- },
4998
- ]),
6646
+ error: derivationRefusal,
6647
+ }),
6648
+ testName,
6649
+ testAppOwner.id,
6650
+ ),
6651
+ noRelays,
4999
6652
  );
5000
- assertNotUndefined(owner);
5001
- return owner.status;
5002
- };
5003
- const failure = (at: number): SyncRouteError => ({
5004
- type: "ProtocolQuotaError",
5005
- at: Millis.orThrow(at),
6653
+ assertEqual(
6654
+ syncStateToOwnerSyncStatus(
6655
+ withTenant({
6656
+ type: "Active",
6657
+ name: testName,
6658
+ owners: [
6659
+ {
6660
+ type: "Readonly",
6661
+ ownerId: testAppOwner.id,
6662
+ transportIds: [openId],
6663
+ },
6664
+ ],
6665
+ }),
6666
+ testName,
6667
+ testAppOwner.id,
6668
+ ),
6669
+ noRelays,
6670
+ );
6671
+ // Routes are briefly empty while the owner's transports are claimed.
6672
+ assertEqual(ownerStatusOf([]), noRelays);
5006
6673
  });
5007
6674
 
5008
- it("derives the status from the routes and their transports", () => {
5009
- assertSame(statusOf([]), "initial");
5010
- assertSame(statusOf([createRoute(closedId)]), "offline");
5011
- assertSame(statusOf([createRoute(connectingId)]), "offline");
5012
- assertSame(statusOf([createRoute(openA)]), "syncing");
5013
- assertSame(statusOf([createRoute(openA, { complete: true })]), "synced");
5014
- // A complete route counts even while another socket is closed.
5015
- assertSame(
5016
- statusOf([createRoute(openA, { complete: true }), createRoute(closedId)]),
5017
- "synced",
5018
- );
5019
- // Work in progress on an open socket shows before a completed route.
5020
- assertSame(
5021
- statusOf([createRoute(openA, { complete: true }), createRoute(openB)]),
5022
- "syncing",
6675
+ it("counts a first connection as Syncing and a lost one as Offline", () => {
6676
+ assertEqual(ownerStatusOf([createPendingRoute(connectingId)]), {
6677
+ type: "Syncing",
6678
+ });
6679
+ assertEqual(ownerStatusOf([createPendingRoute(openId)]), {
6680
+ type: "Syncing",
6681
+ });
6682
+ assertEqual(ownerStatusOf([createPendingRoute(disconnectedId)]), {
6683
+ type: "Offline",
6684
+ });
6685
+ });
6686
+
6687
+ it("ranks Error over Syncing over Synced over Offline", () => {
6688
+ // A complete route counts even while another relay is offline.
6689
+ assertEqual(
6690
+ ownerStatusOf([
6691
+ createPendingRoute(disconnectedId),
6692
+ createCompleteRoute(openId),
6693
+ ]),
6694
+ { type: "Synced" },
5023
6695
  );
5024
- // A failure shows before both.
5025
- assertSame(
5026
- statusOf([
5027
- createRoute(openA, { complete: true }),
5028
- createRoute(openB, { error: failure(1) }),
6696
+ // Work in progress shows before a relay that is already up to date.
6697
+ assertEqual(
6698
+ ownerStatusOf([
6699
+ createCompleteRoute(openId),
6700
+ createPendingRoute(otherOpenId),
5029
6701
  ]),
5030
- "error",
6702
+ { type: "Syncing" },
5031
6703
  );
5032
- });
5033
-
5034
- it("reports the newest completion and error", () => {
5035
- const routes = [
5036
- createRoute(openA, {
5037
- completeAt: Millis.orThrow(3000),
5038
- error: { type: "WriteFailed", at: Millis.orThrow(1000) },
5039
- }),
5040
- createRoute(openB, {
5041
- complete: true,
5042
- completeAt: Millis.orThrow(2000),
5043
- }),
5044
- createRoute(closedId, {
5045
- completeAt: Millis.orThrow(1000),
5046
- error: failure(4000),
5047
- }),
5048
- ];
5049
- const [owner] = syncStateToOwnerSyncStates(
5050
- createState([
5051
- {
5052
- name: testName,
5053
- refused: false,
5054
- owners: [createOwner(testAppOwner.id, routes)],
5055
- },
6704
+ // An error shows before all of them.
6705
+ assertEqual(
6706
+ ownerStatusOf([
6707
+ createCompleteRoute(openId),
6708
+ createPendingRoute(otherOpenId),
6709
+ createPendingRoute(disconnectedId, { failure: quotaFailure(1000) }),
5056
6710
  ]),
6711
+ { type: "Error", error: quotaFailure(1000) },
5057
6712
  );
5058
- assertEqual(owner, {
5059
- name: testName,
5060
- ownerId: testAppOwner.id,
5061
- status: "error",
5062
- syncedAt: Millis.orThrow(3000),
5063
- error: failure(4000),
5064
- relays: [
5065
- { transport: transports[0], route: routes[0], status: "error" },
5066
- { transport: transports[1], route: routes[1], status: "synced" },
5067
- { transport: transports[2], route: routes[2], status: "error" },
5068
- ],
6713
+ // So does a skipped change on a pending route, such as a settled one
6714
+ // whose relay then disconnected, alone or next to a complete relay.
6715
+ const skippedOnPending = createPendingRoute(disconnectedId, {
6716
+ skippedError: skippedChange(1000),
6717
+ });
6718
+ assertEqual(ownerStatusOf([skippedOnPending]), {
6719
+ type: "Error",
6720
+ error: skippedChange(1000),
5069
6721
  });
6722
+ assertEqual(
6723
+ ownerStatusOf([createCompleteRoute(openId), skippedOnPending]),
6724
+ { type: "Error", error: skippedChange(1000) },
6725
+ );
5070
6726
  });
5071
6727
 
5072
- it("pairs each route with its transport and a status", () => {
5073
- const routes = [
5074
- createRoute(connectingId),
5075
- createRoute(openB, { complete: true }),
5076
- createRoute(openA),
5077
- createRoute(closedId),
5078
- ];
5079
- const [owner] = syncStateToOwnerSyncStates(
5080
- createState([
5081
- {
5082
- name: testName,
5083
- refused: false,
5084
- owners: [createOwner(testAppOwner.id, routes)],
5085
- },
6728
+ it("does not hide an older failure behind a newer skipped change on another relay", () => {
6729
+ // A skipped change is stamped again by every reply that skips it, and the
6730
+ // app can act on a quota failure.
6731
+ assertEqual(
6732
+ ownerStatusOf([
6733
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6734
+ createSettledRoute(otherOpenId, skippedChange(5000)),
5086
6735
  ]),
6736
+ { type: "Error", error: quotaFailure(1000) },
6737
+ );
6738
+ // Among failures, and without one among skipped changes, the newest shows.
6739
+ assertEqual(
6740
+ ownerStatusOf([
6741
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6742
+ createPendingRoute(otherOpenId, {
6743
+ failure: { type: "WriteFailed", at: Millis.orThrow(2000) },
6744
+ }),
6745
+ ]),
6746
+ {
6747
+ type: "Error",
6748
+ error: { type: "WriteFailed", at: Millis.orThrow(2000) },
6749
+ },
6750
+ );
6751
+ assertEqual(
6752
+ ownerStatusOf([
6753
+ createSettledRoute(openId, skippedChange(3000)),
6754
+ createPendingRoute(otherOpenId, { skippedError: skippedChange(1000) }),
6755
+ ]),
6756
+ { type: "Error", error: skippedChange(3000) },
6757
+ );
6758
+ // The newest wins in either route order.
6759
+ assertEqual(
6760
+ ownerStatusOf([
6761
+ createPendingRoute(openId, {
6762
+ failure: { type: "WriteFailed", at: Millis.orThrow(2000) },
6763
+ }),
6764
+ createPendingRoute(otherOpenId, { failure: quotaFailure(1000) }),
6765
+ ]),
6766
+ {
6767
+ type: "Error",
6768
+ error: { type: "WriteFailed", at: Millis.orThrow(2000) },
6769
+ },
6770
+ );
6771
+ assertEqual(
6772
+ ownerStatusOf([
6773
+ createPendingRoute(openId, { skippedError: skippedChange(1000) }),
6774
+ createSettledRoute(otherOpenId, skippedChange(3000)),
6775
+ ]),
6776
+ { type: "Error", error: skippedChange(3000) },
5087
6777
  );
5088
- assertNotUndefined(owner);
5089
- // Relays keep the order of the routes, whatever the transport order.
5090
- assertEqual(owner.relays, [
5091
- { transport: transports[3], route: routes[0], status: "offline" },
5092
- { transport: transports[1], route: routes[1], status: "synced" },
5093
- { transport: transports[0], route: routes[2], status: "syncing" },
5094
- { transport: transports[2], route: routes[3], status: "offline" },
5095
- ]);
5096
6778
  });
5097
6779
 
5098
- it("reports each writable owner of each running database", () => {
6780
+ it("finds the owner in the named database", () => {
5099
6781
  const secondName = Name.orThrow("second");
5100
- const states = syncStateToOwnerSyncStates(
5101
- createState([
6782
+ const state: SyncState = {
6783
+ transports: syncTransports,
6784
+ tenants: [
5102
6785
  {
6786
+ type: "Active",
5103
6787
  name: testName,
5104
- refused: false,
5105
6788
  owners: [
5106
- createOwner(testAppOwner.id, [
5107
- createRoute(openA, { complete: true }),
5108
- ]),
5109
- // A readonly registration never synchronizes.
5110
- createOwner(testAppOwner2.id, [], false),
6789
+ {
6790
+ type: "Writable",
6791
+ ownerId: testAppOwner.id,
6792
+ routes: [createCompleteRoute(openId)],
6793
+ },
5111
6794
  ],
5112
6795
  },
5113
6796
  {
6797
+ type: "Active",
5114
6798
  name: secondName,
5115
- refused: false,
5116
- owners: [createOwner(testAppOwner2.id, [createRoute(openB)])],
5117
- },
5118
- // A refused database synchronizes nothing, whatever it registered.
5119
- {
5120
- name: Name.orThrow("refused"),
5121
- refused: true,
5122
- owners: [createOwner(testAppOwner.id, [createRoute(openA)])],
6799
+ owners: [
6800
+ {
6801
+ type: "Writable",
6802
+ ownerId: testAppOwner.id,
6803
+ routes: [createPendingRoute(disconnectedId)],
6804
+ },
6805
+ ],
5123
6806
  },
5124
- ]),
5125
- );
5126
- assertEqual(
5127
- states.map(({ name, ownerId, status }) => ({ name, ownerId, status })),
5128
- [
5129
- { name: testName, ownerId: testAppOwner.id, status: "synced" },
5130
- { name: secondName, ownerId: testAppOwner2.id, status: "syncing" },
5131
6807
  ],
6808
+ };
6809
+ assertEqual(syncStateToOwnerSyncStatus(state, testName, testAppOwner.id), {
6810
+ type: "Synced",
6811
+ });
6812
+ assertEqual(
6813
+ syncStateToOwnerSyncStatus(state, secondName, testAppOwner.id),
6814
+ { type: "Offline" },
6815
+ );
6816
+ });
6817
+
6818
+ it("gives the same status object while the status is unchanged", () => {
6819
+ for (const routes of [
6820
+ [],
6821
+ [createPendingRoute(openId)],
6822
+ [createCompleteRoute(openId)],
6823
+ [createPendingRoute(disconnectedId)],
6824
+ ]) {
6825
+ // Each call derives from a new snapshot, as bindings receive them.
6826
+ assertSame(ownerStatusOf(routes), ownerStatusOf(routes));
6827
+ }
6828
+
6829
+ // An Error status is the same object for the same error object, which the
6830
+ // store keeps between snapshots while it is unchanged, for the owner and
6831
+ // for each relay alike.
6832
+ const failure = quotaFailure(1000);
6833
+ const failedRoute = createPendingRoute(openId, { failure });
6834
+ const errorStatus = ownerStatusOf([failedRoute]);
6835
+ assertSame(
6836
+ ownerStatusOf([createCompleteRoute(otherOpenId), failedRoute]),
6837
+ errorStatus,
6838
+ );
6839
+ assertSame(
6840
+ relaySyncStateToStatus({
6841
+ transport: createSyncTransport(openId, openConnection),
6842
+ route: failedRoute,
6843
+ }),
6844
+ errorStatus,
6845
+ );
6846
+ const skipped = skippedChange(2000);
6847
+ assertSame(
6848
+ ownerStatusOf([createSettledRoute(openId, skipped)]),
6849
+ ownerStatusOf([createPendingRoute(openId, { skippedError: skipped })]),
6850
+ );
6851
+ assertSame(
6852
+ relaySyncStateToStatus({
6853
+ transport: createSyncTransport(openId, openConnection),
6854
+ route: createSettledRoute(openId, skipped),
6855
+ }),
6856
+ ownerStatusOf([createSettledRoute(openId, skipped)]),
6857
+ );
6858
+
6859
+ // An equal error in a new object, which the store never hands over while
6860
+ // the error is unchanged, gives a new status.
6861
+ assertNotSame(
6862
+ ownerStatusOf([
6863
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6864
+ ]),
6865
+ errorStatus,
5132
6866
  );
5133
6867
  });
5134
6868
  });
@@ -5446,66 +7180,215 @@ describe("with one evolu instance", () => {
5446
7180
  });
5447
7181
  await testWaitForWorkerMessage();
5448
7182
 
5449
- assertTrue(
5450
- firstOutputs.some((output) => output.type === "OnPatchesByQuery"),
5451
- );
5452
- assertFalse(
5453
- firstOutputs.some((output) => output.type === "RefreshQueries"),
5454
- );
5455
- assertTrue(
5456
- secondOutputs.some((output) => output.type === "RefreshQueries"),
5457
- );
7183
+ assertTrue(
7184
+ firstOutputs.some((output) => output.type === "OnPatchesByQuery"),
7185
+ );
7186
+ assertFalse(
7187
+ firstOutputs.some((output) => output.type === "RefreshQueries"),
7188
+ );
7189
+ assertTrue(
7190
+ secondOutputs.some((output) => output.type === "RefreshQueries"),
7191
+ );
7192
+ });
7193
+
7194
+ it("drops CreateEvolu when shared worker stops during tenant startup", async () => {
7195
+ await using setup = await setupSharedWorker();
7196
+ const { announceTabLeader, run, sharedWorkerOutputs, worker } = setup;
7197
+ const id: EvoluInstanceId = createId(run.deps);
7198
+ const evoluOutputs: Array<EvoluOutput> = [];
7199
+
7200
+ using evoluChannel = testCreateMessageChannel<EvoluOutput, EvoluInput>();
7201
+ evoluChannel.port2.onMessage = (output) => {
7202
+ evoluOutputs.push(output);
7203
+ };
7204
+
7205
+ await announceTabLeader();
7206
+
7207
+ const outputCount = sharedWorkerOutputs.length;
7208
+ worker.port.postMessage({
7209
+ type: "CreateEvolu",
7210
+ name: testName,
7211
+ id,
7212
+ consoleLevel: "debug",
7213
+ sqliteSchema: testSqliteSchema,
7214
+ encryptionKey: testAppOwner.encryptionKey,
7215
+ memoryOnly: false,
7216
+ evoluPort: evoluChannel.port1.native,
7217
+ });
7218
+
7219
+ await testWaitForWorkerMessage();
7220
+
7221
+ const initDbWorker = getDbWorkerInit(sharedWorkerOutputs[outputCount]);
7222
+
7223
+ using dbWorkerPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
7224
+ initDbWorker.port,
7225
+ );
7226
+ const dbDisposeInputs: Array<
7227
+ Extract<DbWorkerInput, { type: "Dispose" }>
7228
+ > = [];
7229
+ dbWorkerPort.onMessage = (input) => {
7230
+ if (input.type === "Dispose") dbDisposeInputs.push(input);
7231
+ };
7232
+
7233
+ const disposePromise = setup[Symbol.asyncDispose]();
7234
+ dbWorkerPort.postMessage({
7235
+ clock: createTimestamp(),
7236
+ type: "LeaderAcquired",
7237
+ name: testName,
7238
+ });
7239
+ await disposePromise;
7240
+ await testWaitForWorkerMessage();
7241
+
7242
+ assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
7243
+ assertEqual(evoluOutputs, []);
7244
+ });
7245
+
7246
+ it("tells the tab about a failed mutation and runs the next request", async () => {
7247
+ await using setup = await setupSharedWorker();
7248
+ const { createEvolu, run, sharedWorkerOutputs } = setup;
7249
+ const { time } = run.deps;
7250
+ const { dbInputs, dbWorkerPort, evoluChannel, id } = await createEvolu();
7251
+ const outputs: Array<EvoluOutput> = [];
7252
+ evoluChannel.port2.onMessage = (output) => {
7253
+ outputs.push(output);
7254
+ };
7255
+ using errors = testCreateBroadcastChannel<ConsoleEntryOrError>(
7256
+ consoleEntryOrErrorBroadcastChannelName,
7257
+ );
7258
+ const broadcastErrors: Array<ConsoleEntryOrError> = [];
7259
+ errors.onMessage = (output) => {
7260
+ if (output.type === "Error") broadcastErrors.push(output);
7261
+ };
7262
+
7263
+ const onCompleteId = createId(run.deps);
7264
+ evoluChannel.port2.postMessage({
7265
+ type: "Mutate",
7266
+ changes: [
7267
+ {
7268
+ ownerId: testAppOwner.id,
7269
+ ...testCreateCrdtMessage(createId(run.deps), 1, "failed").change,
7270
+ },
7271
+ ],
7272
+ onCompleteIds: [onCompleteId],
7273
+ subscribedQueries: new Set([testQuery]),
7274
+ });
7275
+ evoluChannel.port2.postMessage({
7276
+ type: "Query",
7277
+ queries: createSet([testQuery]),
7278
+ });
7279
+ time.advance("10s");
7280
+ await testWaitForWorkerMessage();
7281
+
7282
+ const mutateInput = dbInputs.at(-1);
7283
+ assertNotUndefined(mutateInput);
7284
+ assertSame(mutateInput.request.message.type, "Mutate");
7285
+ const outputCount = sharedWorkerOutputs.length;
7286
+ const error = createUnknownError(new Error("database or disk is full"));
7287
+
7288
+ dbWorkerPort.postMessage({
7289
+ type: "OnQueuedResponse",
7290
+ attemptId: mutateInput.attemptId,
7291
+ response: {
7292
+ type: "ForEvolu",
7293
+ id,
7294
+ message: { type: "MutateFailed", error },
7295
+ },
7296
+ });
7297
+ await testWaitForWorkerMessage();
7298
+
7299
+ assertEqual(sharedWorkerOutputs.slice(outputCount), [
7300
+ { type: "Error", error },
7301
+ ]);
7302
+ assertEqual(broadcastErrors, []);
7303
+ assertEqual(outputs, [
7304
+ { type: "OnMutateFailed", onCompleteIds: [onCompleteId] },
7305
+ ]);
7306
+ const queryInput = dbInputs.at(-1);
7307
+ assertNotUndefined(queryInput);
7308
+ assertSame(queryInput.request.message.type, "Query");
5458
7309
  });
5459
7310
 
5460
- it("drops CreateEvolu when shared worker stops during tenant startup", async () => {
7311
+ it("tells every tab about a failed mutation whose instance was disposed", async () => {
5461
7312
  await using setup = await setupSharedWorker();
5462
- const { announceTabLeader, run, sharedWorkerOutputs, worker } = setup;
5463
- const id: EvoluInstanceId = createId(run.deps);
5464
- const evoluOutputs: Array<EvoluOutput> = [];
5465
-
5466
- using evoluChannel = testCreateMessageChannel<EvoluOutput, EvoluInput>();
5467
- evoluChannel.port2.onMessage = (output) => {
5468
- evoluOutputs.push(output);
5469
- };
5470
-
5471
- await announceTabLeader();
7313
+ const { createEvolu, run, sharedWorkerOutputs, worker } = setup;
7314
+ const instance = await createEvolu();
5472
7315
 
5473
- const outputCount = sharedWorkerOutputs.length;
7316
+ // A sibling instance writes and is disposed before the write is
7317
+ // answered.
7318
+ const siblingId = createId<"EvoluInstance">(run.deps);
7319
+ const siblingDisposer = new AsyncDisposableStack();
7320
+ siblingDisposer.use(await run.ok(acquireLeaderLock(siblingId)));
7321
+ using siblingChannel = testCreateMessageChannel<
7322
+ EvoluOutput,
7323
+ EvoluInput
7324
+ >();
5474
7325
  worker.port.postMessage({
5475
7326
  type: "CreateEvolu",
7327
+ id: siblingId,
5476
7328
  name: testName,
5477
- id,
5478
7329
  consoleLevel: "debug",
5479
7330
  sqliteSchema: testSqliteSchema,
5480
7331
  encryptionKey: testAppOwner.encryptionKey,
5481
7332
  memoryOnly: false,
5482
- evoluPort: evoluChannel.port1.native,
7333
+ evoluPort: siblingChannel.port1.native,
5483
7334
  });
5484
-
7335
+ const siblingOutputs: Array<EvoluOutput> = [];
7336
+ siblingChannel.port2.onMessage = (output) => {
7337
+ siblingOutputs.push(output);
7338
+ };
7339
+ await testWaitForWorkerMessage();
7340
+ assertEqual(siblingOutputs, [
7341
+ { type: "OnDevicePersistence", devicePersistence: "Unknown" },
7342
+ ]);
7343
+ siblingChannel.port2.postMessage({
7344
+ type: "Mutate",
7345
+ changes: [
7346
+ {
7347
+ ownerId: testAppOwner.id,
7348
+ ...testCreateCrdtMessage(createId(run.deps), 1, "failed").change,
7349
+ },
7350
+ ],
7351
+ onCompleteIds: [createId(run.deps)],
7352
+ subscribedQueries: new Set(),
7353
+ });
7354
+ await testWaitForWorkerMessage();
7355
+ const mutateInput = instance.dbInputs.at(-1);
7356
+ assertNotUndefined(mutateInput);
7357
+ assertSame(mutateInput.request.type, "ForEvolu");
7358
+ assertSame(mutateInput.request.id, siblingId);
7359
+ await siblingDisposer.disposeAsync();
5485
7360
  await testWaitForWorkerMessage();
5486
7361
 
5487
- const initDbWorker = getDbWorkerInit(sharedWorkerOutputs[outputCount]);
5488
-
5489
- using dbWorkerPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
5490
- initDbWorker.port,
7362
+ using errors = testCreateBroadcastChannel<ConsoleEntryOrError>(
7363
+ consoleEntryOrErrorBroadcastChannelName,
5491
7364
  );
5492
- const dbDisposeInputs: Array<
5493
- Extract<DbWorkerInput, { type: "Dispose" }>
5494
- > = [];
5495
- dbWorkerPort.onMessage = (input) => {
5496
- if (input.type === "Dispose") dbDisposeInputs.push(input);
7365
+ const broadcasts: Array<ConsoleEntryOrError> = [];
7366
+ errors.onMessage = (output) => {
7367
+ broadcasts.push(output);
7368
+ };
7369
+ const evoluOutputs: Array<EvoluOutput> = [];
7370
+ instance.evoluChannel.port2.onMessage = (output) => {
7371
+ evoluOutputs.push(output);
7372
+ };
7373
+ siblingChannel.port2.onMessage = (output) => {
7374
+ evoluOutputs.push(output);
5497
7375
  };
7376
+ const outputCount = sharedWorkerOutputs.length;
7377
+ const error = createUnknownError(new Error("database or disk is full"));
5498
7378
 
5499
- const disposePromise = setup[Symbol.asyncDispose]();
5500
- dbWorkerPort.postMessage({
5501
- clock: createTimestamp(),
5502
- type: "LeaderAcquired",
5503
- name: testName,
7379
+ instance.dbWorkerPort.postMessage({
7380
+ type: "OnQueuedResponse",
7381
+ attemptId: mutateInput.attemptId,
7382
+ response: {
7383
+ type: "ForEvolu",
7384
+ id: siblingId,
7385
+ message: { type: "MutateFailed", error },
7386
+ },
5504
7387
  });
5505
- await disposePromise;
5506
7388
  await testWaitForWorkerMessage();
5507
7389
 
5508
- assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
7390
+ assertEqual(broadcasts, [{ type: "Error", error }]);
7391
+ assertEqual(sharedWorkerOutputs.slice(outputCount), []);
5509
7392
  assertEqual(evoluOutputs, []);
5510
7393
  });
5511
7394
 
@@ -6589,6 +8472,8 @@ describe("with one evolu instance", () => {
6589
8472
  it("routes a relay's version-mismatch reply to the tenant", async () => {
6590
8473
  const createWebSocket = testCreateWebSocket();
6591
8474
  await using setup = await setupSharedWorker({ createWebSocket });
8475
+ using disposer = new DisposableStack();
8476
+ const { latest } = setupSyncStates(setup, disposer);
6592
8477
  const instance = await setup.createEvolu();
6593
8478
  const { dbInputs, evoluChannel } = instance;
6594
8479
  const { time } = setup.run.deps;
@@ -6648,10 +8533,16 @@ describe("with one evolu instance", () => {
6648
8533
  ok: false,
6649
8534
  error: versionError,
6650
8535
  });
6651
- assertEqual(consoleEntryOrErrors.at(-1), {
6652
- type: "Error",
6653
- error: versionError,
8536
+ // The route shows it with its details; it is not posted as an error.
8537
+ const [owner] = ownersOf(latest());
8538
+ assert(owner?.type === "Writable", "Expected a writable owner.");
8539
+ const [route] = owner.routes;
8540
+ assertSame(route?.type, "Pending");
8541
+ assertEqual(route.failure, {
8542
+ ...versionError,
8543
+ at: time.now(),
6654
8544
  });
8545
+ assertFalse(consoleEntryOrErrors.some(({ type }) => type === "Error"));
6655
8546
 
6656
8547
  // The failure requests one round; the same reply to it requests none.
6657
8548
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
@@ -7343,6 +9234,7 @@ describe("with one evolu instance", () => {
7343
9234
  type: "ApplySyncMessage",
7344
9235
  ownerId: testAppOwner.id,
7345
9236
  didWriteMessages: true,
9237
+ skippedError: null,
7346
9238
  result: {
7347
9239
  ok: false,
7348
9240
  error: {
@@ -7359,14 +9251,8 @@ describe("with one evolu instance", () => {
7359
9251
  assertTrue(
7360
9252
  evoluOutputs.some((output) => output.type === "RefreshQueries"),
7361
9253
  );
7362
- assertEqual(consoleEntryOrErrors.at(-1), {
7363
- type: "Error",
7364
- error: {
7365
- type: "ProtocolInvalidDataError",
7366
- data: new Uint8Array(),
7367
- error: "boom",
7368
- },
7369
- });
9254
+ // The route shows the failure; it is not posted as an error.
9255
+ assertFalse(consoleEntryOrErrors.some(({ type }) => type === "Error"));
7370
9256
 
7371
9257
  // The failure requested one round through the route.
7372
9258
  const retryRoundInput = dbInputs.at(-1);
@@ -7410,6 +9296,7 @@ describe("with one evolu instance", () => {
7410
9296
  type: "ApplySyncMessage",
7411
9297
  ownerId: testAppOwner.id,
7412
9298
  didWriteMessages: false,
9299
+ skippedError: null,
7413
9300
  result: {
7414
9301
  ok: true,
7415
9302
  value: {
@@ -7535,6 +9422,7 @@ describe("with one evolu instance", () => {
7535
9422
  type: "ApplySyncMessage",
7536
9423
  ownerId: testAppOwner.id,
7537
9424
  didWriteMessages: false,
9425
+ skippedError: null,
7538
9426
  result: {
7539
9427
  ok: false,
7540
9428
  error: { type: "AbortError", reason: { type: "Stop" } },
@@ -7545,6 +9433,7 @@ describe("with one evolu instance", () => {
7545
9433
  type: "ApplySyncMessage",
7546
9434
  ownerId: testAppOwner.id,
7547
9435
  didWriteMessages: false,
9436
+ skippedError: null,
7548
9437
  result: { ok: true, value: { type: "Broadcast" } },
7549
9438
  });
7550
9439
  await runApplySync({
@@ -7552,6 +9441,7 @@ describe("with one evolu instance", () => {
7552
9441
  type: "ApplySyncMessage",
7553
9442
  ownerId: testAppOwner.id,
7554
9443
  didWriteMessages: false,
9444
+ skippedError: null,
7555
9445
  result: { ok: true, value: { type: "Converged" } },
7556
9446
  });
7557
9447
 
@@ -7828,6 +9718,10 @@ describe("with one evolu instance", () => {
7828
9718
  [siblingChannel.port1.native],
7829
9719
  );
7830
9720
  await testWaitForWorkerMessage();
9721
+ // The tenant keeps the first instance's persistent database.
9722
+ assertEqual(siblingOutputs.splice(0), [
9723
+ { type: "OnDevicePersistence", devicePersistence: "Unknown" },
9724
+ ]);
7831
9725
 
7832
9726
  const bytes = new Uint8Array([1, 2, 3]);
7833
9727
  const mutation: ExtractTyped<EvoluInput, "Mutate"> = {
@@ -8333,30 +10227,6 @@ describe("with one evolu instance", () => {
8333
10227
  assertEqual(setup.run.deps.reportDefect.getDefects(), []);
8334
10228
  });
8335
10229
 
8336
- it("waits for DbWorker leader lock during tenant disposal", async () => {
8337
- await using setup = await setupSharedWorker();
8338
- const { createEvolu } = setup;
8339
- const { dbDisposeInputs, releaseDbWorkerLeader } = await createEvolu({
8340
- releaseDbWorkerLeaderOnDispose: false,
8341
- autoDispose: false,
8342
- });
8343
- let disposed = false;
8344
-
8345
- const disposing = setup[Symbol.asyncDispose]().then(() => {
8346
- disposed = true;
8347
- });
8348
-
8349
- await testWaitForWorkerMessage();
8350
-
8351
- assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
8352
- assertFalse(disposed);
8353
-
8354
- await releaseDbWorkerLeader();
8355
- await disposing;
8356
-
8357
- assertTrue(disposed);
8358
- });
8359
-
8360
10230
  it("stops a replacement DbWorker that acquires the lock during tenant disposal", async () => {
8361
10231
  await using setup = await setupSharedWorker();
8362
10232
  await using disposer = new AsyncDisposableStack();
@@ -8396,6 +10266,29 @@ describe("with one evolu instance", () => {
8396
10266
  await disposing;
8397
10267
  });
8398
10268
 
10269
+ it("leads for its ID until disposed, so its DbWorkers stop without Dispose", async () => {
10270
+ await using setup = await setupSharedWorker();
10271
+ await setup.createEvolu();
10272
+ assertSame(
10273
+ getDbWorkerInit(setup.sharedWorkerOutputs.at(0)).sharedWorkerId,
10274
+ setup.workerId,
10275
+ );
10276
+ await using run = testCreateRun({
10277
+ lockManager: setup.run.deps.lockManager,
10278
+ });
10279
+ let isLeader = false;
10280
+ const leading = run.ok(acquireLeaderLock(setup.workerId)).then((lock) => {
10281
+ isLeader = true;
10282
+ return lock;
10283
+ });
10284
+ await testWaitForWorkerMessage();
10285
+ assertFalse(isLeader);
10286
+
10287
+ await setup[Symbol.asyncDispose]();
10288
+
10289
+ await using _lock = await leading;
10290
+ });
10291
+
8399
10292
  it("disposes cleanly when queued sync resumes during tenant disposal", async () => {
8400
10293
  const createWebSocket = testCreateWebSocket({ isOpen: false });
8401
10294
  const creating = Promise.withResolvers<void>();
@@ -8476,6 +10369,178 @@ describe("with one evolu instance", () => {
8476
10369
  assertEqual(createWebSocket.createdUrls, []);
8477
10370
  });
8478
10371
 
10372
+ it("releases a transport claimed while its instance is disposed and drops queued changes", async () => {
10373
+ const createWebSocket = testCreateWebSocket();
10374
+ const creating = Promise.withResolvers<void>();
10375
+ const continueCreating = Promise.withResolvers<void>();
10376
+ await using setup = await setupSharedWorker({
10377
+ createWebSocket: (url, options) => async (run) => {
10378
+ creating.resolve();
10379
+ await continueCreating.promise;
10380
+ return run(createWebSocket(url, options));
10381
+ },
10382
+ });
10383
+ const instance = await setup.createEvolu({ autoDispose: false });
10384
+ const transport = createOwnerWebSocketTransport({
10385
+ url: "wss://claim-during-dispose.example",
10386
+ ownerId: testAppOwner.id,
10387
+ });
10388
+ instance.evoluChannel.port2.postMessage({
10389
+ type: "UseOwner",
10390
+ actions: [
10391
+ {
10392
+ action: "add",
10393
+ owner: { owner: testAppOwner, transports: [transport] },
10394
+ },
10395
+ ],
10396
+ });
10397
+ // Queued behind the claim in flight.
10398
+ instance.evoluChannel.port2.postMessage({
10399
+ type: "UseOwner",
10400
+ actions: [
10401
+ {
10402
+ action: "add",
10403
+ owner: {
10404
+ owner: testAppOwner,
10405
+ transports: [
10406
+ createOwnerWebSocketTransport({
10407
+ url: "wss://queued-during-dispose.example",
10408
+ ownerId: testAppOwner.id,
10409
+ }),
10410
+ ],
10411
+ },
10412
+ },
10413
+ ],
10414
+ });
10415
+ await creating.promise;
10416
+
10417
+ await instance[Symbol.asyncDispose]();
10418
+ continueCreating.resolve();
10419
+ await testWaitForWorkerMessage();
10420
+
10421
+ assertEqual(createWebSocket.sentMessages.at(-1), {
10422
+ url: transport.url,
10423
+ data: createProtocolMessageForUnsubscribe(testAppOwner.id),
10424
+ });
10425
+ assertEqual(createWebSocket.createdUrls, [transport.url]);
10426
+ assertEqual(setup.run.deps.reportDefect.getDefects(), []);
10427
+ });
10428
+
10429
+ it("reports nothing when the root Run is disposed before the worker", async () => {
10430
+ const createWebSocket = testCreateWebSocket();
10431
+ await using setup = await setupSharedWorker({ createWebSocket });
10432
+ const instance = await setup.createEvolu();
10433
+ const transport = createOwnerWebSocketTransport({
10434
+ url: "wss://root-dispose.example",
10435
+ ownerId: testAppOwner.id,
10436
+ });
10437
+ instance.evoluChannel.port2.postMessage({
10438
+ type: "UseOwner",
10439
+ actions: [
10440
+ {
10441
+ action: "add",
10442
+ owner: { owner: testAppOwner, transports: [transport] },
10443
+ },
10444
+ ],
10445
+ });
10446
+ await testWaitForWorkerMessage();
10447
+ assertEqual(createWebSocket.createdUrls, [transport.url]);
10448
+
10449
+ // As a runtime shutting down while the worker's stack is still owned.
10450
+ await setup.run[Symbol.asyncDispose]();
10451
+
10452
+ assertEqual(setup.run.deps.reportDefect.getDefects(), []);
10453
+ });
10454
+
10455
+ it("reports only the panic when it disposes a tenant with an instance", async () => {
10456
+ // The default test WebSocket throws on create, which panics the root.
10457
+ await using setup = await setupSharedWorker();
10458
+ const instance = await setup.createEvolu();
10459
+ instance.evoluChannel.port2.postMessage({
10460
+ type: "UseOwner",
10461
+ actions: [
10462
+ {
10463
+ action: "add",
10464
+ owner: {
10465
+ owner: testAppOwner,
10466
+ transports: [
10467
+ createOwnerWebSocketTransport({
10468
+ url: "wss://root-panic.example",
10469
+ ownerId: testAppOwner.id,
10470
+ }),
10471
+ ],
10472
+ },
10473
+ },
10474
+ ],
10475
+ });
10476
+ const panic = await setup.run.deps.reportDefect.next();
10477
+ await setup.run[Symbol.asyncDispose]();
10478
+
10479
+ assertEqual(setup.run.deps.reportDefect.getDefects(), [panic]);
10480
+ // The tenant still tells its DbWorker to stop.
10481
+ await testWaitForWorkerMessage();
10482
+ assertEqual(instance.dbDisposeInputs, [{ type: "Dispose" }]);
10483
+ assert(AbortError.is(panic), "Expected an AbortError.");
10484
+ assert(panic.reason.type === "PanicAbortReason", "Expected a panic.");
10485
+ assertInstanceOf(panic.reason.defect, Error);
10486
+ assertEqual(
10487
+ panic.reason.defect.message,
10488
+ "testCreateWebSocket is configured to throw on create",
10489
+ );
10490
+ });
10491
+
10492
+ it("starts the DbWorker of a reopened database while the previous one still holds its lock", async () => {
10493
+ await using setup = await setupSharedWorker();
10494
+ await using disposer = new AsyncDisposableStack();
10495
+ const closed = await setup.createEvolu({ autoDispose: false });
10496
+ // As a DbWorker that got Dispose but is still closing the database.
10497
+ await closed.releaseDbWorkerLeader();
10498
+ const previousDbWorkerRun = disposer.use(
10499
+ testCreateRun({ lockManager: setup.run.deps.lockManager }),
10500
+ );
10501
+ const previousDbWorkerLock = await previousDbWorkerRun.ok(
10502
+ acquireLeaderLock(testName),
10503
+ );
10504
+ await closed[Symbol.asyncDispose]();
10505
+ setup.run.deps.time.advance("3s");
10506
+ await testWaitForWorkerMessage();
10507
+
10508
+ let isReopened = false;
10509
+ let isLeader = false;
10510
+ const reopening = setup
10511
+ .createEvoluBeforeDbWorkerLeader({ autoDispose: false })
10512
+ .then((instance) => {
10513
+ isReopened = true;
10514
+ return instance;
10515
+ });
10516
+ disposer.defer(async () => {
10517
+ await previousDbWorkerLock[Symbol.asyncDispose]();
10518
+ const instance = await reopening;
10519
+ if (!isLeader) await instance.acquireDbWorkerLeader();
10520
+ await instance[Symbol.asyncDispose]();
10521
+ });
10522
+ await testWaitForWorkerMessage();
10523
+ await testWaitForWorkerMessage();
10524
+
10525
+ assertTrue(isReopened);
10526
+ const reopened = await reopening;
10527
+ // Its requests wait until its own DbWorker holds the lock.
10528
+ reopened.evoluChannel.port2.postMessage({
10529
+ type: "Query",
10530
+ queries: createSet([testQuery]),
10531
+ });
10532
+ await testWaitForWorkerMessage();
10533
+ assertEqual(reopened.dbInputs, []);
10534
+
10535
+ await previousDbWorkerLock[Symbol.asyncDispose]();
10536
+ await reopened.acquireDbWorkerLeader();
10537
+ isLeader = true;
10538
+ assertEqual(
10539
+ reopened.dbInputs.map(({ type }) => type),
10540
+ ["Request"],
10541
+ );
10542
+ });
10543
+
8479
10544
  it("releases pending instance lock if tenant disposal wins the acquisition race", async () => {
8480
10545
  await using setup = await setupSharedWorker();
8481
10546
  const { createEvolu } = setup;
@@ -8786,7 +10851,7 @@ describe("startup refusal", () => {
8786
10851
  });
8787
10852
  await testWaitForWorkerMessage();
8788
10853
  assertEqual(latest().tenants, [
8789
- { name: testName, refused: true, owners: [] },
10854
+ { type: "Refused", name: testName, error: refusal },
8790
10855
  ]);
8791
10856
  });
8792
10857
 
@@ -8824,10 +10889,13 @@ describe("startup refusal", () => {
8824
10889
  });
8825
10890
  await testWaitForWorkerMessage();
8826
10891
  // The tab is told through its own connection, not the broadcast channel,
8827
- // and its instance is not answered: pending work stays pending.
10892
+ // and its instance learns only its device persistence: pending work stays
10893
+ // pending.
8828
10894
  assertEqual(firstTabOutputs(), [{ type: "Error", error: refusal }]);
8829
10895
  assertEqual(broadcasts, []);
8830
- assertEqual(evoluOutputs, []);
10896
+ assertEqual(evoluOutputs.splice(0), [
10897
+ { type: "OnDevicePersistence", devicePersistence: "Unknown" },
10898
+ ]);
8831
10899
 
8832
10900
  // Later requests are dropped without a response.
8833
10901
  instance.evoluChannel.port2.postMessage({
@@ -8865,7 +10933,9 @@ describe("startup refusal", () => {
8865
10933
  );
8866
10934
  await testWaitForWorkerMessage();
8867
10935
  assertEqual(firstTabOutputs(), [{ type: "Error", error: refusal }]);
8868
- assertEqual(secondOutputs, []);
10936
+ assertEqual(secondOutputs, [
10937
+ { type: "OnDevicePersistence", devicePersistence: "Unknown" },
10938
+ ]);
8869
10939
 
8870
10940
  // A tab that connects later is told once, through its own connection.
8871
10941
  const laterTab = setupTab(setup, disposer);