@evolu/common 8.12.0 → 8.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/src/Callbacks.d.ts +12 -1
  2. package/dist/src/Callbacks.d.ts.map +1 -1
  3. package/dist/src/Callbacks.js +3 -0
  4. package/dist/src/Error.d.ts +10 -4
  5. package/dist/src/Error.d.ts.map +1 -1
  6. package/dist/src/Error.js +10 -4
  7. package/dist/src/Object.d.ts +65 -0
  8. package/dist/src/Object.d.ts.map +1 -1
  9. package/dist/src/Object.js +142 -0
  10. package/dist/src/Resource.d.ts +0 -5
  11. package/dist/src/Resource.d.ts.map +1 -1
  12. package/dist/src/Resource.js +6 -13
  13. package/dist/src/Sqlite.d.ts.map +1 -1
  14. package/dist/src/Sqlite.js +7 -0
  15. package/dist/src/Task.d.ts +10 -8
  16. package/dist/src/Task.d.ts.map +1 -1
  17. package/dist/src/Task.js +41 -5
  18. package/dist/src/Worker.d.ts +3 -3
  19. package/dist/src/local-first/Db.d.ts +8 -3
  20. package/dist/src/local-first/Db.d.ts.map +1 -1
  21. package/dist/src/local-first/Db.js +56 -19
  22. package/dist/src/local-first/Evolu.d.ts +81 -37
  23. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  24. package/dist/src/local-first/Evolu.js +31 -6
  25. package/dist/src/local-first/Owner.d.ts +9 -0
  26. package/dist/src/local-first/Owner.d.ts.map +1 -1
  27. package/dist/src/local-first/Owner.js +9 -0
  28. package/dist/src/local-first/Protocol.d.ts +6 -4
  29. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  30. package/dist/src/local-first/Protocol.js +1 -6
  31. package/dist/src/local-first/Schema.d.ts +7 -1
  32. package/dist/src/local-first/Schema.d.ts.map +1 -1
  33. package/dist/src/local-first/Shared.d.ts +523 -133
  34. package/dist/src/local-first/Shared.d.ts.map +1 -1
  35. package/dist/src/local-first/Shared.js +696 -256
  36. package/dist/src/local-first/Storage.d.ts +16 -12
  37. package/dist/src/local-first/Storage.d.ts.map +1 -1
  38. package/package.json +1 -1
  39. package/src/Callbacks.test.ts +20 -0
  40. package/src/Callbacks.ts +17 -1
  41. package/src/Error.ts +10 -4
  42. package/src/Object.test.ts +296 -0
  43. package/src/Object.ts +163 -0
  44. package/src/Resource.test.ts +20 -16
  45. package/src/Resource.ts +6 -20
  46. package/src/Sqlite.ts +7 -0
  47. package/src/Task.test.ts +233 -62
  48. package/src/Task.ts +47 -13
  49. package/src/Worker.ts +3 -3
  50. package/src/local-first/Db.ts +88 -18
  51. package/src/local-first/Evolu.test.ts +281 -1
  52. package/src/local-first/Evolu.ts +124 -46
  53. package/src/local-first/Owner.ts +9 -0
  54. package/src/local-first/Protocol.test.ts +42 -60
  55. package/src/local-first/Protocol.ts +7 -9
  56. package/src/local-first/Schema.ts +8 -2
  57. package/src/local-first/Shared.test.ts +2633 -646
  58. package/src/local-first/Shared.ts +1192 -364
  59. package/src/local-first/Storage.ts +18 -19
@@ -18,6 +18,9 @@ import {
18
18
  } from "../Assert.ts";
19
19
  import { createBuffer, encodeNonNegativeInt } from "../Bytes.ts";
20
20
  import type { ConsoleEntry, ConsoleLevel } from "../Console.ts";
21
+ import type { DecryptWithXChaCha20Poly1305Error } from "../Crypto.ts";
22
+ import { createUnknownError, UnknownError } from "../Error.ts";
23
+ import { isPlainObject } from "../Object.ts";
21
24
  import type { DbWorkerInit, UnsupportedDbVersionError } from "./Db.ts";
22
25
  import {
23
26
  createAppOwner,
@@ -34,6 +37,9 @@ import {
34
37
  MessageType,
35
38
  ProtocolErrorCode,
36
39
  parseProtocolHeader,
40
+ type ProtocolInvalidDataError,
41
+ type ProtocolSyncError,
42
+ type ProtocolTimestampMismatchError,
37
43
  SubscriptionFlags,
38
44
  } from "./Protocol.ts";
39
45
  import {
@@ -42,8 +48,10 @@ import {
42
48
  type MutationChange,
43
49
  } from "./Schema.ts";
44
50
  import {
45
- syncStateToOwnerSyncStates,
46
51
  consoleEntryOrErrorBroadcastChannelName,
52
+ relaySyncStateToStatus,
53
+ syncStateToOwnerSyncStatus,
54
+ syncStateToRelaySyncStates,
47
55
  type EvoluInstanceId,
48
56
  initSharedWorker,
49
57
  type ConsoleEntryOrError,
@@ -55,6 +63,12 @@ import {
55
63
  type SharedWorkerId,
56
64
  type SharedWorkerInput,
57
65
  type SharedWorkerOutput,
66
+ type CompleteSyncRoute,
67
+ type OwnerSyncStatus,
68
+ type PendingSyncRoute,
69
+ type RelaySyncStatus,
70
+ type SettledSyncRoute,
71
+ type SyncConnection,
58
72
  type SyncRoute,
59
73
  type SyncState,
60
74
  type SyncTransportId,
@@ -168,7 +182,9 @@ const setupSharedWorker = async ({
168
182
  seed?: string;
169
183
  } = {}) => {
170
184
  await using disposer = new AsyncDisposableStack();
171
- const createTestId = testCreateId();
185
+ // Instance IDs name leader locks, as the worker's ID does, so they come from
186
+ // another seed than the worker's randomness, which would repeat one of them.
187
+ const tabDeps = testCreateDeps({ seed: "tab" });
172
188
 
173
189
  const consoleStoreOutputEntry = createStore<ConsoleEntry | null>(null);
174
190
  const worker = disposer.use(
@@ -231,7 +247,7 @@ const setupSharedWorker = async ({
231
247
  initialClock?: Timestamp;
232
248
  } = {}) => {
233
249
  const instanceDisposables = new AsyncDisposableStack();
234
- const id = createTestId<"EvoluInstance">();
250
+ const id = createId<"EvoluInstance">(tabDeps);
235
251
  const dbInputs: Array<Exclude<DbWorkerInput, { type: "Dispose" }>> = [];
236
252
  const dbDisposeInputs: Array<Extract<DbWorkerInput, { type: "Dispose" }>> =
237
253
  [];
@@ -419,12 +435,22 @@ const setupSyncStates = (
419
435
  return { states, latest };
420
436
  };
421
437
 
438
+ /** The owners of a database that has not refused startup. */
439
+ const ownersOf = (
440
+ state: SyncState,
441
+ tenantName: Name = testName,
442
+ ): ReadonlyArray<SyncTenantOwner> => {
443
+ const tenant = state.tenants.find(({ name }) => name === tenantName);
444
+ assert(tenant?.type === "Active", "Expected an active tenant.");
445
+ return tenant.owners;
446
+ };
447
+
422
448
  type TestEvoluInstance = Awaited<ReturnType<SharedWorkerSetup["createEvolu"]>>;
423
449
 
424
- type ApplySyncResult = Extract<
450
+ type ApplySyncResponse = Extract<
425
451
  Extract<DbWorkerQueuedResponse, { type: "ForSharedWorker" }>["message"],
426
452
  { type: "ApplySyncMessage" }
427
- >["result"];
453
+ >;
428
454
 
429
455
  /** Completes the latest queued sync round and returns the URLs it was sent to. */
430
456
  const respondToSyncRound = async (
@@ -466,7 +492,8 @@ const respondToApplySync = async (
466
492
  dbWorkerPort,
467
493
  }: Pick<TestEvoluInstance, "dbInputs" | "dbWorkerPort">,
468
494
  didWriteMessages: boolean,
469
- result: ApplySyncResult,
495
+ result: ApplySyncResponse["result"],
496
+ skippedError: ApplySyncResponse["skippedError"] = null,
470
497
  ): Promise<void> => {
471
498
  const input = dbInputs.at(-1);
472
499
  assertNotUndefined(input);
@@ -482,6 +509,7 @@ const respondToApplySync = async (
482
509
  clock: createTimestamp(),
483
510
  ownerId: input.request.message.owner.id,
484
511
  didWriteMessages,
512
+ skippedError,
485
513
  result,
486
514
  },
487
515
  },
@@ -976,21 +1004,21 @@ describe("sync state", () => {
976
1004
  assertEqual(state.transports[0].label, "/relay");
977
1005
  assertEqual(state.tenants, [
978
1006
  {
1007
+ type: "Active",
979
1008
  name: testName,
980
- refused: false,
981
1009
  owners: [
982
1010
  {
1011
+ type: "Writable",
983
1012
  ownerId: testAppOwner.id,
984
- writable: true,
985
- transportIds: [state.transports[0].id],
986
1013
  routes: [
987
1014
  {
1015
+ type: "Pending",
988
1016
  transportId: state.transports[0].id,
989
- complete: false,
1017
+ failure: null,
1018
+ skippedError: null,
990
1019
  completeAt: null,
991
1020
  lastSentAt: null,
992
1021
  lastReceivedAt: null,
993
- error: null,
994
1022
  },
995
1023
  ],
996
1024
  },
@@ -1016,14 +1044,13 @@ describe("sync state", () => {
1016
1044
  assertLength(tab.syncStates, 1);
1017
1045
  });
1018
1046
 
1019
- it("includes transitions without an event in the next snapshot", async () => {
1020
- const createWebSocket = testCreateWebSocket({ isOpen: false });
1021
- await using setup = await setupSharedWorker({ createWebSocket });
1022
- using disposer = new DisposableStack();
1023
- const { latest } = setupSyncStates(setup, disposer);
1024
- const { evoluChannel } = await setup.createEvolu();
1047
+ /** Registers the app owner through one transport of the instance's database. */
1048
+ const setupAppOwnerTransport = async (
1049
+ { evoluChannel }: Pick<TestEvoluInstance, "evoluChannel">,
1050
+ url: string,
1051
+ ) => {
1025
1052
  const transport = createOwnerWebSocketTransport({
1026
- url: "wss://relay.example",
1053
+ url,
1027
1054
  ownerId: testAppOwner.id,
1028
1055
  });
1029
1056
  evoluChannel.port2.postMessage({
@@ -1036,21 +1063,184 @@ describe("sync state", () => {
1036
1063
  ],
1037
1064
  });
1038
1065
  await testWaitForWorkerMessage();
1066
+ return transport;
1067
+ };
1068
+
1069
+ it("publishes a transport as Connecting until its first connection opens", async () => {
1070
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1071
+ await using setup = await setupSharedWorker({ createWebSocket });
1072
+ using disposer = new DisposableStack();
1073
+ const { latest } = setupSyncStates(setup, disposer);
1074
+ const { time } = setup.run.deps;
1075
+ const instance = await setup.createEvolu();
1076
+ const transport = await setupAppOwnerTransport(
1077
+ instance,
1078
+ "wss://relay.example/sync",
1079
+ );
1080
+
1081
+ const { id } = latest().transports[0];
1082
+ assertEqual(latest().transports, [
1083
+ {
1084
+ type: "WebSocket",
1085
+ id,
1086
+ label: "wss://relay.example/sync",
1087
+ connection: { type: "Connecting" },
1088
+ },
1089
+ ]);
1090
+ // A first connection is work in progress, not an outage.
1091
+ assertEqual(
1092
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
1093
+ { type: "Syncing" },
1094
+ );
1095
+
1096
+ time.advance("1s");
1097
+ createWebSocket.open(transport.url);
1098
+ await testWaitForWorkerMessage();
1099
+ assertEqual(latest().transports[0], {
1100
+ type: "WebSocket",
1101
+ id,
1102
+ label: "wss://relay.example/sync",
1103
+ connection: { type: "Open", openedAt: time.now(), error: null },
1104
+ });
1105
+ });
1106
+
1107
+ it("keeps when a transport disconnected while its reconnects fail", async () => {
1108
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1109
+ await using setup = await setupSharedWorker({ createWebSocket });
1110
+ using disposer = new DisposableStack();
1111
+ const { latest } = setupSyncStates(setup, disposer);
1112
+ const { time } = setup.run.deps;
1113
+ const instance = await setup.createEvolu();
1114
+ const transport = await setupAppOwnerTransport(
1115
+ instance,
1116
+ "wss://relay.example",
1117
+ );
1039
1118
  createWebSocket.open(transport.url);
1040
1119
  await testWaitForWorkerMessage();
1120
+ const openedAt = time.now();
1121
+
1122
+ time.advance("1s");
1041
1123
  createWebSocket.close(transport.url);
1042
1124
  await testWaitForWorkerMessage();
1043
- assertSame(latest().transports[0].readyState, "closed");
1125
+ const disconnectedAt = time.now();
1126
+ assertEqual(latest().transports[0].connection, {
1127
+ type: "Disconnected",
1128
+ disconnectedAt,
1129
+ openedAt,
1130
+ error: null,
1131
+ });
1132
+ assertEqual(
1133
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
1134
+ { type: "Offline" },
1135
+ );
1044
1136
 
1045
- // The socket then starts reconnecting without an event, so nothing
1046
- // publishes it until the next snapshot, here one a connecting tab asks
1047
- // for, which reaches every tab.
1137
+ // A failed reconnect attempt records its error, not a new disconnect.
1138
+ time.advance("1s");
1139
+ createWebSocket.error(transport.url, {
1140
+ type: "WebSocketConnectError",
1141
+ event: new Event("error"),
1142
+ });
1143
+ await testWaitForWorkerMessage();
1144
+ const error = { type: "WebSocketConnectError", at: time.now() } as const;
1145
+ const disconnected = {
1146
+ type: "Disconnected",
1147
+ disconnectedAt,
1148
+ openedAt,
1149
+ error,
1150
+ } as const;
1151
+ assertEqual(latest().transports[0].connection, disconnected);
1152
+
1153
+ // The socket starts reconnecting without an event, and a snapshot a
1154
+ // connecting tab asks for, which reaches every tab, reads the same.
1155
+ const published = latest();
1048
1156
  setupTab(setup, disposer);
1049
1157
  await testWaitForWorkerMessage();
1050
- assertSame(latest().transports[0].readyState, "connecting");
1158
+ assertNotSame(latest(), published);
1159
+ assertEqual(latest().transports[0].connection, disconnected);
1160
+
1161
+ // The error stays after the transport opens again.
1162
+ time.advance("1s");
1163
+ createWebSocket.open(transport.url);
1164
+ await testWaitForWorkerMessage();
1165
+ const reopenedAt = time.now();
1166
+ assertEqual(latest().transports[0].connection, {
1167
+ type: "Open",
1168
+ openedAt: reopenedAt,
1169
+ error,
1170
+ });
1171
+
1172
+ // A close records no error, so the earlier one stays.
1173
+ time.advance("1s");
1174
+ createWebSocket.close(transport.url);
1175
+ await testWaitForWorkerMessage();
1176
+ assertEqual(latest().transports[0].connection, {
1177
+ type: "Disconnected",
1178
+ disconnectedAt: time.now(),
1179
+ openedAt: reopenedAt,
1180
+ error,
1181
+ });
1182
+ });
1183
+
1184
+ it("disconnects an open transport at a connection error before its close", async () => {
1185
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
1186
+ await using setup = await setupSharedWorker({ createWebSocket });
1187
+ using disposer = new DisposableStack();
1188
+ const { latest } = setupSyncStates(setup, disposer);
1189
+ const { time } = setup.run.deps;
1190
+ const instance = await setup.createEvolu();
1191
+ const transport = await setupAppOwnerTransport(
1192
+ instance,
1193
+ "wss://relay.example",
1194
+ );
1195
+ createWebSocket.open(transport.url);
1196
+ await testWaitForWorkerMessage();
1197
+ const openedAt = time.now();
1198
+
1199
+ // The socket is closed when the error arrives, and its close follows.
1200
+ time.advance("1s");
1201
+ createWebSocket.error(transport.url, {
1202
+ type: "WebSocketConnectionError",
1203
+ event: new Event("error"),
1204
+ });
1205
+ await testWaitForWorkerMessage();
1206
+ const disconnected = {
1207
+ type: "Disconnected",
1208
+ disconnectedAt: time.now(),
1209
+ openedAt,
1210
+ error: { type: "WebSocketConnectionError", at: time.now() },
1211
+ } as const;
1212
+ assertEqual(latest().transports[0].connection, disconnected);
1213
+
1214
+ time.advance("1s");
1215
+ createWebSocket.close(transport.url);
1216
+ await testWaitForWorkerMessage();
1217
+ assertEqual(latest().transports[0].connection, disconnected);
1218
+
1219
+ // A reopened connection keeps the error until a later one replaces it.
1220
+ time.advance("1s");
1221
+ createWebSocket.open(transport.url);
1222
+ await testWaitForWorkerMessage();
1223
+ const reopenedAt = time.now();
1224
+ assertEqual(latest().transports[0].connection, {
1225
+ type: "Open",
1226
+ openedAt: reopenedAt,
1227
+ error: disconnected.error,
1228
+ });
1229
+ time.advance("1s");
1230
+ createWebSocket.error(transport.url, {
1231
+ type: "WebSocketConnectionError",
1232
+ event: new Event("error"),
1233
+ });
1234
+ await testWaitForWorkerMessage();
1235
+ assertEqual(latest().transports[0].connection, {
1236
+ type: "Disconnected",
1237
+ disconnectedAt: time.now(),
1238
+ openedAt: reopenedAt,
1239
+ error: { type: "WebSocketConnectionError", at: time.now() },
1240
+ });
1051
1241
  });
1052
1242
 
1053
- it("publishes transport and registration changes", async () => {
1243
+ it("publishes a readonly owner with transports and a writable owner with routes only", async () => {
1054
1244
  const createWebSocket = testCreateWebSocket({ isOpen: false });
1055
1245
  await using setup = await setupSharedWorker({ createWebSocket });
1056
1246
  using disposer = new DisposableStack();
@@ -1088,96 +1278,64 @@ describe("sync state", () => {
1088
1278
  await testWaitForWorkerMessage();
1089
1279
 
1090
1280
  const registered = latest();
1091
- assertEqual(
1092
- registered.transports.map(({ label, readyState, error }) => ({
1093
- label,
1094
- readyState,
1095
- error,
1096
- })),
1097
- [
1098
- {
1099
- label: "wss://relay.example/sync",
1100
- readyState: "connecting",
1101
- error: null,
1102
- },
1103
- {
1104
- label: "wss://relay.example/sync",
1105
- readyState: "connecting",
1106
- error: null,
1107
- },
1108
- ],
1109
- );
1110
1281
  const [writableId, readonlyId] = registered.transports.map(({ id }) => id);
1282
+ assertEqual(registered.transports, [
1283
+ {
1284
+ type: "WebSocket",
1285
+ id: writableId,
1286
+ label: "wss://relay.example/sync",
1287
+ connection: { type: "Connecting" },
1288
+ },
1289
+ {
1290
+ type: "WebSocket",
1291
+ id: readonlyId,
1292
+ label: "wss://relay.example/sync",
1293
+ connection: { type: "Connecting" },
1294
+ },
1295
+ ]);
1296
+ assertEqual(registered.tenants, [
1297
+ {
1298
+ type: "Active",
1299
+ name: testName,
1300
+ owners: [
1301
+ {
1302
+ type: "Writable",
1303
+ ownerId: testAppOwner.id,
1304
+ routes: [
1305
+ {
1306
+ type: "Pending",
1307
+ transportId: writableId,
1308
+ failure: null,
1309
+ skippedError: null,
1310
+ completeAt: null,
1311
+ lastSentAt: null,
1312
+ lastReceivedAt: null,
1313
+ },
1314
+ ],
1315
+ },
1316
+ {
1317
+ type: "Readonly",
1318
+ ownerId: readonlyOwner.id,
1319
+ transportIds: [readonlyId],
1320
+ },
1321
+ ],
1322
+ },
1323
+ ]);
1324
+ // A readonly owner is not synced here, and no relay syncs it.
1111
1325
  assertEqual(
1112
- registered.tenants.map(({ name, refused, owners }) => ({
1113
- name,
1114
- refused,
1115
- owners: owners.map(({ routes, ...owner }) => ({
1116
- ...owner,
1117
- routeCount: routes.length,
1118
- })),
1119
- })),
1120
- [
1121
- {
1122
- name: testName,
1123
- refused: false,
1124
- owners: [
1125
- {
1126
- ownerId: testAppOwner.id,
1127
- writable: true,
1128
- transportIds: [writableId],
1129
- routeCount: 1,
1130
- },
1131
- {
1132
- ownerId: readonlyOwner.id,
1133
- writable: false,
1134
- transportIds: [readonlyId],
1135
- routeCount: 0,
1136
- },
1137
- ],
1138
- },
1139
- ],
1326
+ syncStateToOwnerSyncStatus(registered, testName, readonlyOwner.id),
1327
+ { type: "NoRelays" },
1140
1328
  );
1141
1329
 
1142
- createWebSocket.open(writableTransport.url);
1143
- await testWaitForWorkerMessage();
1144
- assertSame(latest().transports[0].readyState, "open");
1145
- assertEqual(latest().transports[0].openedAt, time.now());
1146
-
1147
- time.advance("1s");
1148
- createWebSocket.close(writableTransport.url, { code: 1006 });
1149
- await testWaitForWorkerMessage();
1150
- // The close is published while the socket is still closed.
1151
- assertSame(latest().transports[0].readyState, "closed");
1152
- assertEqual(latest().transports[0].closedAt, time.now());
1153
-
1154
- const error: WebSocketError = {
1155
- type: "WebSocketConnectError",
1156
- event: new Event("error"),
1157
- };
1158
- createWebSocket.error(writableTransport.url, error);
1159
- await testWaitForWorkerMessage();
1160
- // By the next publish the socket has been dropped and is being retried.
1161
- assertSame(latest().transports[0].readyState, "connecting");
1162
- assertEqual(latest().transports[0].error, {
1163
- type: "WebSocketConnectError",
1164
- at: time.now(),
1165
- });
1166
- const lastError = latest().transports[0].error;
1167
- time.advance("1s");
1168
- createWebSocket.open(writableTransport.url);
1169
- await testWaitForWorkerMessage();
1170
- assertSame(latest().transports[0].readyState, "open");
1171
- assertEqual(latest().transports[0].openedAt, time.now());
1172
- assertEqual(latest().transports[0].error, lastError);
1173
-
1174
1330
  evoluChannel.port2.postMessage({
1175
1331
  type: "UseOwner",
1176
1332
  actions: [{ owner: readonlyRegistration, action: "remove" }],
1177
1333
  });
1178
1334
  await testWaitForWorkerMessage();
1335
+ const [tenant] = latest().tenants;
1336
+ assert(tenant?.type === "Active", "Expected an active tenant.");
1179
1337
  assertEqual(
1180
- latest().tenants[0].owners.map(({ ownerId }) => ownerId),
1338
+ tenant.owners.map(({ ownerId }) => ownerId),
1181
1339
  [testAppOwner.id],
1182
1340
  );
1183
1341
 
@@ -1193,7 +1351,7 @@ describe("sync state", () => {
1193
1351
  // follows after its own idle period.
1194
1352
  await instance[Symbol.asyncDispose]();
1195
1353
  assertEqual(latest().tenants, [
1196
- { name: testName, refused: false, owners: [] },
1354
+ { type: "Active", name: testName, owners: [] },
1197
1355
  ]);
1198
1356
  time.advance("3s");
1199
1357
  await testWaitForWorkerMessage();
@@ -1201,6 +1359,12 @@ describe("sync state", () => {
1201
1359
  });
1202
1360
 
1203
1361
  describe("routes", () => {
1362
+ /** The error of a received message the DbWorker skipped. */
1363
+ const skippedError = {
1364
+ type: "DecryptWithXChaCha20Poly1305Error",
1365
+ error: "wrong encryption key",
1366
+ } satisfies DecryptWithXChaCha20Poly1305Error;
1367
+
1204
1368
  /** A header-only relay Response: nothing to reconcile. */
1205
1369
  const relayResponse = (ownerId = testAppOwner.id): ArrayBuffer =>
1206
1370
  protocolMessageToArrayBuffer(
@@ -1216,15 +1380,23 @@ describe("sync state", () => {
1216
1380
  transportIndex = 0,
1217
1381
  ownerId: OwnerId = testAppOwner.id,
1218
1382
  ): SyncRoute => {
1219
- const tenant = state.tenants.find(({ name }) => name === tenantName);
1220
- assertNotUndefined(tenant);
1221
- const owner = tenant.owners.find((owner) => owner.ownerId === ownerId);
1222
- assertNotUndefined(owner);
1383
+ const owner = ownersOf(state, tenantName).find(
1384
+ (owner) => owner.ownerId === ownerId,
1385
+ );
1386
+ assert(owner?.type === "Writable", "Expected a writable owner.");
1223
1387
  const route = owner.routes[transportIndex];
1224
1388
  assertNotUndefined(route);
1225
1389
  return route;
1226
1390
  };
1227
1391
 
1392
+ /** The route's failure, which only a Pending route has. */
1393
+ const failureOf = (route: SyncRoute): SyncRouteError | null =>
1394
+ route.type === "Pending" ? route.failure : null;
1395
+
1396
+ /** The change the route skipped, which a Complete route never has. */
1397
+ const skippedErrorOf = (route: SyncRoute): SyncRouteError | null =>
1398
+ route.type === "Complete" ? null : route.skippedError;
1399
+
1228
1400
  /** Posts a mutation for the owner without answering it. */
1229
1401
  const queueMutation = async (
1230
1402
  instance: TestEvoluInstance,
@@ -1335,17 +1507,18 @@ describe("sync state", () => {
1335
1507
  });
1336
1508
  await testWaitForWorkerMessage();
1337
1509
  // The first claim requested a round that is not sent yet.
1338
- assertFalse(routeOf(latest()).complete);
1510
+ assertSame(routeOf(latest()).type, "Pending");
1339
1511
 
1340
1512
  await respondToSyncRound(instance, createWebSocket);
1341
1513
  const sentAt = time.now();
1342
1514
  assertEqual(routeOf(latest()), {
1515
+ type: "Pending",
1343
1516
  transportId: latest().transports[0].id,
1344
- complete: false,
1517
+ failure: null,
1518
+ skippedError: null,
1345
1519
  completeAt: null,
1346
1520
  lastSentAt: sentAt,
1347
1521
  lastReceivedAt: null,
1348
- error: null,
1349
1522
  });
1350
1523
 
1351
1524
  // The relay answers with nothing to reconcile.
@@ -1353,35 +1526,35 @@ describe("sync state", () => {
1353
1526
  createWebSocket.message(transport.url, relayResponse());
1354
1527
  await testWaitForWorkerMessage();
1355
1528
  // Received but not applied yet.
1356
- assertFalse(routeOf(latest()).complete);
1529
+ assertSame(routeOf(latest()).type, "Pending");
1357
1530
  await respondToApplySync(instance, false, {
1358
1531
  ok: true,
1359
1532
  value: { type: "Converged" },
1360
1533
  });
1361
1534
  assertEqual(routeOf(latest()), {
1535
+ type: "Complete",
1362
1536
  transportId: latest().transports[0].id,
1363
- complete: true,
1364
1537
  completeAt: time.now(),
1365
1538
  lastSentAt: sentAt,
1366
1539
  lastReceivedAt: time.now(),
1367
- error: null,
1368
1540
  });
1369
1541
  const completeAt = time.now();
1370
1542
 
1371
1543
  // A mutation needs only its upload answered.
1372
1544
  time.advance("1s");
1373
1545
  await postMutation(instance, setup.run);
1374
- assertFalse(routeOf(latest()).complete);
1546
+ assertSame(routeOf(latest()).type, "Pending");
1375
1547
  createWebSocket.message(transport.url, relayResponse());
1376
1548
  await testWaitForWorkerMessage();
1377
1549
  await respondToApplySync(instance, false, {
1378
1550
  ok: true,
1379
1551
  value: { type: "Converged" },
1380
1552
  });
1381
- assertTrue(routeOf(latest()).complete);
1553
+ assertSame(routeOf(latest()).type, "Complete");
1382
1554
  // Completing again after the upload restamps the time.
1383
- assertEqual(routeOf(latest()).completeAt, time.now());
1384
- assertNotSame(routeOf(latest()).completeAt, completeAt);
1555
+ const recompletedAt = time.now();
1556
+ assertEqual(routeOf(latest()).completeAt, recompletedAt);
1557
+ assertNotSame(recompletedAt, completeAt);
1385
1558
 
1386
1559
  // A rejected upload marks the route until a new round converges.
1387
1560
  time.advance("1s");
@@ -1392,17 +1565,21 @@ describe("sync state", () => {
1392
1565
  ok: false,
1393
1566
  error: { type: "ProtocolQuotaError", ownerId: testAppOwner.id },
1394
1567
  });
1395
- assertFalse(routeOf(latest()).complete);
1396
- assertEqual(routeOf(latest()).error, {
1568
+ const quotaFailed = routeOf(latest());
1569
+ assertSame(quotaFailed.type, "Pending");
1570
+ assertEqual(quotaFailed.failure, {
1397
1571
  type: "ProtocolQuotaError",
1572
+ ownerId: testAppOwner.id,
1398
1573
  at: time.now(),
1399
1574
  });
1575
+ // The route keeps when it last became complete.
1576
+ assertSame(quotaFailed.completeAt, recompletedAt);
1400
1577
 
1401
1578
  // The failure requested the round that clears it.
1402
1579
  time.advance("1s");
1403
1580
  await testWaitForWorkerMessage();
1404
1581
  await respondToSyncRound(instance, createWebSocket);
1405
- assertFalse(routeOf(latest()).complete);
1582
+ assertSame(routeOf(latest()).type, "Pending");
1406
1583
  createWebSocket.message(transport.url, relayResponse());
1407
1584
  await testWaitForWorkerMessage();
1408
1585
  await respondToApplySync(instance, false, {
@@ -1410,12 +1587,11 @@ describe("sync state", () => {
1410
1587
  value: { type: "Converged" },
1411
1588
  });
1412
1589
  assertEqual(routeOf(latest()), {
1590
+ type: "Complete",
1413
1591
  transportId: latest().transports[0].id,
1414
- complete: true,
1415
1592
  completeAt: time.now(),
1416
1593
  lastSentAt: time.now(),
1417
1594
  lastReceivedAt: time.now(),
1418
- error: null,
1419
1595
  });
1420
1596
  });
1421
1597
 
@@ -1463,7 +1639,7 @@ describe("sync state", () => {
1463
1639
  value: { type: "Converged" },
1464
1640
  });
1465
1641
  assertTrue(baseTime.now() > completedAt);
1466
- assertTrue(routeOf(latest()).complete);
1642
+ assertSame(routeOf(latest()).type, "Complete");
1467
1643
  assertSame(routeOf(latest()).lastReceivedAt, completedAt);
1468
1644
  assertSame(routeOf(latest()).completeAt, completedAt);
1469
1645
  });
@@ -1496,7 +1672,7 @@ describe("sync state", () => {
1496
1672
  ok: true,
1497
1673
  value: { type: "Converged" },
1498
1674
  });
1499
- assertTrue(routeOf(latest()).complete);
1675
+ assertSame(routeOf(latest()).type, "Complete");
1500
1676
 
1501
1677
  const publish = setupDeferredSyncStatePublication(
1502
1678
  t,
@@ -1541,9 +1717,8 @@ describe("sync state", () => {
1541
1717
  await testWaitForWorkerMessage();
1542
1718
  }
1543
1719
  assertTrue(states.length > publishedCount);
1544
- assertTrue(routeOf(latest()).complete);
1720
+ assertSame(routeOf(latest()).type, "Complete");
1545
1721
  assertSame(routeOf(latest()).completeAt, completedAt);
1546
- assertSame(routeOf(latest()).error, null);
1547
1722
  });
1548
1723
 
1549
1724
  it("forgets released routes before a new registration without reading a snapshot", async (t) => {
@@ -1569,7 +1744,7 @@ describe("sync state", () => {
1569
1744
  ok: true,
1570
1745
  value: { type: "Converged" },
1571
1746
  });
1572
- assertTrue(routeOf(latest()).complete);
1747
+ assertSame(routeOf(latest()).type, "Complete");
1573
1748
  assertNotSame(routeOf(latest()).completeAt, null);
1574
1749
 
1575
1750
  const publish = setupDeferredSyncStatePublication(
@@ -1598,10 +1773,11 @@ describe("sync state", () => {
1598
1773
  // No snapshot observed the registration gap. The next one must still
1599
1774
  // show a fresh route, whose new round has not been sent.
1600
1775
  const route = routeOf(latest());
1601
- assertFalse(route.complete);
1776
+ assertSame(route.type, "Pending");
1602
1777
  assertSame(route.completeAt, null);
1603
1778
  assertSame(route.lastSentAt, null);
1604
- assertSame(route.error, null);
1779
+ assertSame(route.failure, null);
1780
+ assertSame(route.skippedError, null);
1605
1781
  });
1606
1782
 
1607
1783
  for (const removed of [
@@ -1674,10 +1850,12 @@ describe("sync state", () => {
1674
1850
  queries: createSet([testQuery]),
1675
1851
  });
1676
1852
  await testWaitForWorkerMessage();
1853
+ const [ownerAfterRemoval] = ownersOf(latest());
1677
1854
  assertFalse(
1678
- latest().tenants[0].owners[0].routes.some(
1679
- ({ transportId }) => transportId === sourceTransportId,
1680
- ),
1855
+ ownerAfterRemoval?.type === "Writable" &&
1856
+ ownerAfterRemoval.routes.some(
1857
+ ({ transportId }) => transportId === sourceTransportId,
1858
+ ),
1681
1859
  );
1682
1860
  createWebSocket.sentMessages.splice(0);
1683
1861
 
@@ -1685,7 +1863,7 @@ describe("sync state", () => {
1685
1863
  type: "ProtocolInvalidDataError",
1686
1864
  data: Uint8Array.of(255),
1687
1865
  error: "late range decoding failure",
1688
- } satisfies StorageWriteMessagesError;
1866
+ } satisfies ProtocolInvalidDataError;
1689
1867
  const continuation = createProtocolMessageForUnsubscribe(
1690
1868
  testAppOwner.id,
1691
1869
  );
@@ -1701,10 +1879,9 @@ describe("sync state", () => {
1701
1879
  );
1702
1880
 
1703
1881
  // Losing the route must not discard an accepted database response.
1704
- assertEqual(
1705
- errors,
1706
- outcome === "error" ? [{ type: "Error", error }] : [],
1707
- );
1882
+ // The route is gone, so a late relay failure is dropped: it is not
1883
+ // posted as an error, and no remaining route shows it.
1884
+ assertEqual(errors, []);
1708
1885
  assertEqual(outputs, [{ type: "RefreshQueries" }]);
1709
1886
  assertEqual(
1710
1887
  createWebSocket.sentMessages.splice(0),
@@ -1712,15 +1889,17 @@ describe("sync state", () => {
1712
1889
  ? [{ url: sourceTransport.url, data: continuation }]
1713
1890
  : [],
1714
1891
  );
1715
- const owner = latest().tenants[0].owners[0];
1716
- assertSame(owner.writable, keepsWritable);
1717
- assertSame(owner.routes.length, keepsWritable ? 1 : 0);
1892
+ const [owner] = ownersOf(latest());
1893
+ assertSame(owner?.type, keepsWritable ? "Writable" : "Readonly");
1894
+ const routes = owner?.type === "Writable" ? owner.routes : [];
1895
+ assertSame(routes.length, keepsWritable ? 1 : 0);
1718
1896
  assertFalse(
1719
- owner.routes.some(
1720
- ({ transportId }) => transportId === sourceTransportId,
1721
- ),
1897
+ routes.some(({ transportId }) => transportId === sourceTransportId),
1722
1898
  );
1723
- for (const route of owner.routes) assertSame(route.error, null);
1899
+ for (const route of routes) {
1900
+ assertSame(failureOf(route), null);
1901
+ assertSame(skippedErrorOf(route), null);
1902
+ }
1724
1903
 
1725
1904
  const query = instance.dbInputs.at(-1);
1726
1905
  assertNotUndefined(query);
@@ -1786,7 +1965,7 @@ describe("sync state", () => {
1786
1965
  value: { type: "Converged" },
1787
1966
  });
1788
1967
  const otherRoute = routeOf(latest(), testName, 1);
1789
- assertTrue(otherRoute.complete);
1968
+ assertSame(otherRoute.type, "Complete");
1790
1969
  createWebSocket.message(transport.url, relayResponse());
1791
1970
  await testWaitForWorkerMessage();
1792
1971
  const inputCount = instance.dbInputs.length;
@@ -1797,12 +1976,16 @@ describe("sync state", () => {
1797
1976
 
1798
1977
  // Receiving the answer consumed the outstanding request, but the
1799
1978
  // aborted apply did not establish convergence or a domain error.
1800
- assertFalse(routeOf(latest()).complete);
1979
+ assertSame(routeOf(latest()).type, "Pending");
1801
1980
  assertSame(routeOf(latest()).completeAt, null);
1802
1981
  // Nothing was applied, so the route reports no received frame.
1803
1982
  assertSame(routeOf(latest()).lastReceivedAt, null);
1804
- assertSame(routeOf(latest()).error, null);
1805
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "syncing");
1983
+ assertSame(failureOf(routeOf(latest())), null);
1984
+ assertSame(skippedErrorOf(routeOf(latest())), null);
1985
+ assertEqual(
1986
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
1987
+ { type: "Syncing" },
1988
+ );
1806
1989
  // The frame came from one relay, so the other route stays complete.
1807
1990
  assertEqual(routeOf(latest(), testName, 1), otherRoute);
1808
1991
  await testWaitForWorkerMessage();
@@ -1815,7 +1998,7 @@ describe("sync state", () => {
1815
1998
  ok: true,
1816
1999
  value: { type: "Converged" },
1817
2000
  });
1818
- assertFalse(routeOf(latest()).complete);
2001
+ assertSame(routeOf(latest()).type, "Pending");
1819
2002
  });
1820
2003
  }
1821
2004
 
@@ -1846,7 +2029,7 @@ describe("sync state", () => {
1846
2029
  ok: true,
1847
2030
  value: { type: "Converged" },
1848
2031
  });
1849
- assertTrue(routeOf(latest()).complete);
2032
+ assertSame(routeOf(latest()).type, "Complete");
1850
2033
 
1851
2034
  // The database worker logged an exception while creating the round.
1852
2035
  instance.evoluChannel.port2.postMessage({
@@ -1877,12 +2060,28 @@ describe("sync state", () => {
1877
2060
  // Nothing is sent or retried until an explicit request or a reopen.
1878
2061
  assertEqual(createWebSocket.sentMessages, []);
1879
2062
  assertLength(instance.dbInputs, inputCount);
1880
- assertFalse(routeOf(latest()).complete);
1881
- assertEqual(routeOf(latest()).error, {
2063
+ assertSame(routeOf(latest()).type, "Pending");
2064
+ assertEqual(failureOf(routeOf(latest())), {
1882
2065
  type: "SyncFailed",
1883
2066
  at: setup.run.deps.time.now(),
1884
2067
  });
1885
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2068
+ assertSame(
2069
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2070
+ "Error",
2071
+ );
2072
+
2073
+ // SyncFailed counts as the route's failure, so a later failed apply
2074
+ // requests no round either.
2075
+ createWebSocket.message(transport.url, relayResponse());
2076
+ await testWaitForWorkerMessage();
2077
+ const error = {
2078
+ type: "ProtocolSyncError",
2079
+ ownerId: testAppOwner.id,
2080
+ } satisfies ProtocolSyncError;
2081
+ await respondToApplySync(instance, false, { ok: false, error });
2082
+ await testWaitForWorkerMessage();
2083
+ assertLength(instance.dbInputs, inputCount + 1);
2084
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
1886
2085
 
1887
2086
  instance.evoluChannel.port2.postMessage({
1888
2087
  type: "UseOwner",
@@ -1896,8 +2095,7 @@ describe("sync state", () => {
1896
2095
  ok: true,
1897
2096
  value: { type: "Converged" },
1898
2097
  });
1899
- assertTrue(routeOf(latest()).complete);
1900
- assertSame(routeOf(latest()).error, null);
2098
+ assertSame(routeOf(latest()).type, "Complete");
1901
2099
  });
1902
2100
 
1903
2101
  it("reports SyncFailed only on the routes of the failed round's target", async () => {
@@ -1930,8 +2128,8 @@ describe("sync state", () => {
1930
2128
  value: { type: "Converged" },
1931
2129
  });
1932
2130
  }
1933
- assertTrue(routeOf(latest(), testName, 0).complete);
1934
- assertTrue(routeOf(latest(), testName, 1).complete);
2131
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
2132
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
1935
2133
 
1936
2134
  // The first transport reopens, and the database worker fails to create
1937
2135
  // the round through it.
@@ -1958,13 +2156,12 @@ describe("sync state", () => {
1958
2156
  });
1959
2157
  await testWaitForWorkerMessage();
1960
2158
 
1961
- assertFalse(routeOf(latest(), testName, 0).complete);
1962
- assertEqual(routeOf(latest(), testName, 0).error, {
2159
+ assertSame(routeOf(latest(), testName, 0).type, "Pending");
2160
+ assertEqual(failureOf(routeOf(latest(), testName, 0)), {
1963
2161
  type: "SyncFailed",
1964
2162
  at: setup.run.deps.time.now(),
1965
2163
  });
1966
- assertTrue(routeOf(latest(), testName, 1).complete);
1967
- assertSame(routeOf(latest(), testName, 1).error, null);
2164
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
1968
2165
  });
1969
2166
 
1970
2167
  it("keeps SyncFailed when a round fails after an earlier round was sent", async () => {
@@ -2026,14 +2223,14 @@ describe("sync state", () => {
2026
2223
  ok: true,
2027
2224
  value: { type: "Converged" },
2028
2225
  });
2029
- assertFalse(routeOf(latest()).complete);
2030
- assertEqual(routeOf(latest()).error, {
2226
+ assertSame(routeOf(latest()).type, "Pending");
2227
+ assertEqual(failureOf(routeOf(latest())), {
2031
2228
  type: "SyncFailed",
2032
2229
  at: setup.run.deps.time.now(),
2033
2230
  });
2034
2231
  });
2035
2232
 
2036
- it("keeps completeAt while a route stays complete", async () => {
2233
+ it("keeps completeAt in a complete route and carries it into Pending", async () => {
2037
2234
  const createWebSocket = testCreateWebSocket();
2038
2235
  await using setup = await setupSharedWorker({ createWebSocket });
2039
2236
  using disposer = new DisposableStack();
@@ -2060,7 +2257,7 @@ describe("sync state", () => {
2060
2257
  ok: true,
2061
2258
  value: { type: "Converged" },
2062
2259
  });
2063
- assertTrue(routeOf(latest()).complete);
2260
+ assertSame(routeOf(latest()).type, "Complete");
2064
2261
  const { completeAt } = routeOf(latest());
2065
2262
  assertNonNullable(completeAt);
2066
2263
 
@@ -2086,32 +2283,51 @@ describe("sync state", () => {
2086
2283
  });
2087
2284
  await testWaitForWorkerMessage();
2088
2285
  await respondToSyncRound(instance, createWebSocket, testAppOwner2);
2089
- assertTrue(routeOf(latest()).complete);
2286
+ assertSame(routeOf(latest()).type, "Complete");
2090
2287
  assertSame(routeOf(latest()).completeAt, completeAt);
2288
+
2289
+ // A route that becomes Pending again keeps when it last became
2290
+ // complete.
2291
+ setup.run.deps.time.advance("5s");
2292
+ await queueMutation(instance);
2293
+ const pending = routeOf(latest());
2294
+ assertSame(pending.type, "Pending");
2295
+ assertSame(pending.completeAt, completeAt);
2296
+
2297
+ // So does a route that settles with a skipped change, and the Pending
2298
+ // route a request to check it again makes of it.
2299
+ await answerMutation(instance, setup.run);
2300
+ createWebSocket.message(transport.url, relayResponse());
2301
+ await testWaitForWorkerMessage();
2302
+ await respondToApplySync(
2303
+ instance,
2304
+ false,
2305
+ { ok: true, value: { type: "Converged" } },
2306
+ skippedError,
2307
+ );
2308
+ await testWaitForWorkerMessage();
2309
+ const settled = routeOf(latest());
2310
+ assertSame(settled.type, "Settled");
2311
+ assertSame(settled.completeAt, completeAt);
2312
+
2313
+ instance.evoluChannel.port2.postMessage({
2314
+ type: "UseOwner",
2315
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
2316
+ });
2317
+ await testWaitForWorkerMessage();
2318
+ const rechecking = routeOf(latest());
2319
+ assertSame(rechecking.type, "Pending");
2320
+ assertSame(rechecking.completeAt, completeAt);
2091
2321
  });
2092
2322
 
2093
- const writeErrors: ReadonlyArray<StorageWriteMessagesError> = [
2094
- {
2095
- type: "DecryptWithXChaCha20Poly1305Error",
2096
- error: "decryption failed",
2097
- },
2098
- {
2099
- type: "ProtocolInvalidDataError",
2100
- data: Uint8Array.of(255),
2101
- error: "decoding failed",
2102
- },
2323
+ const failedApplies = [
2103
2324
  {
2104
- type: "ProtocolTimestampMismatchError",
2105
- expected: createTimestamp(),
2106
- timestamp: createTimestamp({ millis: Millis.orThrow(1) }),
2325
+ result: {
2326
+ ok: false,
2327
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
2328
+ },
2329
+ errorType: "StorageQuotaError",
2107
2330
  },
2108
- { type: "StorageQuotaError", ownerId: testAppOwner.id },
2109
- ];
2110
- const failedApplies = [
2111
- ...writeErrors.map((error) => ({
2112
- result: { ok: false as const, error },
2113
- errorType: error.type,
2114
- })),
2115
2331
  {
2116
2332
  result: { ok: true, value: { type: "Failed", cause: "Write" } },
2117
2333
  errorType: "WriteFailed",
@@ -2121,11 +2337,11 @@ describe("sync state", () => {
2121
2337
  errorType: "SyncFailed",
2122
2338
  },
2123
2339
  ] satisfies ReadonlyArray<{
2124
- result: ApplySyncResult;
2340
+ result: ApplySyncResponse["result"];
2125
2341
  errorType: SyncRouteError["type"];
2126
2342
  }>;
2127
2343
  for (const { result, errorType } of failedApplies) {
2128
- it(`reports ${errorType} and requests only one round after repeated failure`, async () => {
2344
+ it(`shows ${errorType} on its route and requests only one round after repeated failure`, async () => {
2129
2345
  const createWebSocket = testCreateWebSocket();
2130
2346
  await using setup = await setupSharedWorker({ createWebSocket });
2131
2347
  using disposer = new DisposableStack();
@@ -2137,9 +2353,6 @@ describe("sync state", () => {
2137
2353
  errorChannel.onMessage = (output) => {
2138
2354
  errors.push(output);
2139
2355
  };
2140
- const expectedErrors: ReadonlyArray<ConsoleEntryOrError> = result.ok
2141
- ? []
2142
- : [{ type: "Error", error: result.error }];
2143
2356
  const instance = await setup.createEvolu();
2144
2357
  const transport = createOwnerWebSocketTransport({
2145
2358
  url: "wss://failed-apply.example",
@@ -2161,13 +2374,17 @@ describe("sync state", () => {
2161
2374
  await testWaitForWorkerMessage();
2162
2375
  await respondToApplySync(instance, false, result);
2163
2376
 
2164
- // The failure is reported and one round is requested through the route.
2165
- assertEqual(errors, expectedErrors);
2166
- assertEqual(routeOf(latest()).error, {
2167
- type: errorType,
2377
+ // The route shows the failure, which is not posted as an error, and
2378
+ // one round is requested through the route.
2379
+ assertEqual(errors, []);
2380
+ assertEqual(failureOf(routeOf(latest())), {
2381
+ ...(result.ok ? { type: errorType } : result.error),
2168
2382
  at: setup.run.deps.time.now(),
2169
2383
  });
2170
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2384
+ assertSame(
2385
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2386
+ "Error",
2387
+ );
2171
2388
  await testWaitForWorkerMessage();
2172
2389
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
2173
2390
  transport.url,
@@ -2179,10 +2396,13 @@ describe("sync state", () => {
2179
2396
 
2180
2397
  // A repeated failure waits for an explicit request or a reopen.
2181
2398
  await testWaitForWorkerMessage();
2182
- assertEqual(errors, [...expectedErrors, ...expectedErrors]);
2399
+ assertEqual(errors, []);
2183
2400
  assertLength(instance.dbInputs, inputCount);
2184
- assertFalse(routeOf(latest()).complete);
2185
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "error");
2401
+ assertSame(routeOf(latest()).type, "Pending");
2402
+ assertSame(
2403
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id).type,
2404
+ "Error",
2405
+ );
2186
2406
 
2187
2407
  // An upload is not a round, so its answer does not clear the failure.
2188
2408
  await postMutation(instance, setup.run);
@@ -2196,8 +2416,8 @@ describe("sync state", () => {
2196
2416
  ok: true,
2197
2417
  value: { type: "Converged" },
2198
2418
  });
2199
- assertFalse(routeOf(latest()).complete);
2200
- assertSame(routeOf(latest()).error?.type, errorType);
2419
+ assertSame(routeOf(latest()).type, "Pending");
2420
+ assertSame(failureOf(routeOf(latest()))?.type, errorType);
2201
2421
 
2202
2422
  // An explicit request starts the round that clears the failure.
2203
2423
  instance.evoluChannel.port2.postMessage({
@@ -2212,42 +2432,531 @@ describe("sync state", () => {
2212
2432
  ok: true,
2213
2433
  value: { type: "Converged" },
2214
2434
  });
2215
- assertTrue(routeOf(latest()).complete);
2216
- assertSame(routeOf(latest()).error, null);
2217
- assertEqual(errors, [...expectedErrors, ...expectedErrors]);
2435
+ assertSame(routeOf(latest()).type, "Complete");
2436
+ assertEqual(errors, []);
2218
2437
  });
2219
2438
  }
2220
2439
 
2221
- it("reports a write rejection only for the active attempt after leader replacement", async () => {
2222
- const createWebSocket = testCreateWebSocket();
2223
- await using setup = await setupSharedWorker({ createWebSocket });
2224
- using disposer = new DisposableStack();
2225
- const { latest } = setupSyncStates(setup, disposer);
2226
- const errors: Array<ConsoleEntryOrError> = [];
2227
- using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
2228
- consoleEntryOrErrorBroadcastChannelName,
2229
- );
2230
- errorChannel.onMessage = (output) => {
2231
- errors.push(output);
2440
+ describe("skipped message", () => {
2441
+ /** Opens a route and queues the apply of the relay's first reply. */
2442
+ const setupRelayReply = async (
2443
+ setup: SharedWorkerSetup,
2444
+ disposer: DisposableStack,
2445
+ createWebSocket: TestCreateWebSocket,
2446
+ ) => {
2447
+ const { latest } = setupSyncStates(setup, disposer);
2448
+ const errors: Array<ConsoleEntryOrError> = [];
2449
+ const errorChannel = disposer.use(
2450
+ testCreateBroadcastChannel<ConsoleEntryOrError>(
2451
+ consoleEntryOrErrorBroadcastChannelName,
2452
+ ),
2453
+ );
2454
+ errorChannel.onMessage = (output) => {
2455
+ errors.push(output);
2456
+ };
2457
+ const instance = await setup.createEvolu();
2458
+ const transport = createOwnerWebSocketTransport({
2459
+ url: "wss://skipped-message.example",
2460
+ ownerId: testAppOwner.id,
2461
+ });
2462
+ instance.evoluChannel.port2.postMessage({
2463
+ type: "UseOwner",
2464
+ actions: [
2465
+ {
2466
+ owner: { owner: testAppOwner, transports: [transport] },
2467
+ action: "add",
2468
+ },
2469
+ ],
2470
+ });
2471
+ await testWaitForWorkerMessage();
2472
+ await respondToSyncRound(instance, createWebSocket);
2473
+ createWebSocket.message(transport.url, relayResponse());
2474
+ await testWaitForWorkerMessage();
2475
+ return { errors, instance, latest, transport };
2232
2476
  };
2233
- const first = await setup.createEvolu({
2234
- releaseDbWorkerLeaderOnDispose: false,
2235
- });
2236
- const transport = createOwnerWebSocketTransport({
2237
- url: "wss://replayed-write-rejection.example",
2238
- ownerId: testAppOwner.id,
2239
- });
2240
- first.evoluChannel.port2.postMessage({
2241
- type: "UseOwner",
2242
- actions: [
2243
- {
2244
- owner: { owner: testAppOwner, transports: [transport] },
2245
- action: "add",
2246
- },
2247
- ],
2248
- });
2249
- await testWaitForWorkerMessage();
2250
- await respondToSyncRound(first, createWebSocket);
2477
+
2478
+ it("records the skip on the route while its round continues", async () => {
2479
+ const createWebSocket = testCreateWebSocket();
2480
+ await using setup = await setupSharedWorker({ createWebSocket });
2481
+ using disposer = new DisposableStack();
2482
+ const { errors, instance, latest, transport } = await setupRelayReply(
2483
+ setup,
2484
+ disposer,
2485
+ createWebSocket,
2486
+ );
2487
+ const response = createProtocolMessageForUnsubscribe(testAppOwner.id);
2488
+ const inputCount = instance.dbInputs.length;
2489
+ await respondToApplySync(
2490
+ instance,
2491
+ true,
2492
+ { ok: true, value: { type: "Response", message: response } },
2493
+ skippedError,
2494
+ );
2495
+
2496
+ // The response is sent, so the round continues, while the skipped
2497
+ // message is recorded as the route's skippedError with its details,
2498
+ // which is neither a failure nor an Evolu error. No other round is
2499
+ // requested, because this one continues.
2500
+ assertEqual(createWebSocket.sentMessages.splice(0), [
2501
+ { url: transport.url, data: response },
2502
+ ]);
2503
+ assertEqual(errors, []);
2504
+ const receivedAt = setup.run.deps.time.now();
2505
+ assertEqual(routeOf(latest()), {
2506
+ type: "Pending",
2507
+ transportId: latest().transports[0].id,
2508
+ failure: null,
2509
+ skippedError: {
2510
+ ...skippedError,
2511
+ error: createUnknownError(skippedError.error),
2512
+ at: receivedAt,
2513
+ },
2514
+ completeAt: null,
2515
+ lastSentAt: receivedAt,
2516
+ lastReceivedAt: receivedAt,
2517
+ });
2518
+ await testWaitForWorkerMessage();
2519
+ assertLength(instance.dbInputs, inputCount);
2520
+
2521
+ // The relay converges and skips a message again, whose error replaces
2522
+ // the shown one. Nothing is outstanding, yet the route does not
2523
+ // complete, and no round is requested.
2524
+ setup.run.deps.time.advance("1s");
2525
+ const mismatch = {
2526
+ type: "ProtocolTimestampMismatchError",
2527
+ expected: createTimestamp(),
2528
+ timestamp: createTimestamp({ nodeId: maxNodeId }),
2529
+ } satisfies ProtocolTimestampMismatchError;
2530
+ createWebSocket.message(transport.url, relayResponse());
2531
+ await testWaitForWorkerMessage();
2532
+ await respondToApplySync(
2533
+ instance,
2534
+ false,
2535
+ { ok: true, value: { type: "Converged" } },
2536
+ mismatch,
2537
+ );
2538
+ await testWaitForWorkerMessage();
2539
+ assertLength(instance.dbInputs, inputCount + 1);
2540
+ assertEqual(errors, []);
2541
+ assertEqual(routeOf(latest()), {
2542
+ type: "Settled",
2543
+ transportId: latest().transports[0].id,
2544
+ skippedError: { ...mismatch, at: setup.run.deps.time.now() },
2545
+ completeAt: null,
2546
+ lastSentAt: receivedAt,
2547
+ lastReceivedAt: setup.run.deps.time.now(),
2548
+ });
2549
+
2550
+ // Once the relay stops serving it, an explicit request completes the
2551
+ // route and clears the error.
2552
+ instance.evoluChannel.port2.postMessage({
2553
+ type: "UseOwner",
2554
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
2555
+ });
2556
+ await testWaitForWorkerMessage();
2557
+ await respondToSyncRound(instance, createWebSocket);
2558
+ createWebSocket.message(transport.url, relayResponse());
2559
+ await testWaitForWorkerMessage();
2560
+ await respondToApplySync(instance, false, {
2561
+ ok: true,
2562
+ value: { type: "Converged" },
2563
+ });
2564
+ assertSame(routeOf(latest()).type, "Complete");
2565
+ });
2566
+
2567
+ it("leaves its route the error that ended its round", async () => {
2568
+ const createWebSocket = testCreateWebSocket();
2569
+ await using setup = await setupSharedWorker({ createWebSocket });
2570
+ using disposer = new DisposableStack();
2571
+ const { errors, instance, latest, transport } = await setupRelayReply(
2572
+ setup,
2573
+ disposer,
2574
+ createWebSocket,
2575
+ );
2576
+ const error = {
2577
+ type: "ProtocolInvalidDataError",
2578
+ data: Uint8Array.of(255),
2579
+ error: "decoding failed",
2580
+ } satisfies ProtocolInvalidDataError;
2581
+ await respondToApplySync(
2582
+ instance,
2583
+ true,
2584
+ { ok: false, error },
2585
+ skippedError,
2586
+ );
2587
+
2588
+ // Nothing is posted as an error. The route shows the error that ended
2589
+ // the round, without its data, which requests one round, as any
2590
+ // failure does.
2591
+ assertEqual(errors, []);
2592
+ const failed = routeOf(latest());
2593
+ assertSame(failed.type, "Pending");
2594
+ assertEqual(failed.failure, {
2595
+ type: error.type,
2596
+ error: createUnknownError(error.error),
2597
+ at: setup.run.deps.time.now(),
2598
+ });
2599
+ // The skipped message stays visible beside the failure.
2600
+ assertEqual(failed.skippedError, {
2601
+ ...skippedError,
2602
+ error: createUnknownError(skippedError.error),
2603
+ at: setup.run.deps.time.now(),
2604
+ });
2605
+ await testWaitForWorkerMessage();
2606
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2607
+ transport.url,
2608
+ ]);
2609
+
2610
+ // The retry also checks the skipped message again, so once it skips
2611
+ // nothing, the route completes.
2612
+ createWebSocket.message(transport.url, relayResponse());
2613
+ await testWaitForWorkerMessage();
2614
+ await respondToApplySync(instance, false, {
2615
+ ok: true,
2616
+ value: { type: "Converged" },
2617
+ });
2618
+ assertSame(routeOf(latest()).type, "Complete");
2619
+ });
2620
+
2621
+ it("does not hide a Failed result", async () => {
2622
+ const createWebSocket = testCreateWebSocket();
2623
+ await using setup = await setupSharedWorker({ createWebSocket });
2624
+ using disposer = new DisposableStack();
2625
+ const { errors, instance, latest, transport } = await setupRelayReply(
2626
+ setup,
2627
+ disposer,
2628
+ createWebSocket,
2629
+ );
2630
+ await respondToApplySync(
2631
+ instance,
2632
+ true,
2633
+ { ok: true, value: { type: "Failed", cause: "Sync" } },
2634
+ skippedError,
2635
+ );
2636
+
2637
+ // The route shows the failure that ended the round, which requests
2638
+ // one round, as any failure does. The DbWorker already logged it.
2639
+ assertEqual(errors, []);
2640
+ assertSame(failureOf(routeOf(latest()))?.type, "SyncFailed");
2641
+ await testWaitForWorkerMessage();
2642
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2643
+ transport.url,
2644
+ ]);
2645
+ });
2646
+
2647
+ it("keeps the retry limit when failures also skip a message", async () => {
2648
+ const createWebSocket = testCreateWebSocket();
2649
+ await using setup = await setupSharedWorker({ createWebSocket });
2650
+ using disposer = new DisposableStack();
2651
+ const { instance, latest, transport } = await setupRelayReply(
2652
+ setup,
2653
+ disposer,
2654
+ createWebSocket,
2655
+ );
2656
+ const error = {
2657
+ type: "ProtocolInvalidDataError",
2658
+ data: Uint8Array.of(255),
2659
+ error: "decoding failed",
2660
+ } satisfies ProtocolInvalidDataError;
2661
+
2662
+ // The first failure requests one round, although it also skipped a
2663
+ // message.
2664
+ await respondToApplySync(
2665
+ instance,
2666
+ false,
2667
+ { ok: false, error },
2668
+ skippedError,
2669
+ );
2670
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2671
+ transport.url,
2672
+ ]);
2673
+
2674
+ // The retry's reply fails and skips again, so no further round is
2675
+ // requested.
2676
+ createWebSocket.message(transport.url, relayResponse());
2677
+ await testWaitForWorkerMessage();
2678
+ const inputCount = instance.dbInputs.length;
2679
+ await respondToApplySync(
2680
+ instance,
2681
+ false,
2682
+ { ok: false, error },
2683
+ skippedError,
2684
+ );
2685
+ await testWaitForWorkerMessage();
2686
+ assertLength(instance.dbInputs, inputCount);
2687
+
2688
+ // A converged reply that only skips does not end the wait, so a later
2689
+ // failure requests no round either; only the two applies are added.
2690
+ // The pending failure stays shown over the later skip.
2691
+ createWebSocket.message(transport.url, relayResponse());
2692
+ await testWaitForWorkerMessage();
2693
+ await respondToApplySync(
2694
+ instance,
2695
+ false,
2696
+ { ok: true, value: { type: "Converged" } },
2697
+ skippedError,
2698
+ );
2699
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
2700
+ createWebSocket.message(transport.url, relayResponse());
2701
+ await testWaitForWorkerMessage();
2702
+ await respondToApplySync(instance, false, { ok: false, error });
2703
+ await testWaitForWorkerMessage();
2704
+ assertLength(instance.dbInputs, inputCount + 2);
2705
+ assertSame(routeOf(latest()).type, "Pending");
2706
+ });
2707
+
2708
+ it("leaves a later failure its round", async () => {
2709
+ const createWebSocket = testCreateWebSocket();
2710
+ await using setup = await setupSharedWorker({ createWebSocket });
2711
+ using disposer = new DisposableStack();
2712
+ const { instance, latest, transport } = await setupRelayReply(
2713
+ setup,
2714
+ disposer,
2715
+ createWebSocket,
2716
+ );
2717
+ await respondToApplySync(
2718
+ instance,
2719
+ false,
2720
+ {
2721
+ ok: true,
2722
+ value: {
2723
+ type: "Response",
2724
+ message: createProtocolMessageForUnsubscribe(testAppOwner.id),
2725
+ },
2726
+ },
2727
+ skippedError,
2728
+ );
2729
+ createWebSocket.sentMessages.splice(0);
2730
+ const error = {
2731
+ type: "ProtocolSyncError",
2732
+ ownerId: testAppOwner.id,
2733
+ } satisfies ProtocolSyncError;
2734
+
2735
+ // The route already shows the skipped message, yet the first failure
2736
+ // still requests one round.
2737
+ createWebSocket.message(transport.url, relayResponse());
2738
+ await testWaitForWorkerMessage();
2739
+ await respondToApplySync(instance, false, { ok: false, error });
2740
+ assertSame(failureOf(routeOf(latest()))?.type, error.type);
2741
+ await testWaitForWorkerMessage();
2742
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2743
+ transport.url,
2744
+ ]);
2745
+
2746
+ // A further failure waits for an explicit request or a reopen.
2747
+ createWebSocket.message(transport.url, relayResponse());
2748
+ await testWaitForWorkerMessage();
2749
+ const inputCount = instance.dbInputs.length;
2750
+ await respondToApplySync(instance, false, { ok: false, error });
2751
+ await testWaitForWorkerMessage();
2752
+ assertLength(instance.dbInputs, inputCount);
2753
+ assertSame(routeOf(latest()).type, "Pending");
2754
+ });
2755
+
2756
+ it("lets a failure request a round after a retry skips the message again", async () => {
2757
+ const createWebSocket = testCreateWebSocket();
2758
+ await using setup = await setupSharedWorker({ createWebSocket });
2759
+ using disposer = new DisposableStack();
2760
+ const { instance, latest, transport } = await setupRelayReply(
2761
+ setup,
2762
+ disposer,
2763
+ createWebSocket,
2764
+ );
2765
+ await respondToApplySync(
2766
+ instance,
2767
+ false,
2768
+ { ok: true, value: { type: "Converged" } },
2769
+ skippedError,
2770
+ );
2771
+ const error = {
2772
+ type: "ProtocolSyncError",
2773
+ ownerId: testAppOwner.id,
2774
+ } satisfies ProtocolSyncError;
2775
+
2776
+ // A failure requests one round, whose relay converges but offers the
2777
+ // skipped message again.
2778
+ createWebSocket.message(transport.url, relayResponse());
2779
+ await testWaitForWorkerMessage();
2780
+ await respondToApplySync(instance, false, { ok: false, error });
2781
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2782
+ transport.url,
2783
+ ]);
2784
+ createWebSocket.message(transport.url, relayResponse());
2785
+ await testWaitForWorkerMessage();
2786
+ await respondToApplySync(
2787
+ instance,
2788
+ false,
2789
+ { ok: true, value: { type: "Converged" } },
2790
+ skippedError,
2791
+ );
2792
+ assertSame(routeOf(latest()).type, "Settled");
2793
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2794
+
2795
+ // The retry's reconciliation has ended, so a later failure requests a
2796
+ // round again, although the skipped message keeps the route
2797
+ // incomplete.
2798
+ createWebSocket.message(transport.url, relayResponse());
2799
+ await testWaitForWorkerMessage();
2800
+ await respondToApplySync(instance, false, { ok: false, error });
2801
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
2802
+ transport.url,
2803
+ ]);
2804
+ });
2805
+
2806
+ it("publishes a caught Error inside a skipped change as an UnknownError", async () => {
2807
+ const createWebSocket = testCreateWebSocket();
2808
+ await using setup = await setupSharedWorker({ createWebSocket });
2809
+ using disposer = new DisposableStack();
2810
+ const { instance, latest } = await setupRelayReply(
2811
+ setup,
2812
+ disposer,
2813
+ createWebSocket,
2814
+ );
2815
+ await respondToApplySync(
2816
+ instance,
2817
+ false,
2818
+ { ok: true, value: { type: "Converged" } },
2819
+ {
2820
+ type: "DecryptWithXChaCha20Poly1305Error",
2821
+ error: new Error("invalid tag", { cause: new Error("tag") }),
2822
+ },
2823
+ );
2824
+
2825
+ // Sync state stays plain data, which tabs compare by content.
2826
+ const skipped = skippedErrorOf(routeOf(latest()));
2827
+ assert(
2828
+ skipped?.type === "DecryptWithXChaCha20Poly1305Error",
2829
+ "The route should show the skipped change.",
2830
+ );
2831
+ const { error } = skipped;
2832
+ assert(
2833
+ UnknownError.is(error) &&
2834
+ isPlainObject(error.error) &&
2835
+ isPlainObject(error.error.cause),
2836
+ "The Error and its cause should be plain objects in an UnknownError.",
2837
+ );
2838
+ assertSame(error.error.message, "invalid tag");
2839
+ assertSame(error.error.cause.message, "tag");
2840
+ });
2841
+
2842
+ it("publishes a caught Error inside a failure as an UnknownError", async () => {
2843
+ const createWebSocket = testCreateWebSocket();
2844
+ await using setup = await setupSharedWorker({ createWebSocket });
2845
+ using disposer = new DisposableStack();
2846
+ const { instance, latest } = await setupRelayReply(
2847
+ setup,
2848
+ disposer,
2849
+ createWebSocket,
2850
+ );
2851
+ await respondToApplySync(instance, false, {
2852
+ ok: false,
2853
+ error: {
2854
+ type: "ProtocolInvalidDataError",
2855
+ data: Uint8Array.of(255),
2856
+ error: new Error("invalid varint"),
2857
+ },
2858
+ });
2859
+
2860
+ const failure = failureOf(routeOf(latest()));
2861
+ assert(
2862
+ failure?.type === "ProtocolInvalidDataError",
2863
+ "The route should show the failure.",
2864
+ );
2865
+ const { error } = failure;
2866
+ assert(
2867
+ UnknownError.is(error) && isPlainObject(error.error),
2868
+ "The Error should be a plain object in an UnknownError.",
2869
+ );
2870
+ assertSame(error.error.message, "invalid varint");
2871
+ });
2872
+
2873
+ it("keeps its skip through an aborted apply", async () => {
2874
+ const createWebSocket = testCreateWebSocket();
2875
+ await using setup = await setupSharedWorker({ createWebSocket });
2876
+ using disposer = new DisposableStack();
2877
+ const { errors, instance, latest, transport } = await setupRelayReply(
2878
+ setup,
2879
+ disposer,
2880
+ createWebSocket,
2881
+ );
2882
+ await respondToApplySync(
2883
+ instance,
2884
+ false,
2885
+ { ok: true, value: { type: "Converged" } },
2886
+ skippedError,
2887
+ );
2888
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2889
+
2890
+ // An abort proves nothing about the skipped message, so the route
2891
+ // still shows it, and it is not a failure.
2892
+ createWebSocket.message(transport.url, relayResponse());
2893
+ await testWaitForWorkerMessage();
2894
+ await respondToApplySync(instance, false, {
2895
+ ok: false,
2896
+ error: { type: "AbortError", reason: { type: "Stop" } },
2897
+ });
2898
+ assertEqual(errors, []);
2899
+ assertSame(routeOf(latest()).type, "Pending");
2900
+ assertSame(skippedErrorOf(routeOf(latest()))?.type, skippedError.type);
2901
+ assertSame(failureOf(routeOf(latest())), null);
2902
+ });
2903
+
2904
+ it("is ignored for an aborted apply", async () => {
2905
+ const createWebSocket = testCreateWebSocket();
2906
+ await using setup = await setupSharedWorker({ createWebSocket });
2907
+ using disposer = new DisposableStack();
2908
+ const { errors, instance, latest } = await setupRelayReply(
2909
+ setup,
2910
+ disposer,
2911
+ createWebSocket,
2912
+ );
2913
+ await respondToApplySync(
2914
+ instance,
2915
+ false,
2916
+ {
2917
+ ok: false,
2918
+ error: { type: "AbortError", reason: { type: "Stop" } },
2919
+ },
2920
+ skippedError,
2921
+ );
2922
+
2923
+ assertEqual(errors, []);
2924
+ assertSame(failureOf(routeOf(latest())), null);
2925
+ assertSame(skippedErrorOf(routeOf(latest())), null);
2926
+ assertSame(routeOf(latest()).type, "Pending");
2927
+ });
2928
+ });
2929
+
2930
+ it("applies a write rejection only for the active attempt after leader replacement", async () => {
2931
+ const createWebSocket = testCreateWebSocket();
2932
+ await using setup = await setupSharedWorker({ createWebSocket });
2933
+ using disposer = new DisposableStack();
2934
+ const { latest } = setupSyncStates(setup, disposer);
2935
+ const errors: Array<ConsoleEntryOrError> = [];
2936
+ using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
2937
+ consoleEntryOrErrorBroadcastChannelName,
2938
+ );
2939
+ errorChannel.onMessage = (output) => {
2940
+ errors.push(output);
2941
+ };
2942
+ const first = await setup.createEvolu({
2943
+ releaseDbWorkerLeaderOnDispose: false,
2944
+ });
2945
+ const transport = createOwnerWebSocketTransport({
2946
+ url: "wss://replayed-write-rejection.example",
2947
+ ownerId: testAppOwner.id,
2948
+ });
2949
+ first.evoluChannel.port2.postMessage({
2950
+ type: "UseOwner",
2951
+ actions: [
2952
+ {
2953
+ owner: { owner: testAppOwner, transports: [transport] },
2954
+ action: "add",
2955
+ },
2956
+ ],
2957
+ });
2958
+ await testWaitForWorkerMessage();
2959
+ await respondToSyncRound(first, createWebSocket);
2251
2960
  createWebSocket.message(transport.url, relayResponse());
2252
2961
  await testWaitForWorkerMessage();
2253
2962
  const original = first.dbInputs.at(-1);
@@ -2276,8 +2985,8 @@ describe("sync state", () => {
2276
2985
  assertNotSame(replay.attemptId, original.attemptId);
2277
2986
 
2278
2987
  const error = {
2279
- type: "DecryptWithXChaCha20Poly1305Error",
2280
- error: "wrong encryption key",
2988
+ type: "StorageQuotaError",
2989
+ ownerId: testAppOwner.id,
2281
2990
  } satisfies StorageWriteMessagesError;
2282
2991
  const response = {
2283
2992
  type: "OnQueuedResponse",
@@ -2289,34 +2998,36 @@ describe("sync state", () => {
2289
2998
  clock: createTimestamp(),
2290
2999
  ownerId: testAppOwner.id,
2291
3000
  didWriteMessages: false,
3001
+ skippedError: null,
2292
3002
  result: { ok: false, error },
2293
3003
  },
2294
3004
  },
2295
3005
  } satisfies DbWorkerOutput;
2296
3006
 
2297
3007
  // The retired port and an obsolete attempt on the live port must both
2298
- // leave the active replay and its error reporting untouched.
3008
+ // leave the active replay and its route untouched.
2299
3009
  first.dbWorkerPort.postMessage(response);
2300
3010
  dbWorkerPort.postMessage(response);
2301
3011
  await testWaitForWorkerMessage();
2302
3012
  assertEqual(errors, []);
2303
- assertSame(routeOf(latest()).error, null);
3013
+ assertSame(failureOf(routeOf(latest())), null);
3014
+ assertSame(skippedErrorOf(routeOf(latest())), null);
2304
3015
  assertLength(dbInputs, 1);
2305
3016
 
2306
3017
  const currentResponse = { ...response, attemptId: replay.attemptId };
2307
3018
  dbWorkerPort.postMessage(currentResponse);
2308
3019
  await testWaitForWorkerMessage();
2309
- assertEqual(errors, [{ type: "Error", error }]);
2310
- assertEqual(routeOf(latest()).error, {
2311
- type: error.type,
2312
- at: setup.run.deps.time.now(),
2313
- });
3020
+ const routeError = { ...error, at: setup.run.deps.time.now() };
3021
+ assertEqual(errors, []);
3022
+ assertEqual(failureOf(routeOf(latest())), routeError);
2314
3023
  assertLength(dbInputs, 2);
2315
3024
 
2316
- // A duplicate completion is stale as soon as its attempt is accepted.
3025
+ // A duplicate completion is stale as soon as its attempt is accepted,
3026
+ // so the route keeps the time of the accepted one.
3027
+ setup.run.deps.time.advance("1s");
2317
3028
  dbWorkerPort.postMessage(currentResponse);
2318
3029
  await testWaitForWorkerMessage();
2319
- assertEqual(errors, [{ type: "Error", error }]);
3030
+ assertEqual(failureOf(routeOf(latest())), routeError);
2320
3031
  assertLength(dbInputs, 2);
2321
3032
  const replacement = { dbInputs, dbWorkerPort };
2322
3033
  await respondToSyncRound(replacement, createWebSocket);
@@ -2326,9 +3037,8 @@ describe("sync state", () => {
2326
3037
  ok: true,
2327
3038
  value: { type: "Converged" },
2328
3039
  });
2329
- assertTrue(routeOf(latest()).complete);
2330
- assertSame(routeOf(latest()).error, null);
2331
- assertEqual(errors, [{ type: "Error", error }]);
3040
+ assertSame(routeOf(latest()).type, "Complete");
3041
+ assertEqual(errors, []);
2332
3042
  });
2333
3043
 
2334
3044
  it("keeps an aborted sibling apply incomplete without scheduling recovery", async () => {
@@ -2365,7 +3075,7 @@ describe("sync state", () => {
2365
3075
  });
2366
3076
  }
2367
3077
  }
2368
- assertTrue(routeOf(latest()).complete);
3078
+ assertSame(routeOf(latest()).type, "Complete");
2369
3079
  const completeAt = routeOf(latest()).completeAt;
2370
3080
 
2371
3081
  await postMutation(second, setup.run);
@@ -2392,10 +3102,11 @@ describe("sync state", () => {
2392
3102
  ok: true,
2393
3103
  value: { type: "Converged" },
2394
3104
  });
2395
- assertFalse(routeOf(latest()).complete);
3105
+ assertSame(routeOf(latest()).type, "Pending");
2396
3106
  assertSame(routeOf(latest()).completeAt, completeAt);
2397
- assertSame(routeOf(latest()).error, null);
2398
- assertTrue(routeOf(latest(), secondName).complete);
3107
+ assertSame(failureOf(routeOf(latest())), null);
3108
+ assertSame(skippedErrorOf(routeOf(latest())), null);
3109
+ assertSame(routeOf(latest(), secondName).type, "Complete");
2399
3110
  await testWaitForWorkerMessage();
2400
3111
  assertLength(first.dbInputs, inputCount);
2401
3112
  });
@@ -2464,6 +3175,34 @@ describe("sync state", () => {
2464
3175
  return { source, sibling, siblingName, transports };
2465
3176
  };
2466
3177
 
3178
+ it("completes a route when its sibling continuation copy is applied", async () => {
3179
+ const createWebSocket = testCreateWebSocket();
3180
+ await using setup = await setupSharedWorker({ createWebSocket });
3181
+ using disposer = new DisposableStack();
3182
+ const { latest } = setupSyncStates(setup, disposer);
3183
+ const { sibling, siblingName } = await setupSiblingContinuation(
3184
+ setup,
3185
+ createWebSocket,
3186
+ );
3187
+ const queued = routeOf(latest(), siblingName, 1);
3188
+ assertSame(queued.type, "Pending");
3189
+
3190
+ // The route through the other relay completes when the copy is applied,
3191
+ // which is later than its last frame.
3192
+ setup.run.deps.time.advance("1s");
3193
+ await respondToApplySync(sibling, false, {
3194
+ ok: true,
3195
+ value: { type: "Broadcast" },
3196
+ });
3197
+ assertEqual(routeOf(latest(), siblingName, 1), {
3198
+ type: "Complete",
3199
+ transportId: queued.transportId,
3200
+ completeAt: setup.run.deps.time.now(),
3201
+ lastSentAt: queued.lastSentAt,
3202
+ lastReceivedAt: queued.lastReceivedAt,
3203
+ });
3204
+ });
3205
+
2467
3206
  it("keeps every route incomplete after aborting a sibling continuation copy", async () => {
2468
3207
  const createWebSocket = testCreateWebSocket();
2469
3208
  await using setup = await setupSharedWorker({ createWebSocket });
@@ -2494,10 +3233,11 @@ describe("sync state", () => {
2494
3233
 
2495
3234
  for (const index of [0, 1]) {
2496
3235
  const route = routeOf(latest(), siblingName, index);
2497
- assertFalse(route.complete);
3236
+ assertSame(route.type, "Pending");
2498
3237
  assertSame(route.completeAt, completeAt[index]);
2499
- assertSame(route.error, null);
2500
- assertTrue(routeOf(latest(), testName, index).complete);
3238
+ assertSame(route.failure, null);
3239
+ assertSame(route.skippedError, null);
3240
+ assertSame(routeOf(latest(), testName, index).type, "Complete");
2501
3241
  }
2502
3242
  await testWaitForWorkerMessage();
2503
3243
  assertLength(sibling.dbInputs, inputCount);
@@ -2545,8 +3285,8 @@ describe("sync state", () => {
2545
3285
  ok: true,
2546
3286
  value: { type: "Converged" },
2547
3287
  });
2548
- assertFalse(routeOf(latest()).complete);
2549
- assertFalse(routeOf(latest(), secondName).complete);
3288
+ assertSame(routeOf(latest()).type, "Pending");
3289
+ assertSame(routeOf(latest(), secondName).type, "Pending");
2550
3290
 
2551
3291
  createWebSocket.message(transport.url, relayResponse());
2552
3292
  await testWaitForWorkerMessage();
@@ -2554,13 +3294,13 @@ describe("sync state", () => {
2554
3294
  ok: true,
2555
3295
  value: { type: "Converged" },
2556
3296
  });
2557
- assertFalse(routeOf(latest(), secondName).complete);
3297
+ assertSame(routeOf(latest(), secondName).type, "Pending");
2558
3298
  await respondToApplySync(second, false, {
2559
3299
  ok: true,
2560
3300
  value: { type: "Converged" },
2561
3301
  });
2562
- assertTrue(routeOf(latest()).complete);
2563
- assertTrue(routeOf(latest(), secondName).complete);
3302
+ assertSame(routeOf(latest()).type, "Complete");
3303
+ assertSame(routeOf(latest(), secondName).type, "Complete");
2564
3304
  });
2565
3305
 
2566
3306
  it("keeps routes incomplete until a sibling's continuation copy reconciles the other relays", async () => {
@@ -2591,12 +3331,10 @@ describe("sync state", () => {
2591
3331
  (candidate) => candidate.label === label,
2592
3332
  );
2593
3333
  assertNotUndefined(transport);
2594
- const tenant = state.tenants.find(({ name }) => name === tenantName);
2595
- assertNotUndefined(tenant);
2596
- const owner = tenant.owners.find(
3334
+ const owner = ownersOf(state, tenantName).find(
2597
3335
  ({ ownerId }) => ownerId === testAppOwner.id,
2598
3336
  );
2599
- assertNotUndefined(owner);
3337
+ assert(owner?.type === "Writable", "Expected a writable owner.");
2600
3338
  const route = owner.routes.find(
2601
3339
  ({ transportId }) => transportId === transport.id,
2602
3340
  );
@@ -2629,7 +3367,7 @@ describe("sync state", () => {
2629
3367
  }
2630
3368
  for (const label of [labelA, labelB]) {
2631
3369
  for (const name of [testName, siblingName]) {
2632
- assertTrue(routeAt(label, name).complete);
3370
+ assertSame(routeAt(label, name).type, "Complete");
2633
3371
  }
2634
3372
  }
2635
3373
  const siblingCompleteAt = routeAt(labelB, siblingName).completeAt;
@@ -2661,10 +3399,11 @@ describe("sync state", () => {
2661
3399
 
2662
3400
  // Relay B neither sent nor was sent anything, so only the queued local
2663
3401
  // copy can hold the sibling's route through B.
2664
- assertFalse(routeAt(labelB, siblingName).complete);
3402
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2665
3403
  assertSame(routeAt(labelB, siblingName).completeAt, siblingCompleteAt);
2666
- assertSame(routeAt(labelB, siblingName).error, null);
2667
- assertTrue(routeAt(labelB, testName).complete);
3404
+ assertSame(failureOf(routeAt(labelB, siblingName)), null);
3405
+ assertSame(skippedErrorOf(routeAt(labelB, siblingName)), null);
3406
+ assertSame(routeAt(labelB, testName).type, "Complete");
2668
3407
 
2669
3408
  // The copy was uploaded only through A. Storing it requires a new round
2670
3409
  // through B, and local delivery does not count as a frame received on A.
@@ -2674,13 +3413,13 @@ describe("sync state", () => {
2674
3413
  ok: true,
2675
3414
  value: { type: "Broadcast" },
2676
3415
  });
2677
- assertFalse(routeAt(labelB, siblingName).complete);
3416
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2678
3417
  assertSame(routeAt(labelB, siblingName).completeAt, siblingCompleteAt);
2679
3418
  assertSame(routeAt(labelA, siblingName).lastReceivedAt, lastReceivedAt);
2680
3419
  assertEqual(await respondToSyncRound(sibling, createWebSocket), [
2681
3420
  transports[1].url,
2682
3421
  ]);
2683
- assertFalse(routeAt(labelB, siblingName).complete);
3422
+ assertSame(routeAt(labelB, siblingName).type, "Pending");
2684
3423
  createWebSocket.message(transports[1].url, relayResponse());
2685
3424
  await testWaitForWorkerMessage();
2686
3425
  for (const instance of [source, sibling]) {
@@ -2689,7 +3428,7 @@ describe("sync state", () => {
2689
3428
  value: { type: "Converged" },
2690
3429
  });
2691
3430
  }
2692
- assertTrue(routeAt(labelB, siblingName).complete);
3431
+ assertSame(routeAt(labelB, siblingName).type, "Complete");
2693
3432
  });
2694
3433
 
2695
3434
  for (const fractional of [false, true]) {
@@ -2727,6 +3466,7 @@ describe("sync state", () => {
2727
3466
  await testWaitForWorkerMessage();
2728
3467
  createWebSocket.open(transport.url);
2729
3468
  await testWaitForWorkerMessage();
3469
+ const openedAt = time.now();
2730
3470
  await respondToSyncRound(instance, createWebSocket);
2731
3471
  await postMutation(instance, setup.run);
2732
3472
 
@@ -2745,7 +3485,7 @@ describe("sync state", () => {
2745
3485
  time.advance("5s");
2746
3486
  await testWaitForWorkerMessage();
2747
3487
  assertEqual(createWebSocket.reconnectedUrls, []);
2748
- assertSame(latest().transports[0].closedAt, null);
3488
+ assertSame(latest().transports[0].connection.type, "Open");
2749
3489
 
2750
3490
  // Nothing arrives for a whole timeout: the socket reconnects.
2751
3491
  time.advance(Millis.orThrow(84_999));
@@ -2755,8 +3495,14 @@ describe("sync state", () => {
2755
3495
  await testWaitForWorkerMessage();
2756
3496
  assertEqual(createWebSocket.createdUrls, [transport.url]);
2757
3497
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
2758
- assertSame(latest().transports[0].readyState, "connecting");
2759
- assertEqual(latest().transports[0].closedAt, time.now());
3498
+ // The socket reports no close for a reconnect, so the transport
3499
+ // disconnects when the request times out.
3500
+ assertEqual(latest().transports[0].connection, {
3501
+ type: "Disconnected",
3502
+ disconnectedAt: time.now(),
3503
+ openedAt,
3504
+ error: null,
3505
+ });
2760
3506
 
2761
3507
  // The reopen forgets the unanswered request and starts a round.
2762
3508
  createWebSocket.open(transport.url);
@@ -2768,7 +3514,7 @@ describe("sync state", () => {
2768
3514
  ok: true,
2769
3515
  value: { type: "Converged" },
2770
3516
  });
2771
- assertTrue(routeOf(latest()).complete);
3517
+ assertSame(routeOf(latest()).type, "Complete");
2772
3518
  });
2773
3519
  }
2774
3520
 
@@ -2874,7 +3620,7 @@ describe("sync state", () => {
2874
3620
  time.advance("1s");
2875
3621
  await testWaitForWorkerMessage();
2876
3622
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
2877
- assertSame(latest().transports[0].readyState, "connecting");
3623
+ assertSame(latest().transports[0].connection.type, "Disconnected");
2878
3624
  });
2879
3625
 
2880
3626
  it("stops timing a request when its socket closes", async () => {
@@ -2900,6 +3646,7 @@ describe("sync state", () => {
2900
3646
  await testWaitForWorkerMessage();
2901
3647
  createWebSocket.open(transport.url);
2902
3648
  await testWaitForWorkerMessage();
3649
+ const openedAt = time.now();
2903
3650
  await respondToSyncRound(instance, createWebSocket);
2904
3651
 
2905
3652
  // The request dies with its connection, and the WebSocket's own retry
@@ -2907,12 +3654,17 @@ describe("sync state", () => {
2907
3654
  time.advance("10s");
2908
3655
  createWebSocket.close(transport.url);
2909
3656
  await testWaitForWorkerMessage();
2910
- const { closedAt } = latest().transports[0];
2911
- assertEqual(closedAt, time.now());
3657
+ const disconnected = {
3658
+ type: "Disconnected",
3659
+ disconnectedAt: time.now(),
3660
+ openedAt,
3661
+ error: null,
3662
+ } as const;
3663
+ assertEqual(latest().transports[0].connection, disconnected);
2912
3664
  time.advance("2m");
2913
3665
  await testWaitForWorkerMessage();
2914
3666
  assertEqual(createWebSocket.reconnectedUrls, []);
2915
- assertSame(latest().transports[0].closedAt, closedAt);
3667
+ assertEqual(latest().transports[0].connection, disconnected);
2916
3668
  });
2917
3669
 
2918
3670
  it("leaves a complete route incomplete when its socket closes", async () => {
@@ -2942,17 +3694,66 @@ describe("sync state", () => {
2942
3694
  ok: true,
2943
3695
  value: { type: "Converged" },
2944
3696
  });
2945
- assertTrue(routeOf(latest()).complete);
3697
+ assertSame(routeOf(latest()).type, "Complete");
2946
3698
 
2947
3699
  createWebSocket.close(transport.url);
2948
3700
  await testWaitForWorkerMessage();
2949
- assertSame(latest().transports[0].readyState, "closed");
2950
- assertFalse(routeOf(latest()).complete);
2951
- assertSame(syncStateToOwnerSyncStates(latest())[0]?.status, "offline");
3701
+ assertSame(latest().transports[0].connection.type, "Disconnected");
3702
+ assertSame(routeOf(latest()).type, "Pending");
3703
+ assertEqual(
3704
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3705
+ { type: "Offline" },
3706
+ );
2952
3707
  });
2953
3708
 
2954
- /**
2955
- * Opens the socket and lets the app owner's round time out once per
3709
+ it("disconnects a first connection that fails without a close", async () => {
3710
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
3711
+ await using setup = await setupSharedWorker({ createWebSocket });
3712
+ using disposer = new DisposableStack();
3713
+ const { latest } = setupSyncStates(setup, disposer);
3714
+ const { time } = setup.run.deps;
3715
+ const instance = await setup.createEvolu();
3716
+ const transport = createOwnerWebSocketTransport({
3717
+ url: "wss://first-connection.example",
3718
+ ownerId: testAppOwner.id,
3719
+ });
3720
+ instance.evoluChannel.port2.postMessage({
3721
+ type: "UseOwner",
3722
+ actions: [
3723
+ {
3724
+ owner: { owner: testAppOwner, transports: [transport] },
3725
+ action: "add",
3726
+ },
3727
+ ],
3728
+ });
3729
+ await testWaitForWorkerMessage();
3730
+ assertEqual(
3731
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3732
+ { type: "Syncing" },
3733
+ );
3734
+
3735
+ // A browser reports a failed attempt only as an error, because the
3736
+ // socket stops listening before its close arrives.
3737
+ time.advance("1s");
3738
+ createWebSocket.error(transport.url, {
3739
+ type: "WebSocketConnectError",
3740
+ event: new Event("error"),
3741
+ });
3742
+ await testWaitForWorkerMessage();
3743
+ assertEqual(latest().transports[0].connection, {
3744
+ type: "Disconnected",
3745
+ disconnectedAt: time.now(),
3746
+ openedAt: null,
3747
+ error: { type: "WebSocketConnectError", at: time.now() },
3748
+ });
3749
+ assertEqual(
3750
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
3751
+ { type: "Offline" },
3752
+ );
3753
+ });
3754
+
3755
+ /**
3756
+ * Opens the socket and lets the app owner's round time out once per
2956
3757
  * timeout, asserting it reconnects exactly when each one expires.
2957
3758
  */
2958
3759
  const timeOutRounds = async (
@@ -3085,6 +3886,63 @@ describe("sync state", () => {
3085
3886
  assertLength(createWebSocket.reconnectedUrls, 3);
3086
3887
  });
3087
3888
 
3889
+ it("restores the base timeout once a round settles with a skipped message", async () => {
3890
+ const createWebSocket = testCreateWebSocket({ isOpen: false });
3891
+ await using setup = await setupSharedWorker({ createWebSocket });
3892
+ using disposer = new DisposableStack();
3893
+ const { latest } = setupSyncStates(setup, disposer);
3894
+ const instance = await setup.createEvolu();
3895
+ const { time } = setup.run.deps;
3896
+ const transport = createOwnerWebSocketTransport({
3897
+ url: "wss://skipping.example",
3898
+ ownerId: testAppOwner.id,
3899
+ });
3900
+ instance.evoluChannel.port2.postMessage({
3901
+ type: "UseOwner",
3902
+ actions: [
3903
+ {
3904
+ owner: { owner: testAppOwner, transports: [transport] },
3905
+ action: "add",
3906
+ },
3907
+ ],
3908
+ });
3909
+ await testWaitForWorkerMessage();
3910
+ await timeOutRounds(
3911
+ { setup, instance, createWebSocket, url: transport.url },
3912
+ [90],
3913
+ );
3914
+
3915
+ // The reopened round converges, but the relay offers a message the
3916
+ // database skips, so the route stays incomplete after its
3917
+ // reconciliation has ended.
3918
+ createWebSocket.open(transport.url);
3919
+ await testWaitForWorkerMessage();
3920
+ await respondToSyncRound(instance, createWebSocket);
3921
+ createWebSocket.message(transport.url, relayResponse());
3922
+ await testWaitForWorkerMessage();
3923
+ await respondToApplySync(
3924
+ instance,
3925
+ false,
3926
+ { ok: true, value: { type: "Converged" } },
3927
+ skippedError,
3928
+ );
3929
+ assertSame(routeOf(latest()).type, "Settled");
3930
+
3931
+ // The next request gets the base timeout.
3932
+ instance.evoluChannel.port2.postMessage({
3933
+ type: "UseOwner",
3934
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
3935
+ });
3936
+ await testWaitForWorkerMessage();
3937
+ await respondToSyncRound(instance, createWebSocket);
3938
+ time.advance(Millis.orThrow(89_999));
3939
+ await testWaitForWorkerMessage();
3940
+ assertLength(createWebSocket.reconnectedUrls, 1);
3941
+ time.advance("1ms");
3942
+ await testWaitForWorkerMessage();
3943
+ assertLength(createWebSocket.reconnectedUrls, 2);
3944
+ });
3945
+
3088
3946
  it("keeps a grown timeout while another database using the owner reconciles", async () => {
3089
3947
  const createWebSocket = testCreateWebSocket({ isOpen: false });
3090
3948
  await using setup = await setupSharedWorker({ createWebSocket });
@@ -3141,7 +3999,7 @@ describe("sync state", () => {
3141
3999
  ok: true,
3142
4000
  value: { type: "Converged" },
3143
4001
  });
3144
- assertTrue(routeOf(latest()).complete);
4002
+ assertSame(routeOf(latest()).type, "Complete");
3145
4003
  await respondToApplySync(second, false, {
3146
4004
  ok: true,
3147
4005
  value: {
@@ -3208,8 +4066,8 @@ describe("sync state", () => {
3208
4066
  value: { type: "Converged" },
3209
4067
  });
3210
4068
  }
3211
- assertTrue(routeOf(latest()).complete);
3212
- assertFalse(routeOf(latest(), Name.orThrow("second")).complete);
4069
+ assertSame(routeOf(latest()).type, "Complete");
4070
+ assertSame(routeOf(latest(), Name.orThrow("second")).type, "Pending");
3213
4071
 
3214
4072
  // The second database stops using the owner without completing, which
3215
4073
  // leaves no incomplete route, so the base timeout applies again.
@@ -3319,7 +4177,10 @@ describe("sync state", () => {
3319
4177
  value: { type: "Converged" },
3320
4178
  });
3321
4179
  }
3322
- assertTrue(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4180
+ assertSame(
4181
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4182
+ "Complete",
4183
+ );
3323
4184
 
3324
4185
  // The app owner's round goes unanswered, so the socket is replaced, and
3325
4186
  // the other owner's route through it is no longer reconciled.
@@ -3332,7 +4193,10 @@ describe("sync state", () => {
3332
4193
  time.advance(Millis.orThrow(90_000));
3333
4194
  await testWaitForWorkerMessage();
3334
4195
  assertEqual(createWebSocket.reconnectedUrls, [transport.url]);
3335
- assertFalse(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4196
+ assertSame(
4197
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4198
+ "Pending",
4199
+ );
3336
4200
  });
3337
4201
 
3338
4202
  it("restores a relay's base timeout while another relay of the owner is down", async () => {
@@ -3373,8 +4237,8 @@ describe("sync state", () => {
3373
4237
  ok: true,
3374
4238
  value: { type: "Converged" },
3375
4239
  });
3376
- assertTrue(routeOf(latest(), testName, 0).complete);
3377
- assertFalse(routeOf(latest(), testName, 1).complete);
4240
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
4241
+ assertSame(routeOf(latest(), testName, 1).type, "Pending");
3378
4242
 
3379
4243
  instance.evoluChannel.port2.postMessage({
3380
4244
  type: "UseOwner",
@@ -3459,8 +4323,11 @@ describe("sync state", () => {
3459
4323
  ok: true,
3460
4324
  value: { type: "Converged" },
3461
4325
  });
3462
- assertTrue(routeOf(latest()).complete);
3463
- assertFalse(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4326
+ assertSame(routeOf(latest()).type, "Complete");
4327
+ assertSame(
4328
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4329
+ "Pending",
4330
+ );
3464
4331
 
3465
4332
  instance.evoluChannel.port2.postMessage({
3466
4333
  type: "UseOwner",
@@ -3515,14 +4382,15 @@ describe("sync state", () => {
3515
4382
  using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
3516
4383
  init.port,
3517
4384
  );
4385
+ const error: UnsupportedDbVersionError = {
4386
+ type: "UnsupportedDbVersionError",
4387
+ storedVersion: PositiveInt.orThrow(2),
4388
+ supportedVersion: PositiveInt.orThrow(1),
4389
+ };
3518
4390
  leaderPort.postMessage({
3519
4391
  type: "LeaderRefused",
3520
4392
  name: refusedName,
3521
- error: {
3522
- type: "UnsupportedDbVersionError",
3523
- storedVersion: PositiveInt.orThrow(2),
3524
- supportedVersion: PositiveInt.orThrow(1),
3525
- },
4393
+ error,
3526
4394
  });
3527
4395
  await testWaitForWorkerMessage();
3528
4396
  const context = { setup, instance, createWebSocket, url: transport.url };
@@ -3539,8 +4407,12 @@ describe("sync state", () => {
3539
4407
  ok: true,
3540
4408
  value: { type: "Converged" },
3541
4409
  });
3542
- assertTrue(routeOf(latest()).complete);
3543
- assertFalse(routeOf(latest(), refusedName).complete);
4410
+ assertSame(routeOf(latest()).type, "Complete");
4411
+ // The refused database publishes no route.
4412
+ assertEqual(
4413
+ latest().tenants.find(({ name }) => name === refusedName),
4414
+ { type: "Refused", name: refusedName, error },
4415
+ );
3544
4416
  instance.evoluChannel.port2.postMessage({
3545
4417
  type: "UseOwner",
3546
4418
  actions: [{ ownerId: testAppOwner.id, action: "sync" }],
@@ -3620,7 +4492,10 @@ describe("sync state", () => {
3620
4492
  ok: true,
3621
4493
  value: { type: "Converged" },
3622
4494
  });
3623
- assertTrue(routeOf(latest(), testName, 0, testAppOwner2.id).complete);
4495
+ assertSame(
4496
+ routeOf(latest(), testName, 0, testAppOwner2.id).type,
4497
+ "Complete",
4498
+ );
3624
4499
  createWebSocket.message(transport.url, relayResponse());
3625
4500
  await testWaitForWorkerMessage();
3626
4501
  await respondToApplySync(instance, false, {
@@ -3682,7 +4557,8 @@ describe("sync state", () => {
3682
4557
  const completeRoutes = (): Array<boolean> =>
3683
4558
  owners.flatMap((owner) =>
3684
4559
  transports.map(
3685
- (_, index) => routeOf(latest(), testName, index, owner.id).complete,
4560
+ (_, index) =>
4561
+ routeOf(latest(), testName, index, owner.id).type === "Complete",
3686
4562
  ),
3687
4563
  );
3688
4564
  assertEqual(completeRoutes(), [true, true, true, true]);
@@ -3752,7 +4628,8 @@ describe("sync state", () => {
3752
4628
  }
3753
4629
  const completeRoutes = (): Array<boolean> =>
3754
4630
  [testAppOwner, testAppOwner2].map(
3755
- (owner) => routeOf(latest(), testName, 0, owner.id).complete,
4631
+ (owner) =>
4632
+ routeOf(latest(), testName, 0, owner.id).type === "Complete",
3756
4633
  );
3757
4634
  assertEqual(completeRoutes(), [true, true]);
3758
4635
 
@@ -3865,7 +4742,7 @@ describe("sync state", () => {
3865
4742
  ok: true,
3866
4743
  value: { type: "Converged" },
3867
4744
  });
3868
- assertTrue(routeOf(latest()).complete);
4745
+ assertSame(routeOf(latest()).type, "Complete");
3869
4746
 
3870
4747
  let changeReads = 0;
3871
4748
  const changeHandler: ProxyHandler<MutationChange> = {
@@ -3898,7 +4775,7 @@ describe("sync state", () => {
3898
4775
  });
3899
4776
  }
3900
4777
  await testWaitForWorkerMessage();
3901
- assertTrue(routeOf(latest()).complete);
4778
+ assertSame(routeOf(latest()).type, "Complete");
3902
4779
 
3903
4780
  // A transport error publishes another snapshot without enqueuing or
3904
4781
  // completing database work. The pending queue must not be rescanned.
@@ -3911,7 +4788,7 @@ describe("sync state", () => {
3911
4788
  await testWaitForWorkerMessage();
3912
4789
  assertTrue(states.length > publishedCount);
3913
4790
  assertSame(changeReads, 0);
3914
- assertTrue(routeOf(latest()).complete);
4791
+ assertSame(routeOf(latest()).type, "Complete");
3915
4792
  });
3916
4793
 
3917
4794
  it("publishes no sync state for a local-only mutation", async () => {
@@ -3945,7 +4822,7 @@ describe("sync state", () => {
3945
4822
  ok: true,
3946
4823
  value: { type: "Converged" },
3947
4824
  });
3948
- assertTrue(routeOf(latest()).complete);
4825
+ assertSame(routeOf(latest()).type, "Complete");
3949
4826
  const publishedCount = states.length;
3950
4827
 
3951
4828
  instance.evoluChannel.port2.postMessage({
@@ -3980,7 +4857,7 @@ describe("sync state", () => {
3980
4857
  // The response was handled without publishing.
3981
4858
  assertSame(evoluOutputs.at(-1)?.type, "OnPatchesByQuery");
3982
4859
  assertSame(states.length, publishedCount);
3983
- assertTrue(routeOf(latest()).complete);
4860
+ assertSame(routeOf(latest()).type, "Complete");
3984
4861
  });
3985
4862
 
3986
4863
  it("refreshes routes once per sync round, however many transports it reaches", async () => {
@@ -4067,30 +4944,30 @@ describe("sync state", () => {
4067
4944
  ok: true,
4068
4945
  value: { type: "Converged" },
4069
4946
  });
4070
- assertTrue(routeOf(latest()).complete);
4947
+ assertSame(routeOf(latest()).type, "Complete");
4071
4948
 
4072
4949
  // A write for another owner leaves the route alone.
4073
4950
  await queueMutation(instance, testAppOwner2.id);
4074
- assertTrue(routeOf(latest()).complete);
4951
+ assertSame(routeOf(latest()).type, "Complete");
4075
4952
 
4076
4953
  // The owner's write waits behind it. Its upload is sent only once the
4077
4954
  // database worker answers it, so the route is incomplete from now on.
4078
4955
  await queueMutation(instance);
4079
- assertFalse(routeOf(latest()).complete);
4956
+ assertSame(routeOf(latest()).type, "Pending");
4080
4957
  await answerMutation(instance, setup.run, testAppOwner2.id);
4081
- assertFalse(routeOf(latest()).complete);
4958
+ assertSame(routeOf(latest()).type, "Pending");
4082
4959
  assertLength(createWebSocket.sentMessages.splice(0), 0);
4083
4960
 
4084
4961
  await answerMutation(instance, setup.run);
4085
4962
  assertLength(createWebSocket.sentMessages.splice(0), 1);
4086
- assertFalse(routeOf(latest()).complete);
4963
+ assertSame(routeOf(latest()).type, "Pending");
4087
4964
  createWebSocket.message(transport.url, relayResponse());
4088
4965
  await testWaitForWorkerMessage();
4089
4966
  await respondToApplySync(instance, false, {
4090
4967
  ok: true,
4091
4968
  value: { type: "Converged" },
4092
4969
  });
4093
- assertTrue(routeOf(latest()).complete);
4970
+ assertSame(routeOf(latest()).type, "Complete");
4094
4971
  });
4095
4972
 
4096
4973
  it("keeps a route complete while a local-only mutation is queued and completed", async () => {
@@ -4122,7 +4999,7 @@ describe("sync state", () => {
4122
4999
  value: { type: "Converged" },
4123
5000
  });
4124
5001
  const completedRoute = routeOf(latest());
4125
- assertTrue(completedRoute.complete);
5002
+ assertSame(completedRoute.type, "Complete");
4126
5003
 
4127
5004
  setup.run.deps.time.advance("1s");
4128
5005
  instance.evoluChannel.port2.postMessage({
@@ -4272,13 +5149,13 @@ describe("sync state", () => {
4272
5149
  while (latest().tenants.length > 0) await testWaitForWorkerMessage();
4273
5150
  await disposingSocket.promise;
4274
5151
  // A frame the mid-disposal stalled socket still delivers publishes a
4275
- // snapshot while its transport is registered. The snapshot must not
4276
- // read that socket.
5152
+ // snapshot while its transport is registered. The transport keeps the
5153
+ // connection its last event left, without reading that socket.
4277
5154
  createWebSocket.message(transports[0].url, relayResponse());
4278
5155
  await testWaitForWorkerMessage();
4279
5156
  assertEqual(
4280
- latest().transports.map((transport) => transport.readyState),
4281
- ["connecting"],
5157
+ latest().transports.map(({ connection }) => connection.type),
5158
+ ["Disconnected"],
4282
5159
  );
4283
5160
  } finally {
4284
5161
  continueDisposing.resolve();
@@ -4505,7 +5382,7 @@ describe("sync state", () => {
4505
5382
  ok: true,
4506
5383
  value: { type: "Converged" },
4507
5384
  });
4508
- assertTrue(routeOf(latest()).complete);
5385
+ assertSame(routeOf(latest()).type, "Complete");
4509
5386
 
4510
5387
  // A sibling instance writes and is disposed before the write is
4511
5388
  // answered.
@@ -4567,14 +5444,14 @@ describe("sync state", () => {
4567
5444
  // The surviving registration uploads the write; the route waits for
4568
5445
  // the answer.
4569
5446
  assertLength(createWebSocket.sentMessages.splice(0), 1);
4570
- assertFalse(routeOf(latest()).complete);
5447
+ assertSame(routeOf(latest()).type, "Pending");
4571
5448
  createWebSocket.message(transport.url, relayResponse());
4572
5449
  await testWaitForWorkerMessage();
4573
5450
  await respondToApplySync(instance, false, {
4574
5451
  ok: true,
4575
5452
  value: { type: "Converged" },
4576
5453
  });
4577
- assertTrue(routeOf(latest()).complete);
5454
+ assertSame(routeOf(latest()).type, "Complete");
4578
5455
  });
4579
5456
 
4580
5457
  it("uploads a write from an instance without a registration", async () => {
@@ -4605,7 +5482,7 @@ describe("sync state", () => {
4605
5482
  ok: true,
4606
5483
  value: { type: "Converged" },
4607
5484
  });
4608
- assertTrue(routeOf(latest()).complete);
5485
+ assertSame(routeOf(latest()).type, "Complete");
4609
5486
 
4610
5487
  // A sibling instance of the same database has no registration.
4611
5488
  const siblingId = createId<"EvoluInstance">(setup.run.deps);
@@ -4666,14 +5543,14 @@ describe("sync state", () => {
4666
5543
  createWebSocket.sentMessages.splice(0).map(({ url }) => url),
4667
5544
  [transport.url],
4668
5545
  );
4669
- assertFalse(routeOf(latest()).complete);
5546
+ assertSame(routeOf(latest()).type, "Pending");
4670
5547
  createWebSocket.message(transport.url, relayResponse());
4671
5548
  await testWaitForWorkerMessage();
4672
5549
  await respondToApplySync(instance, false, {
4673
5550
  ok: true,
4674
5551
  value: { type: "Converged" },
4675
5552
  });
4676
- assertTrue(routeOf(latest()).complete);
5553
+ assertSame(routeOf(latest()).type, "Complete");
4677
5554
  });
4678
5555
 
4679
5556
  it("requires a round when a sibling database's messages fail to apply", async () => {
@@ -4712,8 +5589,8 @@ describe("sync state", () => {
4712
5589
  value: { type: "Converged" },
4713
5590
  });
4714
5591
  }
4715
- assertTrue(routeOf(latest()).complete);
4716
- assertTrue(routeOf(latest(), secondName).complete);
5592
+ assertSame(routeOf(latest()).type, "Complete");
5593
+ assertSame(routeOf(latest(), secondName).type, "Complete");
4717
5594
 
4718
5595
  // The second database's write reaches the first one locally but is not
4719
5596
  // stored there. The relay answers the upload before the first database
@@ -4725,20 +5602,17 @@ describe("sync state", () => {
4725
5602
  ok: true,
4726
5603
  value: { type: "Converged" },
4727
5604
  });
4728
- assertTrue(routeOf(latest(), secondName).complete);
5605
+ assertSame(routeOf(latest(), secondName).type, "Complete");
4729
5606
  await respondToApplySync(first, false, {
4730
5607
  ok: false,
4731
- error: {
4732
- type: "DecryptWithXChaCha20Poly1305Error",
4733
- error: "sibling copy rejected",
4734
- },
5608
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
4735
5609
  });
4736
5610
  await respondToApplySync(first, false, {
4737
5611
  ok: true,
4738
5612
  value: { type: "Converged" },
4739
5613
  });
4740
5614
  // Nothing is outstanding, yet the failure requires a round.
4741
- assertFalse(routeOf(latest()).complete);
5615
+ assertSame(routeOf(latest()).type, "Pending");
4742
5616
 
4743
5617
  // The round requested by the failure fetches the messages again.
4744
5618
  await respondToSyncRound(first, createWebSocket);
@@ -4752,7 +5626,7 @@ describe("sync state", () => {
4752
5626
  ok: true,
4753
5627
  value: { type: "Converged" },
4754
5628
  });
4755
- assertTrue(routeOf(latest()).complete);
5629
+ assertSame(routeOf(latest()).type, "Complete");
4756
5630
  });
4757
5631
 
4758
5632
  it("retries every route after a sibling continuation copy fails without blaming its source relay", async () => {
@@ -4777,25 +5651,24 @@ describe("sync state", () => {
4777
5651
  });
4778
5652
  await respondToApplySync(sibling, false, {
4779
5653
  ok: false,
4780
- error: {
4781
- type: "DecryptWithXChaCha20Poly1305Error",
4782
- error: "continuation copy rejected",
4783
- },
5654
+ error: { type: "StorageQuotaError", ownerId: testAppOwner.id },
4784
5655
  });
4785
5656
  for (const index of [0, 1]) {
4786
5657
  const route = routeOf(latest(), siblingName, index);
4787
- assertFalse(route.complete);
4788
- assertSame(route.error, null);
5658
+ assertSame(route.type, "Pending");
5659
+ assertSame(route.failure, null);
5660
+ assertSame(route.skippedError, null);
4789
5661
  assertSame(route.lastReceivedAt, lastReceivedAt[index]);
4790
5662
  }
4791
5663
  await respondToApplySync(sibling, false, {
4792
5664
  ok: true,
4793
5665
  value: { type: "Converged" },
4794
5666
  });
4795
- assertEqual(await respondToSyncRound(sibling, createWebSocket), [
4796
- transports[0].url,
4797
- transports[1].url,
4798
- ]);
5667
+ for (const transport of transports) {
5668
+ assertEqual(await respondToSyncRound(sibling, createWebSocket), [
5669
+ transport.url,
5670
+ ]);
5671
+ }
4799
5672
 
4800
5673
  for (const transport of transports) {
4801
5674
  createWebSocket.message(transport.url, relayResponse());
@@ -4808,8 +5681,181 @@ describe("sync state", () => {
4808
5681
  }
4809
5682
  }
4810
5683
  for (const index of [0, 1]) {
4811
- assertTrue(routeOf(latest(), siblingName, index).complete);
5684
+ assertSame(routeOf(latest(), siblingName, index).type, "Complete");
5685
+ }
5686
+ });
5687
+
5688
+ const siblingCopyOutcomes = [
5689
+ {
5690
+ outcome: "error",
5691
+ result: {
5692
+ ok: false,
5693
+ error: {
5694
+ type: "ProtocolInvalidDataError",
5695
+ data: Uint8Array.of(255),
5696
+ error: "decoding failed",
5697
+ },
5698
+ },
5699
+ skipped: null,
5700
+ posted: {
5701
+ type: "ProtocolInvalidDataError",
5702
+ error: createUnknownError("decoding failed"),
5703
+ },
5704
+ },
5705
+ {
5706
+ outcome: "skip",
5707
+ result: { ok: true, value: { type: "Broadcast" } },
5708
+ skipped: skippedError,
5709
+ posted: {
5710
+ ...skippedError,
5711
+ error: createUnknownError(skippedError.error),
5712
+ },
5713
+ },
5714
+ {
5715
+ outcome: "Failed result",
5716
+ result: { ok: true, value: { type: "Failed", cause: "Write" } },
5717
+ skipped: null,
5718
+ posted: null,
5719
+ },
5720
+ {
5721
+ outcome: "abort",
5722
+ result: {
5723
+ ok: false,
5724
+ error: { type: "AbortError", reason: { type: "Stop" } },
5725
+ },
5726
+ skipped: null,
5727
+ posted: null,
5728
+ },
5729
+ ] satisfies ReadonlyArray<{
5730
+ outcome: string;
5731
+ result: ApplySyncResponse["result"];
5732
+ skipped: DecryptWithXChaCha20Poly1305Error | null;
5733
+ posted:
5734
+ | Omit<ProtocolInvalidDataError, "data">
5735
+ | DecryptWithXChaCha20Poly1305Error
5736
+ | null;
5737
+ }>;
5738
+ for (const { outcome, result, skipped, posted } of siblingCopyOutcomes) {
5739
+ it(`${posted === null ? "does not post" : "posts an UnknownError for"} a sibling continuation copy's ${outcome}`, async () => {
5740
+ const createWebSocket = testCreateWebSocket();
5741
+ await using setup = await setupSharedWorker({ createWebSocket });
5742
+ const errors: Array<ConsoleEntryOrError> = [];
5743
+ using errorChannel = testCreateBroadcastChannel<ConsoleEntryOrError>(
5744
+ consoleEntryOrErrorBroadcastChannelName,
5745
+ );
5746
+ errorChannel.onMessage = (output) => {
5747
+ if (output.type === "Error") errors.push(output);
5748
+ };
5749
+ const { sibling } = await setupSiblingContinuation(
5750
+ setup,
5751
+ createWebSocket,
5752
+ );
5753
+
5754
+ await respondToApplySync(sibling, false, result, skipped);
5755
+
5756
+ // No relay sent the copy, so no route can show its problem. An error
5757
+ // or a skip means a bug and is posted as an unexpected failure,
5758
+ // without the frame, as a route shows it. The protocol already logged
5759
+ // a Failed result, and the tab turns that log into an unexpected
5760
+ // failure, so it is not posted again. An abort is not a failure.
5761
+ assertEqual(
5762
+ errors,
5763
+ posted === null
5764
+ ? []
5765
+ : [
5766
+ {
5767
+ type: "Error",
5768
+ error: createUnknownError({
5769
+ ...posted,
5770
+ at: setup.run.deps.time.now(),
5771
+ }),
5772
+ },
5773
+ ],
5774
+ );
5775
+ });
5776
+ }
5777
+
5778
+ it("retries every route after a sibling continuation copy skips a message", async () => {
5779
+ const createWebSocket = testCreateWebSocket();
5780
+ await using setup = await setupSharedWorker({ createWebSocket });
5781
+ using disposer = new DisposableStack();
5782
+ const { latest } = setupSyncStates(setup, disposer);
5783
+ const { source, sibling, siblingName, transports } =
5784
+ await setupSiblingContinuation(setup, createWebSocket);
5785
+
5786
+ createWebSocket.message(transports[0].url, relayResponse());
5787
+ await testWaitForWorkerMessage();
5788
+ await respondToApplySync(source, false, {
5789
+ ok: true,
5790
+ value: { type: "Converged" },
5791
+ });
5792
+ await respondToApplySync(
5793
+ sibling,
5794
+ false,
5795
+ { ok: true, value: { type: "Broadcast" } },
5796
+ skippedError,
5797
+ );
5798
+ await respondToApplySync(sibling, false, {
5799
+ ok: true,
5800
+ value: { type: "Converged" },
5801
+ });
5802
+ for (const transport of transports) {
5803
+ assertEqual(await respondToSyncRound(sibling, createWebSocket), [
5804
+ transport.url,
5805
+ ]);
5806
+ }
5807
+
5808
+ // The relays offer the skipped message, which fails their routes
5809
+ // without requesting another round, so each relay frame adds only its
5810
+ // apply.
5811
+ const inputCount = sibling.dbInputs.length;
5812
+ for (const transport of transports) {
5813
+ createWebSocket.message(transport.url, relayResponse());
5814
+ await testWaitForWorkerMessage();
5815
+ await respondToApplySync(source, false, {
5816
+ ok: true,
5817
+ value: { type: "Converged" },
5818
+ });
5819
+ await respondToApplySync(
5820
+ sibling,
5821
+ false,
5822
+ { ok: true, value: { type: "Converged" } },
5823
+ skippedError,
5824
+ );
5825
+ }
5826
+ assertLength(sibling.dbInputs, inputCount + transports.length);
5827
+ for (const index of [0, 1]) {
5828
+ const route = routeOf(latest(), siblingName, index);
5829
+ assertSame(route.type, "Settled");
5830
+ assertSame(route.skippedError.type, skippedError.type);
4812
5831
  }
5832
+
5833
+ // Another copy that skips a message requests no round, because each
5834
+ // round would download the skipped messages again.
5835
+ createWebSocket.message(transports[0].url, relayResponse());
5836
+ await testWaitForWorkerMessage();
5837
+ await respondToApplySync(source, false, {
5838
+ ok: true,
5839
+ value: {
5840
+ type: "Response",
5841
+ message: createProtocolMessageForUnsubscribe(testAppOwner.id),
5842
+ broadcast: createProtocolMessageBuffer(testAppOwner.id, {
5843
+ messageType: MessageType.Broadcast,
5844
+ }).unwrap(),
5845
+ },
5846
+ });
5847
+ await respondToApplySync(sibling, false, {
5848
+ ok: true,
5849
+ value: { type: "Converged" },
5850
+ });
5851
+ const copyInputCount = sibling.dbInputs.length;
5852
+ await respondToApplySync(
5853
+ sibling,
5854
+ false,
5855
+ { ok: true, value: { type: "Broadcast" } },
5856
+ skippedError,
5857
+ );
5858
+ assertLength(sibling.dbInputs, copyInputCount);
4813
5859
  });
4814
5860
 
4815
5861
  it("requires a new round through the other transports when a relay stores messages", async () => {
@@ -4846,8 +5892,8 @@ describe("sync state", () => {
4846
5892
  value: { type: "Converged" },
4847
5893
  });
4848
5894
  }
4849
- assertTrue(routeOf(latest(), testName, 0).complete);
4850
- assertTrue(routeOf(latest(), testName, 1).complete);
5895
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5896
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
4851
5897
 
4852
5898
  // Messages stored from the first relay invalidate the second route.
4853
5899
  createWebSocket.message(transports[0].url, relayResponse());
@@ -4856,8 +5902,8 @@ describe("sync state", () => {
4856
5902
  ok: true,
4857
5903
  value: { type: "Converged" },
4858
5904
  });
4859
- assertTrue(routeOf(latest(), testName, 0).complete);
4860
- assertFalse(routeOf(latest(), testName, 1).complete);
5905
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5906
+ assertSame(routeOf(latest(), testName, 1).type, "Pending");
4861
5907
 
4862
5908
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
4863
5909
  transports[1].url,
@@ -4868,267 +5914,888 @@ describe("sync state", () => {
4868
5914
  ok: true,
4869
5915
  value: { type: "Converged" },
4870
5916
  });
4871
- assertTrue(routeOf(latest(), testName, 1).complete);
5917
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
4872
5918
  });
4873
5919
 
4874
- it("never completes a route after a replacement leader refused startup", async () => {
4875
- const createWebSocket = testCreateWebSocket();
4876
- await using setup = await setupSharedWorker({ createWebSocket });
4877
- using disposer = new DisposableStack();
5920
+ /**
5921
+ * Opens routes through two relays and settles the second with a skipped
5922
+ * message.
5923
+ */
5924
+ const setupSkippingSecondRoute = async (
5925
+ setup: SharedWorkerSetup,
5926
+ disposer: DisposableStack,
5927
+ createWebSocket: TestCreateWebSocket,
5928
+ ) => {
4878
5929
  const { latest } = setupSyncStates(setup, disposer);
4879
- const { time } = setup.run.deps;
4880
- const instance = await setup.createEvolu({
4881
- releaseDbWorkerLeaderOnDispose: false,
4882
- });
4883
- const transport = createOwnerWebSocketTransport({
4884
- url: "wss://refused-route.example",
4885
- ownerId: testAppOwner.id,
4886
- });
5930
+ const instance = await setup.createEvolu();
5931
+ const transports = [
5932
+ createOwnerWebSocketTransport({
5933
+ url: "wss://first.example",
5934
+ ownerId: testAppOwner.id,
5935
+ }),
5936
+ createOwnerWebSocketTransport({
5937
+ url: "wss://skipping.example",
5938
+ ownerId: testAppOwner.id,
5939
+ }),
5940
+ ] as const;
5941
+
4887
5942
  instance.evoluChannel.port2.postMessage({
4888
5943
  type: "UseOwner",
4889
5944
  actions: [
4890
- {
4891
- owner: { owner: testAppOwner, transports: [transport] },
4892
- action: "add",
4893
- },
5945
+ { owner: { owner: testAppOwner, transports }, action: "add" },
4894
5946
  ],
4895
5947
  });
4896
5948
  await testWaitForWorkerMessage();
4897
5949
  await respondToSyncRound(instance, createWebSocket);
4898
- createWebSocket.message(transport.url, relayResponse());
5950
+ await respondToSyncRound(instance, createWebSocket);
5951
+ createWebSocket.message(transports[0].url, relayResponse());
4899
5952
  await testWaitForWorkerMessage();
4900
5953
  await respondToApplySync(instance, false, {
4901
5954
  ok: true,
4902
5955
  value: { type: "Converged" },
4903
5956
  });
4904
- assertTrue(routeOf(latest()).complete);
4905
- const completeAt = routeOf(latest()).completeAt;
4906
-
4907
- // The queued write alone keeps the route incomplete.
4908
- time.advance("1s");
4909
- await queueMutation(instance);
4910
- assertFalse(routeOf(latest()).complete);
4911
- await instance.releaseDbWorkerLeader();
4912
-
4913
- const init = await setupTabLeader(setup, disposer);
4914
- using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
4915
- init.port,
5957
+ createWebSocket.message(transports[1].url, relayResponse());
5958
+ await testWaitForWorkerMessage();
5959
+ await respondToApplySync(
5960
+ instance,
5961
+ false,
5962
+ { ok: true, value: { type: "Converged" } },
5963
+ skippedError,
4916
5964
  );
4917
- leaderPort.postMessage({
4918
- type: "LeaderRefused",
4919
- name: testName,
4920
- error: {
4921
- type: "UnsupportedDbVersionError",
4922
- storedVersion: PositiveInt.orThrow(2),
4923
- supportedVersion: PositiveInt.orThrow(1),
5965
+ return { instance, latest, transports };
5966
+ };
5967
+
5968
+ it("requests no round through a route that skipped a message when a relay stores messages", async () => {
5969
+ const createWebSocket = testCreateWebSocket();
5970
+ await using setup = await setupSharedWorker({ createWebSocket });
5971
+ using disposer = new DisposableStack();
5972
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
5973
+ setup,
5974
+ disposer,
5975
+ createWebSocket,
5976
+ );
5977
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5978
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
5979
+
5980
+ // Messages stored from the first relay request no round through the
5981
+ // second, whose relay would offer the skipped message again.
5982
+ createWebSocket.message(transports[0].url, relayResponse());
5983
+ await testWaitForWorkerMessage();
5984
+ const inputCount = instance.dbInputs.length;
5985
+ await respondToApplySync(instance, true, {
5986
+ ok: true,
5987
+ value: { type: "Converged" },
5988
+ });
5989
+ assertLength(instance.dbInputs, inputCount);
5990
+ assertSame(routeOf(latest(), testName, 0).type, "Complete");
5991
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
5992
+
5993
+ // Messages stored from the second relay still reconcile through the
5994
+ // first.
5995
+ createWebSocket.message(transports[1].url, relayResponse());
5996
+ await testWaitForWorkerMessage();
5997
+ await respondToApplySync(
5998
+ instance,
5999
+ true,
6000
+ { ok: true, value: { type: "Converged" } },
6001
+ skippedError,
6002
+ );
6003
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6004
+ transports[0].url,
6005
+ ]);
6006
+
6007
+ // An explicit request checks the second relay again. Messages stored
6008
+ // from the first relay meanwhile request no round through it either,
6009
+ // and the check may not include them, so it no longer completes the
6010
+ // route even when it skips nothing.
6011
+ const sync = () => {
6012
+ instance.evoluChannel.port2.postMessage({
6013
+ type: "UseOwner",
6014
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
6015
+ });
6016
+ };
6017
+ sync();
6018
+ await testWaitForWorkerMessage();
6019
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6020
+ transports[0].url,
6021
+ transports[1].url,
6022
+ ]);
6023
+ createWebSocket.message(transports[0].url, relayResponse());
6024
+ await testWaitForWorkerMessage();
6025
+ const checkInputCount = instance.dbInputs.length;
6026
+ await respondToApplySync(instance, true, {
6027
+ ok: true,
6028
+ value: { type: "Converged" },
6029
+ });
6030
+ assertLength(instance.dbInputs, checkInputCount);
6031
+ createWebSocket.message(transports[1].url, relayResponse());
6032
+ await testWaitForWorkerMessage();
6033
+ await respondToApplySync(instance, false, {
6034
+ ok: true,
6035
+ value: { type: "Converged" },
6036
+ });
6037
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
6038
+
6039
+ // The next request checks it again with those messages and completes
6040
+ // the route once the relay no longer offers the skipped message.
6041
+ sync();
6042
+ await testWaitForWorkerMessage();
6043
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6044
+ transports[0].url,
6045
+ transports[1].url,
6046
+ ]);
6047
+ createWebSocket.message(transports[1].url, relayResponse());
6048
+ await testWaitForWorkerMessage();
6049
+ await respondToApplySync(instance, false, {
6050
+ ok: true,
6051
+ value: { type: "Converged" },
6052
+ });
6053
+ assertSame(routeOf(latest(), testName, 1).type, "Complete");
6054
+ });
6055
+
6056
+ it("keeps a route's skip when the database fails to create a requested round", async () => {
6057
+ const createWebSocket = testCreateWebSocket();
6058
+ await using setup = await setupSharedWorker({ createWebSocket });
6059
+ using disposer = new DisposableStack();
6060
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
6061
+ setup,
6062
+ disposer,
6063
+ createWebSocket,
6064
+ );
6065
+
6066
+ // The database worker fails to create the requested round, so the
6067
+ // second route shows the failure.
6068
+ instance.evoluChannel.port2.postMessage({
6069
+ type: "UseOwner",
6070
+ actions: [{ ownerId: testAppOwner.id, action: "sync" }],
6071
+ });
6072
+ await testWaitForWorkerMessage();
6073
+ const round = instance.dbInputs.at(-1);
6074
+ assertNotUndefined(round);
6075
+ assertSame(round.request.type, "ForSharedWorker");
6076
+ assertSame(round.request.message.type, "CreateSyncMessages");
6077
+ instance.dbWorkerPort.postMessage({
6078
+ type: "OnQueuedResponse",
6079
+ attemptId: round.attemptId,
6080
+ response: {
6081
+ type: "ForSharedWorker",
6082
+ message: {
6083
+ type: "CreateSyncMessages",
6084
+ protocolMessagesByOwnerId: new Map(),
6085
+ failedOwnerIds: new Set([testAppOwner.id]),
6086
+ },
4924
6087
  },
4925
6088
  });
4926
6089
  await testWaitForWorkerMessage();
6090
+ assertSame(failureOf(routeOf(latest(), testName, 1))?.type, "SyncFailed");
6091
+
6092
+ // The failed route keeps its skip, so messages stored from the first
6093
+ // relay still request no round through the second.
6094
+ createWebSocket.message(transports[0].url, relayResponse());
6095
+ await testWaitForWorkerMessage();
6096
+ const inputCount = instance.dbInputs.length;
6097
+ await respondToApplySync(instance, true, {
6098
+ ok: true,
6099
+ value: { type: "Converged" },
6100
+ });
6101
+ assertLength(instance.dbInputs, inputCount);
6102
+ });
6103
+
6104
+ it("shows the skip, not a recovered failure, once a route settles", async () => {
6105
+ const createWebSocket = testCreateWebSocket();
6106
+ await using setup = await setupSharedWorker({ createWebSocket });
6107
+ using disposer = new DisposableStack();
6108
+ const { instance, latest, transports } = await setupSkippingSecondRoute(
6109
+ setup,
6110
+ disposer,
6111
+ createWebSocket,
6112
+ );
6113
+ assertSame(
6114
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6115
+ skippedError.type,
6116
+ );
6117
+
6118
+ // A failure on the second route requests a round that checks the relay
6119
+ // again, but messages stored from the first relay meanwhile keep it from
6120
+ // completing the route.
6121
+ createWebSocket.message(transports[1].url, relayResponse());
6122
+ await testWaitForWorkerMessage();
6123
+ await respondToApplySync(instance, false, {
6124
+ ok: false,
6125
+ error: { type: "ProtocolQuotaError", ownerId: testAppOwner.id },
6126
+ });
6127
+ assertSame(
6128
+ failureOf(routeOf(latest(), testName, 1))?.type,
6129
+ "ProtocolQuotaError",
6130
+ );
6131
+ // The skip stays visible beside the failure.
6132
+ assertSame(
6133
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6134
+ skippedError.type,
6135
+ );
6136
+ assertEqual(await respondToSyncRound(instance, createWebSocket), [
6137
+ transports[1].url,
6138
+ ]);
6139
+ createWebSocket.message(transports[0].url, relayResponse());
6140
+ await testWaitForWorkerMessage();
6141
+ await respondToApplySync(instance, true, {
6142
+ ok: true,
6143
+ value: { type: "Converged" },
6144
+ });
6145
+
6146
+ // The round recovers from the failure, and the route settles showing the
6147
+ // skip that keeps it incomplete.
6148
+ createWebSocket.message(transports[1].url, relayResponse());
6149
+ await testWaitForWorkerMessage();
6150
+ await respondToApplySync(instance, false, {
6151
+ ok: true,
6152
+ value: { type: "Converged" },
6153
+ });
6154
+ assertSame(routeOf(latest(), testName, 1).type, "Settled");
6155
+ assertSame(
6156
+ skippedErrorOf(routeOf(latest(), testName, 1))?.type,
6157
+ skippedError.type,
6158
+ );
6159
+ });
6160
+
6161
+ it("publishes a refused database with its error and no owners", async () => {
6162
+ const createWebSocket = testCreateWebSocket();
6163
+ await using setup = await setupSharedWorker({ createWebSocket });
6164
+ using disposer = new DisposableStack();
6165
+ const { latest } = setupSyncStates(setup, disposer);
6166
+ const instance = await setup.createEvolu({
6167
+ releaseDbWorkerLeaderOnDispose: false,
6168
+ });
6169
+ instance.evoluChannel.port2.postMessage({
6170
+ type: "UseOwner",
6171
+ actions: [
6172
+ {
6173
+ owner: {
6174
+ owner: testAppOwner,
6175
+ transports: [
6176
+ createOwnerWebSocketTransport({
6177
+ url: "wss://refused-route.example",
6178
+ ownerId: testAppOwner.id,
6179
+ }),
6180
+ ],
6181
+ },
6182
+ action: "add",
6183
+ },
6184
+ ],
6185
+ });
6186
+ await testWaitForWorkerMessage();
6187
+ assertSame(routeOf(latest()).type, "Pending");
6188
+ await instance.releaseDbWorkerLeader();
6189
+
6190
+ const init = await setupTabLeader(setup, disposer);
6191
+ using leaderPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
6192
+ init.port,
6193
+ );
6194
+ const error: UnsupportedDbVersionError = {
6195
+ type: "UnsupportedDbVersionError",
6196
+ storedVersion: PositiveInt.orThrow(2),
6197
+ supportedVersion: PositiveInt.orThrow(1),
6198
+ };
6199
+ leaderPort.postMessage({ type: "LeaderRefused", name: testName, error });
6200
+ await testWaitForWorkerMessage();
6201
+
6202
+ // Nothing the refused database holds syncs, so it publishes its error
6203
+ // instead of its registrations, and no relay syncs its owner.
6204
+ const refused: ReadonlyArray<SyncTenant> = [
6205
+ { type: "Refused", name: testName, error },
6206
+ ];
6207
+ assertEqual(latest().tenants, refused);
6208
+ assertEqual(
6209
+ syncStateToOwnerSyncStatus(latest(), testName, testAppOwner.id),
6210
+ { type: "NoRelays" },
6211
+ );
6212
+ assertEqual(
6213
+ syncStateToRelaySyncStates(latest(), testName, testAppOwner.id),
6214
+ [],
6215
+ );
6216
+ });
6217
+ });
6218
+ });
6219
+
6220
+ const derivationDeps = testCreateDeps();
6221
+ const connectingId = createId<"SyncTransport">(derivationDeps);
6222
+ const openId = createId<"SyncTransport">(derivationDeps);
6223
+ const otherOpenId = createId<"SyncTransport">(derivationDeps);
6224
+ const disconnectedId = createId<"SyncTransport">(derivationDeps);
6225
+
6226
+ const connectingConnection: SyncConnection = { type: "Connecting" };
6227
+ const openConnection: SyncConnection = {
6228
+ type: "Open",
6229
+ openedAt: Millis.orThrow(1000),
6230
+ error: null,
6231
+ };
6232
+ const disconnectedConnection: SyncConnection = {
6233
+ type: "Disconnected",
6234
+ disconnectedAt: Millis.orThrow(2000),
6235
+ openedAt: Millis.orThrow(1000),
6236
+ error: null,
6237
+ };
6238
+
6239
+ const createSyncTransport = (
6240
+ id: SyncTransportId,
6241
+ connection: SyncConnection,
6242
+ ): SyncTransport => ({
6243
+ type: "WebSocket",
6244
+ id,
6245
+ label: "wss://relay.example",
6246
+ connection,
6247
+ });
6248
+
6249
+ const syncTransports = [
6250
+ createSyncTransport(connectingId, connectingConnection),
6251
+ createSyncTransport(openId, openConnection),
6252
+ createSyncTransport(otherOpenId, openConnection),
6253
+ createSyncTransport(disconnectedId, disconnectedConnection),
6254
+ ] as const;
6255
+
6256
+ const createPendingRoute = (
6257
+ transportId: SyncTransportId,
6258
+ route: Partial<Omit<PendingSyncRoute, "type" | "transportId">> = {},
6259
+ ): PendingSyncRoute => ({
6260
+ type: "Pending",
6261
+ transportId,
6262
+ failure: null,
6263
+ skippedError: null,
6264
+ completeAt: null,
6265
+ lastSentAt: null,
6266
+ lastReceivedAt: null,
6267
+ ...route,
6268
+ });
6269
+
6270
+ const createSettledRoute = (
6271
+ transportId: SyncTransportId,
6272
+ skippedError: SyncRouteError,
6273
+ ): SettledSyncRoute => ({
6274
+ type: "Settled",
6275
+ transportId,
6276
+ skippedError,
6277
+ completeAt: null,
6278
+ lastSentAt: Millis.orThrow(1500),
6279
+ lastReceivedAt: Millis.orThrow(1600),
6280
+ });
6281
+
6282
+ const createCompleteRoute = (
6283
+ transportId: SyncTransportId,
6284
+ ): CompleteSyncRoute => ({
6285
+ type: "Complete",
6286
+ transportId,
6287
+ completeAt: Millis.orThrow(3000),
6288
+ lastSentAt: Millis.orThrow(2500),
6289
+ lastReceivedAt: Millis.orThrow(3000),
6290
+ });
6291
+
6292
+ const quotaFailure = (at: number): SyncRouteError => ({
6293
+ type: "ProtocolQuotaError",
6294
+ ownerId: testAppOwner.id,
6295
+ at: Millis.orThrow(at),
6296
+ });
6297
+
6298
+ const skippedChange = (at: number): SyncRouteError => ({
6299
+ type: "DecryptWithXChaCha20Poly1305Error",
6300
+ error: "wrong encryption key",
6301
+ at: Millis.orThrow(at),
6302
+ });
6303
+
6304
+ const derivationRefusal: UnsupportedDbVersionError = {
6305
+ type: "UnsupportedDbVersionError",
6306
+ storedVersion: PositiveInt.orThrow(2),
6307
+ supportedVersion: PositiveInt.orThrow(1),
6308
+ };
6309
+
6310
+ /** A snapshot whose database syncs the app owner through `routes`. */
6311
+ const createRoutesState = (routes: ReadonlyArray<SyncRoute>): SyncState => ({
6312
+ transports: syncTransports,
6313
+ tenants: [
6314
+ {
6315
+ type: "Active",
6316
+ name: testName,
6317
+ owners: [{ type: "Writable", ownerId: testAppOwner.id, routes }],
6318
+ },
6319
+ ],
6320
+ });
6321
+
6322
+ const ownerStatusOf = (routes: ReadonlyArray<SyncRoute>): OwnerSyncStatus =>
6323
+ syncStateToOwnerSyncStatus(
6324
+ createRoutesState(routes),
6325
+ testName,
6326
+ testAppOwner.id,
6327
+ );
6328
+
6329
+ describe("SyncState", () => {
6330
+ it("accepts only states the shared worker can be in", () => {
6331
+ const transportId = openId;
6332
+ const ownerId = testAppOwner.id;
6333
+
6334
+ const connections: ReadonlyArray<SyncConnection> = [
6335
+ { type: "Connecting" },
6336
+ {
6337
+ type: "Connecting",
6338
+ // @ts-expect-error A first connection has no error; one that failed is Disconnected.
6339
+ error: null,
6340
+ },
6341
+ { type: "Open", openedAt: Millis.orThrow(1000), error: null },
6342
+ // @ts-expect-error An open connection has openedAt, when it opened.
6343
+ { type: "Open", error: null },
6344
+ ];
6345
+
6346
+ const routes: ReadonlyArray<SyncRoute> = [
6347
+ {
6348
+ type: "Complete",
6349
+ transportId,
6350
+ completeAt: Millis.orThrow(1000),
6351
+ lastSentAt: Millis.orThrow(900),
6352
+ lastReceivedAt: null,
6353
+ },
6354
+ // @ts-expect-error A complete route has when it became complete.
6355
+ {
6356
+ type: "Complete",
6357
+ transportId,
6358
+ completeAt: null,
6359
+ lastSentAt: Millis.orThrow(900),
6360
+ lastReceivedAt: null,
6361
+ },
6362
+ // @ts-expect-error A complete route has sent a round, so it has when it last sent.
6363
+ {
6364
+ type: "Complete",
6365
+ transportId,
6366
+ completeAt: Millis.orThrow(1000),
6367
+ lastSentAt: null,
6368
+ lastReceivedAt: null,
6369
+ },
6370
+ {
6371
+ type: "Complete",
6372
+ transportId,
6373
+ completeAt: Millis.orThrow(1000),
6374
+ lastSentAt: Millis.orThrow(900),
6375
+ lastReceivedAt: null,
6376
+ // @ts-expect-error A complete route has no failure.
6377
+ failure: null,
6378
+ },
6379
+ {
6380
+ type: "Settled",
6381
+ transportId,
6382
+ skippedError: skippedChange(1000),
6383
+ completeAt: null,
6384
+ lastSentAt: Millis.orThrow(900),
6385
+ lastReceivedAt: null,
6386
+ },
6387
+ // @ts-expect-error A settled route has sent a round, so it has when it last sent.
6388
+ {
6389
+ type: "Settled",
6390
+ transportId,
6391
+ skippedError: skippedChange(1000),
6392
+ completeAt: null,
6393
+ lastSentAt: null,
6394
+ lastReceivedAt: null,
6395
+ },
6396
+ {
6397
+ type: "Settled",
6398
+ transportId,
6399
+ // @ts-expect-error A settled route has the skipped change that keeps it incomplete.
6400
+ skippedError: null,
6401
+ completeAt: null,
6402
+ lastSentAt: Millis.orThrow(900),
6403
+ lastReceivedAt: null,
6404
+ },
6405
+ ];
6406
+
6407
+ const owners: ReadonlyArray<SyncTenantOwner> = [
6408
+ { type: "Readonly", ownerId, transportIds: [transportId] },
6409
+ {
6410
+ type: "Readonly",
6411
+ ownerId,
6412
+ transportIds: [transportId],
6413
+ // @ts-expect-error A readonly owner has no routes; its database does not sync it.
6414
+ routes: [],
6415
+ },
6416
+ ];
6417
+
6418
+ const tenants: ReadonlyArray<SyncTenant> = [
6419
+ { type: "Refused", name: testName, error: derivationRefusal },
6420
+ {
6421
+ type: "Refused",
6422
+ name: testName,
6423
+ error: derivationRefusal,
6424
+ // @ts-expect-error A refused database has no owners; nothing it holds syncs.
6425
+ owners: [],
6426
+ },
6427
+ ];
6428
+
6429
+ assertLength(connections, 4);
6430
+ assertLength(routes, 7);
6431
+ assertLength(owners, 2);
6432
+ assertLength(tenants, 2);
6433
+
6434
+ const transport = createSyncTransport(openId, openConnection);
6435
+ // @ts-expect-error A transport has no readyState; its connection tells whether it is open.
6436
+ assertSame(transport.readyState, undefined);
6437
+ });
6438
+ });
6439
+
6440
+ describe("syncStateToRelaySyncStates", () => {
6441
+ it("pairs each route with its transport in route order", () => {
6442
+ const routes = [
6443
+ createPendingRoute(disconnectedId),
6444
+ createCompleteRoute(openId),
6445
+ createPendingRoute(connectingId),
6446
+ ];
6447
+ assertEqual(
6448
+ syncStateToRelaySyncStates(
6449
+ createRoutesState(routes),
6450
+ testName,
6451
+ testAppOwner.id,
6452
+ ),
6453
+ [
6454
+ { transport: syncTransports[3], route: routes[0] },
6455
+ { transport: syncTransports[1], route: routes[1] },
6456
+ { transport: syncTransports[0], route: routes[2] },
6457
+ ],
6458
+ );
6459
+ });
6460
+
6461
+ it("returns none without an active database or a writable owner", () => {
6462
+ const state = createRoutesState([createCompleteRoute(openId)]);
6463
+ const secondName = Name.orThrow("second");
6464
+ const withTenant = (tenant: SyncTenant): SyncState => ({
6465
+ transports: syncTransports,
6466
+ tenants: [tenant],
6467
+ });
6468
+
6469
+ assertEqual(
6470
+ syncStateToRelaySyncStates(null, testName, testAppOwner.id),
6471
+ [],
6472
+ );
6473
+ assertEqual(
6474
+ syncStateToRelaySyncStates(state, secondName, testAppOwner.id),
6475
+ [],
6476
+ );
6477
+ assertEqual(
6478
+ syncStateToRelaySyncStates(state, testName, testAppOwner2.id),
6479
+ [],
6480
+ );
6481
+ assertEqual(
6482
+ syncStateToRelaySyncStates(
6483
+ withTenant({
6484
+ type: "Refused",
6485
+ name: testName,
6486
+ error: derivationRefusal,
6487
+ }),
6488
+ testName,
6489
+ testAppOwner.id,
6490
+ ),
6491
+ [],
6492
+ );
6493
+ assertEqual(
6494
+ syncStateToRelaySyncStates(
6495
+ withTenant({
6496
+ type: "Active",
6497
+ name: testName,
6498
+ owners: [
6499
+ {
6500
+ type: "Readonly",
6501
+ ownerId: testAppOwner.id,
6502
+ transportIds: [openId],
6503
+ },
6504
+ ],
6505
+ }),
6506
+ testName,
6507
+ testAppOwner.id,
6508
+ ),
6509
+ [],
6510
+ );
6511
+ });
6512
+ });
4927
6513
 
4928
- // Refusal discards the write, which is never uploaded.
4929
- assertTrue(latest().tenants[0].refused);
4930
- assertFalse(routeOf(latest()).complete);
4931
- assertSame(routeOf(latest()).completeAt, completeAt);
6514
+ describe("relaySyncStateToStatus", () => {
6515
+ it("tells the status of every route on every connection", () => {
6516
+ for (const transport of [
6517
+ syncTransports[0],
6518
+ syncTransports[1],
6519
+ syncTransports[3],
6520
+ ]) {
6521
+ const statusOf = (route: SyncRoute): RelaySyncStatus =>
6522
+ relaySyncStateToStatus({ transport, route });
6523
+ const { id } = transport;
4932
6524
 
4933
- // Later refreshes, such as a relay frame, keep it incomplete.
4934
- createWebSocket.message(transport.url, relayResponse());
4935
- await testWaitForWorkerMessage();
4936
- assertFalse(routeOf(latest()).complete);
4937
- });
6525
+ assertEqual(statusOf(createCompleteRoute(id)), { type: "Synced" });
6526
+ assertEqual(statusOf(createSettledRoute(id, skippedChange(2000))), {
6527
+ type: "Error",
6528
+ error: skippedChange(2000),
6529
+ });
6530
+ assertEqual(
6531
+ statusOf(createPendingRoute(id)),
6532
+ transport.connection.type === "Disconnected"
6533
+ ? { type: "Offline" }
6534
+ : { type: "Syncing" },
6535
+ );
6536
+ assertEqual(
6537
+ statusOf(createPendingRoute(id, { skippedError: skippedChange(2000) })),
6538
+ { type: "Error", error: skippedChange(2000) },
6539
+ );
6540
+ // A failure shows before a skipped change, even a newer one.
6541
+ assertEqual(
6542
+ statusOf(
6543
+ createPendingRoute(id, {
6544
+ failure: quotaFailure(1000),
6545
+ skippedError: skippedChange(2000),
6546
+ }),
6547
+ ),
6548
+ { type: "Error", error: quotaFailure(1000) },
6549
+ );
6550
+ }
4938
6551
  });
4939
6552
  });
4940
6553
 
4941
- describe("syncStateToOwnerSyncStates", () => {
4942
- const deps = testCreateDeps();
4943
- const openA = createId<"SyncTransport">(deps);
4944
- const openB = createId<"SyncTransport">(deps);
4945
- const closedId = createId<"SyncTransport">(deps);
4946
- const connectingId = createId<"SyncTransport">(deps);
4947
- const createTransport = (
4948
- id: SyncTransportId,
4949
- readyState: SyncTransport["readyState"],
4950
- ): SyncTransport => ({
4951
- id,
4952
- label: "wss://relay.example",
4953
- readyState,
4954
- openedAt: null,
4955
- closedAt: null,
4956
- error: null,
4957
- });
4958
- const transports = [
4959
- createTransport(openA, "open"),
4960
- createTransport(openB, "open"),
4961
- createTransport(closedId, "closed"),
4962
- createTransport(connectingId, "connecting"),
4963
- ];
4964
- const createRoute = (
4965
- transportId: SyncTransportId,
4966
- route: Partial<Omit<SyncRoute, "transportId">> = {},
4967
- ): SyncRoute => ({
4968
- transportId,
4969
- complete: false,
4970
- completeAt: null,
4971
- lastSentAt: null,
4972
- lastReceivedAt: null,
4973
- error: null,
4974
- ...route,
4975
- });
4976
- const createOwner = (
4977
- ownerId: OwnerId,
4978
- routes: ReadonlyArray<SyncRoute>,
4979
- writable = true,
4980
- ): SyncTenantOwner => ({
4981
- ownerId,
4982
- writable,
4983
- transportIds: routes.map(({ transportId }) => transportId),
4984
- routes,
4985
- });
4986
- const createState = (tenants: ReadonlyArray<SyncTenant>): SyncState => ({
4987
- transports,
4988
- tenants,
4989
- });
4990
- const statusOf = (routes: ReadonlyArray<SyncRoute>): string => {
4991
- const [owner] = syncStateToOwnerSyncStates(
4992
- createState([
4993
- {
6554
+ describe("syncStateToOwnerSyncStatus", () => {
6555
+ it("reports NoRelays without a route that syncs the owner", () => {
6556
+ const noRelays = { type: "NoRelays" } as const;
6557
+ const withTenant = (tenant: SyncTenant): SyncState => ({
6558
+ transports: syncTransports,
6559
+ tenants: [tenant],
6560
+ });
6561
+
6562
+ assertEqual(
6563
+ syncStateToOwnerSyncStatus(null, testName, testAppOwner.id),
6564
+ noRelays,
6565
+ );
6566
+ assertEqual(
6567
+ syncStateToOwnerSyncStatus(
6568
+ createRoutesState([createCompleteRoute(openId)]),
6569
+ Name.orThrow("second"),
6570
+ testAppOwner.id,
6571
+ ),
6572
+ noRelays,
6573
+ );
6574
+ assertEqual(
6575
+ syncStateToOwnerSyncStatus(
6576
+ withTenant({
6577
+ type: "Refused",
4994
6578
  name: testName,
4995
- refused: false,
4996
- owners: [createOwner(testAppOwner.id, routes)],
4997
- },
4998
- ]),
6579
+ error: derivationRefusal,
6580
+ }),
6581
+ testName,
6582
+ testAppOwner.id,
6583
+ ),
6584
+ noRelays,
4999
6585
  );
5000
- assertNotUndefined(owner);
5001
- return owner.status;
5002
- };
5003
- const failure = (at: number): SyncRouteError => ({
5004
- type: "ProtocolQuotaError",
5005
- at: Millis.orThrow(at),
6586
+ assertEqual(
6587
+ syncStateToOwnerSyncStatus(
6588
+ withTenant({
6589
+ type: "Active",
6590
+ name: testName,
6591
+ owners: [
6592
+ {
6593
+ type: "Readonly",
6594
+ ownerId: testAppOwner.id,
6595
+ transportIds: [openId],
6596
+ },
6597
+ ],
6598
+ }),
6599
+ testName,
6600
+ testAppOwner.id,
6601
+ ),
6602
+ noRelays,
6603
+ );
6604
+ // Routes are briefly empty while the owner's transports are claimed.
6605
+ assertEqual(ownerStatusOf([]), noRelays);
5006
6606
  });
5007
6607
 
5008
- it("derives the status from the routes and their transports", () => {
5009
- assertSame(statusOf([]), "initial");
5010
- assertSame(statusOf([createRoute(closedId)]), "offline");
5011
- assertSame(statusOf([createRoute(connectingId)]), "offline");
5012
- assertSame(statusOf([createRoute(openA)]), "syncing");
5013
- assertSame(statusOf([createRoute(openA, { complete: true })]), "synced");
5014
- // A complete route counts even while another socket is closed.
5015
- assertSame(
5016
- statusOf([createRoute(openA, { complete: true }), createRoute(closedId)]),
5017
- "synced",
5018
- );
5019
- // Work in progress on an open socket shows before a completed route.
5020
- assertSame(
5021
- statusOf([createRoute(openA, { complete: true }), createRoute(openB)]),
5022
- "syncing",
6608
+ it("counts a first connection as Syncing and a lost one as Offline", () => {
6609
+ assertEqual(ownerStatusOf([createPendingRoute(connectingId)]), {
6610
+ type: "Syncing",
6611
+ });
6612
+ assertEqual(ownerStatusOf([createPendingRoute(openId)]), {
6613
+ type: "Syncing",
6614
+ });
6615
+ assertEqual(ownerStatusOf([createPendingRoute(disconnectedId)]), {
6616
+ type: "Offline",
6617
+ });
6618
+ });
6619
+
6620
+ it("ranks Error over Syncing over Synced over Offline", () => {
6621
+ // A complete route counts even while another relay is offline.
6622
+ assertEqual(
6623
+ ownerStatusOf([
6624
+ createPendingRoute(disconnectedId),
6625
+ createCompleteRoute(openId),
6626
+ ]),
6627
+ { type: "Synced" },
5023
6628
  );
5024
- // A failure shows before both.
5025
- assertSame(
5026
- statusOf([
5027
- createRoute(openA, { complete: true }),
5028
- createRoute(openB, { error: failure(1) }),
6629
+ // Work in progress shows before a relay that is already up to date.
6630
+ assertEqual(
6631
+ ownerStatusOf([
6632
+ createCompleteRoute(openId),
6633
+ createPendingRoute(otherOpenId),
5029
6634
  ]),
5030
- "error",
6635
+ { type: "Syncing" },
5031
6636
  );
5032
- });
5033
-
5034
- it("reports the newest completion and error", () => {
5035
- const routes = [
5036
- createRoute(openA, {
5037
- completeAt: Millis.orThrow(3000),
5038
- error: { type: "WriteFailed", at: Millis.orThrow(1000) },
5039
- }),
5040
- createRoute(openB, {
5041
- complete: true,
5042
- completeAt: Millis.orThrow(2000),
5043
- }),
5044
- createRoute(closedId, {
5045
- completeAt: Millis.orThrow(1000),
5046
- error: failure(4000),
5047
- }),
5048
- ];
5049
- const [owner] = syncStateToOwnerSyncStates(
5050
- createState([
5051
- {
5052
- name: testName,
5053
- refused: false,
5054
- owners: [createOwner(testAppOwner.id, routes)],
5055
- },
6637
+ // An error shows before all of them.
6638
+ assertEqual(
6639
+ ownerStatusOf([
6640
+ createCompleteRoute(openId),
6641
+ createPendingRoute(otherOpenId),
6642
+ createPendingRoute(disconnectedId, { failure: quotaFailure(1000) }),
5056
6643
  ]),
6644
+ { type: "Error", error: quotaFailure(1000) },
5057
6645
  );
5058
- assertEqual(owner, {
5059
- name: testName,
5060
- ownerId: testAppOwner.id,
5061
- status: "error",
5062
- syncedAt: Millis.orThrow(3000),
5063
- error: failure(4000),
5064
- relays: [
5065
- { transport: transports[0], route: routes[0], status: "error" },
5066
- { transport: transports[1], route: routes[1], status: "synced" },
5067
- { transport: transports[2], route: routes[2], status: "error" },
5068
- ],
6646
+ // So does a skipped change on a pending route, such as a settled one
6647
+ // whose relay then disconnected, alone or next to a complete relay.
6648
+ const skippedOnPending = createPendingRoute(disconnectedId, {
6649
+ skippedError: skippedChange(1000),
5069
6650
  });
6651
+ assertEqual(ownerStatusOf([skippedOnPending]), {
6652
+ type: "Error",
6653
+ error: skippedChange(1000),
6654
+ });
6655
+ assertEqual(
6656
+ ownerStatusOf([createCompleteRoute(openId), skippedOnPending]),
6657
+ { type: "Error", error: skippedChange(1000) },
6658
+ );
5070
6659
  });
5071
6660
 
5072
- it("pairs each route with its transport and a status", () => {
5073
- const routes = [
5074
- createRoute(connectingId),
5075
- createRoute(openB, { complete: true }),
5076
- createRoute(openA),
5077
- createRoute(closedId),
5078
- ];
5079
- const [owner] = syncStateToOwnerSyncStates(
5080
- createState([
5081
- {
5082
- name: testName,
5083
- refused: false,
5084
- owners: [createOwner(testAppOwner.id, routes)],
5085
- },
6661
+ it("does not hide an older failure behind a newer skipped change on another relay", () => {
6662
+ // A skipped change is stamped again by every reply that skips it, and the
6663
+ // app can act on a quota failure.
6664
+ assertEqual(
6665
+ ownerStatusOf([
6666
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6667
+ createSettledRoute(otherOpenId, skippedChange(5000)),
5086
6668
  ]),
6669
+ { type: "Error", error: quotaFailure(1000) },
6670
+ );
6671
+ // Among failures, and without one among skipped changes, the newest shows.
6672
+ assertEqual(
6673
+ ownerStatusOf([
6674
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6675
+ createPendingRoute(otherOpenId, {
6676
+ failure: { type: "WriteFailed", at: Millis.orThrow(2000) },
6677
+ }),
6678
+ ]),
6679
+ {
6680
+ type: "Error",
6681
+ error: { type: "WriteFailed", at: Millis.orThrow(2000) },
6682
+ },
6683
+ );
6684
+ assertEqual(
6685
+ ownerStatusOf([
6686
+ createSettledRoute(openId, skippedChange(3000)),
6687
+ createPendingRoute(otherOpenId, { skippedError: skippedChange(1000) }),
6688
+ ]),
6689
+ { type: "Error", error: skippedChange(3000) },
6690
+ );
6691
+ // The newest wins in either route order.
6692
+ assertEqual(
6693
+ ownerStatusOf([
6694
+ createPendingRoute(openId, {
6695
+ failure: { type: "WriteFailed", at: Millis.orThrow(2000) },
6696
+ }),
6697
+ createPendingRoute(otherOpenId, { failure: quotaFailure(1000) }),
6698
+ ]),
6699
+ {
6700
+ type: "Error",
6701
+ error: { type: "WriteFailed", at: Millis.orThrow(2000) },
6702
+ },
6703
+ );
6704
+ assertEqual(
6705
+ ownerStatusOf([
6706
+ createPendingRoute(openId, { skippedError: skippedChange(1000) }),
6707
+ createSettledRoute(otherOpenId, skippedChange(3000)),
6708
+ ]),
6709
+ { type: "Error", error: skippedChange(3000) },
5087
6710
  );
5088
- assertNotUndefined(owner);
5089
- // Relays keep the order of the routes, whatever the transport order.
5090
- assertEqual(owner.relays, [
5091
- { transport: transports[3], route: routes[0], status: "offline" },
5092
- { transport: transports[1], route: routes[1], status: "synced" },
5093
- { transport: transports[0], route: routes[2], status: "syncing" },
5094
- { transport: transports[2], route: routes[3], status: "offline" },
5095
- ]);
5096
6711
  });
5097
6712
 
5098
- it("reports each writable owner of each running database", () => {
6713
+ it("finds the owner in the named database", () => {
5099
6714
  const secondName = Name.orThrow("second");
5100
- const states = syncStateToOwnerSyncStates(
5101
- createState([
6715
+ const state: SyncState = {
6716
+ transports: syncTransports,
6717
+ tenants: [
5102
6718
  {
6719
+ type: "Active",
5103
6720
  name: testName,
5104
- refused: false,
5105
6721
  owners: [
5106
- createOwner(testAppOwner.id, [
5107
- createRoute(openA, { complete: true }),
5108
- ]),
5109
- // A readonly registration never synchronizes.
5110
- createOwner(testAppOwner2.id, [], false),
6722
+ {
6723
+ type: "Writable",
6724
+ ownerId: testAppOwner.id,
6725
+ routes: [createCompleteRoute(openId)],
6726
+ },
5111
6727
  ],
5112
6728
  },
5113
6729
  {
6730
+ type: "Active",
5114
6731
  name: secondName,
5115
- refused: false,
5116
- owners: [createOwner(testAppOwner2.id, [createRoute(openB)])],
5117
- },
5118
- // A refused database synchronizes nothing, whatever it registered.
5119
- {
5120
- name: Name.orThrow("refused"),
5121
- refused: true,
5122
- owners: [createOwner(testAppOwner.id, [createRoute(openA)])],
6732
+ owners: [
6733
+ {
6734
+ type: "Writable",
6735
+ ownerId: testAppOwner.id,
6736
+ routes: [createPendingRoute(disconnectedId)],
6737
+ },
6738
+ ],
5123
6739
  },
5124
- ]),
5125
- );
5126
- assertEqual(
5127
- states.map(({ name, ownerId, status }) => ({ name, ownerId, status })),
5128
- [
5129
- { name: testName, ownerId: testAppOwner.id, status: "synced" },
5130
- { name: secondName, ownerId: testAppOwner2.id, status: "syncing" },
5131
6740
  ],
6741
+ };
6742
+ assertEqual(syncStateToOwnerSyncStatus(state, testName, testAppOwner.id), {
6743
+ type: "Synced",
6744
+ });
6745
+ assertEqual(
6746
+ syncStateToOwnerSyncStatus(state, secondName, testAppOwner.id),
6747
+ { type: "Offline" },
6748
+ );
6749
+ });
6750
+
6751
+ it("gives the same status object while the status is unchanged", () => {
6752
+ for (const routes of [
6753
+ [],
6754
+ [createPendingRoute(openId)],
6755
+ [createCompleteRoute(openId)],
6756
+ [createPendingRoute(disconnectedId)],
6757
+ ]) {
6758
+ // Each call derives from a new snapshot, as bindings receive them.
6759
+ assertSame(ownerStatusOf(routes), ownerStatusOf(routes));
6760
+ }
6761
+
6762
+ // An Error status is the same object for the same error object, which the
6763
+ // store keeps between snapshots while it is unchanged, for the owner and
6764
+ // for each relay alike.
6765
+ const failure = quotaFailure(1000);
6766
+ const failedRoute = createPendingRoute(openId, { failure });
6767
+ const errorStatus = ownerStatusOf([failedRoute]);
6768
+ assertSame(
6769
+ ownerStatusOf([createCompleteRoute(otherOpenId), failedRoute]),
6770
+ errorStatus,
6771
+ );
6772
+ assertSame(
6773
+ relaySyncStateToStatus({
6774
+ transport: createSyncTransport(openId, openConnection),
6775
+ route: failedRoute,
6776
+ }),
6777
+ errorStatus,
6778
+ );
6779
+ const skipped = skippedChange(2000);
6780
+ assertSame(
6781
+ ownerStatusOf([createSettledRoute(openId, skipped)]),
6782
+ ownerStatusOf([createPendingRoute(openId, { skippedError: skipped })]),
6783
+ );
6784
+ assertSame(
6785
+ relaySyncStateToStatus({
6786
+ transport: createSyncTransport(openId, openConnection),
6787
+ route: createSettledRoute(openId, skipped),
6788
+ }),
6789
+ ownerStatusOf([createSettledRoute(openId, skipped)]),
6790
+ );
6791
+
6792
+ // An equal error in a new object, which the store never hands over while
6793
+ // the error is unchanged, gives a new status.
6794
+ assertNotSame(
6795
+ ownerStatusOf([
6796
+ createPendingRoute(openId, { failure: quotaFailure(1000) }),
6797
+ ]),
6798
+ errorStatus,
5132
6799
  );
5133
6800
  });
5134
6801
  });
@@ -5430,82 +7097,224 @@ describe("with one evolu instance", () => {
5430
7097
  },
5431
7098
  });
5432
7099
 
5433
- first.dbWorkerPort.postMessage({
7100
+ first.dbWorkerPort.postMessage({
7101
+ type: "OnQueuedResponse",
7102
+ attemptId: mutateInput.attemptId,
7103
+ response: {
7104
+ type: "ForEvolu",
7105
+ id: first.id,
7106
+ message: {
7107
+ clock: createTimestamp(),
7108
+ type: "Mutate",
7109
+ messagesByOwnerId: new Map(),
7110
+ rowsByQuery: new Map([[testQuery, [{ value: 1 }]]]),
7111
+ },
7112
+ },
7113
+ });
7114
+ await testWaitForWorkerMessage();
7115
+
7116
+ assertTrue(
7117
+ firstOutputs.some((output) => output.type === "OnPatchesByQuery"),
7118
+ );
7119
+ assertFalse(
7120
+ firstOutputs.some((output) => output.type === "RefreshQueries"),
7121
+ );
7122
+ assertTrue(
7123
+ secondOutputs.some((output) => output.type === "RefreshQueries"),
7124
+ );
7125
+ });
7126
+
7127
+ it("drops CreateEvolu when shared worker stops during tenant startup", async () => {
7128
+ await using setup = await setupSharedWorker();
7129
+ const { announceTabLeader, run, sharedWorkerOutputs, worker } = setup;
7130
+ const id: EvoluInstanceId = createId(run.deps);
7131
+ const evoluOutputs: Array<EvoluOutput> = [];
7132
+
7133
+ using evoluChannel = testCreateMessageChannel<EvoluOutput, EvoluInput>();
7134
+ evoluChannel.port2.onMessage = (output) => {
7135
+ evoluOutputs.push(output);
7136
+ };
7137
+
7138
+ await announceTabLeader();
7139
+
7140
+ const outputCount = sharedWorkerOutputs.length;
7141
+ worker.port.postMessage({
7142
+ type: "CreateEvolu",
7143
+ name: testName,
7144
+ id,
7145
+ consoleLevel: "debug",
7146
+ sqliteSchema: testSqliteSchema,
7147
+ encryptionKey: testAppOwner.encryptionKey,
7148
+ memoryOnly: false,
7149
+ evoluPort: evoluChannel.port1.native,
7150
+ });
7151
+
7152
+ await testWaitForWorkerMessage();
7153
+
7154
+ const initDbWorker = getDbWorkerInit(sharedWorkerOutputs[outputCount]);
7155
+
7156
+ using dbWorkerPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
7157
+ initDbWorker.port,
7158
+ );
7159
+ const dbDisposeInputs: Array<
7160
+ Extract<DbWorkerInput, { type: "Dispose" }>
7161
+ > = [];
7162
+ dbWorkerPort.onMessage = (input) => {
7163
+ if (input.type === "Dispose") dbDisposeInputs.push(input);
7164
+ };
7165
+
7166
+ const disposePromise = setup[Symbol.asyncDispose]();
7167
+ dbWorkerPort.postMessage({
7168
+ clock: createTimestamp(),
7169
+ type: "LeaderAcquired",
7170
+ name: testName,
7171
+ });
7172
+ await disposePromise;
7173
+ await testWaitForWorkerMessage();
7174
+
7175
+ assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
7176
+ assertEqual(evoluOutputs, []);
7177
+ });
7178
+
7179
+ it("tells the tab about a failed mutation and runs the next request", async () => {
7180
+ await using setup = await setupSharedWorker();
7181
+ const { createEvolu, run, sharedWorkerOutputs } = setup;
7182
+ const { time } = run.deps;
7183
+ const { dbInputs, dbWorkerPort, evoluChannel, id } = await createEvolu();
7184
+ const outputs: Array<EvoluOutput> = [];
7185
+ evoluChannel.port2.onMessage = (output) => {
7186
+ outputs.push(output);
7187
+ };
7188
+ using errors = testCreateBroadcastChannel<ConsoleEntryOrError>(
7189
+ consoleEntryOrErrorBroadcastChannelName,
7190
+ );
7191
+ const broadcastErrors: Array<ConsoleEntryOrError> = [];
7192
+ errors.onMessage = (output) => {
7193
+ if (output.type === "Error") broadcastErrors.push(output);
7194
+ };
7195
+
7196
+ const onCompleteId = createId(run.deps);
7197
+ evoluChannel.port2.postMessage({
7198
+ type: "Mutate",
7199
+ changes: [
7200
+ {
7201
+ ownerId: testAppOwner.id,
7202
+ ...testCreateCrdtMessage(createId(run.deps), 1, "failed").change,
7203
+ },
7204
+ ],
7205
+ onCompleteIds: [onCompleteId],
7206
+ subscribedQueries: new Set([testQuery]),
7207
+ });
7208
+ evoluChannel.port2.postMessage({
7209
+ type: "Query",
7210
+ queries: createSet([testQuery]),
7211
+ });
7212
+ time.advance("10s");
7213
+ await testWaitForWorkerMessage();
7214
+
7215
+ const mutateInput = dbInputs.at(-1);
7216
+ assertNotUndefined(mutateInput);
7217
+ assertSame(mutateInput.request.message.type, "Mutate");
7218
+ const outputCount = sharedWorkerOutputs.length;
7219
+ const error = createUnknownError(new Error("database or disk is full"));
7220
+
7221
+ dbWorkerPort.postMessage({
5434
7222
  type: "OnQueuedResponse",
5435
7223
  attemptId: mutateInput.attemptId,
5436
7224
  response: {
5437
7225
  type: "ForEvolu",
5438
- id: first.id,
5439
- message: {
5440
- clock: createTimestamp(),
5441
- type: "Mutate",
5442
- messagesByOwnerId: new Map(),
5443
- rowsByQuery: new Map([[testQuery, [{ value: 1 }]]]),
5444
- },
7226
+ id,
7227
+ message: { type: "MutateFailed", error },
5445
7228
  },
5446
7229
  });
5447
7230
  await testWaitForWorkerMessage();
5448
7231
 
5449
- assertTrue(
5450
- firstOutputs.some((output) => output.type === "OnPatchesByQuery"),
5451
- );
5452
- assertFalse(
5453
- firstOutputs.some((output) => output.type === "RefreshQueries"),
5454
- );
5455
- assertTrue(
5456
- secondOutputs.some((output) => output.type === "RefreshQueries"),
5457
- );
7232
+ assertEqual(sharedWorkerOutputs.slice(outputCount), [
7233
+ { type: "Error", error },
7234
+ ]);
7235
+ assertEqual(broadcastErrors, []);
7236
+ assertEqual(outputs, [
7237
+ { type: "OnMutateFailed", onCompleteIds: [onCompleteId] },
7238
+ ]);
7239
+ const queryInput = dbInputs.at(-1);
7240
+ assertNotUndefined(queryInput);
7241
+ assertSame(queryInput.request.message.type, "Query");
5458
7242
  });
5459
7243
 
5460
- it("drops CreateEvolu when shared worker stops during tenant startup", async () => {
7244
+ it("tells every tab about a failed mutation whose instance was disposed", async () => {
5461
7245
  await using setup = await setupSharedWorker();
5462
- const { announceTabLeader, run, sharedWorkerOutputs, worker } = setup;
5463
- const id: EvoluInstanceId = createId(run.deps);
5464
- const evoluOutputs: Array<EvoluOutput> = [];
5465
-
5466
- using evoluChannel = testCreateMessageChannel<EvoluOutput, EvoluInput>();
5467
- evoluChannel.port2.onMessage = (output) => {
5468
- evoluOutputs.push(output);
5469
- };
5470
-
5471
- await announceTabLeader();
7246
+ const { createEvolu, run, sharedWorkerOutputs, worker } = setup;
7247
+ const instance = await createEvolu();
5472
7248
 
5473
- const outputCount = sharedWorkerOutputs.length;
7249
+ // A sibling instance writes and is disposed before the write is
7250
+ // answered.
7251
+ const siblingId = createId<"EvoluInstance">(run.deps);
7252
+ const siblingDisposer = new AsyncDisposableStack();
7253
+ siblingDisposer.use(await run.ok(acquireLeaderLock(siblingId)));
7254
+ using siblingChannel = testCreateMessageChannel<
7255
+ EvoluOutput,
7256
+ EvoluInput
7257
+ >();
5474
7258
  worker.port.postMessage({
5475
7259
  type: "CreateEvolu",
7260
+ id: siblingId,
5476
7261
  name: testName,
5477
- id,
5478
7262
  consoleLevel: "debug",
5479
7263
  sqliteSchema: testSqliteSchema,
5480
7264
  encryptionKey: testAppOwner.encryptionKey,
5481
7265
  memoryOnly: false,
5482
- evoluPort: evoluChannel.port1.native,
7266
+ evoluPort: siblingChannel.port1.native,
5483
7267
  });
5484
-
7268
+ await testWaitForWorkerMessage();
7269
+ siblingChannel.port2.postMessage({
7270
+ type: "Mutate",
7271
+ changes: [
7272
+ {
7273
+ ownerId: testAppOwner.id,
7274
+ ...testCreateCrdtMessage(createId(run.deps), 1, "failed").change,
7275
+ },
7276
+ ],
7277
+ onCompleteIds: [createId(run.deps)],
7278
+ subscribedQueries: new Set(),
7279
+ });
7280
+ await testWaitForWorkerMessage();
7281
+ const mutateInput = instance.dbInputs.at(-1);
7282
+ assertNotUndefined(mutateInput);
7283
+ assertSame(mutateInput.request.type, "ForEvolu");
7284
+ assertSame(mutateInput.request.id, siblingId);
7285
+ await siblingDisposer.disposeAsync();
5485
7286
  await testWaitForWorkerMessage();
5486
7287
 
5487
- const initDbWorker = getDbWorkerInit(sharedWorkerOutputs[outputCount]);
5488
-
5489
- using dbWorkerPort = testCreateMessagePort<DbWorkerOutput, DbWorkerInput>(
5490
- initDbWorker.port,
7288
+ using errors = testCreateBroadcastChannel<ConsoleEntryOrError>(
7289
+ consoleEntryOrErrorBroadcastChannelName,
5491
7290
  );
5492
- const dbDisposeInputs: Array<
5493
- Extract<DbWorkerInput, { type: "Dispose" }>
5494
- > = [];
5495
- dbWorkerPort.onMessage = (input) => {
5496
- if (input.type === "Dispose") dbDisposeInputs.push(input);
7291
+ const broadcasts: Array<ConsoleEntryOrError> = [];
7292
+ errors.onMessage = (output) => {
7293
+ broadcasts.push(output);
7294
+ };
7295
+ const evoluOutputs: Array<EvoluOutput> = [];
7296
+ instance.evoluChannel.port2.onMessage = (output) => {
7297
+ evoluOutputs.push(output);
7298
+ };
7299
+ siblingChannel.port2.onMessage = (output) => {
7300
+ evoluOutputs.push(output);
5497
7301
  };
7302
+ const outputCount = sharedWorkerOutputs.length;
7303
+ const error = createUnknownError(new Error("database or disk is full"));
5498
7304
 
5499
- const disposePromise = setup[Symbol.asyncDispose]();
5500
- dbWorkerPort.postMessage({
5501
- clock: createTimestamp(),
5502
- type: "LeaderAcquired",
5503
- name: testName,
7305
+ instance.dbWorkerPort.postMessage({
7306
+ type: "OnQueuedResponse",
7307
+ attemptId: mutateInput.attemptId,
7308
+ response: {
7309
+ type: "ForEvolu",
7310
+ id: siblingId,
7311
+ message: { type: "MutateFailed", error },
7312
+ },
5504
7313
  });
5505
- await disposePromise;
5506
7314
  await testWaitForWorkerMessage();
5507
7315
 
5508
- assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
7316
+ assertEqual(broadcasts, [{ type: "Error", error }]);
7317
+ assertEqual(sharedWorkerOutputs.slice(outputCount), []);
5509
7318
  assertEqual(evoluOutputs, []);
5510
7319
  });
5511
7320
 
@@ -6589,6 +8398,8 @@ describe("with one evolu instance", () => {
6589
8398
  it("routes a relay's version-mismatch reply to the tenant", async () => {
6590
8399
  const createWebSocket = testCreateWebSocket();
6591
8400
  await using setup = await setupSharedWorker({ createWebSocket });
8401
+ using disposer = new DisposableStack();
8402
+ const { latest } = setupSyncStates(setup, disposer);
6592
8403
  const instance = await setup.createEvolu();
6593
8404
  const { dbInputs, evoluChannel } = instance;
6594
8405
  const { time } = setup.run.deps;
@@ -6648,10 +8459,16 @@ describe("with one evolu instance", () => {
6648
8459
  ok: false,
6649
8460
  error: versionError,
6650
8461
  });
6651
- assertEqual(consoleEntryOrErrors.at(-1), {
6652
- type: "Error",
6653
- error: versionError,
8462
+ // The route shows it with its details; it is not posted as an error.
8463
+ const [owner] = ownersOf(latest());
8464
+ assert(owner?.type === "Writable", "Expected a writable owner.");
8465
+ const [route] = owner.routes;
8466
+ assertSame(route?.type, "Pending");
8467
+ assertEqual(route.failure, {
8468
+ ...versionError,
8469
+ at: time.now(),
6654
8470
  });
8471
+ assertFalse(consoleEntryOrErrors.some(({ type }) => type === "Error"));
6655
8472
 
6656
8473
  // The failure requests one round; the same reply to it requests none.
6657
8474
  assertEqual(await respondToSyncRound(instance, createWebSocket), [
@@ -7343,6 +9160,7 @@ describe("with one evolu instance", () => {
7343
9160
  type: "ApplySyncMessage",
7344
9161
  ownerId: testAppOwner.id,
7345
9162
  didWriteMessages: true,
9163
+ skippedError: null,
7346
9164
  result: {
7347
9165
  ok: false,
7348
9166
  error: {
@@ -7359,14 +9177,8 @@ describe("with one evolu instance", () => {
7359
9177
  assertTrue(
7360
9178
  evoluOutputs.some((output) => output.type === "RefreshQueries"),
7361
9179
  );
7362
- assertEqual(consoleEntryOrErrors.at(-1), {
7363
- type: "Error",
7364
- error: {
7365
- type: "ProtocolInvalidDataError",
7366
- data: new Uint8Array(),
7367
- error: "boom",
7368
- },
7369
- });
9180
+ // The route shows the failure; it is not posted as an error.
9181
+ assertFalse(consoleEntryOrErrors.some(({ type }) => type === "Error"));
7370
9182
 
7371
9183
  // The failure requested one round through the route.
7372
9184
  const retryRoundInput = dbInputs.at(-1);
@@ -7410,6 +9222,7 @@ describe("with one evolu instance", () => {
7410
9222
  type: "ApplySyncMessage",
7411
9223
  ownerId: testAppOwner.id,
7412
9224
  didWriteMessages: false,
9225
+ skippedError: null,
7413
9226
  result: {
7414
9227
  ok: true,
7415
9228
  value: {
@@ -7535,6 +9348,7 @@ describe("with one evolu instance", () => {
7535
9348
  type: "ApplySyncMessage",
7536
9349
  ownerId: testAppOwner.id,
7537
9350
  didWriteMessages: false,
9351
+ skippedError: null,
7538
9352
  result: {
7539
9353
  ok: false,
7540
9354
  error: { type: "AbortError", reason: { type: "Stop" } },
@@ -7545,6 +9359,7 @@ describe("with one evolu instance", () => {
7545
9359
  type: "ApplySyncMessage",
7546
9360
  ownerId: testAppOwner.id,
7547
9361
  didWriteMessages: false,
9362
+ skippedError: null,
7548
9363
  result: { ok: true, value: { type: "Broadcast" } },
7549
9364
  });
7550
9365
  await runApplySync({
@@ -7552,6 +9367,7 @@ describe("with one evolu instance", () => {
7552
9367
  type: "ApplySyncMessage",
7553
9368
  ownerId: testAppOwner.id,
7554
9369
  didWriteMessages: false,
9370
+ skippedError: null,
7555
9371
  result: { ok: true, value: { type: "Converged" } },
7556
9372
  });
7557
9373
 
@@ -8333,30 +10149,6 @@ describe("with one evolu instance", () => {
8333
10149
  assertEqual(setup.run.deps.reportDefect.getDefects(), []);
8334
10150
  });
8335
10151
 
8336
- it("waits for DbWorker leader lock during tenant disposal", async () => {
8337
- await using setup = await setupSharedWorker();
8338
- const { createEvolu } = setup;
8339
- const { dbDisposeInputs, releaseDbWorkerLeader } = await createEvolu({
8340
- releaseDbWorkerLeaderOnDispose: false,
8341
- autoDispose: false,
8342
- });
8343
- let disposed = false;
8344
-
8345
- const disposing = setup[Symbol.asyncDispose]().then(() => {
8346
- disposed = true;
8347
- });
8348
-
8349
- await testWaitForWorkerMessage();
8350
-
8351
- assertEqual(dbDisposeInputs, [{ type: "Dispose" }]);
8352
- assertFalse(disposed);
8353
-
8354
- await releaseDbWorkerLeader();
8355
- await disposing;
8356
-
8357
- assertTrue(disposed);
8358
- });
8359
-
8360
10152
  it("stops a replacement DbWorker that acquires the lock during tenant disposal", async () => {
8361
10153
  await using setup = await setupSharedWorker();
8362
10154
  await using disposer = new AsyncDisposableStack();
@@ -8396,6 +10188,29 @@ describe("with one evolu instance", () => {
8396
10188
  await disposing;
8397
10189
  });
8398
10190
 
10191
+ it("leads for its ID until disposed, so its DbWorkers stop without Dispose", async () => {
10192
+ await using setup = await setupSharedWorker();
10193
+ await setup.createEvolu();
10194
+ assertSame(
10195
+ getDbWorkerInit(setup.sharedWorkerOutputs.at(0)).sharedWorkerId,
10196
+ setup.workerId,
10197
+ );
10198
+ await using run = testCreateRun({
10199
+ lockManager: setup.run.deps.lockManager,
10200
+ });
10201
+ let isLeader = false;
10202
+ const leading = run.ok(acquireLeaderLock(setup.workerId)).then((lock) => {
10203
+ isLeader = true;
10204
+ return lock;
10205
+ });
10206
+ await testWaitForWorkerMessage();
10207
+ assertFalse(isLeader);
10208
+
10209
+ await setup[Symbol.asyncDispose]();
10210
+
10211
+ await using _lock = await leading;
10212
+ });
10213
+
8399
10214
  it("disposes cleanly when queued sync resumes during tenant disposal", async () => {
8400
10215
  const createWebSocket = testCreateWebSocket({ isOpen: false });
8401
10216
  const creating = Promise.withResolvers<void>();
@@ -8476,6 +10291,178 @@ describe("with one evolu instance", () => {
8476
10291
  assertEqual(createWebSocket.createdUrls, []);
8477
10292
  });
8478
10293
 
10294
+ it("releases a transport claimed while its instance is disposed and drops queued changes", async () => {
10295
+ const createWebSocket = testCreateWebSocket();
10296
+ const creating = Promise.withResolvers<void>();
10297
+ const continueCreating = Promise.withResolvers<void>();
10298
+ await using setup = await setupSharedWorker({
10299
+ createWebSocket: (url, options) => async (run) => {
10300
+ creating.resolve();
10301
+ await continueCreating.promise;
10302
+ return run(createWebSocket(url, options));
10303
+ },
10304
+ });
10305
+ const instance = await setup.createEvolu({ autoDispose: false });
10306
+ const transport = createOwnerWebSocketTransport({
10307
+ url: "wss://claim-during-dispose.example",
10308
+ ownerId: testAppOwner.id,
10309
+ });
10310
+ instance.evoluChannel.port2.postMessage({
10311
+ type: "UseOwner",
10312
+ actions: [
10313
+ {
10314
+ action: "add",
10315
+ owner: { owner: testAppOwner, transports: [transport] },
10316
+ },
10317
+ ],
10318
+ });
10319
+ // Queued behind the claim in flight.
10320
+ instance.evoluChannel.port2.postMessage({
10321
+ type: "UseOwner",
10322
+ actions: [
10323
+ {
10324
+ action: "add",
10325
+ owner: {
10326
+ owner: testAppOwner,
10327
+ transports: [
10328
+ createOwnerWebSocketTransport({
10329
+ url: "wss://queued-during-dispose.example",
10330
+ ownerId: testAppOwner.id,
10331
+ }),
10332
+ ],
10333
+ },
10334
+ },
10335
+ ],
10336
+ });
10337
+ await creating.promise;
10338
+
10339
+ await instance[Symbol.asyncDispose]();
10340
+ continueCreating.resolve();
10341
+ await testWaitForWorkerMessage();
10342
+
10343
+ assertEqual(createWebSocket.sentMessages.at(-1), {
10344
+ url: transport.url,
10345
+ data: createProtocolMessageForUnsubscribe(testAppOwner.id),
10346
+ });
10347
+ assertEqual(createWebSocket.createdUrls, [transport.url]);
10348
+ assertEqual(setup.run.deps.reportDefect.getDefects(), []);
10349
+ });
10350
+
10351
+ it("reports nothing when the root Run is disposed before the worker", async () => {
10352
+ const createWebSocket = testCreateWebSocket();
10353
+ await using setup = await setupSharedWorker({ createWebSocket });
10354
+ const instance = await setup.createEvolu();
10355
+ const transport = createOwnerWebSocketTransport({
10356
+ url: "wss://root-dispose.example",
10357
+ ownerId: testAppOwner.id,
10358
+ });
10359
+ instance.evoluChannel.port2.postMessage({
10360
+ type: "UseOwner",
10361
+ actions: [
10362
+ {
10363
+ action: "add",
10364
+ owner: { owner: testAppOwner, transports: [transport] },
10365
+ },
10366
+ ],
10367
+ });
10368
+ await testWaitForWorkerMessage();
10369
+ assertEqual(createWebSocket.createdUrls, [transport.url]);
10370
+
10371
+ // As a runtime shutting down while the worker's stack is still owned.
10372
+ await setup.run[Symbol.asyncDispose]();
10373
+
10374
+ assertEqual(setup.run.deps.reportDefect.getDefects(), []);
10375
+ });
10376
+
10377
+ it("reports only the panic when it disposes a tenant with an instance", async () => {
10378
+ // The default test WebSocket throws on create, which panics the root.
10379
+ await using setup = await setupSharedWorker();
10380
+ const instance = await setup.createEvolu();
10381
+ instance.evoluChannel.port2.postMessage({
10382
+ type: "UseOwner",
10383
+ actions: [
10384
+ {
10385
+ action: "add",
10386
+ owner: {
10387
+ owner: testAppOwner,
10388
+ transports: [
10389
+ createOwnerWebSocketTransport({
10390
+ url: "wss://root-panic.example",
10391
+ ownerId: testAppOwner.id,
10392
+ }),
10393
+ ],
10394
+ },
10395
+ },
10396
+ ],
10397
+ });
10398
+ const panic = await setup.run.deps.reportDefect.next();
10399
+ await setup.run[Symbol.asyncDispose]();
10400
+
10401
+ assertEqual(setup.run.deps.reportDefect.getDefects(), [panic]);
10402
+ // The tenant still tells its DbWorker to stop.
10403
+ await testWaitForWorkerMessage();
10404
+ assertEqual(instance.dbDisposeInputs, [{ type: "Dispose" }]);
10405
+ assert(AbortError.is(panic), "Expected an AbortError.");
10406
+ assert(panic.reason.type === "PanicAbortReason", "Expected a panic.");
10407
+ assertInstanceOf(panic.reason.defect, Error);
10408
+ assertEqual(
10409
+ panic.reason.defect.message,
10410
+ "testCreateWebSocket is configured to throw on create",
10411
+ );
10412
+ });
10413
+
10414
+ it("starts the DbWorker of a reopened database while the previous one still holds its lock", async () => {
10415
+ await using setup = await setupSharedWorker();
10416
+ await using disposer = new AsyncDisposableStack();
10417
+ const closed = await setup.createEvolu({ autoDispose: false });
10418
+ // As a DbWorker that got Dispose but is still closing the database.
10419
+ await closed.releaseDbWorkerLeader();
10420
+ const previousDbWorkerRun = disposer.use(
10421
+ testCreateRun({ lockManager: setup.run.deps.lockManager }),
10422
+ );
10423
+ const previousDbWorkerLock = await previousDbWorkerRun.ok(
10424
+ acquireLeaderLock(testName),
10425
+ );
10426
+ await closed[Symbol.asyncDispose]();
10427
+ setup.run.deps.time.advance("3s");
10428
+ await testWaitForWorkerMessage();
10429
+
10430
+ let isReopened = false;
10431
+ let isLeader = false;
10432
+ const reopening = setup
10433
+ .createEvoluBeforeDbWorkerLeader({ autoDispose: false })
10434
+ .then((instance) => {
10435
+ isReopened = true;
10436
+ return instance;
10437
+ });
10438
+ disposer.defer(async () => {
10439
+ await previousDbWorkerLock[Symbol.asyncDispose]();
10440
+ const instance = await reopening;
10441
+ if (!isLeader) await instance.acquireDbWorkerLeader();
10442
+ await instance[Symbol.asyncDispose]();
10443
+ });
10444
+ await testWaitForWorkerMessage();
10445
+ await testWaitForWorkerMessage();
10446
+
10447
+ assertTrue(isReopened);
10448
+ const reopened = await reopening;
10449
+ // Its requests wait until its own DbWorker holds the lock.
10450
+ reopened.evoluChannel.port2.postMessage({
10451
+ type: "Query",
10452
+ queries: createSet([testQuery]),
10453
+ });
10454
+ await testWaitForWorkerMessage();
10455
+ assertEqual(reopened.dbInputs, []);
10456
+
10457
+ await previousDbWorkerLock[Symbol.asyncDispose]();
10458
+ await reopened.acquireDbWorkerLeader();
10459
+ isLeader = true;
10460
+ assertEqual(
10461
+ reopened.dbInputs.map(({ type }) => type),
10462
+ ["Request"],
10463
+ );
10464
+ });
10465
+
8479
10466
  it("releases pending instance lock if tenant disposal wins the acquisition race", async () => {
8480
10467
  await using setup = await setupSharedWorker();
8481
10468
  const { createEvolu } = setup;
@@ -8786,7 +10773,7 @@ describe("startup refusal", () => {
8786
10773
  });
8787
10774
  await testWaitForWorkerMessage();
8788
10775
  assertEqual(latest().tenants, [
8789
- { name: testName, refused: true, owners: [] },
10776
+ { type: "Refused", name: testName, error: refusal },
8790
10777
  ]);
8791
10778
  });
8792
10779