@evolu/common 8.11.0 → 8.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/src/Bytes.d.ts +39 -2
  2. package/dist/src/Bytes.d.ts.map +1 -1
  3. package/dist/src/Bytes.js +50 -2
  4. package/dist/src/Callbacks.d.ts +12 -1
  5. package/dist/src/Callbacks.d.ts.map +1 -1
  6. package/dist/src/Callbacks.js +3 -0
  7. package/dist/src/Error.d.ts +10 -4
  8. package/dist/src/Error.d.ts.map +1 -1
  9. package/dist/src/Error.js +10 -4
  10. package/dist/src/Object.d.ts +65 -0
  11. package/dist/src/Object.d.ts.map +1 -1
  12. package/dist/src/Object.js +142 -0
  13. package/dist/src/Resource.d.ts +0 -5
  14. package/dist/src/Resource.d.ts.map +1 -1
  15. package/dist/src/Resource.js +6 -13
  16. package/dist/src/Sqlite.d.ts.map +1 -1
  17. package/dist/src/Sqlite.js +7 -0
  18. package/dist/src/Task.d.ts +10 -8
  19. package/dist/src/Task.d.ts.map +1 -1
  20. package/dist/src/Task.js +41 -5
  21. package/dist/src/Worker.d.ts +3 -3
  22. package/dist/src/index.d.ts +1 -1
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +1 -1
  25. package/dist/src/local-first/Db.d.ts +8 -3
  26. package/dist/src/local-first/Db.d.ts.map +1 -1
  27. package/dist/src/local-first/Db.js +56 -19
  28. package/dist/src/local-first/Evolu.d.ts +142 -24
  29. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  30. package/dist/src/local-first/Evolu.js +109 -8
  31. package/dist/src/local-first/Owner.d.ts +9 -0
  32. package/dist/src/local-first/Owner.d.ts.map +1 -1
  33. package/dist/src/local-first/Owner.js +9 -0
  34. package/dist/src/local-first/Protocol.d.ts +18 -7
  35. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  36. package/dist/src/local-first/Protocol.js +45 -28
  37. package/dist/src/local-first/Relay.d.ts.map +1 -1
  38. package/dist/src/local-first/Relay.js +4 -2
  39. package/dist/src/local-first/Schema.d.ts +18 -3
  40. package/dist/src/local-first/Schema.d.ts.map +1 -1
  41. package/dist/src/local-first/Shared.d.ts +523 -133
  42. package/dist/src/local-first/Shared.d.ts.map +1 -1
  43. package/dist/src/local-first/Shared.js +696 -256
  44. package/dist/src/local-first/Storage.d.ts +19 -15
  45. package/dist/src/local-first/Storage.d.ts.map +1 -1
  46. package/dist/src/local-first/Storage.js +4 -2
  47. package/package.json +1 -1
  48. package/src/Bytes.test.ts +27 -0
  49. package/src/Bytes.ts +58 -2
  50. package/src/Callbacks.test.ts +20 -0
  51. package/src/Callbacks.ts +17 -1
  52. package/src/Error.ts +10 -4
  53. package/src/Object.test.ts +296 -0
  54. package/src/Object.ts +163 -0
  55. package/src/Resource.test.ts +20 -16
  56. package/src/Resource.ts +6 -20
  57. package/src/Sqlite.ts +7 -0
  58. package/src/Task.test.ts +233 -62
  59. package/src/Task.ts +47 -13
  60. package/src/Worker.ts +3 -3
  61. package/src/index.ts +6 -1
  62. package/src/local-first/Db.ts +88 -18
  63. package/src/local-first/Evolu.test.ts +589 -2
  64. package/src/local-first/Evolu.ts +285 -36
  65. package/src/local-first/Owner.ts +9 -0
  66. package/src/local-first/Protocol.test.ts +59 -60
  67. package/src/local-first/Protocol.ts +67 -47
  68. package/src/local-first/Relay.ts +4 -2
  69. package/src/local-first/Schema.ts +20 -3
  70. package/src/local-first/Shared.test.ts +2633 -646
  71. package/src/local-first/Shared.ts +1192 -364
  72. package/src/local-first/Storage.ts +24 -23
@@ -53,7 +53,9 @@
53
53
  * still waiting after three seconds reports {@link OtherBuildRunningError} to
54
54
  * its tabs. Every build broadcasts console entries and errors on
55
55
  * {@link consoleEntryOrErrorBroadcastChannelName}, so a waiting tab also prints
56
- * the running build's output and reports its errors as its own `evoluError`.
56
+ * the running build's output, reports each {@link UnknownError} it sends as its
57
+ * own `evoluError`, and only logs any other error, such as an older build's
58
+ * sync error.
57
59
  *
58
60
  * The tabs of one worker elect the host of its DbWorkers among themselves, with
59
61
  * a lock scoped to the worker, so a tab of another worker never hosts them.
@@ -100,7 +102,8 @@
100
102
  * writable registrations for its owner, whichever instance made it, even one
101
103
  * disposed before the database worker answered. When a relay frame stores new
102
104
  * messages, the tenant requests a round through each other transport claimed
103
- * for the owner, so data learned from one relay reaches the others. A closed
105
+ * for the owner, so data learned from one relay reaches the others, except
106
+ * through a route that skipped a message, as described below. A closed
104
107
  * transport reconciles when it opens, and a replacement leader reconciles every
105
108
  * transport again, because a response reporting stored messages may have been
106
109
  * lost.
@@ -109,26 +112,34 @@
109
112
  * also delivers it as local Broadcast frames to every other tenant with
110
113
  * writable access to the owner, even while sockets are closed. A copy keeps the
111
114
  * uploader's target, and a recipient that stores new continuation messages
112
- * reconciles them through the transports outside that target. Local delivery
113
- * forwards uploads, including historical messages sent during reconciliation,
114
- * but does not reconcile local database histories with each other. That waits
115
- * until replication scopes and retention semantics are defined.
115
+ * reconciles them through the transports outside that target, except through a
116
+ * route that skipped a message. Local delivery forwards uploads, including
117
+ * historical messages sent during reconciliation, but does not reconcile local
118
+ * database histories with each other. That waits until replication scopes and
119
+ * retention semantics are defined.
116
120
  *
117
121
  * ## Sync state
118
122
  *
119
123
  * The shared worker publishes one plain snapshot, {@link SyncState}, of every
120
124
  * transport it manages and every database and owner registration it holds, with
121
- * one route per writable registration and transport, as specified below.
122
- * {@link syncStateToOwnerSyncStates} derives one state per database and owner.
125
+ * one route per writable registration and transport, as specified below. Each
126
+ * part is a union of the states the worker keeps for it: a transport's
127
+ * connection, a database that is active or refused startup, a writable or
128
+ * readonly owner, and a pending, settled, or complete route. A transport's
129
+ * events drive its connection, so publishing never reads a socket.
130
+ *
131
+ * Apps show users an owner's {@link OwnerSyncStatus}, which
132
+ * {@link syncStateToOwnerSyncStatus} derives. A view of each relay pairs each
133
+ * route with its transport by {@link syncStateToRelaySyncStates} and tells the
134
+ * relay's status by {@link relaySyncStateToStatus}. Snapshots store neither, so
135
+ * a status never disagrees with the routes it comes from.
123
136
  *
124
137
  * Each worker broadcasts snapshots on its own channel, whose name a connecting
125
138
  * tab receives through its port, so a tab never hears another worker, such as
126
139
  * one of a different app version, and
127
140
  * {@link SyncStateDep.syncState | deps.syncState} keeps the last snapshot. The
128
- * worker publishes after every change it observes; a transition without an
129
- * event, such as a closed socket starting to reconnect, appears with the next
130
- * snapshot. The snapshot lives in worker memory only, so a new worker starts
131
- * empty.
141
+ * worker publishes after every change it observes. The snapshot lives in worker
142
+ * memory only, so a new worker starts empty.
132
143
  *
133
144
  * ## Synchronization completion
134
145
  *
@@ -156,27 +167,51 @@
156
167
  * - The tenant has sent a round through the transport since the last event that
157
168
  * requires one: its first use of the transport for the owner, the socket
158
169
  * opening, an explicit request, a replacement leader, or storing messages
159
- * from another transport. No failed or aborted result has arrived on the
160
- * route since.
170
+ * from another transport, unless the route skipped a message. No failed or
171
+ * aborted result has arrived on the route since.
172
+ * - No result that skipped a received message has arrived on the route since a
173
+ * round was last requested through it, and since that request, no messages
174
+ * have been stored from another transport, directly or through a sibling copy
175
+ * uploaded outside this transport, and no sibling copy has failed to apply or
176
+ * skipped a message.
177
+ *
178
+ * A route is settled when every condition except the last holds, so a complete
179
+ * route is settled too. A settled route that is incomplete has ended its
180
+ * reconciliation, but its relay may offer a message the database skipped, or
181
+ * messages stored elsewhere may not have reached it; it is published as
182
+ * {@link SettledSyncRoute}.
161
183
  *
162
184
  * A reconciliation chain ends only with a converged result, a failure, an
163
185
  * abort, a dropped frame, or a continuation that finds the socket closed, and
164
186
  * relay errors reach every applying tenant, so these conditions mean every
165
187
  * chain, including this tenant's, converged. A local Broadcast from a sibling
166
188
  * tenant holds every route of its owner until it is applied, because it arrives
167
- * without a request of its own; one that fails to apply requires a round
168
- * through every transport.
189
+ * without a request of its own; one that fails to apply or skips a message
190
+ * requires a round through every transport whose route has not skipped one.
169
191
  *
170
192
  * A failed result on a route requests one round through it. Any further failure
171
- * before the route completes waits for an explicit request or a reopen, so a
193
+ * before the route settles waits for an explicit request or a reopen, so a
172
194
  * persistent failure cannot loop; a converged reply in between does not end the
173
195
  * wait, because it may answer another tenant's round on the shared socket. An
174
196
  * aborted apply leaves its routes incomplete without a retry. An exception
175
197
  * while the database worker creates a round is logged there and fails the
176
- * round's routes with `SyncFailed` without a retry; other unexpected SQLite
177
- * exceptions remain unsupported and can panic the database worker. A frame the
178
- * relay silently drops, such as invalid data, leaves the count above zero until
179
- * the liveness rule below replaces the socket.
198
+ * round's routes with `SyncFailed` without a retry. A mutation that throws is
199
+ * rolled back and reported as an {@link UnknownError} to the tab that made it,
200
+ * or to every tab when its Evolu instance was disposed first. Other unexpected
201
+ * SQLite exceptions remain unsupported and can panic the database worker. A
202
+ * frame the relay silently drops, such as invalid data, leaves the count above
203
+ * zero until the liveness rule below replaces the socket.
204
+ *
205
+ * A result that skipped a received message the database could not decrypt,
206
+ * verify, or decode records the error on the route as its `skippedError`,
207
+ * without ending its chain, so it requests no round, and the relay offers the
208
+ * message again in every round through the route until the database stores a
209
+ * message with that timestamp. The messages received elsewhere that the last
210
+ * condition names request no round through such a route, even while a requested
211
+ * round checks it again, because each round would download every skipped
212
+ * message again. The next round that a first use of the transport for the
213
+ * owner, an explicit request, a reopen, a replacement leader, or a failure
214
+ * sends through the route reconciles them.
180
215
  *
181
216
  * ### Liveness
182
217
  *
@@ -196,11 +231,13 @@
196
231
  * The timeout belongs to the transport, not to an owner, because a small reply
197
232
  * for one owner can wait behind another owner's large frame on the socket. A
198
233
  * grown timeout lasts while a request is outstanding on the socket or a
199
- * database that has not refused startup has an incomplete route through it,
200
- * including one waiting after a failure, and ends with the transport. A reply's
201
- * speed proves nothing, because a recovery on a slow link starts with small
202
- * replies that arrive quickly. The reopen resets the counts and starts the open
203
- * rounds, so a dropped frame delays a route instead of stranding it.
234
+ * database that has not refused startup has an unsettled route through it,
235
+ * including one waiting after a failure, and ends with the transport. A settled
236
+ * route that is incomplete does not count, because its reconciliation has
237
+ * ended. A reply's speed proves nothing, because a recovery on a slow link
238
+ * starts with small replies that arrive quickly. The reopen resets the counts
239
+ * and starts the open rounds, so a dropped frame delays a route instead of
240
+ * stranding it.
204
241
  *
205
242
  * The shared worker sends nothing while idle, so a path that dies then may go
206
243
  * unnoticed until its next request; until then, changes from other devices stop
@@ -264,6 +301,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
264
301
  });
265
302
  import { emptyArray, firstInArray, isNonEmptyArray, } from "../Array.js";
266
303
  import { assert, assertNonNullable, assertNotSame, assertNotUndefined, assertSame, } from "../Assert.js";
304
+ import { createUnknownError } from "../Error.js";
267
305
  import { disposable, exhaustiveCheck } from "../Function.js";
268
306
  import { acquireLeaderLock } from "../LockManager.js";
269
307
  import { createLookupMap, structuralLookup, } from "../Lookup.js";
@@ -305,24 +343,200 @@ export const BuildWaitingRequest = /*#__PURE__*/ object({
305
343
  type: /*#__PURE__*/ literal("BuildWaitingRequest"),
306
344
  });
307
345
  /**
308
- * Folds the routes of every writable owner registration of every running
309
- * database in a {@link SyncState} into one {@link OwnerSyncState} per database
310
- * and owner, which pairs each route with its transport as a
311
- * {@link RelaySyncState}. A database that refused startup and a readonly
312
- * registration synchronize nothing, so they are left out.
346
+ * Tells what an app shows about syncing an owner in a database: `Error` when a
347
+ * relay has one, holding the newest failure of any relay or, without one, the
348
+ * newest skipped change; otherwise the first of `Syncing`, `Synced`, and
349
+ * `Offline` that a relay has, or `NoRelays`. Accepts null, the store's value
350
+ * before the first snapshot. See {@link OwnerSyncStatus} for what to show.
351
+ *
352
+ * The same status is the same object: statuses other than `Error` are shared
353
+ * constants, and an `Error` status is the same object for the same error
354
+ * object, which {@link SyncStateDep.syncState} keeps between snapshots while it
355
+ * is unchanged. So compare statuses with `===`.
313
356
  *
314
357
  * ### Example
315
358
  *
316
359
  * ```ts
317
360
  * import {
318
361
  * assertEqual,
362
+ * assertSame,
319
363
  * createId,
364
+ * createUnknownError,
320
365
  * Millis,
321
366
  * testCreateDeps,
322
367
  * testName,
323
368
  * } from "@evolu/common";
324
369
  * import {
325
- * syncStateToOwnerSyncStates,
370
+ * syncStateToOwnerSyncStatus,
371
+ * testAppOwner,
372
+ * type PendingSyncRoute,
373
+ * type SyncConnection,
374
+ * type SyncRoute,
375
+ * type SyncState,
376
+ * type SyncTransportId,
377
+ * } from "@evolu/common/local-first";
378
+ *
379
+ * const deps = testCreateDeps();
380
+ * const primaryId = createId<"SyncTransport">(deps);
381
+ * const backupId = createId<"SyncTransport">(deps);
382
+ *
383
+ * const stateOf = (
384
+ * connections: ReadonlyArray<SyncConnection>,
385
+ * routes: ReadonlyArray<SyncRoute>,
386
+ * ): SyncState => ({
387
+ * transports: connections.map((connection, index) => ({
388
+ * type: "WebSocket",
389
+ * id: index === 0 ? primaryId : backupId,
390
+ * label: "wss://relay.example",
391
+ * connection,
392
+ * })),
393
+ * tenants: [
394
+ * {
395
+ * type: "Active",
396
+ * name: testName,
397
+ * owners: [{ type: "Writable", ownerId: testAppOwner.id, routes }],
398
+ * },
399
+ * ],
400
+ * });
401
+ * const pending = (transportId: SyncTransportId): PendingSyncRoute => ({
402
+ * type: "Pending",
403
+ * transportId,
404
+ * failure: null,
405
+ * skippedError: null,
406
+ * completeAt: null,
407
+ * lastSentAt: null,
408
+ * lastReceivedAt: null,
409
+ * });
410
+ * const statusOf = (state: SyncState | null) =>
411
+ * syncStateToOwnerSyncStatus(state, testName, testAppOwner.id);
412
+ *
413
+ * // Before the first snapshot, nothing syncs the owner.
414
+ * assertEqual(statusOf(null), { type: "NoRelays" });
415
+ *
416
+ * // A first connection is syncing; a lost one is offline.
417
+ * const connecting: SyncConnection = { type: "Connecting" };
418
+ * assertEqual(statusOf(stateOf([connecting], [pending(primaryId)])), {
419
+ * type: "Syncing",
420
+ * });
421
+ * const disconnected: SyncConnection = {
422
+ * type: "Disconnected",
423
+ * disconnectedAt: Millis.orThrow(2000),
424
+ * openedAt: Millis.orThrow(1000),
425
+ * error: null,
426
+ * };
427
+ * assertEqual(statusOf(stateOf([disconnected], [pending(primaryId)])), {
428
+ * type: "Offline",
429
+ * });
430
+ *
431
+ * // A quota failure on one relay shows before a newer skipped change on
432
+ * // another, because the app can act on it.
433
+ * const quotaError = {
434
+ * type: "ProtocolQuotaError",
435
+ * ownerId: testAppOwner.id,
436
+ * at: Millis.orThrow(3000),
437
+ * } as const;
438
+ * const open: SyncConnection = {
439
+ * type: "Open",
440
+ * openedAt: Millis.orThrow(3400),
441
+ * error: null,
442
+ * };
443
+ * assertEqual(
444
+ * statusOf(
445
+ * stateOf(
446
+ * [disconnected, open],
447
+ * [
448
+ * { ...pending(primaryId), failure: quotaError },
449
+ * {
450
+ * type: "Settled",
451
+ * transportId: backupId,
452
+ * skippedError: {
453
+ * type: "DecryptWithXChaCha20Poly1305Error",
454
+ * error: createUnknownError(new Error("invalid tag")),
455
+ * at: Millis.orThrow(4000),
456
+ * },
457
+ * completeAt: null,
458
+ * lastSentAt: Millis.orThrow(3500),
459
+ * lastReceivedAt: Millis.orThrow(4000),
460
+ * },
461
+ * ],
462
+ * ),
463
+ * ),
464
+ * { type: "Error", error: quotaError },
465
+ * );
466
+ *
467
+ * // The same error gives the same status object.
468
+ * const quotaState = stateOf(
469
+ * [disconnected],
470
+ * [{ ...pending(primaryId), failure: quotaError }],
471
+ * );
472
+ * assertSame(statusOf(quotaState), statusOf(quotaState));
473
+ * ```
474
+ */
475
+ export const syncStateToOwnerSyncStatus = (state, name, ownerId) => {
476
+ let failure = null;
477
+ let skippedError = null;
478
+ let isSyncing = false;
479
+ let isSynced = false;
480
+ let isOffline = false;
481
+ for (const relay of syncStateToRelaySyncStates(state, name, ownerId)) {
482
+ const { route } = relay;
483
+ // A failure stops syncing through its relay, and a skipped change is
484
+ // stamped again by every reply that skips it, so a newer skipped change
485
+ // must not hide an older failure the app can act on.
486
+ if (route.type === "Pending" &&
487
+ route.failure &&
488
+ (!failure || route.failure.at > failure.at))
489
+ failure = route.failure;
490
+ if (route.type !== "Complete" &&
491
+ route.skippedError &&
492
+ (!skippedError || route.skippedError.at > skippedError.at))
493
+ skippedError = route.skippedError;
494
+ const status = relaySyncStateToStatus(relay);
495
+ switch (status.type) {
496
+ case "Error":
497
+ break;
498
+ case "Syncing":
499
+ isSyncing = true;
500
+ break;
501
+ case "Synced":
502
+ isSynced = true;
503
+ break;
504
+ case "Offline":
505
+ isOffline = true;
506
+ break;
507
+ default:
508
+ exhaustiveCheck(status);
509
+ }
510
+ }
511
+ const error = failure ?? skippedError;
512
+ return error
513
+ ? syncRouteErrorToSyncStatus(error)
514
+ : isSyncing
515
+ ? syncingSyncStatus
516
+ : isSynced
517
+ ? syncedSyncStatus
518
+ : isOffline
519
+ ? offlineSyncStatus
520
+ : noRelaysSyncStatus;
521
+ };
522
+ /**
523
+ * Pairs each route of an owner in a database with its transport, in route
524
+ * order. Returns none for a null snapshot, a missing or refused database, and a
525
+ * missing or readonly owner.
526
+ *
527
+ * ### Example
528
+ *
529
+ * ```ts
530
+ * import {
531
+ * assertEqual,
532
+ * createId,
533
+ * Millis,
534
+ * testCreateDeps,
535
+ * testName,
536
+ * } from "@evolu/common";
537
+ * import {
538
+ * relaySyncStateToStatus,
539
+ * syncStateToRelaySyncStates,
326
540
  * testAppOwner,
327
541
  * type SyncRoute,
328
542
  * type SyncState,
@@ -331,86 +545,94 @@ export const BuildWaitingRequest = /*#__PURE__*/ object({
331
545
  *
332
546
  * const deps = testCreateDeps();
333
547
  * const transport: SyncTransport = {
548
+ * type: "WebSocket",
334
549
  * id: createId<"SyncTransport">(deps),
335
550
  * label: "wss://relay.example",
336
- * readyState: "open",
337
- * openedAt: null,
338
- * closedAt: null,
339
- * error: null,
551
+ * connection: {
552
+ * type: "Open",
553
+ * openedAt: Millis.orThrow(800),
554
+ * error: null,
555
+ * },
340
556
  * };
341
557
  * const route: SyncRoute = {
558
+ * type: "Complete",
342
559
  * transportId: transport.id,
343
- * complete: true,
344
560
  * completeAt: Millis.orThrow(1000),
345
561
  * lastSentAt: Millis.orThrow(900),
346
562
  * lastReceivedAt: Millis.orThrow(1000),
347
- * error: null,
348
563
  * };
349
564
  * const state: SyncState = {
350
565
  * transports: [transport],
351
566
  * tenants: [
352
567
  * {
568
+ * type: "Active",
353
569
  * name: testName,
354
- * refused: false,
355
570
  * owners: [
356
- * {
357
- * ownerId: testAppOwner.id,
358
- * writable: true,
359
- * transportIds: [transport.id],
360
- * routes: [route],
361
- * },
571
+ * { type: "Writable", ownerId: testAppOwner.id, routes: [route] },
362
572
  * ],
363
573
  * },
364
574
  * ],
365
575
  * };
366
576
  *
367
- * assertEqual(syncStateToOwnerSyncStates(state), [
368
- * {
369
- * name: testName,
370
- * ownerId: testAppOwner.id,
371
- * status: "synced",
372
- * syncedAt: Millis.orThrow(1000),
373
- * error: null,
374
- * relays: [{ transport, route, status: "synced" }],
375
- * },
376
- * ]);
577
+ * const relays = syncStateToRelaySyncStates(
578
+ * state,
579
+ * testName,
580
+ * testAppOwner.id,
581
+ * );
582
+ * assertEqual(relays, [{ transport, route }]);
583
+ * assertEqual(relays.map(relaySyncStateToStatus), [{ type: "Synced" }]);
584
+ * assertEqual(
585
+ * syncStateToRelaySyncStates(null, testName, testAppOwner.id),
586
+ * [],
587
+ * );
377
588
  * ```
378
589
  */
379
- export const syncStateToOwnerSyncStates = (state) => {
380
- const transportById = new Map(state.transports.map((transport) => [transport.id, transport]));
381
- return state.tenants.flatMap(({ name, refused, owners }) => refused
382
- ? []
383
- : owners.flatMap(({ ownerId, writable, routes }) => {
384
- if (!writable)
385
- return [];
386
- let syncedAt = null;
387
- let error = null;
388
- const relays = [];
389
- for (const route of routes) {
390
- if (route.completeAt !== null &&
391
- (syncedAt === null || route.completeAt > syncedAt))
392
- syncedAt = route.completeAt;
393
- if (route.error !== null &&
394
- (error === null || route.error.at > error.at))
395
- error = route.error;
396
- // A snapshot lists the transport of every route.
397
- const transport = transportById.get(route.transportId);
398
- assertNonNullable(transport);
399
- relays.push({
400
- transport,
401
- route,
402
- status: route.error !== null
403
- ? "error"
404
- : route.complete
405
- ? "synced"
406
- : transport.readyState === "open"
407
- ? "syncing"
408
- : "offline",
409
- });
410
- }
411
- const status = ["error", "syncing", "synced", "offline"].find((candidate) => relays.some((relay) => relay.status === candidate)) ?? "initial";
412
- return [{ name, ownerId, status, syncedAt, error, relays }];
413
- }));
590
+ export const syncStateToRelaySyncStates = (state, name, ownerId) => {
591
+ const tenant = state?.tenants.find((tenant) => tenant.name === name);
592
+ if (!state || tenant?.type !== "Active")
593
+ return emptyArray;
594
+ const owner = tenant.owners.find((owner) => owner.ownerId === ownerId);
595
+ if (owner?.type !== "Writable")
596
+ return emptyArray;
597
+ return owner.routes.map((route) => {
598
+ // A snapshot lists the transport of every route.
599
+ const transport = state.transports.find(({ id }) => id === route.transportId);
600
+ assertNonNullable(transport);
601
+ return { transport, route };
602
+ });
603
+ };
604
+ /** Tells the status of one relay; a failure shows before a skipped change. */
605
+ export const relaySyncStateToStatus = ({ transport, route, }) => {
606
+ switch (route.type) {
607
+ case "Complete":
608
+ return syncedSyncStatus;
609
+ case "Settled":
610
+ return syncRouteErrorToSyncStatus(route.skippedError);
611
+ case "Pending": {
612
+ const error = route.failure ?? route.skippedError;
613
+ if (error)
614
+ return syncRouteErrorToSyncStatus(error);
615
+ return transport.connection.type === "Disconnected"
616
+ ? offlineSyncStatus
617
+ : syncingSyncStatus;
618
+ }
619
+ }
620
+ };
621
+ // Every status keeps its reference while it is unchanged, so bindings compare
622
+ // statuses with `===`. The store shares an unchanged error object between
623
+ // snapshots, and an error gives the same Error status object.
624
+ const noRelaysSyncStatus = { type: "NoRelays" };
625
+ const syncingSyncStatus = { type: "Syncing" };
626
+ const syncedSyncStatus = { type: "Synced" };
627
+ const offlineSyncStatus = { type: "Offline" };
628
+ const errorSyncStatusByError = /*#__PURE__*/ new WeakMap();
629
+ const syncRouteErrorToSyncStatus = (error) => {
630
+ let status = errorSyncStatusByError.get(error);
631
+ if (!status) {
632
+ status = { type: "Error", error };
633
+ errorSyncStatusByError.set(error, status);
634
+ }
635
+ return status;
414
636
  };
415
637
  /**
416
638
  * How long an unanswered request keeps a socket before it is replaced.
@@ -426,8 +648,78 @@ const syncRequestTimeout = PositiveMillis.orThrow(90_000);
426
648
  * {@link syncRequestTimeout}, which covers a 1 MB frame at about 6 kbit/s.
427
649
  */
428
650
  const maxSyncRequestTimeout = PositiveMillis.orThrow(16 * syncRequestTimeout);
651
+ const connectingSyncConnection = {
652
+ type: "Connecting",
653
+ };
654
+ /**
655
+ * A connection already disconnected keeps when it disconnected, so failed
656
+ * reconnect attempts only record their error.
657
+ */
658
+ const disconnectSyncConnection = (connection, at, error) => {
659
+ switch (connection.type) {
660
+ case "Connecting":
661
+ return {
662
+ type: "Disconnected",
663
+ disconnectedAt: at,
664
+ openedAt: null,
665
+ error,
666
+ };
667
+ case "Open":
668
+ return {
669
+ type: "Disconnected",
670
+ disconnectedAt: at,
671
+ openedAt: connection.openedAt,
672
+ error: error ?? connection.error,
673
+ };
674
+ case "Disconnected":
675
+ return error ? { ...connection, error } : connection;
676
+ }
677
+ };
429
678
  const allTransports = { type: "AllTransports" };
430
679
  const isTargetTransport = (target, transport) => target.type === "AllTransports" || target.key === structuralLookup(transport);
680
+ const routeToPending = (progress) => {
681
+ switch (progress.type) {
682
+ case "Pending":
683
+ return progress;
684
+ case "Settled":
685
+ return {
686
+ type: "Pending",
687
+ roundRequired: false,
688
+ failure: null,
689
+ skip: { error: progress.error, isRechecking: false },
690
+ completeAt: progress.completeAt,
691
+ };
692
+ case "Complete":
693
+ return {
694
+ type: "Pending",
695
+ roundRequired: false,
696
+ failure: null,
697
+ skip: null,
698
+ completeAt: progress.completeAt,
699
+ };
700
+ }
701
+ };
702
+ /**
703
+ * Converts an error to a {@link SyncRouteError}: a caught value becomes an
704
+ * {@link UnknownError}, and a {@link ProtocolInvalidDataError} leaves out its
705
+ * data.
706
+ */
707
+ const errorToSyncRouteError = (error, at) => {
708
+ // A caught value inside an error becomes an UnknownError, and a
709
+ // ProtocolInvalidDataError leaves out its data, which can be a whole frame.
710
+ if (error.type === "ProtocolInvalidDataError") {
711
+ const { data: _data, ...rest } = error;
712
+ return { ...rest, error: createUnknownError(rest.error), at };
713
+ }
714
+ if (error.type === "DecryptWithXChaCha20Poly1305Error")
715
+ return { ...error, error: createUnknownError(error.error), at };
716
+ return { ...error, at };
717
+ };
718
+ const failRoute = (progress, failure) => ({
719
+ ...routeToPending(progress),
720
+ roundRequired: true,
721
+ failure,
722
+ });
431
723
  // Long enough for another build's tabs to reload and its worker to end.
432
724
  const otherBuildRunningReportDelay = "3s";
433
725
  /**
@@ -519,8 +811,14 @@ export const initSharedWorker = (self) => async (run) => {
519
811
  }
520
812
  });
521
813
  };
522
- // Released after every tenant and DbWorker is disposed. Earlier releases
523
- // take the same lock in their leader tab; see Builds.
814
+ // Held while this worker runs. Its DbWorkers stop once they can take it,
815
+ // because a Dispose posted right before this worker closes can be lost, as
816
+ // in Firefox. Taken before the build lock, so nothing delays the end of
817
+ // starting once that lock is held.
818
+ disposer.use(await run.ok(acquireLeaderLock(workerId)));
819
+ // Released after every tenant is disposed and has told its DbWorker to
820
+ // stop. Earlier releases take the same lock in their leader tab; see
821
+ // Builds.
524
822
  disposer.use(await run.ok(acquireLeaderLock("tab")));
525
823
  starting.dispose();
526
824
  // Checked once, before any DbWorker starts, so every DbWorker of this
@@ -549,51 +847,85 @@ export const initSharedWorker = (self) => async (run) => {
549
847
  isPublishScheduled = false;
550
848
  if (isDisposed)
551
849
  return;
552
- const transports = [...transportsByKey.values()].map(({ id, label, socket, openedAt, closedAt, error, }) => ({
850
+ const transports = [...transportsByKey.values()].map(({ id, label, connection }) => ({
851
+ type: "WebSocket",
553
852
  id,
554
853
  label,
555
- // The transport drops its socket before disposal, so this never
556
- // reads a disposed one.
557
- readyState: socket?.getReadyState() ?? "connecting",
558
- openedAt,
559
- closedAt,
560
- error,
854
+ connection,
561
855
  }));
856
+ // A grown timeout lasts while a request is outstanding on the socket
857
+ // or a database that has not refused startup has an unsettled route
858
+ // through it. Every change to either publishes, including a route that
859
+ // goes away without settling.
860
+ const unsettledTransportIds = new Set();
562
861
  const tenants = [...currentTenantsByName.values()].map((tenant) => {
563
- const { name, refused, owners } = tenant.getSyncTenant();
862
+ const state = tenant.getSyncTenant();
863
+ if (state.type === "Refused")
864
+ return state;
564
865
  return {
565
- name,
566
- refused,
567
- owners: owners.map(({ ownerId, writable, transportKeys, routes }) => ({
568
- ownerId,
569
- writable,
570
- transportIds: transportKeys.flatMap((key) => {
571
- const entry = transportsByKey.get(key);
572
- return entry ? [entry.id] : [];
573
- }),
574
- routes: routes.flatMap(({ transportKey, ...route }) => {
575
- const entry = transportsByKey.get(transportKey);
576
- return entry ? [{ transportId: entry.id, ...route }] : [];
866
+ type: "Active",
867
+ name: state.name,
868
+ owners: state.owners.map((owner) => owner.type === "Readonly"
869
+ ? {
870
+ type: "Readonly",
871
+ ownerId: owner.ownerId,
872
+ transportIds: owner.transportKeys.flatMap((key) => {
873
+ const entry = transportsByKey.get(key);
874
+ return entry ? [entry.id] : [];
875
+ }),
876
+ }
877
+ : {
878
+ type: "Writable",
879
+ ownerId: owner.ownerId,
880
+ routes: owner.routes.flatMap(({ transportKey, progress, lastSentAt, lastReceivedAt, }) => {
881
+ const entry = transportsByKey.get(transportKey);
882
+ if (!entry)
883
+ return [];
884
+ const transportId = entry.id;
885
+ if (progress.type !== "Pending") {
886
+ // Only a sent round settles a route or completes
887
+ // it.
888
+ assertNonNullable(lastSentAt);
889
+ return [
890
+ progress.type === "Complete"
891
+ ? {
892
+ type: "Complete",
893
+ transportId,
894
+ completeAt: progress.completeAt,
895
+ lastSentAt,
896
+ lastReceivedAt,
897
+ }
898
+ : {
899
+ type: "Settled",
900
+ transportId,
901
+ skippedError: progress.error,
902
+ completeAt: progress.completeAt,
903
+ lastSentAt,
904
+ lastReceivedAt,
905
+ },
906
+ ];
907
+ }
908
+ // Only a database that has not refused startup
909
+ // publishes routes.
910
+ unsettledTransportIds.add(transportId);
911
+ return [
912
+ {
913
+ type: "Pending",
914
+ transportId,
915
+ failure: progress.failure,
916
+ skippedError: progress.skip?.error ?? null,
917
+ completeAt: progress.completeAt,
918
+ lastSentAt,
919
+ lastReceivedAt,
920
+ },
921
+ ];
922
+ }),
577
923
  }),
578
- })),
579
924
  };
580
925
  });
581
- // A grown timeout lasts while a request is outstanding on the socket
582
- // or a database that has not refused startup has an incomplete route
583
- // through it. Every change to either publishes, including a route that
584
- // goes away without completing.
585
- const incompleteTransportIds = new Set();
586
- for (const { refused, owners } of tenants) {
587
- if (refused)
588
- continue;
589
- for (const { routes } of owners)
590
- for (const { transportId, complete } of routes)
591
- if (!complete)
592
- incompleteTransportIds.add(transportId);
593
- }
594
926
  for (const entry of transportsByKey.values())
595
927
  if (entry.outstandingByOwnerId.size === 0 &&
596
- !incompleteTransportIds.has(entry.id))
928
+ !unsettledTransportIds.has(entry.id))
597
929
  entry.timeout = syncRequestTimeout;
598
930
  syncStateBroadcastChannel.postMessage({ transports, tenants });
599
931
  });
@@ -642,10 +974,10 @@ export const initSharedWorker = (self) => async (run) => {
642
974
  entry.timeout = PositiveMillis.orThrow(Math.min(entry.timeout * 2, maxSyncRequestTimeout));
643
975
  // Reconnecting abandons the connection and starts a fresh retry
644
976
  // schedule. The socket reports no close for it, so the transport
645
- // records the moment here.
977
+ // disconnects here.
646
978
  entry.socket?.reconnect();
647
- // `now` is monotonic; the reported close time is wall clock.
648
- entry.closedAt = deps.time.now();
979
+ // `now` is monotonic; the reported time is wall clock.
980
+ entry.connection = disconnectSyncConnection(entry.connection, deps.time.now(), null);
649
981
  refreshAllSyncRoutes();
650
982
  publishSyncState();
651
983
  }, delay);
@@ -684,9 +1016,7 @@ export const initSharedWorker = (self) => async (run) => {
684
1016
  outstandingByOwnerId: new Map(),
685
1017
  timeout: syncRequestTimeout,
686
1018
  socket: null,
687
- openedAt: null,
688
- closedAt: null,
689
- error: null,
1019
+ connection: connectingSyncConnection,
690
1020
  timeoutId: null,
691
1021
  };
692
1022
  const disposer = __addDisposableResource(env_2, new AsyncDisposableStack(), true);
@@ -706,7 +1036,15 @@ export const initSharedWorker = (self) => async (run) => {
706
1036
  // answered.
707
1037
  entry.outstandingByOwnerId.clear();
708
1038
  clearSyncRequestTimeout(entry);
709
- entry.openedAt = run.deps.time.now();
1039
+ // A connection that opens keeps the last error of an earlier
1040
+ // one.
1041
+ entry.connection = {
1042
+ type: "Open",
1043
+ openedAt: run.deps.time.now(),
1044
+ error: entry.connection.type === "Connecting"
1045
+ ? null
1046
+ : entry.connection.error,
1047
+ };
710
1048
  publishSyncState();
711
1049
  const ownerIds = transports.getClaimsForResource(transport);
712
1050
  console.debug("transportOpen", {
@@ -728,7 +1066,7 @@ export const initSharedWorker = (self) => async (run) => {
728
1066
  code: event.code,
729
1067
  wasClean: event.wasClean,
730
1068
  });
731
- entry.closedAt = run.deps.time.now();
1069
+ entry.connection = disconnectSyncConnection(entry.connection, run.deps.time.now(), null);
732
1070
  clearSyncRequestTimeout(entry);
733
1071
  refreshAllSyncRoutes();
734
1072
  publishSyncState();
@@ -738,10 +1076,13 @@ export const initSharedWorker = (self) => async (run) => {
738
1076
  url: transport.url,
739
1077
  type: error.type,
740
1078
  });
741
- entry.error = {
742
- type: error.type,
743
- at: run.deps.time.now(),
744
- };
1079
+ // Every error disconnects. A browser reports a failed
1080
+ // attempt only as an error, because the socket stops
1081
+ // listening before its close arrives. A connection error
1082
+ // arrives once the socket is closed, before its close, and
1083
+ // exhausted retries end reconnecting.
1084
+ const now = run.deps.time.now();
1085
+ entry.connection = disconnectSyncConnection(entry.connection, now, { type: error.type, at: now });
745
1086
  refreshAllSyncRoutes();
746
1087
  publishSyncState();
747
1088
  },
@@ -794,9 +1135,8 @@ export const initSharedWorker = (self) => async (run) => {
794
1135
  // LIFO: the transport drops its timer and its socket reference
795
1136
  // before the socket is disposed. `disposable` guards every method
796
1137
  // of the socket the claims lease, and disposing the socket awaits
797
- // its retry, so a `publishSyncState` microtask can run while this
798
- // entry is still registered. It must find no socket rather than
799
- // read a disposed one.
1138
+ // its retry, so this entry stays registered meanwhile. It keeps its
1139
+ // last connection and holds no socket to reconnect.
800
1140
  disposer.defer(() => {
801
1141
  clearSyncRequestTimeout(entry);
802
1142
  entry.socket = null;
@@ -855,7 +1195,7 @@ export const initSharedWorker = (self) => async (run) => {
855
1195
  }));
856
1196
  const tenantsByName = disposer.use(await sharedWorkerRun.ok(createSharedResourceByKey((message) => createEvoluTenant(isPersistentStorageUnavailable
857
1197
  ? { ...message, memoryOnly: true }
858
- : message, currentTenantsByName), {
1198
+ : message, currentTenantsByName, workerId), {
859
1199
  idleDisposeAfter: "3s",
860
1200
  lookup: (message) => message.name,
861
1201
  })));
@@ -876,7 +1216,7 @@ export const initSharedWorker = (self) => async (run) => {
876
1216
  await result_1;
877
1217
  }
878
1218
  };
879
- const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, memoryOnly, }, currentTenantsByName) => async (run) => {
1219
+ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, memoryOnly, }, currentTenantsByName, workerId) => async (run) => {
880
1220
  const env_3 = { stack: [], error: void 0, hasError: false };
881
1221
  try {
882
1222
  const disposer = __addDisposableResource(env_3, new AsyncDisposableStack(), true);
@@ -1026,6 +1366,7 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1026
1366
  sqliteSchema,
1027
1367
  encryptionKey,
1028
1368
  memoryOnly,
1369
+ sharedWorkerId: workerId,
1029
1370
  port: dbWorkerChannel.port1.native,
1030
1371
  }, [dbWorkerChannel.port1.native]);
1031
1372
  };
@@ -1126,31 +1467,16 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1126
1467
  });
1127
1468
  }
1128
1469
  };
1129
- disposer.defer(async () => {
1130
- const env_5 = { stack: [], error: void 0, hasError: false };
1131
- try {
1132
- isDisposing = true;
1133
- dbWorkerPort?.postMessage({ type: "Dispose" });
1134
- dbWorkerPort = null;
1135
- activeDispatch = null;
1136
- // The DbWorker holds this tenant leader lock while it is alive. Tenant
1137
- // disposal sends Dispose, then acquires the same lock to wait until the
1138
- // DbWorker releases it: either because Dispose was delivered or because
1139
- // the hosting tab closed. A worker requested from a later tab leader may
1140
- // be queued for the lock first; it is told to stop when it reports in.
1141
- // The wait is unabortable because tenant disposal must finish even after
1142
- // tenantRun receives an abort request.
1143
- const _ = __addDisposableResource(env_5, await tenantRun.ok(acquireLeaderLock(name)), true);
1144
- }
1145
- catch (e_5) {
1146
- env_5.error = e_5;
1147
- env_5.hasError = true;
1148
- }
1149
- finally {
1150
- const result_5 = __disposeResources(env_5);
1151
- if (result_5)
1152
- await result_5;
1153
- }
1470
+ // Disposal does not wait for the DbWorker. It holds the database lock
1471
+ // until Dispose arrives, its tab closes, or this worker ends, and the next
1472
+ // DbWorker for this database, of this worker or another build, waits for
1473
+ // that lock before it reads the clock. A requested DbWorker that reports
1474
+ // in later gets Dispose from the isDisposing check.
1475
+ disposer.defer(() => {
1476
+ isDisposing = true;
1477
+ dbWorkerPort?.postMessage({ type: "Dispose" });
1478
+ dbWorkerPort = null;
1479
+ activeDispatch = null;
1154
1480
  });
1155
1481
  const handleResponseForEvolu = (response, first) => {
1156
1482
  // A disposed instance gets no patches, but its committed write still
@@ -1198,9 +1524,36 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1198
1524
  }
1199
1525
  break;
1200
1526
  }
1527
+ case "MutateFailed": {
1528
+ // The tab shows it, and the instance releases the mutation's
1529
+ // onComplete callbacks without running them. A write outlives its
1530
+ // instance, so without one, every tab is told.
1531
+ //
1532
+ // A replay after leader replacement can fail although the earlier
1533
+ // leader committed the write and only its answer was lost. The tab
1534
+ // then shows an error for a stored write, and its onComplete
1535
+ // callbacks never run. Nothing is lost or reused: the replacement
1536
+ // adopted the stored clock, refreshed every instance's queries, and
1537
+ // reconciles every used owner.
1538
+ const { error } = response.message;
1539
+ if (instance) {
1540
+ assertSame(first.message.type, "Mutate");
1541
+ instance.tabPort.postMessage({ type: "Error", error });
1542
+ instance.port.postMessage({
1543
+ type: "OnMutateFailed",
1544
+ onCompleteIds: first.message.onCompleteIds,
1545
+ });
1546
+ }
1547
+ else {
1548
+ deps.postConsoleEntryOrError({ type: "Error", error });
1549
+ }
1550
+ break;
1551
+ }
1201
1552
  case "Export":
1202
1553
  instance?.port.postMessage({ type: "OnExport", file: response.message.file }, [response.message.file.buffer]);
1203
1554
  break;
1555
+ default:
1556
+ exhaustiveCheck(response.message);
1204
1557
  }
1205
1558
  };
1206
1559
  const routesByOwnerIdByKey = new Map();
@@ -1213,12 +1566,15 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1213
1566
  let route = routesByOwnerId.get(ownerId);
1214
1567
  if (!route) {
1215
1568
  route = {
1216
- roundRequired: true,
1217
- complete: false,
1218
- completeAt: null,
1569
+ progress: {
1570
+ type: "Pending",
1571
+ roundRequired: true,
1572
+ failure: null,
1573
+ skip: null,
1574
+ completeAt: null,
1575
+ },
1219
1576
  lastSentAt: null,
1220
1577
  lastReceivedAt: null,
1221
- error: null,
1222
1578
  };
1223
1579
  routesByOwnerId.set(ownerId, route);
1224
1580
  }
@@ -1277,19 +1633,27 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1277
1633
  // worker answers it. Local-only changes create no synchronization
1278
1634
  // work.
1279
1635
  const hasQueuedWrite = pendingWriteCountByOwnerId.has(ownerId);
1280
- // A refused database synchronizes nothing, and refusal discards its
1281
- // queued writes without uploading them.
1282
- const complete = startupError === null &&
1283
- isOpen &&
1636
+ const isReconciled = isOpen &&
1284
1637
  deps.syncRequests.getOutstanding(ownerId, key) === 0 &&
1285
1638
  !hasQueuedApply &&
1286
- !hasQueuedWrite &&
1287
- !route.roundRequired;
1288
- if (complete && !route.complete) {
1289
- route.completeAt = deps.time.now();
1290
- route.error = null;
1291
- }
1292
- route.complete = complete;
1639
+ !hasQueuedWrite;
1640
+ // Settling drops the failure, so the next one requests a round
1641
+ // again. A route with a skip it is not rechecking settles without
1642
+ // completing, because its relay may offer the skipped message or
1643
+ // lack messages stored elsewhere.
1644
+ const pending = routeToPending(route.progress);
1645
+ route.progress =
1646
+ !isReconciled || pending.roundRequired
1647
+ ? pending
1648
+ : pending.skip && !pending.skip.isRechecking
1649
+ ? {
1650
+ type: "Settled",
1651
+ error: pending.skip.error,
1652
+ completeAt: pending.completeAt,
1653
+ }
1654
+ : route.progress.type === "Complete"
1655
+ ? route.progress
1656
+ : { type: "Complete", completeAt: deps.time.now() };
1293
1657
  }
1294
1658
  }
1295
1659
  };
@@ -1313,8 +1677,10 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1313
1677
  ?.get(ownerId);
1314
1678
  if (!route)
1315
1679
  return;
1316
- route.roundRequired = true;
1317
- route.error = { type: "SyncFailed", at: now };
1680
+ route.progress = failRoute(route.progress, {
1681
+ type: "SyncFailed",
1682
+ at: now,
1683
+ });
1318
1684
  });
1319
1685
  }
1320
1686
  deps.publishSyncState();
@@ -1327,6 +1693,8 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1327
1693
  const { ownerId, result } = response.message;
1328
1694
  const error = result.ok ? null : result.error;
1329
1695
  const isAborted = error?.type === "AbortError";
1696
+ // An aborted apply reports no skipped message.
1697
+ const skippedError = isAborted ? null : response.message.skippedError;
1330
1698
  let failure = null;
1331
1699
  if (isAborted) {
1332
1700
  // An abort proves no convergence. A local sibling copy affects
@@ -1338,19 +1706,22 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1338
1706
  return;
1339
1707
  const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
1340
1708
  if (route)
1341
- route.roundRequired = true;
1709
+ route.progress = {
1710
+ ...routeToPending(route.progress),
1711
+ roundRequired: true,
1712
+ };
1342
1713
  });
1343
1714
  }
1344
1715
  else if (error !== null) {
1345
- failure = error.type;
1346
- deps.postConsoleEntryOrError({
1347
- type: "Error",
1348
- error,
1349
- });
1716
+ // A relay's error shows on its route, not as an EvoluError,
1717
+ // because it belongs to one relay and often repeats in every
1718
+ // round. A sibling copy's error is reported below.
1719
+ failure = error;
1350
1720
  }
1351
1721
  else if (result.ok && result.value.type === "Failed") {
1352
- failure =
1353
- result.value.cause === "Write" ? "WriteFailed" : "SyncFailed";
1722
+ failure = {
1723
+ type: result.value.cause === "Write" ? "WriteFailed" : "SyncFailed",
1724
+ };
1354
1725
  }
1355
1726
  if (source.type === "Transport") {
1356
1727
  // A registration or claim may have been removed while this apply
@@ -1361,36 +1732,63 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1361
1732
  // An aborted apply applied nothing.
1362
1733
  if (!isAborted)
1363
1734
  route.lastReceivedAt = now;
1735
+ // A skipped message is recorded as the route's skip, not a
1736
+ // failure, and does not end the round, whose response below is
1737
+ // still sent, so it requests no round. The relay offers the
1738
+ // message again in every later round, so the route stays
1739
+ // incomplete until a round requested through it settles without
1740
+ // skipping a message and without messages stored elsewhere since
1741
+ // that request.
1742
+ if (skippedError !== null)
1743
+ route.progress = {
1744
+ ...routeToPending(route.progress),
1745
+ skip: {
1746
+ error: errorToSyncRouteError(skippedError, now),
1747
+ isRechecking: false,
1748
+ },
1749
+ };
1364
1750
  if (failure !== null) {
1365
- // The first failure since the route completed requests one
1751
+ // The first failure since the route settled requests one
1366
1752
  // round. Further failures wait for an explicit request or a
1367
1753
  // reopen, even after a converged reply, which may answer
1368
1754
  // another request.
1369
- if (route.error === null)
1755
+ const requestsRetry = route.progress.type !== "Pending" ||
1756
+ route.progress.failure === null;
1757
+ route.progress = failRoute(route.progress, errorToSyncRouteError(failure, now));
1758
+ if (requestsRetry)
1370
1759
  requestCreateSyncMessages(new Set([ownerId]), source);
1371
- route.roundRequired = true;
1372
- route.error = { type: failure, at: now };
1373
1760
  }
1374
1761
  }
1375
1762
  }
1376
- else if (failure !== null) {
1377
- // A sibling's messages were not stored; rounds fetch them from
1378
- // the relays.
1379
- requestCreateSyncMessages(new Set([ownerId]), allTransports);
1763
+ else if (failure !== null || skippedError !== null) {
1764
+ // A sibling's copy comes from this worker, not from a relay, so no
1765
+ // route shows its failure. It is a Broadcast, which carries no
1766
+ // relay error, so an error or a skip means a bug, such as
1767
+ // databases holding different keys for the owner, and is reported
1768
+ // as an unexpected failure. A Failed result was logged, which
1769
+ // reports it already. An error is the failure, which is never an
1770
+ // abort, and like a route, the report leaves out its frame.
1771
+ const unexpected = error !== null ? failure : skippedError;
1772
+ if (unexpected !== null)
1773
+ deps.postConsoleEntryOrError({
1774
+ type: "Error",
1775
+ error: createUnknownError(errorToSyncRouteError(unexpected, deps.time.now())),
1776
+ });
1777
+ // Some of a sibling's messages were not stored; rounds fetch them
1778
+ // from the relays, whose routes then record a skip for any message
1779
+ // this database skips.
1780
+ requestRoundsForReceivedMessages(ownerId, {
1781
+ except: null,
1782
+ afterQueuedWrites: true,
1783
+ });
1380
1784
  }
1381
1785
  if (response.message.didWriteMessages) {
1382
1786
  refreshQueries();
1383
1787
  // Reconcile newly stored messages through each other transport.
1384
- // Rounds toward the same transport coalesce whatever their source.
1385
- const keys = [];
1386
- deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1387
- if (isTargetTransport(target, transport))
1388
- return;
1389
- keys.push(structuralLookup(transport));
1788
+ requestRoundsForReceivedMessages(ownerId, {
1789
+ except: target,
1790
+ afterQueuedWrites: false,
1390
1791
  });
1391
- for (const key of keys) {
1392
- requestCreateSyncMessages(new Set([ownerId]), { type: "Transport", key }, { afterQueuedWrites: false });
1393
- }
1394
1792
  }
1395
1793
  if (result.ok) {
1396
1794
  switch (result.value.type) {
@@ -1443,13 +1841,49 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1443
1841
  const route = getRoute(ownerId, key);
1444
1842
  route.lastSentAt = deps.time.now();
1445
1843
  if (isRound)
1446
- route.roundRequired = false;
1844
+ route.progress = {
1845
+ ...routeToPending(route.progress),
1846
+ roundRequired: false,
1847
+ };
1447
1848
  });
1448
1849
  }
1449
1850
  // Sends raise counters shared by every tenant; one refresh covers them.
1450
1851
  if (protocolMessagesByOwnerId.size > 0)
1451
1852
  deps.refreshAllSyncRoutes();
1452
1853
  };
1854
+ /**
1855
+ * Requests a round through each transport claimed for the owner outside
1856
+ * `except`, after messages from elsewhere were stored or a sibling copy was
1857
+ * not fully stored. Rounds toward the same transport coalesce whatever
1858
+ * their source. A route that skipped a message gets none, because its relay
1859
+ * would offer that message again, and a route rechecking one stops
1860
+ * rechecking, because its round may have read the database before this
1861
+ * event. A later requested round through such a route reconciles it.
1862
+ */
1863
+ const requestRoundsForReceivedMessages = (ownerId, { except, afterQueuedWrites, }) => {
1864
+ const keys = [];
1865
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1866
+ if (except && isTargetTransport(except, transport))
1867
+ return;
1868
+ const key = structuralLookup(transport);
1869
+ const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
1870
+ if (route?.progress.type === "Settled")
1871
+ return;
1872
+ if (route?.progress.type === "Pending" && route.progress.skip) {
1873
+ // A round already requested may have read the database before these
1874
+ // messages were stored, so it can no longer complete the route.
1875
+ route.progress = {
1876
+ ...route.progress,
1877
+ skip: { ...route.progress.skip, isRechecking: false },
1878
+ };
1879
+ return;
1880
+ }
1881
+ keys.push(key);
1882
+ });
1883
+ for (const key of keys) {
1884
+ requestCreateSyncMessages(new Set([ownerId]), { type: "Transport", key }, { afterQueuedWrites });
1885
+ }
1886
+ };
1453
1887
  /** The keys of every transport claimed for the owner, by any database. */
1454
1888
  const getClaimedKeys = (ownerId) => [...deps.transports.getResourceKeysForClaim(ownerId)].map(structuralLookup);
1455
1889
  const getUsedOwnersById = (ownerIds) => {
@@ -1468,12 +1902,19 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1468
1902
  return;
1469
1903
  const usedOwnersById = getUsedOwnersById(ownerIds);
1470
1904
  // Opening, storing messages from another transport, a failure, and an
1471
- // explicit request each require a new round before completion.
1905
+ // explicit request each require a new round before completion. The
1906
+ // round also checks again whether the relay holds a skipped message.
1472
1907
  for (const ownerId of usedOwnersById.keys()) {
1473
1908
  deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1474
1909
  if (!isTargetTransport(target, transport))
1475
1910
  return;
1476
- getRoute(ownerId, structuralLookup(transport)).roundRequired = true;
1911
+ const route = getRoute(ownerId, structuralLookup(transport));
1912
+ const pending = routeToPending(route.progress);
1913
+ route.progress = {
1914
+ ...pending,
1915
+ roundRequired: true,
1916
+ skip: pending.skip && { ...pending.skip, isRechecking: true },
1917
+ };
1477
1918
  });
1478
1919
  }
1479
1920
  refreshSyncRoutes();
@@ -1599,31 +2040,28 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1599
2040
  deps.publishSyncState();
1600
2041
  });
1601
2042
  const tenant = disposable({
1602
- getSyncTenant: () => ({
1603
- name,
1604
- refused: startupError !== null,
1605
- owners: getSyncOwners().map(({ ownerId, writable, transportKeys }) => ({
1606
- ownerId,
1607
- writable,
1608
- transportKeys,
1609
- routes: writable
1610
- ? transportKeys.map((key) => {
1611
- const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
1612
- // Registration and claim changes refresh routes before yielding.
1613
- // Snapshot reads must not create missing routes.
1614
- assertNotUndefined(route);
1615
- return {
1616
- transportKey: key,
1617
- complete: route.complete,
1618
- completeAt: route.completeAt,
1619
- lastSentAt: route.lastSentAt,
1620
- lastReceivedAt: route.lastReceivedAt,
1621
- error: route.error,
1622
- };
1623
- })
1624
- : [],
1625
- })),
1626
- }),
2043
+ getSyncTenant: () => startupError
2044
+ ? { type: "Refused", name, error: startupError }
2045
+ : {
2046
+ type: "Active",
2047
+ name,
2048
+ owners: getSyncOwners().map(({ ownerId, writable, transportKeys }) => writable
2049
+ ? {
2050
+ type: "Writable",
2051
+ ownerId,
2052
+ routes: transportKeys.map((key) => {
2053
+ const route = routesByOwnerIdByKey
2054
+ .get(key)
2055
+ ?.get(ownerId);
2056
+ // Registration and claim changes refresh routes
2057
+ // before yielding. Snapshot reads must not create
2058
+ // missing routes.
2059
+ assertNotUndefined(route);
2060
+ return { transportKey: key, ...route };
2061
+ }),
2062
+ }
2063
+ : { type: "Readonly", ownerId, transportKeys }),
2064
+ },
1627
2065
  refreshSyncRoutes,
1628
2066
  addInstance: (message, tabPort, onDisposed) => {
1629
2067
  const disposer = new AsyncDisposableStack();
@@ -1646,18 +2084,20 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1646
2084
  };
1647
2085
  instancesById.set(instance.id, instance);
1648
2086
  disposer.defer(instance.onDisposed);
1649
- disposer.defer(async () => {
1650
- await tenantRun(instance.useOwnerMutex.withLock(() => {
1651
- for (const leases of instance.ownerRegistrations.values()) {
1652
- for (const lease of leases)
1653
- lease?.release();
1654
- }
1655
- instance.ownerRegistrations.clear();
1656
- deps.refreshAllSyncRoutes();
1657
- deps.publishSyncState();
1658
- return ok();
1659
- }));
2087
+ disposer.defer(() => {
2088
+ for (const leases of instance.ownerRegistrations.values()) {
2089
+ for (const lease of leases)
2090
+ lease?.release();
2091
+ }
2092
+ instance.ownerRegistrations.clear();
2093
+ deps.refreshAllSyncRoutes();
2094
+ deps.publishSyncState();
1660
2095
  });
2096
+ // Cleanup starts no Task, because a root abort may dispose every Run
2097
+ // first. Disposed before the claims above are released, this Run
2098
+ // aborts queued UseOwner batches and waits for the running one, whose
2099
+ // transport claim cannot be aborted, so the release sees every lease.
2100
+ const instanceRun = disposer.use(tenantRun.create());
1661
2101
  disposer.defer(() => {
1662
2102
  instancesById.delete(instance.id);
1663
2103
  refreshSyncRoutes();
@@ -1671,7 +2111,7 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1671
2111
  // while it is alive. Acquiring the same lock here means the main
1672
2112
  // thread instance was disposed or its tab closed, so the tenant-side
1673
2113
  // instance must dispose itself.
1674
- void tenantRun
2114
+ void instanceRun
1675
2115
  .abortable(acquireLeaderLock(message.id))
1676
2116
  .then((lock) => {
1677
2117
  if (!lock.ok)
@@ -1712,7 +2152,7 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1712
2152
  break;
1713
2153
  }
1714
2154
  case "UseOwner": {
1715
- void tenantRun(instance.useOwnerMutex.withLock(async (run) => {
2155
+ void instanceRun(instance.useOwnerMutex.withLock(async (run) => {
1716
2156
  for (const action of message.actions) {
1717
2157
  switch (action.action) {
1718
2158
  case "sync":
@@ -1808,14 +2248,14 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
1808
2248
  // readonly reload: boolean;
1809
2249
  // })
1810
2250
  // TODO: SharedWorker follow-ups.
1811
- // - Complete the queue head when a DbWorker mutation returns an error.
1812
2251
  // - Rotate the node ID when a copied database is detected; see the Duplicate
1813
2252
  // node IDs section in the Timestamp module.
1814
2253
  // - Detect DbWorker and port liveness so a worker-only crash resumes the queue.
1815
2254
  // Defer panicked-worker restart until failure detection and recovery are
1816
- // defined, accounting for SQLite WASM's detection limits. Normal SQLite
1817
- // operations are expected not to throw; user-defined UNIQUE indexes, which
1818
- // can make replicated writes fail, are planned to be forbidden.
2255
+ // defined, accounting for SQLite WASM's detection limits. A mutation that
2256
+ // throws is answered, but other SQLite operations are expected not to throw;
2257
+ // user-defined UNIQUE indexes, which can make replicated writes fail, are
2258
+ // planned to be forbidden.
1819
2259
  // - Split worker protocol types and the EvoluTenant implementation into focused
1820
2260
  // modules.
1821
2261
  // - Remove the obsolete commented protocol block above.