@evolu/common 8.11.0 → 8.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/src/Bytes.d.ts +39 -2
  2. package/dist/src/Bytes.d.ts.map +1 -1
  3. package/dist/src/Bytes.js +50 -2
  4. package/dist/src/Callbacks.d.ts +12 -1
  5. package/dist/src/Callbacks.d.ts.map +1 -1
  6. package/dist/src/Callbacks.js +3 -0
  7. package/dist/src/Error.d.ts +10 -4
  8. package/dist/src/Error.d.ts.map +1 -1
  9. package/dist/src/Error.js +10 -4
  10. package/dist/src/Object.d.ts +65 -0
  11. package/dist/src/Object.d.ts.map +1 -1
  12. package/dist/src/Object.js +142 -0
  13. package/dist/src/Resource.d.ts +0 -5
  14. package/dist/src/Resource.d.ts.map +1 -1
  15. package/dist/src/Resource.js +6 -13
  16. package/dist/src/Sqlite.d.ts.map +1 -1
  17. package/dist/src/Sqlite.js +7 -0
  18. package/dist/src/Task.d.ts +10 -8
  19. package/dist/src/Task.d.ts.map +1 -1
  20. package/dist/src/Task.js +41 -5
  21. package/dist/src/Worker.d.ts +3 -3
  22. package/dist/src/index.d.ts +1 -1
  23. package/dist/src/index.d.ts.map +1 -1
  24. package/dist/src/index.js +1 -1
  25. package/dist/src/local-first/Db.d.ts +8 -3
  26. package/dist/src/local-first/Db.d.ts.map +1 -1
  27. package/dist/src/local-first/Db.js +56 -19
  28. package/dist/src/local-first/Evolu.d.ts +142 -24
  29. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  30. package/dist/src/local-first/Evolu.js +109 -8
  31. package/dist/src/local-first/Owner.d.ts +9 -0
  32. package/dist/src/local-first/Owner.d.ts.map +1 -1
  33. package/dist/src/local-first/Owner.js +9 -0
  34. package/dist/src/local-first/Protocol.d.ts +18 -7
  35. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  36. package/dist/src/local-first/Protocol.js +45 -28
  37. package/dist/src/local-first/Relay.d.ts.map +1 -1
  38. package/dist/src/local-first/Relay.js +4 -2
  39. package/dist/src/local-first/Schema.d.ts +18 -3
  40. package/dist/src/local-first/Schema.d.ts.map +1 -1
  41. package/dist/src/local-first/Shared.d.ts +523 -133
  42. package/dist/src/local-first/Shared.d.ts.map +1 -1
  43. package/dist/src/local-first/Shared.js +696 -256
  44. package/dist/src/local-first/Storage.d.ts +19 -15
  45. package/dist/src/local-first/Storage.d.ts.map +1 -1
  46. package/dist/src/local-first/Storage.js +4 -2
  47. package/package.json +1 -1
  48. package/src/Bytes.test.ts +27 -0
  49. package/src/Bytes.ts +58 -2
  50. package/src/Callbacks.test.ts +20 -0
  51. package/src/Callbacks.ts +17 -1
  52. package/src/Error.ts +10 -4
  53. package/src/Object.test.ts +296 -0
  54. package/src/Object.ts +163 -0
  55. package/src/Resource.test.ts +20 -16
  56. package/src/Resource.ts +6 -20
  57. package/src/Sqlite.ts +7 -0
  58. package/src/Task.test.ts +233 -62
  59. package/src/Task.ts +47 -13
  60. package/src/Worker.ts +3 -3
  61. package/src/index.ts +6 -1
  62. package/src/local-first/Db.ts +88 -18
  63. package/src/local-first/Evolu.test.ts +589 -2
  64. package/src/local-first/Evolu.ts +285 -36
  65. package/src/local-first/Owner.ts +9 -0
  66. package/src/local-first/Protocol.test.ts +59 -60
  67. package/src/local-first/Protocol.ts +67 -47
  68. package/src/local-first/Relay.ts +4 -2
  69. package/src/local-first/Schema.ts +20 -3
  70. package/src/local-first/Shared.test.ts +2633 -646
  71. package/src/local-first/Shared.ts +1192 -364
  72. package/src/local-first/Storage.ts +24 -23
@@ -53,7 +53,9 @@
53
53
  * still waiting after three seconds reports {@link OtherBuildRunningError} to
54
54
  * its tabs. Every build broadcasts console entries and errors on
55
55
  * {@link consoleEntryOrErrorBroadcastChannelName}, so a waiting tab also prints
56
- * the running build's output and reports its errors as its own `evoluError`.
56
+ * the running build's output, reports each {@link UnknownError} it sends as its
57
+ * own `evoluError`, and only logs any other error, such as an older build's
58
+ * sync error.
57
59
  *
58
60
  * The tabs of one worker elect the host of its DbWorkers among themselves, with
59
61
  * a lock scoped to the worker, so a tab of another worker never hosts them.
@@ -100,7 +102,8 @@
100
102
  * writable registrations for its owner, whichever instance made it, even one
101
103
  * disposed before the database worker answered. When a relay frame stores new
102
104
  * messages, the tenant requests a round through each other transport claimed
103
- * for the owner, so data learned from one relay reaches the others. A closed
105
+ * for the owner, so data learned from one relay reaches the others, except
106
+ * through a route that skipped a message, as described below. A closed
104
107
  * transport reconciles when it opens, and a replacement leader reconciles every
105
108
  * transport again, because a response reporting stored messages may have been
106
109
  * lost.
@@ -109,26 +112,34 @@
109
112
  * also delivers it as local Broadcast frames to every other tenant with
110
113
  * writable access to the owner, even while sockets are closed. A copy keeps the
111
114
  * uploader's target, and a recipient that stores new continuation messages
112
- * reconciles them through the transports outside that target. Local delivery
113
- * forwards uploads, including historical messages sent during reconciliation,
114
- * but does not reconcile local database histories with each other. That waits
115
- * until replication scopes and retention semantics are defined.
115
+ * reconciles them through the transports outside that target, except through a
116
+ * route that skipped a message. Local delivery forwards uploads, including
117
+ * historical messages sent during reconciliation, but does not reconcile local
118
+ * database histories with each other. That waits until replication scopes and
119
+ * retention semantics are defined.
116
120
  *
117
121
  * ## Sync state
118
122
  *
119
123
  * The shared worker publishes one plain snapshot, {@link SyncState}, of every
120
124
  * transport it manages and every database and owner registration it holds, with
121
- * one route per writable registration and transport, as specified below.
122
- * {@link syncStateToOwnerSyncStates} derives one state per database and owner.
125
+ * one route per writable registration and transport, as specified below. Each
126
+ * part is a union of the states the worker keeps for it: a transport's
127
+ * connection, a database that is active or refused startup, a writable or
128
+ * readonly owner, and a pending, settled, or complete route. A transport's
129
+ * events drive its connection, so publishing never reads a socket.
130
+ *
131
+ * Apps show users an owner's {@link OwnerSyncStatus}, which
132
+ * {@link syncStateToOwnerSyncStatus} derives. A view of each relay pairs each
133
+ * route with its transport by {@link syncStateToRelaySyncStates} and tells the
134
+ * relay's status by {@link relaySyncStateToStatus}. Snapshots store neither, so
135
+ * a status never disagrees with the routes it comes from.
123
136
  *
124
137
  * Each worker broadcasts snapshots on its own channel, whose name a connecting
125
138
  * tab receives through its port, so a tab never hears another worker, such as
126
139
  * one of a different app version, and
127
140
  * {@link SyncStateDep.syncState | deps.syncState} keeps the last snapshot. The
128
- * worker publishes after every change it observes; a transition without an
129
- * event, such as a closed socket starting to reconnect, appears with the next
130
- * snapshot. The snapshot lives in worker memory only, so a new worker starts
131
- * empty.
141
+ * worker publishes after every change it observes. The snapshot lives in worker
142
+ * memory only, so a new worker starts empty.
132
143
  *
133
144
  * ## Synchronization completion
134
145
  *
@@ -156,27 +167,51 @@
156
167
  * - The tenant has sent a round through the transport since the last event that
157
168
  * requires one: its first use of the transport for the owner, the socket
158
169
  * opening, an explicit request, a replacement leader, or storing messages
159
- * from another transport. No failed or aborted result has arrived on the
160
- * route since.
170
+ * from another transport, unless the route skipped a message. No failed or
171
+ * aborted result has arrived on the route since.
172
+ * - No result that skipped a received message has arrived on the route since a
173
+ * round was last requested through it, and since that request, no messages
174
+ * have been stored from another transport, directly or through a sibling copy
175
+ * uploaded outside this transport, and no sibling copy has failed to apply or
176
+ * skipped a message.
177
+ *
178
+ * A route is settled when every condition except the last holds, so a complete
179
+ * route is settled too. A settled route that is incomplete has ended its
180
+ * reconciliation, but its relay may offer a message the database skipped, or
181
+ * messages stored elsewhere may not have reached it; it is published as
182
+ * {@link SettledSyncRoute}.
161
183
  *
162
184
  * A reconciliation chain ends only with a converged result, a failure, an
163
185
  * abort, a dropped frame, or a continuation that finds the socket closed, and
164
186
  * relay errors reach every applying tenant, so these conditions mean every
165
187
  * chain, including this tenant's, converged. A local Broadcast from a sibling
166
188
  * tenant holds every route of its owner until it is applied, because it arrives
167
- * without a request of its own; one that fails to apply requires a round
168
- * through every transport.
189
+ * without a request of its own; one that fails to apply or skips a message
190
+ * requires a round through every transport whose route has not skipped one.
169
191
  *
170
192
  * A failed result on a route requests one round through it. Any further failure
171
- * before the route completes waits for an explicit request or a reopen, so a
193
+ * before the route settles waits for an explicit request or a reopen, so a
172
194
  * persistent failure cannot loop; a converged reply in between does not end the
173
195
  * wait, because it may answer another tenant's round on the shared socket. An
174
196
  * aborted apply leaves its routes incomplete without a retry. An exception
175
197
  * while the database worker creates a round is logged there and fails the
176
- * round's routes with `SyncFailed` without a retry; other unexpected SQLite
177
- * exceptions remain unsupported and can panic the database worker. A frame the
178
- * relay silently drops, such as invalid data, leaves the count above zero until
179
- * the liveness rule below replaces the socket.
198
+ * round's routes with `SyncFailed` without a retry. A mutation that throws is
199
+ * rolled back and reported as an {@link UnknownError} to the tab that made it,
200
+ * or to every tab when its Evolu instance was disposed first. Other unexpected
201
+ * SQLite exceptions remain unsupported and can panic the database worker. A
202
+ * frame the relay silently drops, such as invalid data, leaves the count above
203
+ * zero until the liveness rule below replaces the socket.
204
+ *
205
+ * A result that skipped a received message the database could not decrypt,
206
+ * verify, or decode records the error on the route as its `skippedError`,
207
+ * without ending its chain, so it requests no round, and the relay offers the
208
+ * message again in every round through the route until the database stores a
209
+ * message with that timestamp. The messages received elsewhere that the last
210
+ * condition names request no round through such a route, even while a requested
211
+ * round checks it again, because each round would download every skipped
212
+ * message again. The next round that a first use of the transport for the
213
+ * owner, an explicit request, a reopen, a replacement leader, or a failure
214
+ * sends through the route reconciles them.
180
215
  *
181
216
  * ### Liveness
182
217
  *
@@ -196,11 +231,13 @@
196
231
  * The timeout belongs to the transport, not to an owner, because a small reply
197
232
  * for one owner can wait behind another owner's large frame on the socket. A
198
233
  * grown timeout lasts while a request is outstanding on the socket or a
199
- * database that has not refused startup has an incomplete route through it,
200
- * including one waiting after a failure, and ends with the transport. A reply's
201
- * speed proves nothing, because a recovery on a slow link starts with small
202
- * replies that arrive quickly. The reopen resets the counts and starts the open
203
- * rounds, so a dropped frame delays a route instead of stranding it.
234
+ * database that has not refused startup has an unsettled route through it,
235
+ * including one waiting after a failure, and ends with the transport. A settled
236
+ * route that is incomplete does not count, because its reconciliation has
237
+ * ended. A reply's speed proves nothing, because a recovery on a slow link
238
+ * starts with small replies that arrive quickly. The reopen resets the counts
239
+ * and starts the open rounds, so a dropped frame delays a route instead of
240
+ * stranding it.
204
241
  *
205
242
  * The shared worker sends nothing while idle, so a path that dies then may go
206
243
  * unnoticed until its next request; until then, changes from other devices stop
@@ -226,7 +263,11 @@ import {
226
263
  } from "../Assert.ts";
227
264
  import type { Brand } from "../Brand.ts";
228
265
  import type { ConsoleEntry, ConsoleLevel } from "../Console.ts";
229
- import type { EncryptionKey } from "../Crypto.ts";
266
+ import type {
267
+ DecryptWithXChaCha20Poly1305Error,
268
+ EncryptionKey,
269
+ } from "../Crypto.ts";
270
+ import { createUnknownError, type UnknownError } from "../Error.ts";
230
271
  import { disposable, exhaustiveCheck } from "../Function.ts";
231
272
  import { acquireLeaderLock, type LockManagerDep } from "../LockManager.ts";
232
273
  import {
@@ -281,7 +322,6 @@ import type {
281
322
  CreateWebSocketDep,
282
323
  WebSocket,
283
324
  WebSocketError,
284
- WebSocketReadyState,
285
325
  } from "../WebSocket.ts";
286
326
  import type {
287
327
  SharedWorker as CommonSharedWorker,
@@ -293,7 +333,7 @@ import type {
293
333
  WorkerDeps,
294
334
  } from "../Worker.ts";
295
335
  import type { DbWorkerInit, UnsupportedDbVersionError } from "./Db.ts";
296
- import type { EvoluError, SyncStateDep } from "./Evolu.ts";
336
+ import type { Evolu, SyncStateDep } from "./Evolu.ts";
297
337
  import type { Owner, OwnerId, OwnerTransport, SyncOwner } from "./Owner.ts";
298
338
  import {
299
339
  createProtocolBroadcastMessagesFromCrdtMessages,
@@ -303,7 +343,10 @@ import {
303
343
  parseProtocolHeader,
304
344
  type ApplyProtocolMessageAsClientResult,
305
345
  type ProtocolError,
346
+ type ProtocolInvalidDataError,
306
347
  type ProtocolMessage,
348
+ type ProtocolQuotaError,
349
+ type ProtocolTimestampMismatchError,
307
350
  } from "./Protocol.ts";
308
351
  import {
309
352
  makePatches,
@@ -352,11 +395,12 @@ export type SharedWorkerOutput =
352
395
  | DbWorkerInit
353
396
  | {
354
397
  /**
355
- * Sent to one tab only: its database refused startup, or another build
356
- * keeps this worker waiting.
398
+ * Sent to one tab only: its database refused startup, a mutation it made
399
+ * could not be stored, or another build keeps this worker waiting.
357
400
  */
358
401
  readonly type: "Error";
359
- readonly error: UnsupportedDbVersionError | OtherBuildRunningError;
402
+ readonly error:
403
+ OtherBuildRunningError | UnknownError | UnsupportedDbVersionError;
360
404
  }
361
405
  | {
362
406
  /**
@@ -393,7 +437,7 @@ export type ConsoleEntryOrError =
393
437
  }
394
438
  | {
395
439
  readonly type: "Error";
396
- readonly error: EvoluError;
440
+ readonly error: UnknownError;
397
441
  };
398
442
 
399
443
  export const consoleEntryOrErrorBroadcastChannelName =
@@ -451,31 +495,31 @@ export interface BuildWaitingRequest extends InferType<
451
495
  export interface OtherBuildRunningError extends Typed<"OtherBuildRunningError"> {}
452
496
 
453
497
  /**
454
- * A snapshot of the transports and databases the shared worker manages.
455
- *
456
- * See the Sync state section of this module's documentation.
498
+ * A snapshot of the transports and databases the shared worker manages. Each
499
+ * part is a union of the states the worker keeps for it, so each part holds
500
+ * only the fields valid for its state. Apps derive what to show with
501
+ * {@link syncStateToOwnerSyncStatus}; see Sync state in this module's
502
+ * documentation.
457
503
  */
458
504
  export interface SyncState {
459
505
  readonly transports: ReadonlyArray<SyncTransport>;
460
506
  readonly tenants: ReadonlyArray<SyncTenant>;
461
507
  }
462
508
 
463
- /** One WebSocket, shared by every owner and database claiming it. */
464
- export interface SyncTransport {
465
- /** Opaque and stable for the transport's lifetime, across socket replacements. */
509
+ /**
510
+ * One transport, shared by every owner and database claiming it. Its `type` is
511
+ * the kind of transport, and its {@link SyncConnection} tells whether it is
512
+ * connected.
513
+ */
514
+ export type SyncTransport = WebSocketSyncTransport;
515
+
516
+ /** A WebSocket {@link SyncTransport}. */
517
+ export interface WebSocketSyncTransport extends Typed<"WebSocket"> {
518
+ /** Opaque and stable for the transport's lifetime, across reconnects. */
466
519
  readonly id: SyncTransportId;
467
- /** The URL without its query, which carries the owner ID. */
520
+ /** The relay URL without the owner-specific query. */
468
521
  readonly label: string;
469
- readonly readyState: WebSocketReadyState;
470
- /** When the connection last opened, or null before its first open. */
471
- readonly openedAt: Millis | null;
472
- /** When the connection last closed, or null before its first close. */
473
- readonly closedAt: Millis | null;
474
- /**
475
- * The last error, retained after a successful reconnect; null if none. Errors
476
- * while reconnecting are routine.
477
- */
478
- readonly error: SyncTransportError | null;
522
+ readonly connection: SyncConnection;
479
523
  }
480
524
 
481
525
  export type SyncTransportId = Id & Brand<"SyncTransport">;
@@ -485,109 +529,518 @@ export interface SyncTransportError {
485
529
  readonly at: Millis;
486
530
  }
487
531
 
488
- /** One named local database and the owners it registered. */
489
- export interface SyncTenant {
532
+ /**
533
+ * The connection of a {@link SyncTransport}: `Connecting` until its first
534
+ * connection opens or fails, `Open` while a connection is open, and
535
+ * `Disconnected` from a close, a failure, or an unanswered request that
536
+ * replaced the connection, until a connection opens again. It never returns to
537
+ * `Connecting`. The transport's events drive these states.
538
+ */
539
+ export type SyncConnection =
540
+ ConnectingSyncConnection | OpenSyncConnection | DisconnectedSyncConnection;
541
+
542
+ /**
543
+ * A first connection, which has neither opened nor failed. A connection to a
544
+ * host that drops packets stays here until the operating system or browser
545
+ * gives up on it, which can take a minute or more.
546
+ */
547
+ export interface ConnectingSyncConnection extends Typed<"Connecting"> {}
548
+
549
+ /** An open connection. */
550
+ export interface OpenSyncConnection extends Typed<"Open"> {
551
+ /** When this connection opened. */
552
+ readonly openedAt: Millis;
553
+ /** The last error of an earlier connection or attempt, or null. */
554
+ readonly error: SyncTransportError | null;
555
+ }
556
+
557
+ /**
558
+ * No connection: a connection closed or failed, or the relay did not answer a
559
+ * request in time. The transport reconnects by itself.
560
+ */
561
+ export interface DisconnectedSyncConnection extends Typed<"Disconnected"> {
562
+ /** When it disconnected. Failed reconnect attempts leave it unchanged. */
563
+ readonly disconnectedAt: Millis;
564
+ /** When the last connection opened, or null when none has. */
565
+ readonly openedAt: Millis | null;
566
+ /**
567
+ * The last error, or null. A close or an unanswered request records none, so
568
+ * it can come from an earlier connection. Errors while reconnecting are
569
+ * routine.
570
+ */
571
+ readonly error: SyncTransportError | null;
572
+ }
573
+
574
+ /** One named local database: `Active`, or `Refused` when it refused startup. */
575
+ export type SyncTenant = ActiveSyncTenant | RefusedSyncTenant;
576
+
577
+ /**
578
+ * A database that has not refused startup, including one still starting, with
579
+ * the owners its instances registered.
580
+ */
581
+ export interface ActiveSyncTenant extends Typed<"Active"> {
490
582
  readonly name: Name;
491
- /** The database refused startup, so nothing it holds synchronizes. */
492
- readonly refused: boolean;
493
583
  readonly owners: ReadonlyArray<SyncTenantOwner>;
494
584
  }
495
585
 
496
- export interface SyncTenantOwner {
586
+ /**
587
+ * A database that refused startup, so nothing it holds syncs. Its tabs also
588
+ * receive the error through `evoluError`.
589
+ */
590
+ export interface RefusedSyncTenant extends Typed<"Refused"> {
591
+ readonly name: Name;
592
+ readonly error: UnsupportedDbVersionError;
593
+ }
594
+
595
+ /**
596
+ * An owner registered by any instance of an {@link ActiveSyncTenant}: `Writable`
597
+ * when any registration holds its write key, otherwise `Readonly`.
598
+ */
599
+ export type SyncTenantOwner = WritableSyncTenantOwner | ReadonlySyncTenantOwner;
600
+
601
+ /**
602
+ * An owner the database syncs, with one route per transport claimed for it by
603
+ * this or any other database. Routes are briefly empty while the owner's
604
+ * transports are being claimed.
605
+ */
606
+ export interface WritableSyncTenantOwner extends Typed<"Writable"> {
607
+ readonly ownerId: OwnerId;
608
+ readonly routes: ReadonlyArray<SyncRoute>;
609
+ }
610
+
611
+ /**
612
+ * An owner registered only without its write key. Its registrations hold
613
+ * transports, which other databases' routes for the owner use, but this
614
+ * database does not sync it.
615
+ */
616
+ export interface ReadonlySyncTenantOwner extends Typed<"Readonly"> {
497
617
  readonly ownerId: OwnerId;
498
- /** A readonly registration holds transports but never synchronizes. */
499
- readonly writable: boolean;
500
618
  /** Every transport claimed for the owner, by any database. */
501
619
  readonly transportIds: ReadonlyArray<SyncTransportId>;
502
- /** One route per transport for a writable owner; none for a readonly one. */
503
- readonly routes: ReadonlyArray<SyncRoute>;
504
620
  }
505
621
 
506
622
  /**
507
- * One database's use of one owner through one transport. See the
508
- * Synchronization completion section of this module's documentation.
623
+ * One database's use of one owner through one transport: `Pending` until its
624
+ * reconciliation ends, then `Complete`, or `Settled` while its relay offers a
625
+ * change the database skipped. See Synchronization completion in this module's
626
+ * documentation.
509
627
  */
510
- export interface SyncRoute {
628
+ export type SyncRoute = PendingSyncRoute | SettledSyncRoute | CompleteSyncRoute;
629
+
630
+ /**
631
+ * A route whose reconciliation has not ended: its transport is not open, a
632
+ * request or a received frame is outstanding, a replicated write is queued, or
633
+ * a round must still be sent through it.
634
+ */
635
+ export interface PendingSyncRoute extends Typed<"Pending"> {
511
636
  readonly transportId: SyncTransportId;
512
- /** Whether the database is reconciled with the relay for the owner. */
513
- readonly complete: boolean;
637
+ /**
638
+ * The failure since the route last settled, or null. The first one requests a
639
+ * round; a further one waits for {@link Evolu.requestSync} or a reopen.
640
+ */
641
+ readonly failure: SyncRouteError | null;
642
+ /**
643
+ * The first change skipped in the latest reply that skipped one, or null. The
644
+ * relay offers it again in every round through the route until this database
645
+ * stores a change with that timestamp.
646
+ */
647
+ readonly skippedError: SyncRouteError | null;
514
648
  /** When the route last became complete, or null. */
515
649
  readonly completeAt: Millis | null;
516
650
  /** When this database last sent a request through the route, or null. */
517
651
  readonly lastSentAt: Millis | null;
518
652
  /**
519
653
  * When processing a frame from the route last finished, successfully or with
520
- * a failure, or null. Aborted processing does not update this timestamp.
654
+ * a failure, or null. Aborted processing does not update it.
521
655
  */
522
656
  readonly lastReceivedAt: Millis | null;
523
- /** The last failed result on the route; cleared when the route completes. */
524
- readonly error: SyncRouteError | null;
525
657
  }
526
658
 
527
- export interface SyncRouteError {
528
- readonly type: SyncRouteErrorType;
529
- readonly at: Millis;
659
+ /**
660
+ * A route whose reconciliation ended while its relay offers a change the
661
+ * database skipped, so it is incomplete. Changes stored from other relays
662
+ * request no round through it; the next round requested through it, such as by
663
+ * {@link Evolu.requestSync} or a reopen, checks it again.
664
+ */
665
+ export interface SettledSyncRoute extends Typed<"Settled"> {
666
+ readonly transportId: SyncTransportId;
667
+ /** The first change skipped in the latest reply that skipped one. */
668
+ readonly skippedError: SyncRouteError;
669
+ /** When the route last became complete, or null. */
670
+ readonly completeAt: Millis | null;
671
+ /** When this database last sent a request through the route. */
672
+ readonly lastSentAt: Millis;
673
+ /** When processing a frame from the route last finished, or null. */
674
+ readonly lastReceivedAt: Millis | null;
675
+ }
676
+
677
+ /** A route on which the database is reconciled with the relay for the owner. */
678
+ export interface CompleteSyncRoute extends Typed<"Complete"> {
679
+ readonly transportId: SyncTransportId;
680
+ /** When the route became complete. */
681
+ readonly completeAt: Millis;
682
+ /** When this database last sent a request through the route. */
683
+ readonly lastSentAt: Millis;
684
+ /** When processing a frame from the route last finished, or null. */
685
+ readonly lastReceivedAt: Millis | null;
530
686
  }
531
687
 
532
688
  /**
533
- * A {@link ProtocolError} or the original {@link StorageWriteMessagesError} type
534
- * for a rejected write. `WriteFailed` means a `writeMessages` call that threw,
535
- * logged by the protocol, and `SyncFailed` means a logged failure while
536
- * creating a round or reconciling ranges.
689
+ * A failure or a skipped change of a {@link SyncRoute}, with the time it
690
+ * arrived.
691
+ *
692
+ * It is the error itself, so it carries its details, such as the expected and
693
+ * actual timestamps of a {@link ProtocolTimestampMismatchError}. A skipped
694
+ * message adds {@link DecryptWithXChaCha20Poly1305Error}. A
695
+ * {@link ProtocolInvalidDataError} leaves out its data, which can be a whole
696
+ * frame. The caught value in the `error` of either is an {@link UnknownError}.
697
+ * `WriteFailed` means a `writeMessages` call that threw, logged by the
698
+ * protocol, and `SyncFailed` means a logged failure while creating a round or
699
+ * reconciling ranges.
537
700
  */
538
- export type SyncRouteErrorType =
539
- | ProtocolError["type"]
540
- | StorageWriteMessagesError["type"]
541
- | "WriteFailed"
542
- | "SyncFailed";
701
+ export type SyncRouteError = (
702
+ | Exclude<ProtocolError, ProtocolInvalidDataError>
703
+ | Omit<ProtocolInvalidDataError, "data">
704
+ | StorageWriteMessagesError
705
+ | DecryptWithXChaCha20Poly1305Error
706
+ | Typed<"WriteFailed">
707
+ | Typed<"SyncFailed">
708
+ ) & { readonly at: Millis };
709
+
710
+ /** The type of a {@link SyncRouteError}. */
711
+ export type SyncRouteErrorType = SyncRouteError["type"];
543
712
 
544
713
  /**
545
- * One owner's standing with its relays in one database, derived from
546
- * {@link SyncState} by {@link syncStateToOwnerSyncStates}.
714
+ * One relay of an owner in one database: a transport and the database's route
715
+ * through it, from {@link syncStateToRelaySyncStates}. Its status comes from
716
+ * {@link relaySyncStateToStatus} and is not stored beside them.
547
717
  */
548
- export interface OwnerSyncState {
549
- readonly name: Name;
550
- readonly ownerId: OwnerId;
551
- readonly status: OwnerSyncStatus;
552
- /** When a route of the owner last became complete, or null. */
553
- readonly syncedAt: Millis | null;
554
- /** The newest route error of the owner, or null. */
555
- readonly error: SyncRouteError | null;
556
- /** Each relay of the owner, in the order of its routes. */
557
- readonly relays: ReadonlyArray<RelaySyncState>;
718
+ export interface RelaySyncState {
719
+ readonly transport: SyncTransport;
720
+ readonly route: SyncRoute;
558
721
  }
559
722
 
560
723
  /**
561
- * The status of an {@link OwnerSyncState}: the first of `error`, `syncing`,
562
- * `synced`, and `offline` that one of its relays has, or `initial` before the
563
- * owner has a relay. Work in progress on an open transport shows before a relay
564
- * that is already up to date.
724
+ * What an app shows users about syncing one owner of one database, from
725
+ * {@link syncStateToOwnerSyncStatus} or the React and Vue `useOwnerSyncStatus`.
726
+ *
727
+ * Its variants:
728
+ *
729
+ * - `NoRelays`: nothing syncs the owner here. There is no snapshot yet, the
730
+ * owner's transports are still being set up, the app uses no relays for it,
731
+ * it is registered only as readonly, or the database refused startup, which
732
+ * `evoluError` reports.
733
+ * - `Syncing`: a relay connects for the first time or reconciles. A first
734
+ * connection to a host that drops packets stays `Syncing` until the platform
735
+ * gives up on it, which can take a minute.
736
+ * - `Synced`: a relay is up to date, and none syncs.
737
+ * - `Offline`: every relay is disconnected. Evolu keeps reconnecting, up to 30
738
+ * seconds apart, so `Offline` can briefly outlast the outage.
739
+ * - `Error`: a relay failed or offers a change this database skipped. `error` is
740
+ * the newest failure, or without one, the newest skipped change: a failure
741
+ * stops syncing through its relay, while a skipped change leaves out only
742
+ * that change.
743
+ *
744
+ * Evolu saves changes on the device before they sync, so sync needs no UI while
745
+ * it works: show nothing for `NoRelays`, `Syncing`, and `Synced`. An indicator
746
+ * that changes with every edit distracts, and screen readers announce each
747
+ * change.
748
+ *
749
+ * For `Offline` and `Error`, show one quiet line that lasts as long as the
750
+ * status, not a dialog, which interrupts, or a toast, which disappears while
751
+ * the problem lasts. Say that changes are saved on this device. Evolu reports
752
+ * `Offline` at once; an app may wait a few seconds before showing it, because
753
+ * brief disconnections, such as waking from sleep, reconnect quickly. For
754
+ * `Error`, write actionable text for the error types the app can act on, such
755
+ * as a {@link ProtocolQuotaError}: the relay stores no more data for the owner,
756
+ * so offer more quota, such as a plan upgrade, then call
757
+ * {@link Evolu.requestSync} with the owner's ID. For any other error, show
758
+ * generic text that names the error type, which helps when the user reports
759
+ * it.
760
+ *
761
+ * Render the line inside one element with `role="status"` that stays mounted:
762
+ * screen readers announce changes only in a live region that already exists,
763
+ * and a polite announcement fits a status that loses nothing. Do not use
764
+ * `role="alert"`.
765
+ *
766
+ * Show the app owner's status once for the whole app, such as below the header,
767
+ * and another owner's status where the app shows that owner's data. Each Evolu
768
+ * instance finds its own status by {@link Evolu.name}, so an app with several
769
+ * databases shows the status of the one the user works in.
770
+ *
771
+ * ### Example
772
+ *
773
+ * ```ts
774
+ * import { assertEqual, Millis } from "@evolu/common";
775
+ * import {
776
+ * testAppOwner,
777
+ * type OwnerSyncStatus,
778
+ * } from "@evolu/common/local-first";
779
+ *
780
+ * // What to tell the user, or null while sync works or is not used.
781
+ * const syncStatusToMessage = (status: OwnerSyncStatus): string | null => {
782
+ * switch (status.type) {
783
+ * case "NoRelays":
784
+ * case "Syncing":
785
+ * case "Synced":
786
+ * return null;
787
+ * case "Offline":
788
+ * return "Offline. Your changes are saved on this device.";
789
+ * case "Error":
790
+ * return status.error.type === "ProtocolQuotaError"
791
+ * ? "Sync is paused because the sync server is full. Your changes are saved on this device."
792
+ * : `Sync error: ${status.error.type}. Your changes are saved on this device.`;
793
+ * }
794
+ * };
795
+ *
796
+ * assertEqual(syncStatusToMessage({ type: "Synced" }), null);
797
+ * assertEqual(
798
+ * syncStatusToMessage({ type: "Offline" }),
799
+ * "Offline. Your changes are saved on this device.",
800
+ * );
801
+ * assertEqual(
802
+ * syncStatusToMessage({
803
+ * type: "Error",
804
+ * error: {
805
+ * type: "ProtocolQuotaError",
806
+ * ownerId: testAppOwner.id,
807
+ * at: Millis.orThrow(1000),
808
+ * },
809
+ * }),
810
+ * "Sync is paused because the sync server is full. Your changes are saved on this device.",
811
+ * );
812
+ * assertEqual(
813
+ * syncStatusToMessage({
814
+ * type: "Error",
815
+ * error: { type: "SyncFailed", at: Millis.orThrow(1000) },
816
+ * }),
817
+ * "Sync error: SyncFailed. Your changes are saved on this device.",
818
+ * );
819
+ * ```
565
820
  */
566
- export type OwnerSyncStatus = "initial" | RelaySyncStatus;
821
+ export type OwnerSyncStatus = NoRelaysSyncStatus | RelaySyncStatus;
567
822
 
568
823
  /**
569
- * One relay of an {@link OwnerSyncState}: a transport and the database's route
570
- * through it.
824
+ * The status of a {@link RelaySyncState}, from {@link relaySyncStateToStatus}:
825
+ * `Error` when its route has a failure or a skipped change, the failure first;
826
+ * otherwise `Synced` when the route is complete, `Syncing` while the
827
+ * transport's connection is `Connecting` or `Open`, and `Offline` while it is
828
+ * `Disconnected`.
571
829
  */
572
- export interface RelaySyncState {
573
- readonly transport: SyncTransport;
574
- readonly route: SyncRoute;
575
- readonly status: RelaySyncStatus;
830
+ export type RelaySyncStatus =
831
+ SyncingSyncStatus | SyncedSyncStatus | OfflineSyncStatus | ErrorSyncStatus;
832
+
833
+ /** Evolu reports no relay syncing the owner in the database. */
834
+ export interface NoRelaysSyncStatus extends Typed<"NoRelays"> {}
835
+
836
+ /** A relay makes its first connection or reconciles. */
837
+ export interface SyncingSyncStatus extends Typed<"Syncing"> {}
838
+
839
+ /** A relay is reconciled with the database for the owner. */
840
+ export interface SyncedSyncStatus extends Typed<"Synced"> {}
841
+
842
+ /** Relays are disconnected and reconnecting, so changes wait on this device. */
843
+ export interface OfflineSyncStatus extends Typed<"Offline"> {}
844
+
845
+ /** A route failed or holds a change the database skipped. */
846
+ export interface ErrorSyncStatus extends Typed<"Error"> {
847
+ /**
848
+ * The failure, or without one, the skipped change. For an owner, the newest
849
+ * failure of any relay, or without one, the newest skipped change.
850
+ */
851
+ readonly error: SyncRouteError;
576
852
  }
577
853
 
578
854
  /**
579
- * The status of a {@link RelaySyncState}: `error` when its route failed and has
580
- * not completed since, `synced` when the route is complete, `syncing` while the
581
- * transport is open, and `offline` otherwise.
855
+ * Tells what an app shows about syncing an owner in a database: `Error` when a
856
+ * relay has one, holding the newest failure of any relay or, without one, the
857
+ * newest skipped change; otherwise the first of `Syncing`, `Synced`, and
858
+ * `Offline` that a relay has, or `NoRelays`. Accepts null, the store's value
859
+ * before the first snapshot. See {@link OwnerSyncStatus} for what to show.
860
+ *
861
+ * The same status is the same object: statuses other than `Error` are shared
862
+ * constants, and an `Error` status is the same object for the same error
863
+ * object, which {@link SyncStateDep.syncState} keeps between snapshots while it
864
+ * is unchanged. So compare statuses with `===`.
865
+ *
866
+ * ### Example
867
+ *
868
+ * ```ts
869
+ * import {
870
+ * assertEqual,
871
+ * assertSame,
872
+ * createId,
873
+ * createUnknownError,
874
+ * Millis,
875
+ * testCreateDeps,
876
+ * testName,
877
+ * } from "@evolu/common";
878
+ * import {
879
+ * syncStateToOwnerSyncStatus,
880
+ * testAppOwner,
881
+ * type PendingSyncRoute,
882
+ * type SyncConnection,
883
+ * type SyncRoute,
884
+ * type SyncState,
885
+ * type SyncTransportId,
886
+ * } from "@evolu/common/local-first";
887
+ *
888
+ * const deps = testCreateDeps();
889
+ * const primaryId = createId<"SyncTransport">(deps);
890
+ * const backupId = createId<"SyncTransport">(deps);
891
+ *
892
+ * const stateOf = (
893
+ * connections: ReadonlyArray<SyncConnection>,
894
+ * routes: ReadonlyArray<SyncRoute>,
895
+ * ): SyncState => ({
896
+ * transports: connections.map((connection, index) => ({
897
+ * type: "WebSocket",
898
+ * id: index === 0 ? primaryId : backupId,
899
+ * label: "wss://relay.example",
900
+ * connection,
901
+ * })),
902
+ * tenants: [
903
+ * {
904
+ * type: "Active",
905
+ * name: testName,
906
+ * owners: [{ type: "Writable", ownerId: testAppOwner.id, routes }],
907
+ * },
908
+ * ],
909
+ * });
910
+ * const pending = (transportId: SyncTransportId): PendingSyncRoute => ({
911
+ * type: "Pending",
912
+ * transportId,
913
+ * failure: null,
914
+ * skippedError: null,
915
+ * completeAt: null,
916
+ * lastSentAt: null,
917
+ * lastReceivedAt: null,
918
+ * });
919
+ * const statusOf = (state: SyncState | null) =>
920
+ * syncStateToOwnerSyncStatus(state, testName, testAppOwner.id);
921
+ *
922
+ * // Before the first snapshot, nothing syncs the owner.
923
+ * assertEqual(statusOf(null), { type: "NoRelays" });
924
+ *
925
+ * // A first connection is syncing; a lost one is offline.
926
+ * const connecting: SyncConnection = { type: "Connecting" };
927
+ * assertEqual(statusOf(stateOf([connecting], [pending(primaryId)])), {
928
+ * type: "Syncing",
929
+ * });
930
+ * const disconnected: SyncConnection = {
931
+ * type: "Disconnected",
932
+ * disconnectedAt: Millis.orThrow(2000),
933
+ * openedAt: Millis.orThrow(1000),
934
+ * error: null,
935
+ * };
936
+ * assertEqual(statusOf(stateOf([disconnected], [pending(primaryId)])), {
937
+ * type: "Offline",
938
+ * });
939
+ *
940
+ * // A quota failure on one relay shows before a newer skipped change on
941
+ * // another, because the app can act on it.
942
+ * const quotaError = {
943
+ * type: "ProtocolQuotaError",
944
+ * ownerId: testAppOwner.id,
945
+ * at: Millis.orThrow(3000),
946
+ * } as const;
947
+ * const open: SyncConnection = {
948
+ * type: "Open",
949
+ * openedAt: Millis.orThrow(3400),
950
+ * error: null,
951
+ * };
952
+ * assertEqual(
953
+ * statusOf(
954
+ * stateOf(
955
+ * [disconnected, open],
956
+ * [
957
+ * { ...pending(primaryId), failure: quotaError },
958
+ * {
959
+ * type: "Settled",
960
+ * transportId: backupId,
961
+ * skippedError: {
962
+ * type: "DecryptWithXChaCha20Poly1305Error",
963
+ * error: createUnknownError(new Error("invalid tag")),
964
+ * at: Millis.orThrow(4000),
965
+ * },
966
+ * completeAt: null,
967
+ * lastSentAt: Millis.orThrow(3500),
968
+ * lastReceivedAt: Millis.orThrow(4000),
969
+ * },
970
+ * ],
971
+ * ),
972
+ * ),
973
+ * { type: "Error", error: quotaError },
974
+ * );
975
+ *
976
+ * // The same error gives the same status object.
977
+ * const quotaState = stateOf(
978
+ * [disconnected],
979
+ * [{ ...pending(primaryId), failure: quotaError }],
980
+ * );
981
+ * assertSame(statusOf(quotaState), statusOf(quotaState));
982
+ * ```
582
983
  */
583
- export type RelaySyncStatus = "syncing" | "synced" | "offline" | "error";
984
+ export const syncStateToOwnerSyncStatus = (
985
+ state: SyncState | null,
986
+ name: Name,
987
+ ownerId: OwnerId,
988
+ ): OwnerSyncStatus => {
989
+ let failure: SyncRouteError | null = null;
990
+ let skippedError: SyncRouteError | null = null;
991
+ let isSyncing = false;
992
+ let isSynced = false;
993
+ let isOffline = false;
994
+ for (const relay of syncStateToRelaySyncStates(state, name, ownerId)) {
995
+ const { route } = relay;
996
+ // A failure stops syncing through its relay, and a skipped change is
997
+ // stamped again by every reply that skips it, so a newer skipped change
998
+ // must not hide an older failure the app can act on.
999
+ if (
1000
+ route.type === "Pending" &&
1001
+ route.failure &&
1002
+ (!failure || route.failure.at > failure.at)
1003
+ )
1004
+ failure = route.failure;
1005
+ if (
1006
+ route.type !== "Complete" &&
1007
+ route.skippedError &&
1008
+ (!skippedError || route.skippedError.at > skippedError.at)
1009
+ )
1010
+ skippedError = route.skippedError;
1011
+ const status = relaySyncStateToStatus(relay);
1012
+ switch (status.type) {
1013
+ case "Error":
1014
+ break;
1015
+ case "Syncing":
1016
+ isSyncing = true;
1017
+ break;
1018
+ case "Synced":
1019
+ isSynced = true;
1020
+ break;
1021
+ case "Offline":
1022
+ isOffline = true;
1023
+ break;
1024
+ default:
1025
+ exhaustiveCheck(status);
1026
+ }
1027
+ }
1028
+ const error = failure ?? skippedError;
1029
+ return error
1030
+ ? syncRouteErrorToSyncStatus(error)
1031
+ : isSyncing
1032
+ ? syncingSyncStatus
1033
+ : isSynced
1034
+ ? syncedSyncStatus
1035
+ : isOffline
1036
+ ? offlineSyncStatus
1037
+ : noRelaysSyncStatus;
1038
+ };
584
1039
 
585
1040
  /**
586
- * Folds the routes of every writable owner registration of every running
587
- * database in a {@link SyncState} into one {@link OwnerSyncState} per database
588
- * and owner, which pairs each route with its transport as a
589
- * {@link RelaySyncState}. A database that refused startup and a readonly
590
- * registration synchronize nothing, so they are left out.
1041
+ * Pairs each route of an owner in a database with its transport, in route
1042
+ * order. Returns none for a null snapshot, a missing or refused database, and a
1043
+ * missing or readonly owner.
591
1044
  *
592
1045
  * ### Example
593
1046
  *
@@ -600,7 +1053,8 @@ export type RelaySyncStatus = "syncing" | "synced" | "offline" | "error";
600
1053
  * testName,
601
1054
  * } from "@evolu/common";
602
1055
  * import {
603
- * syncStateToOwnerSyncStates,
1056
+ * relaySyncStateToStatus,
1057
+ * syncStateToRelaySyncStates,
604
1058
  * testAppOwner,
605
1059
  * type SyncRoute,
606
1060
  * type SyncState,
@@ -609,99 +1063,106 @@ export type RelaySyncStatus = "syncing" | "synced" | "offline" | "error";
609
1063
  *
610
1064
  * const deps = testCreateDeps();
611
1065
  * const transport: SyncTransport = {
1066
+ * type: "WebSocket",
612
1067
  * id: createId<"SyncTransport">(deps),
613
1068
  * label: "wss://relay.example",
614
- * readyState: "open",
615
- * openedAt: null,
616
- * closedAt: null,
617
- * error: null,
1069
+ * connection: {
1070
+ * type: "Open",
1071
+ * openedAt: Millis.orThrow(800),
1072
+ * error: null,
1073
+ * },
618
1074
  * };
619
1075
  * const route: SyncRoute = {
1076
+ * type: "Complete",
620
1077
  * transportId: transport.id,
621
- * complete: true,
622
1078
  * completeAt: Millis.orThrow(1000),
623
1079
  * lastSentAt: Millis.orThrow(900),
624
1080
  * lastReceivedAt: Millis.orThrow(1000),
625
- * error: null,
626
1081
  * };
627
1082
  * const state: SyncState = {
628
1083
  * transports: [transport],
629
1084
  * tenants: [
630
1085
  * {
1086
+ * type: "Active",
631
1087
  * name: testName,
632
- * refused: false,
633
1088
  * owners: [
634
- * {
635
- * ownerId: testAppOwner.id,
636
- * writable: true,
637
- * transportIds: [transport.id],
638
- * routes: [route],
639
- * },
1089
+ * { type: "Writable", ownerId: testAppOwner.id, routes: [route] },
640
1090
  * ],
641
1091
  * },
642
1092
  * ],
643
1093
  * };
644
1094
  *
645
- * assertEqual(syncStateToOwnerSyncStates(state), [
646
- * {
647
- * name: testName,
648
- * ownerId: testAppOwner.id,
649
- * status: "synced",
650
- * syncedAt: Millis.orThrow(1000),
651
- * error: null,
652
- * relays: [{ transport, route, status: "synced" }],
653
- * },
654
- * ]);
1095
+ * const relays = syncStateToRelaySyncStates(
1096
+ * state,
1097
+ * testName,
1098
+ * testAppOwner.id,
1099
+ * );
1100
+ * assertEqual(relays, [{ transport, route }]);
1101
+ * assertEqual(relays.map(relaySyncStateToStatus), [{ type: "Synced" }]);
1102
+ * assertEqual(
1103
+ * syncStateToRelaySyncStates(null, testName, testAppOwner.id),
1104
+ * [],
1105
+ * );
655
1106
  * ```
656
1107
  */
657
- export const syncStateToOwnerSyncStates = (
658
- state: SyncState,
659
- ): ReadonlyArray<OwnerSyncState> => {
660
- const transportById = new Map(
661
- state.transports.map((transport) => [transport.id, transport]),
662
- );
663
- return state.tenants.flatMap(({ name, refused, owners }) =>
664
- refused
665
- ? []
666
- : owners.flatMap(({ ownerId, writable, routes }) => {
667
- if (!writable) return [];
668
- let syncedAt: Millis | null = null;
669
- let error: SyncRouteError | null = null;
670
- const relays: Array<RelaySyncState> = [];
671
- for (const route of routes) {
672
- if (
673
- route.completeAt !== null &&
674
- (syncedAt === null || route.completeAt > syncedAt)
675
- )
676
- syncedAt = route.completeAt;
677
- if (
678
- route.error !== null &&
679
- (error === null || route.error.at > error.at)
680
- )
681
- error = route.error;
682
- // A snapshot lists the transport of every route.
683
- const transport = transportById.get(route.transportId);
684
- assertNonNullable(transport);
685
- relays.push({
686
- transport,
687
- route,
688
- status:
689
- route.error !== null
690
- ? "error"
691
- : route.complete
692
- ? "synced"
693
- : transport.readyState === "open"
694
- ? "syncing"
695
- : "offline",
696
- });
697
- }
698
- const status: OwnerSyncStatus =
699
- (["error", "syncing", "synced", "offline"] as const).find(
700
- (candidate) => relays.some((relay) => relay.status === candidate),
701
- ) ?? "initial";
702
- return [{ name, ownerId, status, syncedAt, error, relays }];
703
- }),
704
- );
1108
+ export const syncStateToRelaySyncStates = (
1109
+ state: SyncState | null,
1110
+ name: Name,
1111
+ ownerId: OwnerId,
1112
+ ): ReadonlyArray<RelaySyncState> => {
1113
+ const tenant = state?.tenants.find((tenant) => tenant.name === name);
1114
+ if (!state || tenant?.type !== "Active") return emptyArray;
1115
+ const owner = tenant.owners.find((owner) => owner.ownerId === ownerId);
1116
+ if (owner?.type !== "Writable") return emptyArray;
1117
+ return owner.routes.map((route) => {
1118
+ // A snapshot lists the transport of every route.
1119
+ const transport = state.transports.find(
1120
+ ({ id }) => id === route.transportId,
1121
+ );
1122
+ assertNonNullable(transport);
1123
+ return { transport, route };
1124
+ });
1125
+ };
1126
+
1127
+ /** Tells the status of one relay; a failure shows before a skipped change. */
1128
+ export const relaySyncStateToStatus = ({
1129
+ transport,
1130
+ route,
1131
+ }: RelaySyncState): RelaySyncStatus => {
1132
+ switch (route.type) {
1133
+ case "Complete":
1134
+ return syncedSyncStatus;
1135
+ case "Settled":
1136
+ return syncRouteErrorToSyncStatus(route.skippedError);
1137
+ case "Pending": {
1138
+ const error = route.failure ?? route.skippedError;
1139
+ if (error) return syncRouteErrorToSyncStatus(error);
1140
+ return transport.connection.type === "Disconnected"
1141
+ ? offlineSyncStatus
1142
+ : syncingSyncStatus;
1143
+ }
1144
+ }
1145
+ };
1146
+
1147
+ // Every status keeps its reference while it is unchanged, so bindings compare
1148
+ // statuses with `===`. The store shares an unchanged error object between
1149
+ // snapshots, and an error gives the same Error status object.
1150
+ const noRelaysSyncStatus: NoRelaysSyncStatus = { type: "NoRelays" };
1151
+ const syncingSyncStatus: SyncingSyncStatus = { type: "Syncing" };
1152
+ const syncedSyncStatus: SyncedSyncStatus = { type: "Synced" };
1153
+ const offlineSyncStatus: OfflineSyncStatus = { type: "Offline" };
1154
+ const errorSyncStatusByError = /*#__PURE__*/ new WeakMap<
1155
+ SyncRouteError,
1156
+ ErrorSyncStatus
1157
+ >();
1158
+
1159
+ const syncRouteErrorToSyncStatus = (error: SyncRouteError): ErrorSyncStatus => {
1160
+ let status = errorSyncStatusByError.get(error);
1161
+ if (!status) {
1162
+ status = { type: "Error", error };
1163
+ errorSyncStatusByError.set(error, status);
1164
+ }
1165
+ return status;
705
1166
  };
706
1167
 
707
1168
  export type EvoluInput =
@@ -744,6 +1205,11 @@ export type EvoluOutput =
744
1205
  | {
745
1206
  readonly type: "OnExport";
746
1207
  readonly file: Uint8Array<ArrayBuffer>;
1208
+ }
1209
+ | {
1210
+ /** The mutation with these onComplete callbacks could not be stored. */
1211
+ readonly type: "OnMutateFailed";
1212
+ readonly onCompleteIds: ReadonlyArray<Id>;
747
1213
  };
748
1214
 
749
1215
  export type DbWorkerInput =
@@ -820,6 +1286,11 @@ export type DbWorkerQueuedResponse =
820
1286
  >;
821
1287
  readonly rowsByQuery: RowsByQueryMap;
822
1288
  }
1289
+ | {
1290
+ /** The mutation threw, so it rolled back and nothing was stored. */
1291
+ readonly type: "MutateFailed";
1292
+ readonly error: UnknownError;
1293
+ }
823
1294
  | {
824
1295
  readonly type: "Query";
825
1296
  readonly rowsByQuery: RowsByQueryMap;
@@ -846,6 +1317,15 @@ export type DbWorkerQueuedResponse =
846
1317
  readonly clock: Timestamp;
847
1318
  readonly ownerId: OwnerId;
848
1319
  readonly didWriteMessages: boolean;
1320
+ /**
1321
+ * The first error of a received message the DbWorker skipped while
1322
+ * storing the rest, or null. It does not end the round.
1323
+ */
1324
+ readonly skippedError:
1325
+ | DecryptWithXChaCha20Poly1305Error
1326
+ | ProtocolInvalidDataError
1327
+ | ProtocolTimestampMismatchError
1328
+ | null;
849
1329
  readonly result: Result<
850
1330
  ApplyProtocolMessageAsClientResult,
851
1331
  ProtocolError | StorageWriteMessagesError | AbortError
@@ -898,19 +1378,31 @@ interface EvoluTenant extends AsyncDisposable {
898
1378
  }
899
1379
 
900
1380
  /** A tenant's part of {@link SyncState}, with its transports still keyed. */
901
- interface TenantSyncState {
1381
+ type TenantSyncState = ActiveTenantSyncState | RefusedSyncTenant;
1382
+
1383
+ interface ActiveTenantSyncState extends Typed<"Active"> {
902
1384
  readonly name: Name;
903
- readonly refused: boolean;
904
- readonly owners: ReadonlyArray<{
905
- readonly ownerId: OwnerId;
906
- readonly writable: boolean;
907
- readonly transportKeys: ReadonlyArray<StructuralLookupKey>;
908
- readonly routes: ReadonlyArray<TenantSyncRoute>;
909
- }>;
1385
+ readonly owners: ReadonlyArray<TenantSyncOwner>;
1386
+ }
1387
+
1388
+ type TenantSyncOwner = WritableTenantSyncOwner | ReadonlyTenantSyncOwner;
1389
+
1390
+ interface WritableTenantSyncOwner extends Typed<"Writable"> {
1391
+ readonly ownerId: OwnerId;
1392
+ readonly routes: ReadonlyArray<TenantSyncRoute>;
1393
+ }
1394
+
1395
+ interface ReadonlyTenantSyncOwner extends Typed<"Readonly"> {
1396
+ readonly ownerId: OwnerId;
1397
+ readonly transportKeys: ReadonlyArray<StructuralLookupKey>;
910
1398
  }
911
1399
 
912
- interface TenantSyncRoute extends Omit<SyncRoute, "transportId"> {
1400
+ /** A tenant's route, with its transport still keyed. */
1401
+ interface TenantSyncRoute {
913
1402
  readonly transportKey: StructuralLookupKey;
1403
+ readonly progress: RouteProgress;
1404
+ readonly lastSentAt: Millis | null;
1405
+ readonly lastReceivedAt: Millis | null;
914
1406
  }
915
1407
 
916
1408
  /**
@@ -929,6 +1421,39 @@ const syncRequestTimeout = PositiveMillis.orThrow(90_000);
929
1421
  */
930
1422
  const maxSyncRequestTimeout = PositiveMillis.orThrow(16 * syncRequestTimeout);
931
1423
 
1424
+ const connectingSyncConnection: ConnectingSyncConnection = {
1425
+ type: "Connecting",
1426
+ };
1427
+
1428
+ /**
1429
+ * A connection already disconnected keeps when it disconnected, so failed
1430
+ * reconnect attempts only record their error.
1431
+ */
1432
+ const disconnectSyncConnection = (
1433
+ connection: SyncConnection,
1434
+ at: Millis,
1435
+ error: SyncTransportError | null,
1436
+ ): DisconnectedSyncConnection => {
1437
+ switch (connection.type) {
1438
+ case "Connecting":
1439
+ return {
1440
+ type: "Disconnected",
1441
+ disconnectedAt: at,
1442
+ openedAt: null,
1443
+ error,
1444
+ };
1445
+ case "Open":
1446
+ return {
1447
+ type: "Disconnected",
1448
+ disconnectedAt: at,
1449
+ openedAt: connection.openedAt,
1450
+ error: error ?? connection.error,
1451
+ };
1452
+ case "Disconnected":
1453
+ return error ? { ...connection, error } : connection;
1454
+ }
1455
+ };
1456
+
932
1457
  /** Where the protocol messages produced by a queued sync request are sent. */
933
1458
  type SyncTarget =
934
1459
  | { readonly type: "AllTransports" }
@@ -955,6 +1480,118 @@ const isTargetTransport = (
955
1480
  ): boolean =>
956
1481
  target.type === "AllTransports" || target.key === structuralLookup(transport);
957
1482
 
1483
+ /**
1484
+ * A tenant's progress toward completing a route, per the Synchronization
1485
+ * completion rules. Events move the route to `Pending`, which keeps what the
1486
+ * route must remember until it settles, except that messages stored elsewhere
1487
+ * leave a `Settled` route settled. Refreshing the routes settles a `Pending`
1488
+ * route once the tenant's reconciliation conditions hold, and returns a settled
1489
+ * or complete route to `Pending` when they no longer hold.
1490
+ */
1491
+ type RouteProgress = PendingRoute | SettledRoute | CompleteRoute;
1492
+
1493
+ /**
1494
+ * A route whose reconciliation has not ended, or has not been evaluated since
1495
+ * an event.
1496
+ */
1497
+ interface PendingRoute extends Typed<"Pending"> {
1498
+ /** A round must be sent through the route before it can settle. */
1499
+ readonly roundRequired: boolean;
1500
+ /**
1501
+ * The failure since the route settled, or null. A further failure requests no
1502
+ * round until the route settles.
1503
+ */
1504
+ readonly failure: SyncRouteError | null;
1505
+ /** The route's skipped message, or null. */
1506
+ readonly skip: RouteSkip | null;
1507
+ /** When the route last became complete, or null. */
1508
+ readonly completeAt: Millis | null;
1509
+ }
1510
+
1511
+ /**
1512
+ * A message a route skipped. Its relay offers the message again in every round,
1513
+ * so messages received elsewhere request no round through the route.
1514
+ */
1515
+ interface RouteSkip {
1516
+ readonly error: SyncRouteError;
1517
+ /**
1518
+ * A round was requested through the route since the skip, and no messages
1519
+ * received elsewhere have been stored since that request, so the route
1520
+ * completes if it settles first.
1521
+ */
1522
+ readonly isRechecking: boolean;
1523
+ }
1524
+
1525
+ /**
1526
+ * A route whose reconciliation has ended, but whose relay may offer a skipped
1527
+ * message or lack messages stored elsewhere, so it is incomplete.
1528
+ */
1529
+ interface SettledRoute extends Typed<"Settled"> {
1530
+ readonly error: SyncRouteError;
1531
+ readonly completeAt: Millis | null;
1532
+ }
1533
+
1534
+ interface CompleteRoute extends Typed<"Complete"> {
1535
+ readonly completeAt: Millis;
1536
+ }
1537
+
1538
+ const routeToPending = (progress: RouteProgress): PendingRoute => {
1539
+ switch (progress.type) {
1540
+ case "Pending":
1541
+ return progress;
1542
+ case "Settled":
1543
+ return {
1544
+ type: "Pending",
1545
+ roundRequired: false,
1546
+ failure: null,
1547
+ skip: { error: progress.error, isRechecking: false },
1548
+ completeAt: progress.completeAt,
1549
+ };
1550
+ case "Complete":
1551
+ return {
1552
+ type: "Pending",
1553
+ roundRequired: false,
1554
+ failure: null,
1555
+ skip: null,
1556
+ completeAt: progress.completeAt,
1557
+ };
1558
+ }
1559
+ };
1560
+
1561
+ /**
1562
+ * Converts an error to a {@link SyncRouteError}: a caught value becomes an
1563
+ * {@link UnknownError}, and a {@link ProtocolInvalidDataError} leaves out its
1564
+ * data.
1565
+ */
1566
+ const errorToSyncRouteError = (
1567
+ error:
1568
+ | ProtocolError
1569
+ | StorageWriteMessagesError
1570
+ | DecryptWithXChaCha20Poly1305Error
1571
+ | Typed<"WriteFailed">
1572
+ | Typed<"SyncFailed">,
1573
+ at: Millis,
1574
+ ): SyncRouteError => {
1575
+ // A caught value inside an error becomes an UnknownError, and a
1576
+ // ProtocolInvalidDataError leaves out its data, which can be a whole frame.
1577
+ if (error.type === "ProtocolInvalidDataError") {
1578
+ const { data: _data, ...rest } = error;
1579
+ return { ...rest, error: createUnknownError(rest.error), at };
1580
+ }
1581
+ if (error.type === "DecryptWithXChaCha20Poly1305Error")
1582
+ return { ...error, error: createUnknownError(error.error), at };
1583
+ return { ...error, at };
1584
+ };
1585
+
1586
+ const failRoute = (
1587
+ progress: RouteProgress,
1588
+ failure: SyncRouteError,
1589
+ ): PendingRoute => ({
1590
+ ...routeToPending(progress),
1591
+ roundRequired: true,
1592
+ failure,
1593
+ });
1594
+
958
1595
  type EvoluTenantDeps = SharedWorkerDeps &
959
1596
  PostConsoleEntryOrErrorDep &
960
1597
  PublishSyncStateDep &
@@ -1117,8 +1754,15 @@ export const initSharedWorker =
1117
1754
  });
1118
1755
  };
1119
1756
 
1120
- // Released after every tenant and DbWorker is disposed. Earlier releases
1121
- // take the same lock in their leader tab; see Builds.
1757
+ // Held while this worker runs. Its DbWorkers stop once they can take it,
1758
+ // because a Dispose posted right before this worker closes can be lost, as
1759
+ // in Firefox. Taken before the build lock, so nothing delays the end of
1760
+ // starting once that lock is held.
1761
+ disposer.use(await run.ok(acquireLeaderLock(workerId)));
1762
+
1763
+ // Released after every tenant is disposed and has told its DbWorker to
1764
+ // stop. Earlier releases take the same lock in their leader tab; see
1765
+ // Builds.
1122
1766
  disposer.use(await run.ok(acquireLeaderLock("tab")));
1123
1767
  starting.dispose();
1124
1768
 
@@ -1156,13 +1800,12 @@ export const initSharedWorker =
1156
1800
  * another, so a reply can wait behind another owner's large frame. It
1157
1801
  * doubles after each timeout and survives reconnects until nothing is
1158
1802
  * outstanding and every route through the transport of a database that
1159
- * has not refused startup is complete.
1803
+ * has not refused startup has settled.
1160
1804
  */
1161
1805
  timeout: PositiveMillis;
1162
1806
  socket: WebSocket | null;
1163
- openedAt: Millis | null;
1164
- closedAt: Millis | null;
1165
- error: SyncTransportError | null;
1807
+ /** Socket events drive it, so publishing never reads the socket. */
1808
+ connection: SyncConnection;
1166
1809
  /** Armed while a request is outstanding on an open socket. */
1167
1810
  timeoutId: TimeoutId | null;
1168
1811
  }
@@ -1181,62 +1824,96 @@ export const initSharedWorker =
1181
1824
  isPublishScheduled = false;
1182
1825
  if (isDisposed) return;
1183
1826
  const transports = [...transportsByKey.values()].map(
1184
- ({
1185
- id,
1186
- label,
1187
- socket,
1188
- openedAt,
1189
- closedAt,
1190
- error,
1191
- }): SyncTransport => ({
1827
+ ({ id, label, connection }): SyncTransport => ({
1828
+ type: "WebSocket",
1192
1829
  id,
1193
1830
  label,
1194
- // The transport drops its socket before disposal, so this never
1195
- // reads a disposed one.
1196
- readyState: socket?.getReadyState() ?? "connecting",
1197
- openedAt,
1198
- closedAt,
1199
- error,
1831
+ connection,
1200
1832
  }),
1201
1833
  );
1834
+ // A grown timeout lasts while a request is outstanding on the socket
1835
+ // or a database that has not refused startup has an unsettled route
1836
+ // through it. Every change to either publishes, including a route that
1837
+ // goes away without settling.
1838
+ const unsettledTransportIds = new Set<SyncTransportId>();
1202
1839
  const tenants = [...currentTenantsByName.values()].map(
1203
1840
  (tenant): SyncTenant => {
1204
- const { name, refused, owners } = tenant.getSyncTenant();
1841
+ const state = tenant.getSyncTenant();
1842
+ if (state.type === "Refused") return state;
1205
1843
  return {
1206
- name,
1207
- refused,
1208
- owners: owners.map(
1209
- ({ ownerId, writable, transportKeys, routes }) => ({
1210
- ownerId,
1211
- writable,
1212
- transportIds: transportKeys.flatMap((key) => {
1213
- const entry = transportsByKey.get(key);
1214
- return entry ? [entry.id] : [];
1215
- }),
1216
- routes: routes.flatMap(({ transportKey, ...route }) => {
1217
- const entry = transportsByKey.get(transportKey);
1218
- return entry ? [{ transportId: entry.id, ...route }] : [];
1219
- }),
1220
- }),
1844
+ type: "Active",
1845
+ name: state.name,
1846
+ owners: state.owners.map((owner): SyncTenantOwner =>
1847
+ owner.type === "Readonly"
1848
+ ? {
1849
+ type: "Readonly",
1850
+ ownerId: owner.ownerId,
1851
+ transportIds: owner.transportKeys.flatMap((key) => {
1852
+ const entry = transportsByKey.get(key);
1853
+ return entry ? [entry.id] : [];
1854
+ }),
1855
+ }
1856
+ : {
1857
+ type: "Writable",
1858
+ ownerId: owner.ownerId,
1859
+ routes: owner.routes.flatMap(
1860
+ ({
1861
+ transportKey,
1862
+ progress,
1863
+ lastSentAt,
1864
+ lastReceivedAt,
1865
+ }): Array<SyncRoute> => {
1866
+ const entry = transportsByKey.get(transportKey);
1867
+ if (!entry) return [];
1868
+ const transportId = entry.id;
1869
+ if (progress.type !== "Pending") {
1870
+ // Only a sent round settles a route or completes
1871
+ // it.
1872
+ assertNonNullable(lastSentAt);
1873
+ return [
1874
+ progress.type === "Complete"
1875
+ ? {
1876
+ type: "Complete",
1877
+ transportId,
1878
+ completeAt: progress.completeAt,
1879
+ lastSentAt,
1880
+ lastReceivedAt,
1881
+ }
1882
+ : {
1883
+ type: "Settled",
1884
+ transportId,
1885
+ skippedError: progress.error,
1886
+ completeAt: progress.completeAt,
1887
+ lastSentAt,
1888
+ lastReceivedAt,
1889
+ },
1890
+ ];
1891
+ }
1892
+ // Only a database that has not refused startup
1893
+ // publishes routes.
1894
+ unsettledTransportIds.add(transportId);
1895
+ return [
1896
+ {
1897
+ type: "Pending",
1898
+ transportId,
1899
+ failure: progress.failure,
1900
+ skippedError: progress.skip?.error ?? null,
1901
+ completeAt: progress.completeAt,
1902
+ lastSentAt,
1903
+ lastReceivedAt,
1904
+ },
1905
+ ];
1906
+ },
1907
+ ),
1908
+ },
1221
1909
  ),
1222
1910
  };
1223
1911
  },
1224
1912
  );
1225
- // A grown timeout lasts while a request is outstanding on the socket
1226
- // or a database that has not refused startup has an incomplete route
1227
- // through it. Every change to either publishes, including a route that
1228
- // goes away without completing.
1229
- const incompleteTransportIds = new Set<SyncTransportId>();
1230
- for (const { refused, owners } of tenants) {
1231
- if (refused) continue;
1232
- for (const { routes } of owners)
1233
- for (const { transportId, complete } of routes)
1234
- if (!complete) incompleteTransportIds.add(transportId);
1235
- }
1236
1913
  for (const entry of transportsByKey.values())
1237
1914
  if (
1238
1915
  entry.outstandingByOwnerId.size === 0 &&
1239
- !incompleteTransportIds.has(entry.id)
1916
+ !unsettledTransportIds.has(entry.id)
1240
1917
  )
1241
1918
  entry.timeout = syncRequestTimeout;
1242
1919
  syncStateBroadcastChannel.postMessage({ transports, tenants });
@@ -1295,10 +1972,14 @@ export const initSharedWorker =
1295
1972
  );
1296
1973
  // Reconnecting abandons the connection and starts a fresh retry
1297
1974
  // schedule. The socket reports no close for it, so the transport
1298
- // records the moment here.
1975
+ // disconnects here.
1299
1976
  entry.socket?.reconnect();
1300
- // `now` is monotonic; the reported close time is wall clock.
1301
- entry.closedAt = deps.time.now();
1977
+ // `now` is monotonic; the reported time is wall clock.
1978
+ entry.connection = disconnectSyncConnection(
1979
+ entry.connection,
1980
+ deps.time.now(),
1981
+ null,
1982
+ );
1302
1983
  refreshAllSyncRoutes();
1303
1984
  publishSyncState();
1304
1985
  }, delay);
@@ -1346,9 +2027,7 @@ export const initSharedWorker =
1346
2027
  outstandingByOwnerId: new Map(),
1347
2028
  timeout: syncRequestTimeout,
1348
2029
  socket: null,
1349
- openedAt: null,
1350
- closedAt: null,
1351
- error: null,
2030
+ connection: connectingSyncConnection,
1352
2031
  timeoutId: null,
1353
2032
  };
1354
2033
  await using disposer = new AsyncDisposableStack();
@@ -1371,7 +2050,16 @@ export const initSharedWorker =
1371
2050
  // answered.
1372
2051
  entry.outstandingByOwnerId.clear();
1373
2052
  clearSyncRequestTimeout(entry);
1374
- entry.openedAt = run.deps.time.now();
2053
+ // A connection that opens keeps the last error of an earlier
2054
+ // one.
2055
+ entry.connection = {
2056
+ type: "Open",
2057
+ openedAt: run.deps.time.now(),
2058
+ error:
2059
+ entry.connection.type === "Connecting"
2060
+ ? null
2061
+ : entry.connection.error,
2062
+ };
1375
2063
  publishSyncState();
1376
2064
  const ownerIds = transports.getClaimsForResource(transport);
1377
2065
  console.debug("transportOpen", {
@@ -1395,7 +2083,11 @@ export const initSharedWorker =
1395
2083
  code: event.code,
1396
2084
  wasClean: event.wasClean,
1397
2085
  });
1398
- entry.closedAt = run.deps.time.now();
2086
+ entry.connection = disconnectSyncConnection(
2087
+ entry.connection,
2088
+ run.deps.time.now(),
2089
+ null,
2090
+ );
1399
2091
  clearSyncRequestTimeout(entry);
1400
2092
  refreshAllSyncRoutes();
1401
2093
  publishSyncState();
@@ -1406,10 +2098,17 @@ export const initSharedWorker =
1406
2098
  url: transport.url,
1407
2099
  type: error.type,
1408
2100
  });
1409
- entry.error = {
1410
- type: error.type,
1411
- at: run.deps.time.now(),
1412
- };
2101
+ // Every error disconnects. A browser reports a failed
2102
+ // attempt only as an error, because the socket stops
2103
+ // listening before its close arrives. A connection error
2104
+ // arrives once the socket is closed, before its close, and
2105
+ // exhausted retries end reconnecting.
2106
+ const now = run.deps.time.now();
2107
+ entry.connection = disconnectSyncConnection(
2108
+ entry.connection,
2109
+ now,
2110
+ { type: error.type, at: now },
2111
+ );
1413
2112
  refreshAllSyncRoutes();
1414
2113
  publishSyncState();
1415
2114
  },
@@ -1474,9 +2173,8 @@ export const initSharedWorker =
1474
2173
  // LIFO: the transport drops its timer and its socket reference
1475
2174
  // before the socket is disposed. `disposable` guards every method
1476
2175
  // of the socket the claims lease, and disposing the socket awaits
1477
- // its retry, so a `publishSyncState` microtask can run while this
1478
- // entry is still registered. It must find no socket rather than
1479
- // read a disposed one.
2176
+ // its retry, so this entry stays registered meanwhile. It keeps its
2177
+ // last connection and holds no socket to reconnect.
1480
2178
  disposer.defer(() => {
1481
2179
  clearSyncRequestTimeout(entry);
1482
2180
  entry.socket = null;
@@ -1541,6 +2239,7 @@ export const initSharedWorker =
1541
2239
  ? { ...message, memoryOnly: true }
1542
2240
  : message,
1543
2241
  currentTenantsByName,
2242
+ workerId,
1544
2243
  ),
1545
2244
  {
1546
2245
  idleDisposeAfter: "3s",
@@ -1571,6 +2270,7 @@ const createEvoluTenant =
1571
2270
  memoryOnly,
1572
2271
  }: ExtractTyped<SharedWorkerInput, "CreateEvolu">,
1573
2272
  currentTenantsByName: Map<Name, BorrowedResource<EvoluTenant>>,
2273
+ workerId: SharedWorkerId,
1574
2274
  ): Task<EvoluTenant, never, EvoluTenantDeps> =>
1575
2275
  async (run) => {
1576
2276
  await using disposer = new AsyncDisposableStack();
@@ -1741,6 +2441,7 @@ const createEvoluTenant =
1741
2441
  sqliteSchema,
1742
2442
  encryptionKey,
1743
2443
  memoryOnly,
2444
+ sharedWorkerId: workerId,
1744
2445
  port: dbWorkerChannel.port1.native,
1745
2446
  },
1746
2447
  [dbWorkerChannel.port1.native],
@@ -1896,20 +2597,16 @@ const createEvoluTenant =
1896
2597
  }
1897
2598
  };
1898
2599
 
1899
- disposer.defer(async () => {
2600
+ // Disposal does not wait for the DbWorker. It holds the database lock
2601
+ // until Dispose arrives, its tab closes, or this worker ends, and the next
2602
+ // DbWorker for this database, of this worker or another build, waits for
2603
+ // that lock before it reads the clock. A requested DbWorker that reports
2604
+ // in later gets Dispose from the isDisposing check.
2605
+ disposer.defer(() => {
1900
2606
  isDisposing = true;
1901
2607
  dbWorkerPort?.postMessage({ type: "Dispose" });
1902
2608
  dbWorkerPort = null;
1903
2609
  activeDispatch = null;
1904
-
1905
- // The DbWorker holds this tenant leader lock while it is alive. Tenant
1906
- // disposal sends Dispose, then acquires the same lock to wait until the
1907
- // DbWorker releases it: either because Dispose was delivered or because
1908
- // the hosting tab closed. A worker requested from a later tab leader may
1909
- // be queued for the lock first; it is told to stop when it reports in.
1910
- // The wait is unabortable because tenant disposal must finish even after
1911
- // tenantRun receives an abort request.
1912
- await using _ = await tenantRun.ok(acquireLeaderLock(name));
1913
2610
  });
1914
2611
 
1915
2612
  const handleResponseForEvolu = (
@@ -1996,23 +2693,47 @@ const createEvoluTenant =
1996
2693
  break;
1997
2694
  }
1998
2695
 
2696
+ case "MutateFailed": {
2697
+ // The tab shows it, and the instance releases the mutation's
2698
+ // onComplete callbacks without running them. A write outlives its
2699
+ // instance, so without one, every tab is told.
2700
+ //
2701
+ // A replay after leader replacement can fail although the earlier
2702
+ // leader committed the write and only its answer was lost. The tab
2703
+ // then shows an error for a stored write, and its onComplete
2704
+ // callbacks never run. Nothing is lost or reused: the replacement
2705
+ // adopted the stored clock, refreshed every instance's queries, and
2706
+ // reconciles every used owner.
2707
+ const { error } = response.message;
2708
+ if (instance) {
2709
+ assertSame(first.message.type, "Mutate");
2710
+ instance.tabPort.postMessage({ type: "Error", error });
2711
+ instance.port.postMessage({
2712
+ type: "OnMutateFailed",
2713
+ onCompleteIds: first.message.onCompleteIds,
2714
+ });
2715
+ } else {
2716
+ deps.postConsoleEntryOrError({ type: "Error", error });
2717
+ }
2718
+ break;
2719
+ }
2720
+
1999
2721
  case "Export":
2000
2722
  instance?.port.postMessage(
2001
2723
  { type: "OnExport", file: response.message.file },
2002
2724
  [response.message.file.buffer],
2003
2725
  );
2004
2726
  break;
2727
+
2728
+ default:
2729
+ exhaustiveCheck(response.message);
2005
2730
  }
2006
2731
  };
2007
2732
 
2008
2733
  interface RouteState {
2009
- /** A round must be sent through the route before it can be complete. */
2010
- roundRequired: boolean;
2011
- complete: boolean;
2012
- completeAt: Millis | null;
2734
+ progress: RouteProgress;
2013
2735
  lastSentAt: Millis | null;
2014
2736
  lastReceivedAt: Millis | null;
2015
- error: SyncRouteError | null;
2016
2737
  }
2017
2738
  const routesByOwnerIdByKey = new Map<
2018
2739
  StructuralLookupKey,
@@ -2031,12 +2752,15 @@ const createEvoluTenant =
2031
2752
  let route = routesByOwnerId.get(ownerId);
2032
2753
  if (!route) {
2033
2754
  route = {
2034
- roundRequired: true,
2035
- complete: false,
2036
- completeAt: null,
2755
+ progress: {
2756
+ type: "Pending",
2757
+ roundRequired: true,
2758
+ failure: null,
2759
+ skip: null,
2760
+ completeAt: null,
2761
+ },
2037
2762
  lastSentAt: null,
2038
2763
  lastReceivedAt: null,
2039
- error: null,
2040
2764
  };
2041
2765
  routesByOwnerId.set(ownerId, route);
2042
2766
  }
@@ -2099,20 +2823,28 @@ const createEvoluTenant =
2099
2823
  // worker answers it. Local-only changes create no synchronization
2100
2824
  // work.
2101
2825
  const hasQueuedWrite = pendingWriteCountByOwnerId.has(ownerId);
2102
- // A refused database synchronizes nothing, and refusal discards its
2103
- // queued writes without uploading them.
2104
- const complete =
2105
- startupError === null &&
2826
+ const isReconciled =
2106
2827
  isOpen &&
2107
2828
  deps.syncRequests.getOutstanding(ownerId, key) === 0 &&
2108
2829
  !hasQueuedApply &&
2109
- !hasQueuedWrite &&
2110
- !route.roundRequired;
2111
- if (complete && !route.complete) {
2112
- route.completeAt = deps.time.now();
2113
- route.error = null;
2114
- }
2115
- route.complete = complete;
2830
+ !hasQueuedWrite;
2831
+ // Settling drops the failure, so the next one requests a round
2832
+ // again. A route with a skip it is not rechecking settles without
2833
+ // completing, because its relay may offer the skipped message or
2834
+ // lack messages stored elsewhere.
2835
+ const pending = routeToPending(route.progress);
2836
+ route.progress =
2837
+ !isReconciled || pending.roundRequired
2838
+ ? pending
2839
+ : pending.skip && !pending.skip.isRechecking
2840
+ ? {
2841
+ type: "Settled",
2842
+ error: pending.skip.error,
2843
+ completeAt: pending.completeAt,
2844
+ }
2845
+ : route.progress.type === "Complete"
2846
+ ? route.progress
2847
+ : { type: "Complete", completeAt: deps.time.now() };
2116
2848
  }
2117
2849
  }
2118
2850
  };
@@ -2144,8 +2876,10 @@ const createEvoluTenant =
2144
2876
  .get(structuralLookup(transport))
2145
2877
  ?.get(ownerId);
2146
2878
  if (!route) return;
2147
- route.roundRequired = true;
2148
- route.error = { type: "SyncFailed", at: now };
2879
+ route.progress = failRoute(route.progress, {
2880
+ type: "SyncFailed",
2881
+ at: now,
2882
+ });
2149
2883
  });
2150
2884
  }
2151
2885
  deps.publishSyncState();
@@ -2159,7 +2893,14 @@ const createEvoluTenant =
2159
2893
  const { ownerId, result } = response.message;
2160
2894
  const error = result.ok ? null : result.error;
2161
2895
  const isAborted = error?.type === "AbortError";
2162
- let failure: SyncRouteErrorType | null = null;
2896
+ // An aborted apply reports no skipped message.
2897
+ const skippedError = isAborted ? null : response.message.skippedError;
2898
+ let failure:
2899
+ | ProtocolError
2900
+ | StorageWriteMessagesError
2901
+ | Typed<"WriteFailed">
2902
+ | Typed<"SyncFailed">
2903
+ | null = null;
2163
2904
  if (isAborted) {
2164
2905
  // An abort proves no convergence. A local sibling copy affects
2165
2906
  // every route; a relay frame affects only its source route.
@@ -2168,17 +2909,22 @@ const createEvoluTenant =
2168
2909
  const key = structuralLookup(transport);
2169
2910
  if (source.type === "Transport" && source.key !== key) return;
2170
2911
  const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
2171
- if (route) route.roundRequired = true;
2912
+ if (route)
2913
+ route.progress = {
2914
+ ...routeToPending(route.progress),
2915
+ roundRequired: true,
2916
+ };
2172
2917
  });
2173
2918
  } else if (error !== null) {
2174
- failure = error.type;
2175
- deps.postConsoleEntryOrError({
2176
- type: "Error",
2177
- error,
2178
- });
2919
+ // A relay's error shows on its route, not as an EvoluError,
2920
+ // because it belongs to one relay and often repeats in every
2921
+ // round. A sibling copy's error is reported below.
2922
+ failure = error;
2179
2923
  } else if (result.ok && result.value.type === "Failed") {
2180
- failure =
2181
- result.value.cause === "Write" ? "WriteFailed" : "SyncFailed";
2924
+ failure = {
2925
+ type:
2926
+ result.value.cause === "Write" ? "WriteFailed" : "SyncFailed",
2927
+ };
2182
2928
  }
2183
2929
  if (source.type === "Transport") {
2184
2930
  // A registration or claim may have been removed while this apply
@@ -2188,39 +2934,69 @@ const createEvoluTenant =
2188
2934
  const now = deps.time.now();
2189
2935
  // An aborted apply applied nothing.
2190
2936
  if (!isAborted) route.lastReceivedAt = now;
2937
+ // A skipped message is recorded as the route's skip, not a
2938
+ // failure, and does not end the round, whose response below is
2939
+ // still sent, so it requests no round. The relay offers the
2940
+ // message again in every later round, so the route stays
2941
+ // incomplete until a round requested through it settles without
2942
+ // skipping a message and without messages stored elsewhere since
2943
+ // that request.
2944
+ if (skippedError !== null)
2945
+ route.progress = {
2946
+ ...routeToPending(route.progress),
2947
+ skip: {
2948
+ error: errorToSyncRouteError(skippedError, now),
2949
+ isRechecking: false,
2950
+ },
2951
+ };
2191
2952
  if (failure !== null) {
2192
- // The first failure since the route completed requests one
2953
+ // The first failure since the route settled requests one
2193
2954
  // round. Further failures wait for an explicit request or a
2194
2955
  // reopen, even after a converged reply, which may answer
2195
2956
  // another request.
2196
- if (route.error === null)
2957
+ const requestsRetry =
2958
+ route.progress.type !== "Pending" ||
2959
+ route.progress.failure === null;
2960
+ route.progress = failRoute(
2961
+ route.progress,
2962
+ errorToSyncRouteError(failure, now),
2963
+ );
2964
+ if (requestsRetry)
2197
2965
  requestCreateSyncMessages(new Set([ownerId]), source);
2198
- route.roundRequired = true;
2199
- route.error = { type: failure, at: now };
2200
2966
  }
2201
2967
  }
2202
- } else if (failure !== null) {
2203
- // A sibling's messages were not stored; rounds fetch them from
2204
- // the relays.
2205
- requestCreateSyncMessages(new Set([ownerId]), allTransports);
2968
+ } else if (failure !== null || skippedError !== null) {
2969
+ // A sibling's copy comes from this worker, not from a relay, so no
2970
+ // route shows its failure. It is a Broadcast, which carries no
2971
+ // relay error, so an error or a skip means a bug, such as
2972
+ // databases holding different keys for the owner, and is reported
2973
+ // as an unexpected failure. A Failed result was logged, which
2974
+ // reports it already. An error is the failure, which is never an
2975
+ // abort, and like a route, the report leaves out its frame.
2976
+ const unexpected = error !== null ? failure : skippedError;
2977
+ if (unexpected !== null)
2978
+ deps.postConsoleEntryOrError({
2979
+ type: "Error",
2980
+ error: createUnknownError(
2981
+ errorToSyncRouteError(unexpected, deps.time.now()),
2982
+ ),
2983
+ });
2984
+ // Some of a sibling's messages were not stored; rounds fetch them
2985
+ // from the relays, whose routes then record a skip for any message
2986
+ // this database skips.
2987
+ requestRoundsForReceivedMessages(ownerId, {
2988
+ except: null,
2989
+ afterQueuedWrites: true,
2990
+ });
2206
2991
  }
2207
2992
 
2208
2993
  if (response.message.didWriteMessages) {
2209
2994
  refreshQueries();
2210
2995
  // Reconcile newly stored messages through each other transport.
2211
- // Rounds toward the same transport coalesce whatever their source.
2212
- const keys: Array<StructuralLookupKey> = [];
2213
- deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
2214
- if (isTargetTransport(target, transport)) return;
2215
- keys.push(structuralLookup(transport));
2996
+ requestRoundsForReceivedMessages(ownerId, {
2997
+ except: target,
2998
+ afterQueuedWrites: false,
2216
2999
  });
2217
- for (const key of keys) {
2218
- requestCreateSyncMessages(
2219
- new Set([ownerId]),
2220
- { type: "Transport", key },
2221
- { afterQueuedWrites: false },
2222
- );
2223
- }
2224
3000
  }
2225
3001
 
2226
3002
  if (result.ok) {
@@ -2299,7 +3075,11 @@ const createEvoluTenant =
2299
3075
  deps.syncRequests.noteSent(ownerId, key);
2300
3076
  const route = getRoute(ownerId, key);
2301
3077
  route.lastSentAt = deps.time.now();
2302
- if (isRound) route.roundRequired = false;
3078
+ if (isRound)
3079
+ route.progress = {
3080
+ ...routeToPending(route.progress),
3081
+ roundRequired: false,
3082
+ };
2303
3083
  },
2304
3084
  );
2305
3085
  }
@@ -2307,6 +3087,48 @@ const createEvoluTenant =
2307
3087
  if (protocolMessagesByOwnerId.size > 0) deps.refreshAllSyncRoutes();
2308
3088
  };
2309
3089
 
3090
+ /**
3091
+ * Requests a round through each transport claimed for the owner outside
3092
+ * `except`, after messages from elsewhere were stored or a sibling copy was
3093
+ * not fully stored. Rounds toward the same transport coalesce whatever
3094
+ * their source. A route that skipped a message gets none, because its relay
3095
+ * would offer that message again, and a route rechecking one stops
3096
+ * rechecking, because its round may have read the database before this
3097
+ * event. A later requested round through such a route reconciles it.
3098
+ */
3099
+ const requestRoundsForReceivedMessages = (
3100
+ ownerId: OwnerId,
3101
+ {
3102
+ except,
3103
+ afterQueuedWrites,
3104
+ }: { except: SyncTarget | null; afterQueuedWrites: boolean },
3105
+ ): void => {
3106
+ const keys: Array<StructuralLookupKey> = [];
3107
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
3108
+ if (except && isTargetTransport(except, transport)) return;
3109
+ const key = structuralLookup(transport);
3110
+ const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
3111
+ if (route?.progress.type === "Settled") return;
3112
+ if (route?.progress.type === "Pending" && route.progress.skip) {
3113
+ // A round already requested may have read the database before these
3114
+ // messages were stored, so it can no longer complete the route.
3115
+ route.progress = {
3116
+ ...route.progress,
3117
+ skip: { ...route.progress.skip, isRechecking: false },
3118
+ };
3119
+ return;
3120
+ }
3121
+ keys.push(key);
3122
+ });
3123
+ for (const key of keys) {
3124
+ requestCreateSyncMessages(
3125
+ new Set([ownerId]),
3126
+ { type: "Transport", key },
3127
+ { afterQueuedWrites },
3128
+ );
3129
+ }
3130
+ };
3131
+
2310
3132
  /** The keys of every transport claimed for the owner, by any database. */
2311
3133
  const getClaimedKeys = (
2312
3134
  ownerId: OwnerId,
@@ -2341,11 +3163,18 @@ const createEvoluTenant =
2341
3163
  if (startupError) return;
2342
3164
  const usedOwnersById = getUsedOwnersById(ownerIds);
2343
3165
  // Opening, storing messages from another transport, a failure, and an
2344
- // explicit request each require a new round before completion.
3166
+ // explicit request each require a new round before completion. The
3167
+ // round also checks again whether the relay holds a skipped message.
2345
3168
  for (const ownerId of usedOwnersById.keys()) {
2346
3169
  deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
2347
3170
  if (!isTargetTransport(target, transport)) return;
2348
- getRoute(ownerId, structuralLookup(transport)).roundRequired = true;
3171
+ const route = getRoute(ownerId, structuralLookup(transport));
3172
+ const pending = routeToPending(route.progress);
3173
+ route.progress = {
3174
+ ...pending,
3175
+ roundRequired: true,
3176
+ skip: pending.skip && { ...pending.skip, isRechecking: true },
3177
+ };
2349
3178
  });
2350
3179
  }
2351
3180
  refreshSyncRoutes();
@@ -2493,33 +3322,32 @@ const createEvoluTenant =
2493
3322
  });
2494
3323
  const tenant = disposable<EvoluTenant>(
2495
3324
  {
2496
- getSyncTenant: () => ({
2497
- name,
2498
- refused: startupError !== null,
2499
- owners: getSyncOwners().map(
2500
- ({ ownerId, writable, transportKeys }) => ({
2501
- ownerId,
2502
- writable,
2503
- transportKeys,
2504
- routes: writable
2505
- ? transportKeys.map((key): TenantSyncRoute => {
2506
- const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
2507
- // Registration and claim changes refresh routes before yielding.
2508
- // Snapshot reads must not create missing routes.
2509
- assertNotUndefined(route);
2510
- return {
2511
- transportKey: key,
2512
- complete: route.complete,
2513
- completeAt: route.completeAt,
2514
- lastSentAt: route.lastSentAt,
2515
- lastReceivedAt: route.lastReceivedAt,
2516
- error: route.error,
2517
- };
2518
- })
2519
- : [],
2520
- }),
2521
- ),
2522
- }),
3325
+ getSyncTenant: () =>
3326
+ startupError
3327
+ ? { type: "Refused", name, error: startupError }
3328
+ : {
3329
+ type: "Active",
3330
+ name,
3331
+ owners: getSyncOwners().map(
3332
+ ({ ownerId, writable, transportKeys }): TenantSyncOwner =>
3333
+ writable
3334
+ ? {
3335
+ type: "Writable",
3336
+ ownerId,
3337
+ routes: transportKeys.map((key): TenantSyncRoute => {
3338
+ const route = routesByOwnerIdByKey
3339
+ .get(key)
3340
+ ?.get(ownerId);
3341
+ // Registration and claim changes refresh routes
3342
+ // before yielding. Snapshot reads must not create
3343
+ // missing routes.
3344
+ assertNotUndefined(route);
3345
+ return { transportKey: key, ...route };
3346
+ }),
3347
+ }
3348
+ : { type: "Readonly", ownerId, transportKeys },
3349
+ ),
3350
+ },
2523
3351
 
2524
3352
  refreshSyncRoutes,
2525
3353
 
@@ -2553,19 +3381,19 @@ const createEvoluTenant =
2553
3381
 
2554
3382
  disposer.defer(instance.onDisposed);
2555
3383
 
2556
- disposer.defer(async () => {
2557
- await tenantRun(
2558
- instance.useOwnerMutex.withLock(() => {
2559
- for (const leases of instance.ownerRegistrations.values()) {
2560
- for (const lease of leases) lease?.release();
2561
- }
2562
- instance.ownerRegistrations.clear();
2563
- deps.refreshAllSyncRoutes();
2564
- deps.publishSyncState();
2565
- return ok();
2566
- }),
2567
- );
3384
+ disposer.defer(() => {
3385
+ for (const leases of instance.ownerRegistrations.values()) {
3386
+ for (const lease of leases) lease?.release();
3387
+ }
3388
+ instance.ownerRegistrations.clear();
3389
+ deps.refreshAllSyncRoutes();
3390
+ deps.publishSyncState();
2568
3391
  });
3392
+ // Cleanup starts no Task, because a root abort may dispose every Run
3393
+ // first. Disposed before the claims above are released, this Run
3394
+ // aborts queued UseOwner batches and waits for the running one, whose
3395
+ // transport claim cannot be aborted, so the release sees every lease.
3396
+ const instanceRun = disposer.use(tenantRun.create());
2569
3397
 
2570
3398
  disposer.defer(() => {
2571
3399
  instancesById.delete(instance.id);
@@ -2581,7 +3409,7 @@ const createEvoluTenant =
2581
3409
  // while it is alive. Acquiring the same lock here means the main
2582
3410
  // thread instance was disposed or its tab closed, so the tenant-side
2583
3411
  // instance must dispose itself.
2584
- void tenantRun
3412
+ void instanceRun
2585
3413
  .abortable(acquireLeaderLock(message.id))
2586
3414
  .then((lock) => {
2587
3415
  if (!lock.ok) return;
@@ -2622,7 +3450,7 @@ const createEvoluTenant =
2622
3450
  break;
2623
3451
  }
2624
3452
  case "UseOwner": {
2625
- void tenantRun(
3453
+ void instanceRun(
2626
3454
  instance.useOwnerMutex.withLock(async (run) => {
2627
3455
  for (const action of message.actions) {
2628
3456
  switch (action.action) {
@@ -2729,14 +3557,14 @@ const createEvoluTenant =
2729
3557
  // })
2730
3558
 
2731
3559
  // TODO: SharedWorker follow-ups.
2732
- // - Complete the queue head when a DbWorker mutation returns an error.
2733
3560
  // - Rotate the node ID when a copied database is detected; see the Duplicate
2734
3561
  // node IDs section in the Timestamp module.
2735
3562
  // - Detect DbWorker and port liveness so a worker-only crash resumes the queue.
2736
3563
  // Defer panicked-worker restart until failure detection and recovery are
2737
- // defined, accounting for SQLite WASM's detection limits. Normal SQLite
2738
- // operations are expected not to throw; user-defined UNIQUE indexes, which
2739
- // can make replicated writes fail, are planned to be forbidden.
3564
+ // defined, accounting for SQLite WASM's detection limits. A mutation that
3565
+ // throws is answered, but other SQLite operations are expected not to throw;
3566
+ // user-defined UNIQUE indexes, which can make replicated writes fail, are
3567
+ // planned to be forbidden.
2740
3568
  // - Split worker protocol types and the EvoluTenant implementation into focused
2741
3569
  // modules.
2742
3570
  // - Remove the obsolete commented protocol block above.