@evolu/common 8.10.0 → 8.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/dist/src/Bytes.d.ts +39 -2
  2. package/dist/src/Bytes.d.ts.map +1 -1
  3. package/dist/src/Bytes.js +50 -2
  4. package/dist/src/Config.d.ts +22 -22
  5. package/dist/src/Config.d.ts.map +1 -1
  6. package/dist/src/Console.d.ts +62 -7
  7. package/dist/src/Console.d.ts.map +1 -1
  8. package/dist/src/Console.js +20 -4
  9. package/dist/src/Crypto.d.ts +76 -4
  10. package/dist/src/Crypto.d.ts.map +1 -1
  11. package/dist/src/Crypto.js +55 -4
  12. package/dist/src/Error.d.ts +45 -0
  13. package/dist/src/Error.d.ts.map +1 -1
  14. package/dist/src/Error.js +69 -0
  15. package/dist/src/Fs.d.ts +92 -18
  16. package/dist/src/Fs.d.ts.map +1 -1
  17. package/dist/src/Fs.js +2 -0
  18. package/dist/src/Identicon.d.ts +2 -2
  19. package/dist/src/Identicon.js +2 -2
  20. package/dist/src/LeakDetector.d.ts +22 -3
  21. package/dist/src/LeakDetector.d.ts.map +1 -1
  22. package/dist/src/LeakDetector.js +12 -2
  23. package/dist/src/LockManager.d.ts +8 -0
  24. package/dist/src/LockManager.d.ts.map +1 -1
  25. package/dist/src/LockManager.js +6 -0
  26. package/dist/src/Object.d.ts.map +1 -1
  27. package/dist/src/Object.js +5 -0
  28. package/dist/src/Platform.d.ts +47 -7
  29. package/dist/src/Platform.d.ts.map +1 -1
  30. package/dist/src/Platform.js +24 -5
  31. package/dist/src/Random.d.ts +25 -2
  32. package/dist/src/Random.d.ts.map +1 -1
  33. package/dist/src/Random.js +14 -2
  34. package/dist/src/Resource.d.ts +156 -1
  35. package/dist/src/Resource.d.ts.map +1 -1
  36. package/dist/src/Resource.js +201 -72
  37. package/dist/src/Schedule.d.ts +11 -10
  38. package/dist/src/Schedule.d.ts.map +1 -1
  39. package/dist/src/Schedule.js +1 -1
  40. package/dist/src/Sqlite.d.ts +132 -16
  41. package/dist/src/Sqlite.d.ts.map +1 -1
  42. package/dist/src/Sqlite.js +63 -9
  43. package/dist/src/Task.d.ts +15 -4
  44. package/dist/src/Task.d.ts.map +1 -1
  45. package/dist/src/Task.js +41 -15
  46. package/dist/src/Test.d.ts +9 -0
  47. package/dist/src/Test.d.ts.map +1 -1
  48. package/dist/src/Test.js +4 -0
  49. package/dist/src/Time.d.ts +106 -9
  50. package/dist/src/Time.d.ts.map +1 -1
  51. package/dist/src/Time.js +55 -4
  52. package/dist/src/Type.d.ts +1455 -1310
  53. package/dist/src/Type.d.ts.map +1 -1
  54. package/dist/src/Type.js +1274 -517
  55. package/dist/src/WebSocket.d.ts +164 -13
  56. package/dist/src/WebSocket.d.ts.map +1 -1
  57. package/dist/src/WebSocket.js +133 -24
  58. package/dist/src/Worker.d.ts +90 -8
  59. package/dist/src/Worker.d.ts.map +1 -1
  60. package/dist/src/Worker.js +28 -2
  61. package/dist/src/index.d.ts +6 -7
  62. package/dist/src/index.d.ts.map +1 -1
  63. package/dist/src/index.js +2 -3
  64. package/dist/src/local-first/Db.d.ts +52 -3
  65. package/dist/src/local-first/Db.d.ts.map +1 -1
  66. package/dist/src/local-first/Db.js +412 -137
  67. package/dist/src/local-first/Evolu.d.ts +412 -213
  68. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  69. package/dist/src/local-first/Evolu.js +181 -18
  70. package/dist/src/local-first/Owner.d.ts +13 -30
  71. package/dist/src/local-first/Owner.d.ts.map +1 -1
  72. package/dist/src/local-first/Owner.js +13 -30
  73. package/dist/src/local-first/Protocol.d.ts +106 -19
  74. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  75. package/dist/src/local-first/Protocol.js +162 -60
  76. package/dist/src/local-first/Query.d.ts +8 -15
  77. package/dist/src/local-first/Query.d.ts.map +1 -1
  78. package/dist/src/local-first/Relay.d.ts.map +1 -1
  79. package/dist/src/local-first/Relay.js +4 -2
  80. package/dist/src/local-first/Schema.d.ts +346 -23
  81. package/dist/src/local-first/Schema.d.ts.map +1 -1
  82. package/dist/src/local-first/Schema.js +214 -17
  83. package/dist/src/local-first/Shared.d.ts +537 -22
  84. package/dist/src/local-first/Shared.d.ts.map +1 -1
  85. package/dist/src/local-first/Shared.js +1437 -234
  86. package/dist/src/local-first/Storage.d.ts +195 -17
  87. package/dist/src/local-first/Storage.d.ts.map +1 -1
  88. package/dist/src/local-first/Storage.js +85 -22
  89. package/dist/src/local-first/Timestamp.d.ts +392 -41
  90. package/dist/src/local-first/Timestamp.d.ts.map +1 -1
  91. package/dist/src/local-first/Timestamp.js +403 -81
  92. package/dist/src/local-first/index.d.ts +0 -1
  93. package/dist/src/local-first/index.d.ts.map +1 -1
  94. package/dist/src/local-first/index.js +0 -1
  95. package/package.json +1 -1
  96. package/src/Assert.test.ts +2 -5
  97. package/src/Bytes.test.ts +27 -0
  98. package/src/Bytes.ts +58 -2
  99. package/src/Config.test.ts +2 -6
  100. package/src/Config.ts +133 -133
  101. package/src/Console.ts +62 -7
  102. package/src/Crypto.ts +76 -4
  103. package/src/Eq.test.ts +2 -3
  104. package/src/Error.test.ts +76 -3
  105. package/src/Error.ts +71 -0
  106. package/src/Fs.ts +92 -18
  107. package/src/Identicon.ts +2 -2
  108. package/src/LeakDetector.ts +22 -3
  109. package/src/LockManager.ts +8 -0
  110. package/src/Object.test.ts +27 -12
  111. package/src/Object.ts +5 -0
  112. package/src/Platform.ts +50 -8
  113. package/src/Random.ts +25 -2
  114. package/src/Resource.test.ts +837 -0
  115. package/src/Resource.ts +235 -15
  116. package/src/Schedule.test.ts +50 -12
  117. package/src/Schedule.ts +24 -14
  118. package/src/Sqlite.ts +137 -17
  119. package/src/Task.test.ts +189 -8
  120. package/src/Task.ts +56 -17
  121. package/src/Test.ts +9 -0
  122. package/src/Time.ts +106 -9
  123. package/src/Type.test.ts +946 -1028
  124. package/src/Type.ts +4195 -3136
  125. package/src/Types.test.ts +4 -14
  126. package/src/WebSocket.ts +313 -40
  127. package/src/Worker.ts +90 -8
  128. package/src/index.ts +20 -6
  129. package/src/local-first/Db.ts +644 -339
  130. package/src/local-first/Evolu.test.ts +994 -22
  131. package/src/local-first/Evolu.ts +625 -232
  132. package/src/local-first/Owner.ts +13 -30
  133. package/src/local-first/Protocol.test.ts +634 -10
  134. package/src/local-first/Protocol.ts +255 -109
  135. package/src/local-first/Query.ts +8 -15
  136. package/src/local-first/Relay.ts +4 -2
  137. package/src/local-first/Schema.test.ts +143 -0
  138. package/src/local-first/Schema.ts +376 -26
  139. package/src/local-first/Shared.test.ts +7731 -559
  140. package/src/local-first/Shared.ts +2036 -267
  141. package/src/local-first/Storage.ts +224 -36
  142. package/src/local-first/Timestamp.test.ts +344 -70
  143. package/src/local-first/Timestamp.ts +434 -118
  144. package/src/local-first/index.ts +0 -1
  145. package/dist/src/local-first/Error.d.ts +0 -12
  146. package/dist/src/local-first/Error.d.ts.map +0 -1
  147. package/dist/src/local-first/Error.js +0 -6
  148. package/dist/src/local-first/LocalAuth.d.ts +0 -150
  149. package/dist/src/local-first/LocalAuth.d.ts.map +0 -1
  150. package/dist/src/local-first/LocalAuth.js +0 -179
  151. package/src/local-first/Error.ts +0 -17
  152. package/src/local-first/LocalAuth.ts +0 -457
@@ -1,6 +1,213 @@
1
1
  /**
2
2
  * Platform-agnostic Evolu SharedWorker.
3
3
  *
4
+ * ## Builds
5
+ *
6
+ * Tabs of different app builds can be open at once, for example an old tab
7
+ * during a deploy. Bundlers derive the worker script's URL from its content, so
8
+ * every build whose worker code differs gets its own SharedWorker, and all of
9
+ * them open the same databases. Only one may use them at a time: a request
10
+ * retried after another worker wrote from the same stored clock can reuse that
11
+ * write's timestamps and then is skipped as already stored.
12
+ *
13
+ * The worker therefore takes an origin-wide lock before it answers any tab and
14
+ * holds it for its lifetime. A worker of another build waits, with its tabs'
15
+ * messages buffered, until every tab of the first one is closed or reloaded and
16
+ * the browser ends it. The lock is the one that earlier releases take in their
17
+ * leader tab, so they are excluded too. An earlier release's worker can outlive
18
+ * its leader tab and resume once the lock is free; it computes new timestamps
19
+ * when it retries, so it cannot reuse another worker's. A tab of an earlier
20
+ * release that opens while a worker of this release runs gets no response to
21
+ * its database requests until it reloads, and until then it also blocks workers
22
+ * that start after this one ends.
23
+ *
24
+ * On the web, the wait is usually short, because tabs of the running build
25
+ * reload to load the build the server now serves:
26
+ *
27
+ * 1. A worker tells the tabs that connect before it holds the lock that they wait,
28
+ * and such a tab announces the worker with {@link BuildWaiting}. It announces
29
+ * again when a tab that connects asks with {@link BuildWaitingRequest}, so a
30
+ * tab that started or connected after the first announcement learns of it
31
+ * too.
32
+ * 2. A tab connected to another worker reloads with {@link ReloadApp}: at once if
33
+ * the user is not in it, otherwise once they leave it, so a tab never
34
+ * reloads while the user works in it. Focus can leave a page from a frame
35
+ * without a window event, so such a tab checks once a second whether it
36
+ * still has focus. A tab that still waits itself keeps the announcements it
37
+ * receives and handles them once it connects, because the lock can pass to
38
+ * its worker first.
39
+ * 3. A page that such a reload loaded never announces, and a tab reloads at most
40
+ * once for each waiting worker, so two builds cannot keep reloading each
41
+ * other, even when a reload loads the running build again. A tab without
42
+ * session storage cannot record its reloads, so it does not reload.
43
+ *
44
+ * Only the web platform does this, because only there do builds coexist. A
45
+ * reload loses UI state the app did not persist, so apps keep drafts in
46
+ * local-only tables. A write a background tab has in flight can be lost, as
47
+ * when a tab crashes.
48
+ *
49
+ * The wait lasts while a tab of the running build does not reload: a tab of an
50
+ * earlier release, a tab Safari has frozen, a tab without session storage, or a
51
+ * tab whose reload loads the running build again, for example because another
52
+ * Evolu app shares the origin or a cache still serves the old build. A worker
53
+ * still waiting after three seconds reports {@link OtherBuildRunningError} to
54
+ * its tabs. Every build broadcasts console entries and errors on
55
+ * {@link consoleEntryOrErrorBroadcastChannelName}, so a waiting tab also prints
56
+ * the running build's output and reports its errors as its own `evoluError`.
57
+ *
58
+ * The tabs of one worker elect the host of its DbWorkers among themselves, with
59
+ * a lock scoped to the worker, so a tab of another worker never hosts them.
60
+ * Builds that differ only in DbWorker code share one SharedWorker, and a tab of
61
+ * either can host its DbWorkers.
62
+ *
63
+ * Safari suspends, rather than ends, a worker whose tabs are all in its
64
+ * back-forward cache, and the suspended worker keeps the lock. There, a waiting
65
+ * build also waits until Safari drops those pages, or until the user goes back
66
+ * to one of them, which reloads it on the web.
67
+ *
68
+ * ## Storage
69
+ *
70
+ * A platform that can lack persistent storage, as a browser does in Safari's
71
+ * Private Browsing, provides {@link PersistentStorageDep}. The worker checks it
72
+ * once, after it takes the build lock and before any DbWorker starts. Without
73
+ * persistent storage, every DbWorker it starts keeps its database in memory,
74
+ * replacements included, and each tab that connects is told with
75
+ * `StorageUnavailable`. The decision holds for the worker's lifetime, so all
76
+ * its tabs see one mode, and the next worker checks again.
77
+ *
78
+ * The check cannot tell a private session from a storage failure, but memory
79
+ * loses nothing that refusing to start would have kept, and the persistent
80
+ * database stays untouched. Each DbWorker keeps its own memory, so when the tab
81
+ * hosting it closes, or on the web navigates away, data that exists only
82
+ * locally or has not synced yet is lost, even though its replacement starts in
83
+ * memory too.
84
+ *
85
+ * ## Synchronization routing
86
+ *
87
+ * WebSocket transports are shared resources keyed by their configuration and
88
+ * claimed by owner ID, so every tenant (one named local database) using an
89
+ * owner shares that owner's sockets, whichever tenant claimed them. An incoming
90
+ * frame is offered to every tenant with a writable registration for its owner,
91
+ * and each tenant reconciles independently: the protocol exchange is stateless
92
+ * per message and message writes are idempotent, so a response that answered
93
+ * another tenant's request is still a valid reconciliation step.
94
+ *
95
+ * Traffic goes only where it is needed. A continuation returns to the transport
96
+ * that produced the response, and a round started by a socket opening or by a
97
+ * tenant's first use of a transport for an owner goes through that transport.
98
+ * Explicit synchronization requests and mutation uploads go to every open
99
+ * transport claimed for the owner. A write uploads through the database's
100
+ * writable registrations for its owner, whichever instance made it, even one
101
+ * disposed before the database worker answered. When a relay frame stores new
102
+ * messages, the tenant requests a round through each other transport claimed
103
+ * for the owner, so data learned from one relay reaches the others. A closed
104
+ * transport reconciles when it opens, and a replacement leader reconciles every
105
+ * transport again, because a response reporting stored messages may have been
106
+ * lost.
107
+ *
108
+ * Relays omit the sending socket when broadcasting an upload, so the uploader
109
+ * also delivers it as local Broadcast frames to every other tenant with
110
+ * writable access to the owner, even while sockets are closed. A copy keeps the
111
+ * uploader's target, and a recipient that stores new continuation messages
112
+ * reconciles them through the transports outside that target. Local delivery
113
+ * forwards uploads, including historical messages sent during reconciliation,
114
+ * but does not reconcile local database histories with each other. That waits
115
+ * until replication scopes and retention semantics are defined.
116
+ *
117
+ * ## Sync state
118
+ *
119
+ * The shared worker publishes one plain snapshot, {@link SyncState}, of every
120
+ * transport it manages and every database and owner registration it holds, with
121
+ * one route per writable registration and transport, as specified below.
122
+ * {@link syncStateToOwnerSyncStates} derives one state per database and owner.
123
+ *
124
+ * Each worker broadcasts snapshots on its own channel, whose name a connecting
125
+ * tab receives through its port, so a tab never hears another worker, such as
126
+ * one of a different app version, and
127
+ * {@link SyncStateDep.syncState | deps.syncState} keeps the last snapshot. The
128
+ * worker publishes after every change it observes; a transition without an
129
+ * event, such as a closed socket starting to reconnect, appears with the next
130
+ * snapshot. The snapshot lives in worker memory only, so a new worker starts
131
+ * empty.
132
+ *
133
+ * ## Synchronization completion
134
+ *
135
+ * A protocol frame carries the owner ID and the message type but nothing that
136
+ * correlates it with a request, and a relay answers a converged round, an
137
+ * upload that fits one frame, and an Unsubscribe alike with a header-only
138
+ * Response. Every tenant with a writable registration applies every frame for
139
+ * its owner, so a tenant cannot tell which response answered its own request.
140
+ *
141
+ * Completion is therefore counted, not attributed. The shared worker counts
142
+ * outstanding requests per owner and socket: every Request sent on an open
143
+ * socket, including an Unsubscribe, increments the count; every Response,
144
+ * including a relay's version-mismatch reply, which has no message type,
145
+ * decrements it; and an opening socket resets it, because requests on the
146
+ * previous connection are never answered. A route, one tenant's use of one
147
+ * owner through one transport, is complete when:
148
+ *
149
+ * - The tenant has not refused startup and the socket is open.
150
+ * - The count is zero.
151
+ * - The tenant has no apply for the owner queued for that transport or for every
152
+ * transport, because a frame is applied asynchronously, so the count reads
153
+ * zero in the middle of a chain.
154
+ * - The tenant has no replicated write for the owner queued, because its upload
155
+ * is sent only after the database worker answers it.
156
+ * - The tenant has sent a round through the transport since the last event that
157
+ * requires one: its first use of the transport for the owner, the socket
158
+ * opening, an explicit request, a replacement leader, or storing messages
159
+ * from another transport. No failed or aborted result has arrived on the
160
+ * route since.
161
+ *
162
+ * A reconciliation chain ends only with a converged result, a failure, an
163
+ * abort, a dropped frame, or a continuation that finds the socket closed, and
164
+ * relay errors reach every applying tenant, so these conditions mean every
165
+ * chain, including this tenant's, converged. A local Broadcast from a sibling
166
+ * tenant holds every route of its owner until it is applied, because it arrives
167
+ * without a request of its own; one that fails to apply requires a round
168
+ * through every transport.
169
+ *
170
+ * A failed result on a route requests one round through it. Any further failure
171
+ * before the route completes waits for an explicit request or a reopen, so a
172
+ * persistent failure cannot loop; a converged reply in between does not end the
173
+ * wait, because it may answer another tenant's round on the shared socket. An
174
+ * aborted apply leaves its routes incomplete without a retry. An exception
175
+ * while the database worker creates a round is logged there and fails the
176
+ * round's routes with `SyncFailed` without a retry; other unexpected SQLite
177
+ * exceptions remain unsupported and can panic the database worker. A frame the
178
+ * relay silently drops, such as invalid data, leaves the count above zero until
179
+ * the liveness rule below replaces the socket.
180
+ *
181
+ * ### Liveness
182
+ *
183
+ * An open socket can be dead without a close event, when a NAT drops an idle
184
+ * mapping or the path fails while nothing is sent. The relay pings every
185
+ * connection and terminates one from which nothing has arrived since the
186
+ * previous ping, which also keeps NAT mappings alive. The shared worker
187
+ * reconnects a socket when a request for an owner has been outstanding for
188
+ * ninety seconds, enough for a 1 MB frame at about 90 kbit/s, with no Response
189
+ * for that owner since. Frames for other owners do not count: they prove the
190
+ * socket alive, not that the request was received.
191
+ *
192
+ * A slow link looks like a dead one, because the browser reports no transfer
193
+ * progress and a large frame saves nothing until it arrives whole. Each timeout
194
+ * therefore doubles the transport's timeout, up to twenty-four minutes: a
195
+ * connection that reopens but times out again is more likely slow than dead.
196
+ * The timeout belongs to the transport, not to an owner, because a small reply
197
+ * for one owner can wait behind another owner's large frame on the socket. A
198
+ * grown timeout lasts while a request is outstanding on the socket or a
199
+ * database that has not refused startup has an incomplete route through it,
200
+ * including one waiting after a failure, and ends with the transport. A reply's
201
+ * speed proves nothing, because a recovery on a slow link starts with small
202
+ * replies that arrive quickly. The reopen resets the counts and starts the open
203
+ * rounds, so a dropped frame delays a route instead of stranding it.
204
+ *
205
+ * The shared worker sends nothing while idle, so a path that dies then may go
206
+ * unnoticed until its next request; until then, changes from other devices stop
207
+ * arriving while routes still read complete. A periodic empty request would
208
+ * notice it without waiting for the app to send. That is deferred, because
209
+ * relay pings already keep NAT mappings alive, the common cause.
210
+ *
4
211
  * @module
5
212
  */
6
213
  var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
@@ -56,20 +263,180 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
56
263
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
57
264
  });
58
265
  import { emptyArray, firstInArray, isNonEmptyArray, } from "../Array.js";
59
- import { assertNonNullable, assertNotSame, assertNotUndefined, } from "../Assert.js";
60
- import { createCallbacks } from "../Callbacks.js";
61
- import { disposable } from "../Function.js";
266
+ import { assert, assertNonNullable, assertNotSame, assertNotUndefined, assertSame, } from "../Assert.js";
267
+ import { disposable, exhaustiveCheck } from "../Function.js";
62
268
  import { acquireLeaderLock } from "../LockManager.js";
63
269
  import { createLookupMap, structuralLookup, } from "../Lookup.js";
64
- import { createRefCountByKey } from "../RefCount.js";
270
+ import { createRefCountedRelation } from "../Relation.js";
65
271
  import { createSharedResourceByKey, createSharedResourceByKeyWithClaims, } from "../Resource.js";
66
272
  import { ok } from "../Result.js";
67
273
  import { createStore } from "../Store.js";
68
274
  import { AbortError, createMutex, unabortable, } from "../Task.js";
69
- import { createProtocolMessageForUnsubscribe, createProtocolMessageFromCrdtMessages, parseProtocolHeader, } from "./Protocol.js";
275
+ import { performanceDurationBetween, PositiveMillis, } from "../Time.js";
276
+ import { createId, id, literal, object, } from "../Type.js";
277
+ import { createProtocolBroadcastMessagesFromCrdtMessages, createProtocolMessageForUnsubscribe, createProtocolMessageFromCrdtMessages, MessageType, parseProtocolHeader, } from "./Protocol.js";
70
278
  import { makePatches, } from "./Query.js";
279
+ import { isLocalOnlyTable } from "./Schema.js";
280
+ import { orderTimestamp } from "./Timestamp.js";
71
281
  export const consoleEntryOrErrorBroadcastChannelName = "evolu:console-entry-or-error";
72
- /** Initializes the platform-agnostic Evolu SharedWorker. */
282
+ /** Identifies one running SharedWorker instance. */
283
+ export const SharedWorkerId = /*#__PURE__*/ id("SharedWorker");
284
+ /**
285
+ * The channel on which a tab announces its waiting worker with
286
+ * {@link BuildWaiting}; see Builds. Builds of different releases share it, so
287
+ * its name and messages never change.
288
+ */
289
+ export const buildsBroadcastChannelName = "evolu:builds";
290
+ /**
291
+ * Posted on {@link buildsBroadcastChannelName} by a tab whose worker waits for
292
+ * the build lock, unless an automatic reload loaded the page, when it starts
293
+ * waiting and for each {@link BuildWaitingRequest}. A tab connected to another
294
+ * worker reloads for it; see Builds.
295
+ */
296
+ export const BuildWaiting = /*#__PURE__*/ object({
297
+ type: /*#__PURE__*/ literal("BuildWaiting"),
298
+ workerId: SharedWorkerId,
299
+ });
300
+ /**
301
+ * Posted on {@link buildsBroadcastChannelName} by a tab once it connects, asking
302
+ * tabs whose worker waits to post {@link BuildWaiting} again; see Builds.
303
+ */
304
+ export const BuildWaitingRequest = /*#__PURE__*/ object({
305
+ type: /*#__PURE__*/ literal("BuildWaitingRequest"),
306
+ });
307
+ /**
308
+ * Folds the routes of every writable owner registration of every running
309
+ * database in a {@link SyncState} into one {@link OwnerSyncState} per database
310
+ * and owner, which pairs each route with its transport as a
311
+ * {@link RelaySyncState}. A database that refused startup and a readonly
312
+ * registration synchronize nothing, so they are left out.
313
+ *
314
+ * ### Example
315
+ *
316
+ * ```ts
317
+ * import {
318
+ * assertEqual,
319
+ * createId,
320
+ * Millis,
321
+ * testCreateDeps,
322
+ * testName,
323
+ * } from "@evolu/common";
324
+ * import {
325
+ * syncStateToOwnerSyncStates,
326
+ * testAppOwner,
327
+ * type SyncRoute,
328
+ * type SyncState,
329
+ * type SyncTransport,
330
+ * } from "@evolu/common/local-first";
331
+ *
332
+ * const deps = testCreateDeps();
333
+ * const transport: SyncTransport = {
334
+ * id: createId<"SyncTransport">(deps),
335
+ * label: "wss://relay.example",
336
+ * readyState: "open",
337
+ * openedAt: null,
338
+ * closedAt: null,
339
+ * error: null,
340
+ * };
341
+ * const route: SyncRoute = {
342
+ * transportId: transport.id,
343
+ * complete: true,
344
+ * completeAt: Millis.orThrow(1000),
345
+ * lastSentAt: Millis.orThrow(900),
346
+ * lastReceivedAt: Millis.orThrow(1000),
347
+ * error: null,
348
+ * };
349
+ * const state: SyncState = {
350
+ * transports: [transport],
351
+ * tenants: [
352
+ * {
353
+ * name: testName,
354
+ * refused: false,
355
+ * owners: [
356
+ * {
357
+ * ownerId: testAppOwner.id,
358
+ * writable: true,
359
+ * transportIds: [transport.id],
360
+ * routes: [route],
361
+ * },
362
+ * ],
363
+ * },
364
+ * ],
365
+ * };
366
+ *
367
+ * assertEqual(syncStateToOwnerSyncStates(state), [
368
+ * {
369
+ * name: testName,
370
+ * ownerId: testAppOwner.id,
371
+ * status: "synced",
372
+ * syncedAt: Millis.orThrow(1000),
373
+ * error: null,
374
+ * relays: [{ transport, route, status: "synced" }],
375
+ * },
376
+ * ]);
377
+ * ```
378
+ */
379
+ export const syncStateToOwnerSyncStates = (state) => {
380
+ const transportById = new Map(state.transports.map((transport) => [transport.id, transport]));
381
+ return state.tenants.flatMap(({ name, refused, owners }) => refused
382
+ ? []
383
+ : owners.flatMap(({ ownerId, writable, routes }) => {
384
+ if (!writable)
385
+ return [];
386
+ let syncedAt = null;
387
+ let error = null;
388
+ const relays = [];
389
+ for (const route of routes) {
390
+ if (route.completeAt !== null &&
391
+ (syncedAt === null || route.completeAt > syncedAt))
392
+ syncedAt = route.completeAt;
393
+ if (route.error !== null &&
394
+ (error === null || route.error.at > error.at))
395
+ error = route.error;
396
+ // A snapshot lists the transport of every route.
397
+ const transport = transportById.get(route.transportId);
398
+ assertNonNullable(transport);
399
+ relays.push({
400
+ transport,
401
+ route,
402
+ status: route.error !== null
403
+ ? "error"
404
+ : route.complete
405
+ ? "synced"
406
+ : transport.readyState === "open"
407
+ ? "syncing"
408
+ : "offline",
409
+ });
410
+ }
411
+ const status = ["error", "syncing", "synced", "offline"].find((candidate) => relays.some((relay) => relay.status === candidate)) ?? "initial";
412
+ return [{ name, ownerId, status, syncedAt, error, relays }];
413
+ }));
414
+ };
415
+ /**
416
+ * How long an unanswered request keeps a socket before it is replaced.
417
+ *
418
+ * A reply cannot arrive before the whole request has been received, and a reply
419
+ * can itself be a large frame. The browser reports no transfer progress, so the
420
+ * timeout must cover a 1 MB frame on a slow link: 90 seconds allows about 90
421
+ * kbit/s.
422
+ */
423
+ const syncRequestTimeout = PositiveMillis.orThrow(90_000);
424
+ /**
425
+ * The longest a transport's timeout grows after repeated timeouts: 16 times
426
+ * {@link syncRequestTimeout}, which covers a 1 MB frame at about 6 kbit/s.
427
+ */
428
+ const maxSyncRequestTimeout = PositiveMillis.orThrow(16 * syncRequestTimeout);
429
+ const allTransports = { type: "AllTransports" };
430
+ const isTargetTransport = (target, transport) => target.type === "AllTransports" || target.key === structuralLookup(transport);
431
+ // Long enough for another build's tabs to reload and its worker to end.
432
+ const otherBuildRunningReportDelay = "3s";
433
+ /**
434
+ * Initializes the platform-agnostic Evolu SharedWorker.
435
+ *
436
+ * The worker holds the build lock until it is disposed, so a platform connects
437
+ * every `createEvoluDeps` call in a JS runtime to one worker, as React Native
438
+ * does; see Builds.
439
+ */
73
440
  export const initSharedWorker = (self) => async (run) => {
74
441
  const env_1 = { stack: [], error: void 0, hasError: false };
75
442
  try {
@@ -81,9 +448,39 @@ export const initSharedWorker = (self) => async (run) => {
81
448
  const postConsoleEntryOrError = (output) => {
82
449
  consoleEntryOrErrorBroadcastChannel.postMessage(output);
83
450
  };
451
+ const workerId = createId(deps);
452
+ // Each worker broadcasts on its own channel, so a tab hears only the
453
+ // worker its port connects to.
454
+ const syncStateChannelName = `evolu:sync-state:${workerId}`;
455
+ const syncStateBroadcastChannel = disposer.use(deps.createBroadcastChannel(syncStateChannelName));
84
456
  const sharedWorkerReady = Promise.withResolvers();
457
+ // Until this worker holds the build lock, it tells connecting tabs that they
458
+ // wait, and reports a lasting wait to them; see Builds.
459
+ const starting = disposer.use(new DisposableStack());
460
+ const waitingTabPorts = [];
461
+ let isOtherBuildRunning = false;
462
+ const reportOtherBuildRunning = (port) => {
463
+ port.postMessage({
464
+ type: "Error",
465
+ error: { type: "OtherBuildRunningError" },
466
+ });
467
+ };
468
+ const otherBuildRunningTimeoutId = deps.time.setTimeout(() => {
469
+ isOtherBuildRunning = true;
470
+ for (const port of waitingTabPorts)
471
+ reportOtherBuildRunning(port);
472
+ }, otherBuildRunningReportDelay);
473
+ starting.defer(() => {
474
+ deps.time.clearTimeout(otherBuildRunningTimeoutId);
475
+ });
85
476
  // Register ASAP so the worker does not miss connections.
86
477
  self.onConnect = (port) => {
478
+ if (!starting.disposed) {
479
+ port.postMessage({ type: "Waiting", workerId });
480
+ waitingTabPorts.push(port);
481
+ if (isOtherBuildRunning)
482
+ reportOtherBuildRunning(port);
483
+ }
87
484
  void sharedWorkerReady.promise.then(() => {
88
485
  // The underlying port buffers messages until onMessage is assigned.
89
486
  port.onMessage = (message) => {
@@ -94,10 +491,14 @@ export const initSharedWorker = (self) => async (run) => {
94
491
  console.info("tabLeaderAnnounced");
95
492
  break;
96
493
  }
494
+ case "RequestSyncState": {
495
+ publishSyncState();
496
+ break;
497
+ }
97
498
  case "CreateEvolu": {
98
499
  void sharedWorkerRun(async (run) => {
99
500
  const tenantLease = await run.ok(unabortable(tenantsByName.acquire(message)));
100
- tenantLease.resource.addInstance(message, () => {
501
+ tenantLease.resource.addInstance(message, port, () => {
101
502
  tenantLease.release();
102
503
  });
103
504
  return ok();
@@ -108,69 +509,353 @@ export const initSharedWorker = (self) => async (run) => {
108
509
  console.error("Unknown shared worker input", message);
109
510
  }
110
511
  };
512
+ port.postMessage({
513
+ type: "Connected",
514
+ workerId,
515
+ syncStateChannelName,
516
+ });
517
+ if (isPersistentStorageUnavailable) {
518
+ port.postMessage({ type: "StorageUnavailable" });
519
+ }
111
520
  });
112
521
  };
522
+ // Released after every tenant and DbWorker is disposed. Earlier releases
523
+ // take the same lock in their leader tab; see Builds.
524
+ disposer.use(await run.ok(acquireLeaderLock("tab")));
525
+ starting.dispose();
526
+ // Checked once, before any DbWorker starts, so every DbWorker of this
527
+ // worker, replacements included, keeps its database in memory; see
528
+ // Storage.
529
+ const isPersistentStorageUnavailable = deps.isPersistentStorageAvailable !== undefined &&
530
+ !(await deps.isPersistentStorageAvailable());
113
531
  disposer.defer(deps.consoleStoreOutputEntry.subscribe(() => {
114
532
  const entry = deps.consoleStoreOutputEntry.get();
115
533
  if (entry)
116
534
  postConsoleEntryOrError({ type: "ConsoleEntry", entry });
117
535
  }));
118
536
  const currentTenantsByName = new Map();
119
- const transports = disposer.use(await run.ok(createSharedResourceByKeyWithClaims((transport) => deps.createWebSocket(transport.url, {
120
- binaryType: "arraybuffer",
121
- onOpen: () => {
122
- const ownerIds = transports.getClaimsForResource(transport);
123
- console.debug("transportOpen", {
124
- url: transport.url,
125
- ownerIds: [...ownerIds],
126
- });
127
- forEachTenant((tenant) => {
128
- tenant.requestCreateSyncMessages(ownerIds);
129
- });
130
- },
131
- onMessage(data) {
132
- if (!(data instanceof ArrayBuffer))
537
+ const transportsByKey = new Map();
538
+ let isDisposed = false;
539
+ disposer.defer(() => {
540
+ isDisposed = true;
541
+ });
542
+ // Changes within one task publish once.
543
+ let isPublishScheduled = false;
544
+ const publishSyncState = () => {
545
+ if (isDisposed || isPublishScheduled)
546
+ return;
547
+ isPublishScheduled = true;
548
+ queueMicrotask(() => {
549
+ isPublishScheduled = false;
550
+ if (isDisposed)
133
551
  return;
134
- const message = new Uint8Array(data);
135
- const headerResult = parseProtocolHeader(message);
136
- if (!headerResult.ok) {
137
- console.debug("transportInvalidProtocolMessage", {
138
- url: transport.url,
139
- byteLength: message.byteLength,
140
- });
552
+ const transports = [...transportsByKey.values()].map(({ id, label, socket, openedAt, closedAt, error, }) => ({
553
+ id,
554
+ label,
555
+ // The transport drops its socket before disposal, so this never
556
+ // reads a disposed one.
557
+ readyState: socket?.getReadyState() ?? "connecting",
558
+ openedAt,
559
+ closedAt,
560
+ error,
561
+ }));
562
+ const tenants = [...currentTenantsByName.values()].map((tenant) => {
563
+ const { name, refused, owners } = tenant.getSyncTenant();
564
+ return {
565
+ name,
566
+ refused,
567
+ owners: owners.map(({ ownerId, writable, transportKeys, routes }) => ({
568
+ ownerId,
569
+ writable,
570
+ transportIds: transportKeys.flatMap((key) => {
571
+ const entry = transportsByKey.get(key);
572
+ return entry ? [entry.id] : [];
573
+ }),
574
+ routes: routes.flatMap(({ transportKey, ...route }) => {
575
+ const entry = transportsByKey.get(transportKey);
576
+ return entry ? [{ transportId: entry.id, ...route }] : [];
577
+ }),
578
+ })),
579
+ };
580
+ });
581
+ // A grown timeout lasts while a request is outstanding on the socket
582
+ // or a database that has not refused startup has an incomplete route
583
+ // through it. Every change to either publishes, including a route that
584
+ // goes away without completing.
585
+ const incompleteTransportIds = new Set();
586
+ for (const { refused, owners } of tenants) {
587
+ if (refused)
588
+ continue;
589
+ for (const { routes } of owners)
590
+ for (const { transportId, complete } of routes)
591
+ if (!complete)
592
+ incompleteTransportIds.add(transportId);
593
+ }
594
+ for (const entry of transportsByKey.values())
595
+ if (entry.outstandingByOwnerId.size === 0 &&
596
+ !incompleteTransportIds.has(entry.id))
597
+ entry.timeout = syncRequestTimeout;
598
+ syncStateBroadcastChannel.postMessage({ transports, tenants });
599
+ });
600
+ };
601
+ // Socket counters and owner claims are shared by all tenants. Refresh
602
+ // after their complete event, independently of snapshot publication.
603
+ const refreshAllSyncRoutes = () => {
604
+ for (const tenant of currentTenantsByName.values())
605
+ tenant.refreshSyncRoutes();
606
+ };
607
+ const clearSyncRequestTimeout = (entry) => {
608
+ if (entry.timeoutId === null)
609
+ return;
610
+ deps.time.clearTimeout(entry.timeoutId);
611
+ entry.timeoutId = null;
612
+ };
613
+ const armSyncRequestTimeout = (entry, delay) => {
614
+ entry.timeoutId = deps.time.setTimeout(() => {
615
+ entry.timeoutId = null;
616
+ // Frames for other owners prove the socket alive, not that this
617
+ // owner's request was received, so each owner's silence is measured
618
+ // separately.
619
+ const now = deps.time.performance.now();
620
+ let shortestRemaining = null;
621
+ let isTimedOut = false;
622
+ for (const { silentSince } of entry.outstandingByOwnerId.values()) {
623
+ const remaining = entry.timeout - performanceDurationBetween(silentSince, now);
624
+ if (remaining <= 0)
625
+ isTimedOut = true;
626
+ else if (shortestRemaining === null || remaining < shortestRemaining)
627
+ shortestRemaining = remaining;
628
+ }
629
+ if (!isTimedOut) {
630
+ if (shortestRemaining === null)
631
+ return;
632
+ // Timer delays use integer milliseconds; round up to wait at least
633
+ // the remaining fractional duration.
634
+ armSyncRequestTimeout(entry, PositiveMillis.orThrow(Math.ceil(shortestRemaining)));
141
635
  return;
142
636
  }
143
- console.debug("transportProtocolMessage", {
144
- url: transport.url,
145
- ownerId: headerResult.value.ownerId,
146
- byteLength: message.byteLength,
147
- });
148
- forEachTenant((tenant) => {
149
- tenant.requestApplySyncMessage(headerResult.value.ownerId, message);
150
- });
151
- },
152
- }), {
153
- onFirstClaimAdded: (ownerId, webSocket) => {
154
- if (!webSocket.isOpen())
637
+ // A request went unanswered for the whole timeout: it was lost, the
638
+ // connection is dead, or the link is too slow for the frames ahead of
639
+ // its reply. A frame that never arrives whole saves nothing, so on a
640
+ // slow link the same reply would time out forever. Doubling the
641
+ // timeout lets it arrive after a reconnect that opens fine.
642
+ entry.timeout = PositiveMillis.orThrow(Math.min(entry.timeout * 2, maxSyncRequestTimeout));
643
+ // Reconnecting abandons the connection and starts a fresh retry
644
+ // schedule. The socket reports no close for it, so the transport
645
+ // records the moment here.
646
+ entry.socket?.reconnect();
647
+ // `now` is monotonic; the reported close time is wall clock.
648
+ entry.closedAt = deps.time.now();
649
+ refreshAllSyncRoutes();
650
+ publishSyncState();
651
+ }, delay);
652
+ };
653
+ const syncRequests = {
654
+ noteSent: (ownerId, key) => {
655
+ const entry = transportsByKey.get(key);
656
+ if (!entry)
155
657
  return;
156
- forEachTenant((tenant) => {
157
- tenant.requestCreateSyncMessages(new Set([ownerId]));
658
+ const outstanding = entry.outstandingByOwnerId.get(ownerId);
659
+ if (outstanding)
660
+ outstanding.count++;
661
+ else
662
+ entry.outstandingByOwnerId.set(ownerId, {
663
+ count: 1,
664
+ silentSince: deps.time.performance.now(),
665
+ });
666
+ if (entry.timeoutId === null)
667
+ armSyncRequestTimeout(entry, entry.timeout);
668
+ publishSyncState();
669
+ },
670
+ getOutstanding: (ownerId, key) => transportsByKey.get(key)?.outstandingByOwnerId.get(ownerId)?.count ?? 0,
671
+ };
672
+ const transports = disposer.use(await run.ok(createSharedResourceByKeyWithClaims((transport) => async (run) => {
673
+ const env_2 = { stack: [], error: void 0, hasError: false };
674
+ try {
675
+ const key = structuralLookup(transport);
676
+ // The query carries the owner ID; WebSocket accepts URLs that
677
+ // URL() rejects, so the label is derived without parsing.
678
+ const queryIndex = transport.url.indexOf("?");
679
+ const entry = {
680
+ id: createId(run.deps),
681
+ label: queryIndex === -1
682
+ ? transport.url
683
+ : transport.url.slice(0, queryIndex),
684
+ outstandingByOwnerId: new Map(),
685
+ timeout: syncRequestTimeout,
686
+ socket: null,
687
+ openedAt: null,
688
+ closedAt: null,
689
+ error: null,
690
+ timeoutId: null,
691
+ };
692
+ const disposer = __addDisposableResource(env_2, new AsyncDisposableStack(), true);
693
+ // Registered before the socket, so an aborted creation also
694
+ // forgets the transport.
695
+ disposer.defer(() => {
696
+ transportsByKey.delete(key);
697
+ refreshAllSyncRoutes();
698
+ publishSyncState();
699
+ });
700
+ transportsByKey.set(key, entry);
701
+ publishSyncState();
702
+ const socket = await run.ok(deps.createWebSocket(transport.url, {
703
+ binaryType: "arraybuffer",
704
+ onOpen: () => {
705
+ // Requests in flight on the previous connection are never
706
+ // answered.
707
+ entry.outstandingByOwnerId.clear();
708
+ clearSyncRequestTimeout(entry);
709
+ entry.openedAt = run.deps.time.now();
710
+ publishSyncState();
711
+ const ownerIds = transports.getClaimsForResource(transport);
712
+ console.debug("transportOpen", {
713
+ url: transport.url,
714
+ ownerIds: [...ownerIds],
715
+ });
716
+ const target = {
717
+ type: "Transport",
718
+ key: structuralLookup(transport),
719
+ };
720
+ forEachTenant((tenant) => {
721
+ tenant.requestCreateSyncMessages(ownerIds, target);
722
+ });
723
+ refreshAllSyncRoutes();
724
+ },
725
+ onClose: (event) => {
726
+ console.debug("transportClose", {
727
+ url: transport.url,
728
+ code: event.code,
729
+ wasClean: event.wasClean,
730
+ });
731
+ entry.closedAt = run.deps.time.now();
732
+ clearSyncRequestTimeout(entry);
733
+ refreshAllSyncRoutes();
734
+ publishSyncState();
735
+ },
736
+ onError: (error) => {
737
+ console.debug("transportError", {
738
+ url: transport.url,
739
+ type: error.type,
740
+ });
741
+ entry.error = {
742
+ type: error.type,
743
+ at: run.deps.time.now(),
744
+ };
745
+ refreshAllSyncRoutes();
746
+ publishSyncState();
747
+ },
748
+ onMessage(data) {
749
+ if (!(data instanceof ArrayBuffer))
750
+ return;
751
+ const message = new Uint8Array(data);
752
+ const headerResult = parseProtocolHeader(message);
753
+ if (!headerResult.ok) {
754
+ console.debug("transportInvalidProtocolMessage", {
755
+ url: transport.url,
756
+ byteLength: message.byteLength,
757
+ });
758
+ return;
759
+ }
760
+ console.debug("transportProtocolMessage", {
761
+ url: transport.url,
762
+ ownerId: headerResult.value.ownerId,
763
+ byteLength: message.byteLength,
764
+ });
765
+ // A Response, or a version-mismatch reply without a message
766
+ // type, answers a request.
767
+ const { messageType } = headerResult.value;
768
+ if (messageType === MessageType.Response ||
769
+ messageType === undefined) {
770
+ const { ownerId } = headerResult.value;
771
+ const outstanding = entry.outstandingByOwnerId.get(ownerId);
772
+ if (outstanding) {
773
+ outstanding.count--;
774
+ outstanding.silentSince = run.deps.time.performance.now();
775
+ if (outstanding.count === 0)
776
+ entry.outstandingByOwnerId.delete(ownerId);
777
+ }
778
+ if (entry.outstandingByOwnerId.size === 0)
779
+ clearSyncRequestTimeout(entry);
780
+ publishSyncState();
781
+ }
782
+ forEachTenant((tenant) => {
783
+ tenant.requestApplySyncMessage(headerResult.value.ownerId, message, {
784
+ type: "Transport",
785
+ key: structuralLookup(transport),
786
+ });
787
+ });
788
+ // Do not complete a sibling after decrementing the shared
789
+ // counter but before its apply has been queued.
790
+ refreshAllSyncRoutes();
791
+ },
792
+ }));
793
+ disposer.use(socket);
794
+ // LIFO: the transport drops its timer and its socket reference
795
+ // before the socket is disposed. `disposable` guards every method
796
+ // of the socket the claims lease, and disposing the socket awaits
797
+ // its retry, so a `publishSyncState` microtask can run while this
798
+ // entry is still registered. It must find no socket rather than
799
+ // read a disposed one.
800
+ disposer.defer(() => {
801
+ clearSyncRequestTimeout(entry);
802
+ entry.socket = null;
803
+ refreshAllSyncRoutes();
158
804
  });
805
+ const disposables = disposer.move();
806
+ entry.socket = socket;
807
+ refreshAllSyncRoutes();
808
+ publishSyncState();
809
+ // `disposable` replaces the socket's disposal method in place, but
810
+ // `disposer.use` above already captured the original, so disposing
811
+ // `disposables` disposes the socket rather than recursing.
812
+ return ok(disposable(socket, disposables));
813
+ }
814
+ catch (e_2) {
815
+ env_2.error = e_2;
816
+ env_2.hasError = true;
817
+ }
818
+ finally {
819
+ const result_2 = __disposeResources(env_2);
820
+ if (result_2)
821
+ await result_2;
822
+ }
823
+ }, {
824
+ onFirstClaimAdded: (ownerId, webSocket, transport) => {
825
+ if (webSocket.isOpen()) {
826
+ const target = {
827
+ type: "Transport",
828
+ key: structuralLookup(transport),
829
+ };
830
+ forEachTenant((tenant) => {
831
+ tenant.requestCreateSyncMessages(new Set([ownerId]), target);
832
+ });
833
+ }
834
+ refreshAllSyncRoutes();
159
835
  },
160
- onLastClaimRemoved: (ownerId, webSocket) => {
161
- webSocket.send(createProtocolMessageForUnsubscribe(ownerId));
836
+ onLastClaimRemoved: (ownerId, webSocket, transport) => {
837
+ if (webSocket.isOpen()) {
838
+ webSocket.send(createProtocolMessageForUnsubscribe(ownerId));
839
+ syncRequests.noteSent(ownerId, structuralLookup(transport));
840
+ }
841
+ refreshAllSyncRoutes();
162
842
  },
163
843
  // Keep sockets alive briefly across short owner churn.
164
844
  idleDisposeAfter: "3s",
165
845
  resourceLookup: structuralLookup,
166
846
  })));
167
- const sharedWorkerRun = run.create({
847
+ const sharedWorkerRun = disposer.use(run.create({
168
848
  ...deps,
169
849
  postConsoleEntryOrError,
850
+ publishSyncState,
851
+ refreshAllSyncRoutes,
852
+ syncRequests,
170
853
  tabLeaderPortStore,
171
854
  transports,
172
- });
173
- const tenantsByName = disposer.use(await sharedWorkerRun.ok(createSharedResourceByKey((message) => createEvoluTenant(message, currentTenantsByName), {
855
+ }));
856
+ const tenantsByName = disposer.use(await sharedWorkerRun.ok(createSharedResourceByKey((message) => createEvoluTenant(isPersistentStorageUnavailable
857
+ ? { ...message, memoryOnly: true }
858
+ : message, currentTenantsByName), {
174
859
  idleDisposeAfter: "3s",
175
860
  lookup: (message) => message.name,
176
861
  })));
@@ -192,51 +877,144 @@ export const initSharedWorker = (self) => async (run) => {
192
877
  }
193
878
  };
194
879
  const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, memoryOnly, }, currentTenantsByName) => async (run) => {
195
- const env_2 = { stack: [], error: void 0, hasError: false };
880
+ const env_3 = { stack: [], error: void 0, hasError: false };
196
881
  try {
197
- const disposer = __addDisposableResource(env_2, new AsyncDisposableStack(), true);
882
+ const disposer = __addDisposableResource(env_3, new AsyncDisposableStack(), true);
198
883
  const tenantRun = disposer.use(run.create());
199
884
  const { deps } = run;
200
885
  const console = deps.console.child(name).child("SharedWorker");
201
886
  const instancesById = disposer.adopt(new Map(), async (instancesById) => {
202
- const env_3 = { stack: [], error: void 0, hasError: false };
887
+ const env_4 = { stack: [], error: void 0, hasError: false };
203
888
  try {
204
- const disposer = __addDisposableResource(env_3, new AsyncDisposableStack(), true);
889
+ const disposer = __addDisposableResource(env_4, new AsyncDisposableStack(), true);
205
890
  for (const instance of instancesById.values()) {
206
891
  disposer.use(instance);
207
892
  }
208
893
  }
209
- catch (e_3) {
210
- env_3.error = e_3;
211
- env_3.hasError = true;
894
+ catch (e_4) {
895
+ env_4.error = e_4;
896
+ env_4.hasError = true;
212
897
  }
213
898
  finally {
214
- const result_3 = __disposeResources(env_3);
215
- if (result_3)
216
- await result_3;
899
+ const result_4 = __disposeResources(env_4);
900
+ if (result_4)
901
+ await result_4;
217
902
  }
218
903
  });
219
904
  let dbWorkerPort = null;
220
905
  const dbWorkerInited = Promise.withResolvers();
221
906
  const initDbWorker = () => {
907
+ // Without a tab leader yet, the store subscription starts the DbWorker
908
+ // once a tab announces itself.
222
909
  const tabLeaderPort = deps.tabLeaderPortStore.get();
223
- assertNonNullable(tabLeaderPort);
910
+ if (startupError || !tabLeaderPort)
911
+ return;
224
912
  const dbWorkerChannel = deps.createMessageChannel();
225
913
  const currentDbWorkerPort = dbWorkerChannel.port2;
226
914
  currentDbWorkerPort.onMessage = (message) => {
227
915
  switch (message.type) {
228
916
  case "LeaderAcquired": {
229
917
  assertNotSame(dbWorkerPort, currentDbWorkerPort);
918
+ if (startupError || isDisposing) {
919
+ // This worker was requested before the refusal or before
920
+ // disposal started. The tenant will not use it, so let it
921
+ // release the database lock.
922
+ currentDbWorkerPort.postMessage({ type: "Dispose" });
923
+ currentDbWorkerPort[Symbol.dispose]();
924
+ break;
925
+ }
926
+ const replacesLeader = dbWorkerPort !== null;
230
927
  dbWorkerPort?.[Symbol.dispose]();
231
928
  dbWorkerPort = currentDbWorkerPort;
232
- queueRequestInFlight = false;
929
+ activeDispatch = null;
930
+ // A replacement may advance the clock by releasing quarantine, or
931
+ // start behind it with an empty memoryOnly database. Keep the
932
+ // greater clock; pending writes retain their captured inputs.
933
+ if (sessionClock === null ||
934
+ orderTimestamp(sessionClock, message.clock) === -1) {
935
+ sessionClock = message.clock;
936
+ }
937
+ // A replacement leader may have committed writes whose responses
938
+ // were lost, and may have released quarantine at startup. A lost
939
+ // response may also have reported stored owner messages, so the
940
+ // owners reconcile through every transport again.
941
+ if (replacesLeader) {
942
+ refreshQueries();
943
+ const usedOwnerIds = new Set();
944
+ for (const instance of instancesById.values()) {
945
+ for (const { owner } of instance.ownerRegistrations.keys()) {
946
+ usedOwnerIds.add(owner.id);
947
+ }
948
+ }
949
+ requestCreateSyncMessages(usedOwnerIds, allTransports);
950
+ }
233
951
  console.info("leaderAcquired");
234
952
  dbWorkerInited.resolve();
235
953
  runQueue();
236
954
  break;
237
955
  }
956
+ case "LeaderRefused": {
957
+ assertNotSame(dbWorkerPort, currentDbWorkerPort);
958
+ // The worker refused startup and is releasing its resources.
959
+ // Requests stay unanswered until their instances are disposed.
960
+ // Keep the tenant unavailable, tell each connected tab once, and
961
+ // tell tabs that connect later without starting another worker.
962
+ dbWorkerPort?.[Symbol.dispose]();
963
+ dbWorkerPort = null;
964
+ currentDbWorkerPort[Symbol.dispose]();
965
+ activeDispatch = null;
966
+ queue.length = 0;
967
+ pendingWriteCountByOwnerId.clear();
968
+ pendingApplyCountForAllRoutesByOwnerId.clear();
969
+ ownerTransportApplyRelation.clear();
970
+ startupError = message.error;
971
+ refreshSyncRoutes();
972
+ deps.publishSyncState();
973
+ console.info("leaderRefused", message.error);
974
+ for (const instance of instancesById.values()) {
975
+ reportRefusal(instance.tabPort, message.error);
976
+ }
977
+ dbWorkerInited.resolve();
978
+ break;
979
+ }
238
980
  case "OnQueuedResponse": {
239
- callbacks.execute(message.callbackId, message);
981
+ if (activeDispatch?.attemptId !== message.attemptId)
982
+ return;
983
+ const { entry } = activeDispatch;
984
+ const { response } = message;
985
+ if (response.message.type === "Mutate" ||
986
+ response.message.type === "ApplySyncMessage") {
987
+ // Replays report their computed clock, which a replacement
988
+ // leader may have passed at startup. Keep the greater clock.
989
+ assertNonNullable(sessionClock);
990
+ if (orderTimestamp(sessionClock, response.message.clock) === -1) {
991
+ sessionClock = response.message.clock;
992
+ }
993
+ }
994
+ switch (entry.type) {
995
+ case "Read":
996
+ case "Write":
997
+ assertSame(response.type, "ForEvolu");
998
+ handleResponseForEvolu(response, entry.request);
999
+ break;
1000
+ case "CreateSyncMessages":
1001
+ case "ApplySyncMessage":
1002
+ assertSame(response.type, "ForSharedWorker");
1003
+ handleResponseForSharedWorker(response, entry);
1004
+ break;
1005
+ default:
1006
+ exhaustiveCheck(entry);
1007
+ }
1008
+ const head = queue.shift();
1009
+ assertNonNullable(head);
1010
+ updatePendingWork(head, -1);
1011
+ activeDispatch = null;
1012
+ // Follow-up uploads and retries are already queued or sent. Only
1013
+ // now can removing the completed head establish convergence.
1014
+ refreshSyncRoutes();
1015
+ if (entry.type === "Write" && entry.replicatedOwnerIds.size > 0)
1016
+ deps.publishSyncState();
1017
+ runQueue();
240
1018
  break;
241
1019
  }
242
1020
  }
@@ -251,148 +1029,408 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
251
1029
  port: dbWorkerChannel.port1.native,
252
1030
  }, [dbWorkerChannel.port1.native]);
253
1031
  };
1032
+ const refreshQueries = (exceptInstanceId) => {
1033
+ for (const [id, instance] of instancesById) {
1034
+ if (id === exceptInstanceId)
1035
+ continue;
1036
+ instance.port.postMessage({ type: "RefreshQueries" });
1037
+ }
1038
+ };
254
1039
  const queue = [];
255
- const callbacks = disposer.use(createCallbacks(run.deps));
256
- let queueRequestInFlight = false;
1040
+ const pendingWriteCountByOwnerId = new Map();
1041
+ const pendingApplyCountForAllRoutesByOwnerId = new Map();
1042
+ // Queued applies of relay frames, counted per owner and source transport.
1043
+ const ownerTransportApplyRelation = createRefCountedRelation();
1044
+ const updatePendingCount = (counts, key, delta) => {
1045
+ const count = (counts.get(key) ?? 0) + delta;
1046
+ assert(count >= 0, "Pending queue count must not become negative");
1047
+ if (count === 0)
1048
+ counts.delete(key);
1049
+ else
1050
+ counts.set(key, count);
1051
+ };
1052
+ // Count each queued entry once, including the dispatched head. Dispatch
1053
+ // and leader replacement leave it queued, so neither changes the counts.
1054
+ const updatePendingWork = (entry, delta) => {
1055
+ switch (entry.type) {
1056
+ case "Read":
1057
+ case "CreateSyncMessages":
1058
+ break;
1059
+ case "Write":
1060
+ for (const ownerId of entry.replicatedOwnerIds)
1061
+ updatePendingCount(pendingWriteCountByOwnerId, ownerId, delta);
1062
+ break;
1063
+ case "ApplySyncMessage": {
1064
+ const ownerId = entry.request.message.owner.id;
1065
+ // A sibling's local copy holds every route, even when its upload
1066
+ // target was a single transport.
1067
+ if (entry.source.type === "Local") {
1068
+ updatePendingCount(pendingApplyCountForAllRoutesByOwnerId, ownerId, delta);
1069
+ }
1070
+ else if (delta === 1) {
1071
+ ownerTransportApplyRelation.increment(ownerId, entry.source.key);
1072
+ }
1073
+ else {
1074
+ ownerTransportApplyRelation.decrement(ownerId, entry.source.key);
1075
+ }
1076
+ break;
1077
+ }
1078
+ default:
1079
+ exhaustiveCheck(entry);
1080
+ }
1081
+ };
1082
+ // Every queue mutation updates the pending-work counts with it.
1083
+ const enqueueRequest = (entry) => {
1084
+ updatePendingWork(entry, 1);
1085
+ queue.push(entry);
1086
+ };
1087
+ let sessionClock = null;
1088
+ let startupError = null;
1089
+ let isDisposing = false;
1090
+ // Each tab is told once during this tenant's lifetime, through its own
1091
+ // connection. Recreating the tenant after idle disposal retries startup
1092
+ // and may report the refusal again.
1093
+ const refusedTabPorts = new WeakSet();
1094
+ const reportRefusal = (tabPort, error) => {
1095
+ if (refusedTabPorts.has(tabPort))
1096
+ return;
1097
+ refusedTabPorts.add(tabPort);
1098
+ tabPort.postMessage({ type: "Error", error });
1099
+ };
1100
+ let activeDispatch = null;
257
1101
  const runQueue = () => {
258
- if (queueRequestInFlight || !isNonEmptyArray(queue) || !dbWorkerPort) {
1102
+ if (activeDispatch || !isNonEmptyArray(queue) || !dbWorkerPort)
259
1103
  return;
1104
+ assertNonNullable(sessionClock);
1105
+ const entry = firstInArray(queue);
1106
+ const attemptId = createId(run.deps);
1107
+ activeDispatch = { entry, attemptId };
1108
+ if (entry.type === "Write" || entry.type === "ApplySyncMessage") {
1109
+ // A write captures its inputs on first dispatch, so a retry after
1110
+ // leader replacement reproduces the same timestamps.
1111
+ entry.now ??= run.deps.time.now();
1112
+ entry.clock ??= sessionClock;
1113
+ dbWorkerPort.postMessage({
1114
+ type: "Request",
1115
+ attemptId,
1116
+ request: entry.request,
1117
+ clock: entry.clock,
1118
+ now: entry.now,
1119
+ });
1120
+ }
1121
+ else {
1122
+ dbWorkerPort.postMessage({
1123
+ type: "Request",
1124
+ attemptId,
1125
+ request: entry.request,
1126
+ });
260
1127
  }
261
- const request = firstInArray(queue);
262
- const callbackId = callbacks.register(({ response }) => {
263
- switch (response.type) {
264
- case "ForEvolu": {
265
- handleResponseForEvolu(response, request);
266
- break;
267
- }
268
- case "ForSharedWorker":
269
- handleResponseForSharedWorker(response);
270
- break;
271
- }
272
- // Complete the current queue item and continue with the next one.
273
- queue.shift();
274
- queueRequestInFlight = false;
275
- runQueue();
276
- });
277
- queueRequestInFlight = true;
278
- dbWorkerPort.postMessage({ type: "Request", callbackId, request });
279
1128
  };
280
1129
  disposer.defer(async () => {
281
- const env_4 = { stack: [], error: void 0, hasError: false };
1130
+ const env_5 = { stack: [], error: void 0, hasError: false };
282
1131
  try {
1132
+ isDisposing = true;
283
1133
  dbWorkerPort?.postMessage({ type: "Dispose" });
284
1134
  dbWorkerPort = null;
285
- queueRequestInFlight = false;
1135
+ activeDispatch = null;
286
1136
  // The DbWorker holds this tenant leader lock while it is alive. Tenant
287
1137
  // disposal sends Dispose, then acquires the same lock to wait until the
288
1138
  // DbWorker releases it: either because Dispose was delivered or because
289
- // the hosting tab closed. The wait is unabortable because tenant disposal
290
- // must finish even after tenantRun receives an abort request.
291
- const _ = __addDisposableResource(env_4, await tenantRun.ok(acquireLeaderLock(name)), true);
1139
+ // the hosting tab closed. A worker requested from a later tab leader may
1140
+ // be queued for the lock first; it is told to stop when it reports in.
1141
+ // The wait is unabortable because tenant disposal must finish even after
1142
+ // tenantRun receives an abort request.
1143
+ const _ = __addDisposableResource(env_5, await tenantRun.ok(acquireLeaderLock(name)), true);
292
1144
  }
293
- catch (e_4) {
294
- env_4.error = e_4;
295
- env_4.hasError = true;
1145
+ catch (e_5) {
1146
+ env_5.error = e_5;
1147
+ env_5.hasError = true;
296
1148
  }
297
1149
  finally {
298
- const result_4 = __disposeResources(env_4);
299
- if (result_4)
300
- await result_4;
1150
+ const result_5 = __disposeResources(env_5);
1151
+ if (result_5)
1152
+ await result_5;
301
1153
  }
302
1154
  });
303
- disposer.defer(deps.tabLeaderPortStore.subscribe(initDbWorker));
304
- initDbWorker();
305
- await dbWorkerInited.promise;
306
1155
  const handleResponseForEvolu = (response, first) => {
1156
+ // A disposed instance gets no patches, but its committed write still
1157
+ // synchronizes.
307
1158
  const instance = instancesById.get(response.id);
308
- if (!instance)
309
- return;
310
1159
  switch (response.message.type) {
311
1160
  case "Mutate":
312
1161
  case "Query": {
313
- const nextRowsByQuery = new Map(instance.rowsByQuery);
314
- const patchesByQuery = new Map();
315
- for (const [query, rows] of response.message.rowsByQuery) {
316
- nextRowsByQuery.set(query, rows);
317
- patchesByQuery.set(query, makePatches(instance.rowsByQuery.get(query), rows));
1162
+ if (instance) {
1163
+ const nextRowsByQuery = new Map(instance.rowsByQuery);
1164
+ const patchesByQuery = new Map();
1165
+ for (const [query, rows] of response.message.rowsByQuery) {
1166
+ nextRowsByQuery.set(query, rows);
1167
+ patchesByQuery.set(query, makePatches(instance.rowsByQuery.get(query), rows));
1168
+ }
1169
+ instance.rowsByQuery = nextRowsByQuery;
1170
+ instance.port.postMessage({
1171
+ type: "OnPatchesByQuery",
1172
+ patchesByQuery,
1173
+ onCompleteIds: first.message.type === "Mutate"
1174
+ ? first.message.onCompleteIds
1175
+ : emptyArray,
1176
+ });
318
1177
  }
319
- instance.rowsByQuery = nextRowsByQuery;
320
- instance.port.postMessage({
321
- type: "OnPatchesByQuery",
322
- patchesByQuery,
323
- onCompleteIds: first.message.type === "Mutate"
324
- ? first.message.onCompleteIds
325
- : emptyArray,
326
- });
327
1178
  if (response.message.type === "Mutate") {
328
- for (const [instanceId, instance] of instancesById) {
329
- if (instanceId === response.id)
330
- continue;
331
- instance.port.postMessage({
332
- type: "RefreshQueries",
333
- });
334
- }
1179
+ refreshQueries(response.id);
335
1180
  const protocolMessagesByOwnerId = new Map();
336
- for (const syncOwner of instance.usedSyncOwners.keys()) {
337
- const { owner } = syncOwner;
338
- const messages = response.message.messagesByOwnerId.get(owner.id);
339
- // Skip owners this instance does not currently sync for
340
- // writing. Read-only owners cannot produce protocol
341
- // messages because they do not have a write key.
342
- if (!messages || !("writeKey" in owner))
1181
+ // The database's writable registrations upload the write,
1182
+ // whichever instance made it and whether it is still alive.
1183
+ const writersById = getUsedOwnersById(new Set(response.message.messagesByOwnerId.keys()));
1184
+ for (const [ownerId, messages] of response.message
1185
+ .messagesByOwnerId) {
1186
+ // Uploading requires the write key. A write for an owner without
1187
+ // a writable registration waits for the round that its first
1188
+ // writable registration starts.
1189
+ const owner = writersById.get(ownerId);
1190
+ if (!owner)
343
1191
  continue;
344
- protocolMessagesByOwnerId.set(owner.id, createProtocolMessageFromCrdtMessages(run.deps)(owner, messages));
1192
+ protocolMessagesByOwnerId.set(ownerId, createProtocolMessageFromCrdtMessages(run.deps)(owner, messages));
1193
+ if (currentTenantsByName.size > 1) {
1194
+ broadcastProtocolMessages(ownerId, createProtocolBroadcastMessagesFromCrdtMessages(run.deps)(owner, messages), allTransports);
1195
+ }
345
1196
  }
346
- sendProtocolMessagesByOwnerId(protocolMessagesByOwnerId);
1197
+ sendProtocolMessagesByOwnerId(protocolMessagesByOwnerId, allTransports);
347
1198
  }
348
1199
  break;
349
1200
  }
350
1201
  case "Export":
351
- instance.port.postMessage({ type: "OnExport", file: response.message.file }, [response.message.file.buffer]);
1202
+ instance?.port.postMessage({ type: "OnExport", file: response.message.file }, [response.message.file.buffer]);
352
1203
  break;
353
1204
  }
354
1205
  };
355
- const handleResponseForSharedWorker = (response) => {
356
- switch (response.message.type) {
357
- case "CreateSyncMessages":
358
- sendProtocolMessagesByOwnerId(response.message.protocolMessagesByOwnerId);
1206
+ const routesByOwnerIdByKey = new Map();
1207
+ const getRoute = (ownerId, key) => {
1208
+ let routesByOwnerId = routesByOwnerIdByKey.get(key);
1209
+ if (!routesByOwnerId) {
1210
+ routesByOwnerId = new Map();
1211
+ routesByOwnerIdByKey.set(key, routesByOwnerId);
1212
+ }
1213
+ let route = routesByOwnerId.get(ownerId);
1214
+ if (!route) {
1215
+ route = {
1216
+ roundRequired: true,
1217
+ complete: false,
1218
+ completeAt: null,
1219
+ lastSentAt: null,
1220
+ lastReceivedAt: null,
1221
+ error: null,
1222
+ };
1223
+ routesByOwnerId.set(ownerId, route);
1224
+ }
1225
+ return route;
1226
+ };
1227
+ const getSyncOwners = () => {
1228
+ const ownersById = new Map();
1229
+ for (const instance of instancesById.values()) {
1230
+ for (const { owner } of instance.ownerRegistrations.keys()) {
1231
+ const entry = ownersById.get(owner.id) ?? { writable: false };
1232
+ entry.writable ||= "writeKey" in owner;
1233
+ ownersById.set(owner.id, entry);
1234
+ }
1235
+ }
1236
+ return [...ownersById].map(([ownerId, { writable }]) => ({
1237
+ ownerId,
1238
+ writable,
1239
+ transportKeys: getClaimedKeys(ownerId),
1240
+ }));
1241
+ };
1242
+ // State-changing handlers own route transitions and claim cleanup.
1243
+ // Reading or delaying a snapshot must not affect protocol retries.
1244
+ // TODO: If profiling warrants it, remove quadratic transport scans per owner:
1245
+ // traverse resources once and use a Set for cleanup membership.
1246
+ const refreshSyncRoutes = () => {
1247
+ const owners = getSyncOwners();
1248
+ const ownerById = new Map(owners.map((owner) => [owner.ownerId, owner]));
1249
+ for (const [key, routesByOwnerId] of routesByOwnerIdByKey) {
1250
+ for (const ownerId of routesByOwnerId.keys()) {
1251
+ const owner = ownerById.get(ownerId);
1252
+ if (!owner?.writable || !owner.transportKeys.includes(key))
1253
+ routesByOwnerId.delete(ownerId);
1254
+ }
1255
+ if (routesByOwnerId.size === 0)
1256
+ routesByOwnerIdByKey.delete(key);
1257
+ }
1258
+ // Evaluate each route per the Synchronization completion rules.
1259
+ for (const { ownerId, writable, transportKeys } of owners) {
1260
+ if (!writable)
1261
+ continue;
1262
+ for (const key of transportKeys) {
1263
+ const route = getRoute(ownerId, key);
1264
+ let isOpen = false;
1265
+ deps.transports.forEachResourceForClaim(ownerId, (webSocket, transport) => {
1266
+ if (structuralLookup(transport) === key && webSocket.isOpen())
1267
+ isOpen = true;
1268
+ });
1269
+ // A received frame is applied asynchronously, so the counter alone
1270
+ // reads zero in the middle of a chain. A dispatched entry stays
1271
+ // queued until it is answered. A sibling's local Broadcast contains
1272
+ // messages that are unstored until it is applied, so it leaves every
1273
+ // route of the owner incomplete.
1274
+ const hasQueuedApply = pendingApplyCountForAllRoutesByOwnerId.has(ownerId) ||
1275
+ ownerTransportApplyRelation.getCount(ownerId, key) > 0;
1276
+ // A replicated write's upload is sent only after the database
1277
+ // worker answers it. Local-only changes create no synchronization
1278
+ // work.
1279
+ const hasQueuedWrite = pendingWriteCountByOwnerId.has(ownerId);
1280
+ // A refused database synchronizes nothing, and refusal discards its
1281
+ // queued writes without uploading them.
1282
+ const complete = startupError === null &&
1283
+ isOpen &&
1284
+ deps.syncRequests.getOutstanding(ownerId, key) === 0 &&
1285
+ !hasQueuedApply &&
1286
+ !hasQueuedWrite &&
1287
+ !route.roundRequired;
1288
+ if (complete && !route.complete) {
1289
+ route.completeAt = deps.time.now();
1290
+ route.error = null;
1291
+ }
1292
+ route.complete = complete;
1293
+ }
1294
+ }
1295
+ };
1296
+ const handleResponseForSharedWorker = (response, entry) => {
1297
+ switch (entry.type) {
1298
+ case "CreateSyncMessages": {
1299
+ assertSame(response.message.type, "CreateSyncMessages");
1300
+ sendProtocolMessagesByOwnerId(response.message.protocolMessagesByOwnerId, entry.target, { isRound: true });
1301
+ const { failedOwnerIds } = response.message;
1302
+ if (failedOwnerIds.size === 0)
1303
+ break;
1304
+ // A retry would likely fail the same way, so the routes wait for an
1305
+ // explicit request or a reopen.
1306
+ const now = deps.time.now();
1307
+ for (const ownerId of failedOwnerIds) {
1308
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1309
+ if (!isTargetTransport(entry.target, transport))
1310
+ return;
1311
+ const route = routesByOwnerIdByKey
1312
+ .get(structuralLookup(transport))
1313
+ ?.get(ownerId);
1314
+ if (!route)
1315
+ return;
1316
+ route.roundRequired = true;
1317
+ route.error = { type: "SyncFailed", at: now };
1318
+ });
1319
+ }
1320
+ deps.publishSyncState();
359
1321
  break;
360
- case "ApplySyncMessage":
361
- if (response.message.didWriteMessages) {
362
- for (const instance of instancesById.values()) {
363
- instance.port.postMessage({ type: "RefreshQueries" });
1322
+ }
1323
+ case "ApplySyncMessage": {
1324
+ assertSame(response.message.type, "ApplySyncMessage");
1325
+ const { source } = entry;
1326
+ const target = source.type === "Local" ? source.uploadTarget : source;
1327
+ const { ownerId, result } = response.message;
1328
+ const error = result.ok ? null : result.error;
1329
+ const isAborted = error?.type === "AbortError";
1330
+ let failure = null;
1331
+ if (isAborted) {
1332
+ // An abort proves no convergence. A local sibling copy affects
1333
+ // every route; a relay frame affects only its source route.
1334
+ // Recovery of a panicked DbWorker remains deferred.
1335
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1336
+ const key = structuralLookup(transport);
1337
+ if (source.type === "Transport" && source.key !== key)
1338
+ return;
1339
+ const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
1340
+ if (route)
1341
+ route.roundRequired = true;
1342
+ });
1343
+ }
1344
+ else if (error !== null) {
1345
+ failure = error.type;
1346
+ deps.postConsoleEntryOrError({
1347
+ type: "Error",
1348
+ error,
1349
+ });
1350
+ }
1351
+ else if (result.ok && result.value.type === "Failed") {
1352
+ failure =
1353
+ result.value.cause === "Write" ? "WriteFailed" : "SyncFailed";
1354
+ }
1355
+ if (source.type === "Transport") {
1356
+ // A registration or claim may have been removed while this apply
1357
+ // was queued.
1358
+ const route = routesByOwnerIdByKey.get(source.key)?.get(ownerId);
1359
+ if (route) {
1360
+ const now = deps.time.now();
1361
+ // An aborted apply applied nothing.
1362
+ if (!isAborted)
1363
+ route.lastReceivedAt = now;
1364
+ if (failure !== null) {
1365
+ // The first failure since the route completed requests one
1366
+ // round. Further failures wait for an explicit request or a
1367
+ // reopen, even after a converged reply, which may answer
1368
+ // another request.
1369
+ if (route.error === null)
1370
+ requestCreateSyncMessages(new Set([ownerId]), source);
1371
+ route.roundRequired = true;
1372
+ route.error = { type: failure, at: now };
1373
+ }
364
1374
  }
365
1375
  }
366
- if (!response.message.result.ok) {
367
- if (response.message.result.error.type !== "AbortError") {
368
- deps.postConsoleEntryOrError({
369
- type: "Error",
370
- error: response.message.result.error,
371
- });
1376
+ else if (failure !== null) {
1377
+ // A sibling's messages were not stored; rounds fetch them from
1378
+ // the relays.
1379
+ requestCreateSyncMessages(new Set([ownerId]), allTransports);
1380
+ }
1381
+ if (response.message.didWriteMessages) {
1382
+ refreshQueries();
1383
+ // Reconcile newly stored messages through each other transport.
1384
+ // Rounds toward the same transport coalesce whatever their source.
1385
+ const keys = [];
1386
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1387
+ if (isTargetTransport(target, transport))
1388
+ return;
1389
+ keys.push(structuralLookup(transport));
1390
+ });
1391
+ for (const key of keys) {
1392
+ requestCreateSyncMessages(new Set([ownerId]), { type: "Transport", key }, { afterQueuedWrites: false });
372
1393
  }
373
1394
  }
374
- else {
375
- switch (response.message.result.value.type) {
1395
+ if (result.ok) {
1396
+ switch (result.value.type) {
376
1397
  case "Response":
377
- sendProtocolMessagesByOwnerId(new Map([
378
- [
379
- response.message.ownerId,
380
- response.message.result.value.message,
381
- ],
382
- ]));
1398
+ if (result.value.broadcast) {
1399
+ broadcastProtocolMessages(ownerId, [result.value.broadcast], target);
1400
+ }
1401
+ sendProtocolMessagesByOwnerId(new Map([[ownerId, result.value.message]]), target);
383
1402
  break;
384
1403
  case "Broadcast":
385
- case "NoResponse":
1404
+ case "Converged":
1405
+ case "Readonly":
1406
+ case "Failed":
386
1407
  break;
1408
+ default:
1409
+ exhaustiveCheck(result.value);
387
1410
  }
388
1411
  }
1412
+ deps.publishSyncState();
389
1413
  break;
1414
+ }
1415
+ default:
1416
+ exhaustiveCheck(entry);
1417
+ }
1418
+ };
1419
+ const broadcastProtocolMessages = (ownerId, messages, target) => {
1420
+ for (const [tenantName, tenant] of currentTenantsByName) {
1421
+ if (tenantName === name)
1422
+ continue;
1423
+ for (const message of messages)
1424
+ tenant.requestApplySyncMessage(ownerId, message, {
1425
+ type: "Local",
1426
+ uploadTarget: target,
1427
+ });
390
1428
  }
391
1429
  };
392
- const sendProtocolMessagesByOwnerId = (protocolMessagesByOwnerId) => {
1430
+ const sendProtocolMessagesByOwnerId = (protocolMessagesByOwnerId, target, { isRound = false, } = {}) => {
393
1431
  for (const [ownerId, protocolMessage] of protocolMessagesByOwnerId) {
394
1432
  deps.transports.forEachResourceForClaim(ownerId, (webSocket, transport) => {
395
- if (!webSocket.isOpen())
1433
+ if (!isTargetTransport(target, transport) || !webSocket.isOpen())
396
1434
  return;
397
1435
  console.debug("sendProtocolMessage", {
398
1436
  ownerId,
@@ -400,13 +1438,24 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
400
1438
  byteLength: protocolMessage.byteLength,
401
1439
  });
402
1440
  webSocket.send(protocolMessage);
1441
+ const key = structuralLookup(transport);
1442
+ deps.syncRequests.noteSent(ownerId, key);
1443
+ const route = getRoute(ownerId, key);
1444
+ route.lastSentAt = deps.time.now();
1445
+ if (isRound)
1446
+ route.roundRequired = false;
403
1447
  });
404
1448
  }
1449
+ // Sends raise counters shared by every tenant; one refresh covers them.
1450
+ if (protocolMessagesByOwnerId.size > 0)
1451
+ deps.refreshAllSyncRoutes();
405
1452
  };
1453
+ /** The keys of every transport claimed for the owner, by any database. */
1454
+ const getClaimedKeys = (ownerId) => [...deps.transports.getResourceKeysForClaim(ownerId)].map(structuralLookup);
406
1455
  const getUsedOwnersById = (ownerIds) => {
407
1456
  const ownersById = new Map();
408
1457
  for (const instance of instancesById.values()) {
409
- for (const { owner } of instance.usedSyncOwners.keys()) {
1458
+ for (const { owner } of instance.ownerRegistrations.keys()) {
410
1459
  if (!ownerIds.has(owner.id) || !("writeKey" in owner))
411
1460
  continue;
412
1461
  ownersById.set(owner.id, owner);
@@ -414,84 +1463,204 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
414
1463
  }
415
1464
  return ownersById;
416
1465
  };
1466
+ const requestCreateSyncMessages = (ownerIds, target, { afterQueuedWrites = true, } = {}) => {
1467
+ if (startupError)
1468
+ return;
1469
+ const usedOwnersById = getUsedOwnersById(ownerIds);
1470
+ // Opening, storing messages from another transport, a failure, and an
1471
+ // explicit request each require a new round before completion.
1472
+ for (const ownerId of usedOwnersById.keys()) {
1473
+ deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
1474
+ if (!isTargetTransport(target, transport))
1475
+ return;
1476
+ getRoute(ownerId, structuralLookup(transport)).roundRequired = true;
1477
+ });
1478
+ }
1479
+ refreshSyncRoutes();
1480
+ deps.publishSyncState();
1481
+ const ownersToSync = [...usedOwnersById.values()].filter(({ id }) => {
1482
+ let hasOpenTransport = false;
1483
+ deps.transports.forEachResourceForClaim(id, (webSocket, transport) => {
1484
+ if (isTargetTransport(target, transport) && webSocket.isOpen())
1485
+ hasOpenTransport = true;
1486
+ });
1487
+ return hasOpenTransport;
1488
+ });
1489
+ if (!isNonEmptyArray(ownersToSync))
1490
+ return;
1491
+ // A queued round reads the database when it is dispatched, so it covers
1492
+ // this request unless the request must also read writes queued behind
1493
+ // that round. A dispatched round covers neither.
1494
+ const ownerIdsToSync = new Set(ownersToSync.map(({ id }) => id));
1495
+ const lastWriteIndex = afterQueuedWrites
1496
+ ? queue.findLastIndex(({ type }) => type === "Write" || type === "ApplySyncMessage")
1497
+ : -1;
1498
+ const isQueued = queue.some((entry, index) => index > lastWriteIndex &&
1499
+ entry !== activeDispatch?.entry &&
1500
+ entry.type === "CreateSyncMessages" &&
1501
+ (entry.target.type === "AllTransports" ||
1502
+ (target.type === "Transport" && entry.target.key === target.key)) &&
1503
+ entry.request.message.owners.length === ownerIdsToSync.size &&
1504
+ entry.request.message.owners.every(({ id }) => ownerIdsToSync.has(id)));
1505
+ console.debug("requestCreateSyncMessages", {
1506
+ ownerIds: [...ownerIdsToSync],
1507
+ target,
1508
+ isQueued,
1509
+ });
1510
+ if (isQueued)
1511
+ return;
1512
+ enqueueRequest({
1513
+ type: "CreateSyncMessages",
1514
+ request: {
1515
+ type: "ForSharedWorker",
1516
+ message: { type: "CreateSyncMessages", owners: ownersToSync },
1517
+ },
1518
+ target,
1519
+ });
1520
+ runQueue();
1521
+ };
417
1522
  const toggleSyncOwner = (instance, syncOwner, action) => async (run) => {
418
1523
  if (action === "add") {
419
- const env_5 = { stack: [], error: void 0, hasError: false };
420
- try {
421
- instance.usedSyncOwners.increment(syncOwner);
422
- let succeeded = false;
423
- const compensation = __addDisposableResource(env_5, new DisposableStack(), false);
424
- compensation.defer(() => {
425
- if (!succeeded)
426
- instance.usedSyncOwners.decrement(syncOwner);
427
- });
428
- const claimLease = await run.ok(deps.transports.claim(syncOwner.owner.id, syncOwner.transports));
429
- instance.claimLeasesBySyncOwner
430
- .getOrInsertComputed(syncOwner, () => [])
431
- .push(claimLease);
432
- succeeded = true;
1524
+ const ownerId = syncOwner.owner.id;
1525
+ const isFirstWritableUse = "writeKey" in syncOwner.owner &&
1526
+ !getUsedOwnersById(new Set([ownerId])).has(ownerId);
1527
+ // A transport first claimed for the owner starts every tenant's
1528
+ // round through onFirstClaimAdded. A joining tenant also reconciles
1529
+ // its existing history through the owner's already claimed transports.
1530
+ // Later registrations reconcile only transports newly used here:
1531
+ // earlier rounds may predate writes from an unregistered instance.
1532
+ const claimedKeys = new Set(getClaimedKeys(ownerId));
1533
+ const usedKeys = new Set();
1534
+ for (const { ownerRegistrations } of instancesById.values()) {
1535
+ for (const [usedSyncOwner, leases] of ownerRegistrations) {
1536
+ if (usedSyncOwner.owner.id !== ownerId)
1537
+ continue;
1538
+ if (!leases.some((lease) => lease !== null))
1539
+ continue;
1540
+ for (const transport of usedSyncOwner.transports) {
1541
+ usedKeys.add(structuralLookup(transport));
1542
+ }
1543
+ }
433
1544
  }
434
- catch (e_5) {
435
- env_5.error = e_5;
436
- env_5.hasError = true;
1545
+ const leases = instance.ownerRegistrations.getOrInsertComputed(syncOwner, () => []);
1546
+ // First-claim callbacks must see the owner before acquisition finishes.
1547
+ const index = leases.push(null) - 1;
1548
+ refreshSyncRoutes();
1549
+ try {
1550
+ leases[index] = await run.ok(deps.transports.claim(ownerId, syncOwner.transports));
437
1551
  }
438
1552
  finally {
439
- __disposeResources(env_5);
1553
+ if (leases[index] === null) {
1554
+ leases.splice(index, 1);
1555
+ if (leases.length === 0)
1556
+ instance.ownerRegistrations.delete(syncOwner);
1557
+ }
1558
+ deps.refreshAllSyncRoutes();
1559
+ }
1560
+ const keysToSync = isFirstWritableUse
1561
+ ? claimedKeys
1562
+ : syncOwner.transports
1563
+ .map(structuralLookup)
1564
+ .filter((key) => claimedKeys.has(key) && !usedKeys.has(key));
1565
+ for (const key of keysToSync) {
1566
+ requestCreateSyncMessages(new Set([ownerId]), {
1567
+ type: "Transport",
1568
+ key,
1569
+ });
440
1570
  }
441
1571
  }
442
1572
  else {
443
- instance.usedSyncOwners.decrement(syncOwner);
444
- const claimLeases = instance.claimLeasesBySyncOwner.get(syncOwner);
1573
+ const claimLeases = instance.ownerRegistrations.get(syncOwner);
445
1574
  assertNotUndefined(claimLeases);
446
1575
  const claimLease = claimLeases.pop();
447
- assertNotUndefined(claimLease);
1576
+ assertNonNullable(claimLease);
448
1577
  claimLease.release();
449
1578
  if (claimLeases.length === 0) {
450
- instance.claimLeasesBySyncOwner.delete(syncOwner);
1579
+ instance.ownerRegistrations.delete(syncOwner);
451
1580
  }
452
1581
  }
1582
+ deps.refreshAllSyncRoutes();
1583
+ deps.publishSyncState();
453
1584
  return ok();
454
1585
  };
1586
+ // Response handlers can refresh routes as soon as the worker answers.
1587
+ // Initialize their state and helpers before starting it.
1588
+ disposer.defer(deps.tabLeaderPortStore.subscribe(initDbWorker));
1589
+ initDbWorker();
1590
+ // Without a tab leader, requests queue until one announces itself and its
1591
+ // DbWorker reports in. Waiting for that here would stall the registry's
1592
+ // disposal, which cannot abort a resource still being created.
1593
+ if (deps.tabLeaderPortStore.get())
1594
+ await dbWorkerInited.promise;
1595
+ // Remove the tenant before any asynchronous disposal step can yield to a
1596
+ // snapshot or another tenant's route refresh.
455
1597
  disposer.defer(() => {
456
1598
  currentTenantsByName.delete(name);
1599
+ deps.publishSyncState();
457
1600
  });
458
1601
  const tenant = disposable({
459
- addInstance: (message, onDisposed) => {
1602
+ getSyncTenant: () => ({
1603
+ name,
1604
+ refused: startupError !== null,
1605
+ owners: getSyncOwners().map(({ ownerId, writable, transportKeys }) => ({
1606
+ ownerId,
1607
+ writable,
1608
+ transportKeys,
1609
+ routes: writable
1610
+ ? transportKeys.map((key) => {
1611
+ const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
1612
+ // Registration and claim changes refresh routes before yielding.
1613
+ // Snapshot reads must not create missing routes.
1614
+ assertNotUndefined(route);
1615
+ return {
1616
+ transportKey: key,
1617
+ complete: route.complete,
1618
+ completeAt: route.completeAt,
1619
+ lastSentAt: route.lastSentAt,
1620
+ lastReceivedAt: route.lastReceivedAt,
1621
+ error: route.error,
1622
+ };
1623
+ })
1624
+ : [],
1625
+ })),
1626
+ }),
1627
+ refreshSyncRoutes,
1628
+ addInstance: (message, tabPort, onDisposed) => {
460
1629
  const disposer = new AsyncDisposableStack();
461
1630
  const instance = {
462
1631
  id: message.id,
463
- claimLeasesBySyncOwner: createLookupMap({
1632
+ ownerRegistrations: createLookupMap({
464
1633
  lookup: (syncOwner) => structuralLookup({
465
- ownerId: syncOwner.owner.id,
1634
+ owner: syncOwner.owner,
466
1635
  transports: syncOwner.transports
467
1636
  .map(structuralLookup)
468
1637
  .toSorted(),
469
1638
  }),
470
1639
  }),
471
1640
  port: deps.createMessagePort(message.evoluPort),
1641
+ tabPort,
472
1642
  onDisposed,
473
1643
  rowsByQuery: new Map(),
474
1644
  useOwnerMutex: createMutex(),
475
- usedSyncOwners: createRefCountByKey({
476
- lookup: (syncOwner) => syncOwner.owner.id,
477
- }),
478
1645
  [Symbol.asyncDispose]: () => disposer.disposeAsync(),
479
1646
  };
480
1647
  instancesById.set(instance.id, instance);
481
1648
  disposer.defer(instance.onDisposed);
482
- disposer.use(instance.usedSyncOwners);
483
1649
  disposer.defer(async () => {
484
- await tenantRun(instance.useOwnerMutex.withLock(async (run) => {
485
- for (const syncOwner of instance.usedSyncOwners.keys()) {
486
- while (instance.usedSyncOwners.has(syncOwner)) {
487
- await run(toggleSyncOwner(instance, syncOwner, "remove"));
488
- }
1650
+ await tenantRun(instance.useOwnerMutex.withLock(() => {
1651
+ for (const leases of instance.ownerRegistrations.values()) {
1652
+ for (const lease of leases)
1653
+ lease?.release();
489
1654
  }
1655
+ instance.ownerRegistrations.clear();
1656
+ deps.refreshAllSyncRoutes();
1657
+ deps.publishSyncState();
490
1658
  return ok();
491
1659
  }));
492
1660
  });
493
1661
  disposer.defer(() => {
494
1662
  instancesById.delete(instance.id);
1663
+ refreshSyncRoutes();
495
1664
  console.info("evoluDispose", { name, id: instance.id });
496
1665
  });
497
1666
  disposer.use(instance.port);
@@ -511,29 +1680,61 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
511
1680
  return instance[Symbol.asyncDispose]();
512
1681
  });
513
1682
  instance.port.onMessage = (message) => {
1683
+ if (startupError)
1684
+ return;
514
1685
  switch (message.type) {
515
1686
  case "Query":
516
1687
  case "Export": {
517
- queue.push({ type: "ForEvolu", id: instance.id, message });
1688
+ enqueueRequest({
1689
+ type: "Read",
1690
+ request: { type: "ForEvolu", id: instance.id, message },
1691
+ });
518
1692
  runQueue();
519
1693
  break;
520
1694
  }
521
1695
  case "Mutate": {
522
- // TODO: Delegate do vsech evolu instances, co to pouzivaji
523
- queue.push({ type: "ForEvolu", id: instance.id, message });
1696
+ const replicatedOwnerIds = new Set();
1697
+ for (const change of message.changes) {
1698
+ if (!isLocalOnlyTable(change.table))
1699
+ replicatedOwnerIds.add(change.ownerId);
1700
+ }
1701
+ enqueueRequest({
1702
+ type: "Write",
1703
+ request: { type: "ForEvolu", id: instance.id, message },
1704
+ replicatedOwnerIds,
1705
+ });
1706
+ // Local-only changes create no synchronization work.
1707
+ if (replicatedOwnerIds.size > 0) {
1708
+ refreshSyncRoutes();
1709
+ deps.publishSyncState();
1710
+ }
524
1711
  runQueue();
525
1712
  break;
526
1713
  }
527
1714
  case "UseOwner": {
528
1715
  void tenantRun(instance.useOwnerMutex.withLock(async (run) => {
529
- for (const { owner, action } of message.actions) {
530
- console.debug("useOwner", {
531
- id: instance.id,
532
- action,
533
- ownerId: owner.owner.id,
534
- transportUrls: owner.transports.map(({ url }) => url),
535
- });
536
- await run(toggleSyncOwner(instance, owner, action));
1716
+ for (const action of message.actions) {
1717
+ switch (action.action) {
1718
+ case "sync":
1719
+ console.debug("requestSync", {
1720
+ id: instance.id,
1721
+ ownerId: action.ownerId,
1722
+ });
1723
+ requestCreateSyncMessages(new Set([action.ownerId]), allTransports);
1724
+ break;
1725
+ case "add":
1726
+ case "remove":
1727
+ console.debug("useOwner", {
1728
+ id: instance.id,
1729
+ action: action.action,
1730
+ ownerId: action.owner.owner.id,
1731
+ transportUrls: action.owner.transports.map(({ url }) => url),
1732
+ });
1733
+ await run(toggleSyncOwner(instance, action.owner, action.action));
1734
+ break;
1735
+ default:
1736
+ exhaustiveCheck(action);
1737
+ }
537
1738
  }
538
1739
  return ok();
539
1740
  }));
@@ -541,53 +1742,47 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
541
1742
  }
542
1743
  }
543
1744
  };
1745
+ if (startupError)
1746
+ reportRefusal(instance.tabPort, startupError);
544
1747
  },
545
- requestCreateSyncMessages: (ownerIds) => {
546
- const ownersToSync = [...getUsedOwnersById(ownerIds).values()];
547
- if (!isNonEmptyArray(ownersToSync))
1748
+ requestCreateSyncMessages,
1749
+ requestApplySyncMessage: (ownerId, inputMessage, source) => {
1750
+ if (startupError)
548
1751
  return;
549
- console.debug("requestCreateSyncMessages", {
550
- ownerIds: ownersToSync.map(({ id }) => id),
551
- });
552
- queue.push({
553
- type: "ForSharedWorker",
554
- message: {
555
- type: "CreateSyncMessages",
556
- owners: ownersToSync,
557
- },
558
- });
559
- runQueue();
560
- },
561
- requestApplySyncMessage: (ownerId, inputMessage) => {
562
1752
  const owner = getUsedOwnersById(new Set([ownerId])).get(ownerId);
563
1753
  if (!owner)
564
1754
  return;
565
1755
  console.debug("requestApplySyncMessage", {
566
1756
  ownerId,
1757
+ source,
567
1758
  byteLength: inputMessage.byteLength,
568
1759
  });
569
- queue.push({
570
- type: "ForSharedWorker",
571
- message: {
572
- type: "ApplySyncMessage",
573
- owner,
574
- inputMessage,
1760
+ const entry = {
1761
+ type: "ApplySyncMessage",
1762
+ request: {
1763
+ type: "ForSharedWorker",
1764
+ message: { type: "ApplySyncMessage", owner, inputMessage },
575
1765
  },
576
- });
1766
+ source,
1767
+ };
1768
+ enqueueRequest(entry);
1769
+ refreshSyncRoutes();
1770
+ deps.publishSyncState();
577
1771
  runQueue();
578
1772
  },
579
1773
  }, disposer);
580
1774
  currentTenantsByName.set(name, tenant);
1775
+ deps.publishSyncState();
581
1776
  return ok(tenant);
582
1777
  }
583
- catch (e_2) {
584
- env_2.error = e_2;
585
- env_2.hasError = true;
1778
+ catch (e_3) {
1779
+ env_3.error = e_3;
1780
+ env_3.hasError = true;
586
1781
  }
587
1782
  finally {
588
- const result_2 = __disposeResources(env_2);
589
- if (result_2)
590
- await result_2;
1783
+ const result_3 = __disposeResources(env_3);
1784
+ if (result_3)
1785
+ await result_3;
591
1786
  }
592
1787
  };
593
1788
  // | (Typed<"reset"> & {
@@ -614,15 +1809,23 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, me
614
1809
  // })
615
1810
  // TODO: SharedWorker follow-ups.
616
1811
  // - Complete the queue head when a DbWorker mutation returns an error.
617
- // - Make retried DbWorker requests deterministic by materializing clocks and
618
- // timestamps in the SharedWorker before enqueueing them.
619
- // - Replace the callback registry and queueRequestInFlight flag with one
620
- // explicit in-flight request state.
1812
+ // - Rotate the node ID when a copied database is detected; see the Duplicate
1813
+ // node IDs section in the Timestamp module.
621
1814
  // - Detect DbWorker and port liveness so a worker-only crash resumes the queue.
622
- // - Consolidate usedSyncOwners and claimLeasesBySyncOwner into one owner-use
623
- // state abstraction without changing repeated-use semantics.
1815
+ // Defer panicked-worker restart until failure detection and recovery are
1816
+ // defined, accounting for SQLite WASM's detection limits. Normal SQLite
1817
+ // operations are expected not to throw; user-defined UNIQUE indexes, which
1818
+ // can make replicated writes fail, are planned to be forbidden.
624
1819
  // - Split worker protocol types and the EvoluTenant implementation into focused
625
1820
  // modules.
626
1821
  // - Remove the obsolete commented protocol block above.
627
- // - Replace the SyncState placeholder with actual sync monitoring state.
628
1822
  // - Propagate invalid protocol messages to sync state.
1823
+ // - Bound sync state publishing during a bulk catch-up: every sent and applied
1824
+ // frame changes a route timestamp, so each frame broadcasts a full snapshot
1825
+ // to every tab. Throttling needs a wall-clock policy and a deterministic way
1826
+ // to test it.
1827
+ // - Measure sync traffic before bounding it: several tenants using one owner
1828
+ // multiply rounds, and a bulk catch-up from one relay starts up to one round
1829
+ // to each other relay per frame that stores new messages, depending on queued
1830
+ // round coalescing. Forwarding the stored messages the way mutation uploads
1831
+ // do would replace those rounds.