@evolu/common 8.12.0 → 8.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Callbacks.d.ts +12 -1
- package/dist/src/Callbacks.d.ts.map +1 -1
- package/dist/src/Callbacks.js +3 -0
- package/dist/src/Error.d.ts +10 -4
- package/dist/src/Error.d.ts.map +1 -1
- package/dist/src/Error.js +10 -4
- package/dist/src/Object.d.ts +65 -0
- package/dist/src/Object.d.ts.map +1 -1
- package/dist/src/Object.js +142 -0
- package/dist/src/Resource.d.ts +0 -5
- package/dist/src/Resource.d.ts.map +1 -1
- package/dist/src/Resource.js +6 -13
- package/dist/src/Sqlite.d.ts.map +1 -1
- package/dist/src/Sqlite.js +7 -0
- package/dist/src/Task.d.ts +10 -8
- package/dist/src/Task.d.ts.map +1 -1
- package/dist/src/Task.js +41 -5
- package/dist/src/Worker.d.ts +3 -3
- package/dist/src/local-first/Db.d.ts +8 -3
- package/dist/src/local-first/Db.d.ts.map +1 -1
- package/dist/src/local-first/Db.js +56 -19
- package/dist/src/local-first/Evolu.d.ts +81 -37
- package/dist/src/local-first/Evolu.d.ts.map +1 -1
- package/dist/src/local-first/Evolu.js +31 -6
- package/dist/src/local-first/Owner.d.ts +9 -0
- package/dist/src/local-first/Owner.d.ts.map +1 -1
- package/dist/src/local-first/Owner.js +9 -0
- package/dist/src/local-first/Protocol.d.ts +6 -4
- package/dist/src/local-first/Protocol.d.ts.map +1 -1
- package/dist/src/local-first/Protocol.js +1 -6
- package/dist/src/local-first/Schema.d.ts +7 -1
- package/dist/src/local-first/Schema.d.ts.map +1 -1
- package/dist/src/local-first/Shared.d.ts +523 -133
- package/dist/src/local-first/Shared.d.ts.map +1 -1
- package/dist/src/local-first/Shared.js +696 -256
- package/dist/src/local-first/Storage.d.ts +16 -12
- package/dist/src/local-first/Storage.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/Callbacks.test.ts +20 -0
- package/src/Callbacks.ts +17 -1
- package/src/Error.ts +10 -4
- package/src/Object.test.ts +296 -0
- package/src/Object.ts +163 -0
- package/src/Resource.test.ts +20 -16
- package/src/Resource.ts +6 -20
- package/src/Sqlite.ts +7 -0
- package/src/Task.test.ts +233 -62
- package/src/Task.ts +47 -13
- package/src/Worker.ts +3 -3
- package/src/local-first/Db.ts +88 -18
- package/src/local-first/Evolu.test.ts +281 -1
- package/src/local-first/Evolu.ts +124 -46
- package/src/local-first/Owner.ts +9 -0
- package/src/local-first/Protocol.test.ts +42 -60
- package/src/local-first/Protocol.ts +7 -9
- package/src/local-first/Schema.ts +8 -2
- package/src/local-first/Shared.test.ts +2633 -646
- package/src/local-first/Shared.ts +1192 -364
- package/src/local-first/Storage.ts +18 -19
|
@@ -53,7 +53,9 @@
|
|
|
53
53
|
* still waiting after three seconds reports {@link OtherBuildRunningError} to
|
|
54
54
|
* its tabs. Every build broadcasts console entries and errors on
|
|
55
55
|
* {@link consoleEntryOrErrorBroadcastChannelName}, so a waiting tab also prints
|
|
56
|
-
* the running build's output
|
|
56
|
+
* the running build's output, reports each {@link UnknownError} it sends as its
|
|
57
|
+
* own `evoluError`, and only logs any other error, such as an older build's
|
|
58
|
+
* sync error.
|
|
57
59
|
*
|
|
58
60
|
* The tabs of one worker elect the host of its DbWorkers among themselves, with
|
|
59
61
|
* a lock scoped to the worker, so a tab of another worker never hosts them.
|
|
@@ -100,7 +102,8 @@
|
|
|
100
102
|
* writable registrations for its owner, whichever instance made it, even one
|
|
101
103
|
* disposed before the database worker answered. When a relay frame stores new
|
|
102
104
|
* messages, the tenant requests a round through each other transport claimed
|
|
103
|
-
* for the owner, so data learned from one relay reaches the others
|
|
105
|
+
* for the owner, so data learned from one relay reaches the others, except
|
|
106
|
+
* through a route that skipped a message, as described below. A closed
|
|
104
107
|
* transport reconciles when it opens, and a replacement leader reconciles every
|
|
105
108
|
* transport again, because a response reporting stored messages may have been
|
|
106
109
|
* lost.
|
|
@@ -109,26 +112,34 @@
|
|
|
109
112
|
* also delivers it as local Broadcast frames to every other tenant with
|
|
110
113
|
* writable access to the owner, even while sockets are closed. A copy keeps the
|
|
111
114
|
* uploader's target, and a recipient that stores new continuation messages
|
|
112
|
-
* reconciles them through the transports outside that target
|
|
113
|
-
*
|
|
114
|
-
* but does not reconcile local
|
|
115
|
-
* until replication scopes and
|
|
115
|
+
* reconciles them through the transports outside that target, except through a
|
|
116
|
+
* route that skipped a message. Local delivery forwards uploads, including
|
|
117
|
+
* historical messages sent during reconciliation, but does not reconcile local
|
|
118
|
+
* database histories with each other. That waits until replication scopes and
|
|
119
|
+
* retention semantics are defined.
|
|
116
120
|
*
|
|
117
121
|
* ## Sync state
|
|
118
122
|
*
|
|
119
123
|
* The shared worker publishes one plain snapshot, {@link SyncState}, of every
|
|
120
124
|
* transport it manages and every database and owner registration it holds, with
|
|
121
|
-
* one route per writable registration and transport, as specified below.
|
|
122
|
-
*
|
|
125
|
+
* one route per writable registration and transport, as specified below. Each
|
|
126
|
+
* part is a union of the states the worker keeps for it: a transport's
|
|
127
|
+
* connection, a database that is active or refused startup, a writable or
|
|
128
|
+
* readonly owner, and a pending, settled, or complete route. A transport's
|
|
129
|
+
* events drive its connection, so publishing never reads a socket.
|
|
130
|
+
*
|
|
131
|
+
* Apps show users an owner's {@link OwnerSyncStatus}, which
|
|
132
|
+
* {@link syncStateToOwnerSyncStatus} derives. A view of each relay pairs each
|
|
133
|
+
* route with its transport by {@link syncStateToRelaySyncStates} and tells the
|
|
134
|
+
* relay's status by {@link relaySyncStateToStatus}. Snapshots store neither, so
|
|
135
|
+
* a status never disagrees with the routes it comes from.
|
|
123
136
|
*
|
|
124
137
|
* Each worker broadcasts snapshots on its own channel, whose name a connecting
|
|
125
138
|
* tab receives through its port, so a tab never hears another worker, such as
|
|
126
139
|
* one of a different app version, and
|
|
127
140
|
* {@link SyncStateDep.syncState | deps.syncState} keeps the last snapshot. The
|
|
128
|
-
* worker publishes after every change it observes
|
|
129
|
-
*
|
|
130
|
-
* snapshot. The snapshot lives in worker memory only, so a new worker starts
|
|
131
|
-
* empty.
|
|
141
|
+
* worker publishes after every change it observes. The snapshot lives in worker
|
|
142
|
+
* memory only, so a new worker starts empty.
|
|
132
143
|
*
|
|
133
144
|
* ## Synchronization completion
|
|
134
145
|
*
|
|
@@ -156,27 +167,51 @@
|
|
|
156
167
|
* - The tenant has sent a round through the transport since the last event that
|
|
157
168
|
* requires one: its first use of the transport for the owner, the socket
|
|
158
169
|
* opening, an explicit request, a replacement leader, or storing messages
|
|
159
|
-
* from another transport
|
|
160
|
-
* route since.
|
|
170
|
+
* from another transport, unless the route skipped a message. No failed or
|
|
171
|
+
* aborted result has arrived on the route since.
|
|
172
|
+
* - No result that skipped a received message has arrived on the route since a
|
|
173
|
+
* round was last requested through it, and since that request, no messages
|
|
174
|
+
* have been stored from another transport, directly or through a sibling copy
|
|
175
|
+
* uploaded outside this transport, and no sibling copy has failed to apply or
|
|
176
|
+
* skipped a message.
|
|
177
|
+
*
|
|
178
|
+
* A route is settled when every condition except the last holds, so a complete
|
|
179
|
+
* route is settled too. A settled route that is incomplete has ended its
|
|
180
|
+
* reconciliation, but its relay may offer a message the database skipped, or
|
|
181
|
+
* messages stored elsewhere may not have reached it; it is published as
|
|
182
|
+
* {@link SettledSyncRoute}.
|
|
161
183
|
*
|
|
162
184
|
* A reconciliation chain ends only with a converged result, a failure, an
|
|
163
185
|
* abort, a dropped frame, or a continuation that finds the socket closed, and
|
|
164
186
|
* relay errors reach every applying tenant, so these conditions mean every
|
|
165
187
|
* chain, including this tenant's, converged. A local Broadcast from a sibling
|
|
166
188
|
* tenant holds every route of its owner until it is applied, because it arrives
|
|
167
|
-
* without a request of its own; one that fails to apply
|
|
168
|
-
* through every transport.
|
|
189
|
+
* without a request of its own; one that fails to apply or skips a message
|
|
190
|
+
* requires a round through every transport whose route has not skipped one.
|
|
169
191
|
*
|
|
170
192
|
* A failed result on a route requests one round through it. Any further failure
|
|
171
|
-
* before the route
|
|
193
|
+
* before the route settles waits for an explicit request or a reopen, so a
|
|
172
194
|
* persistent failure cannot loop; a converged reply in between does not end the
|
|
173
195
|
* wait, because it may answer another tenant's round on the shared socket. An
|
|
174
196
|
* aborted apply leaves its routes incomplete without a retry. An exception
|
|
175
197
|
* while the database worker creates a round is logged there and fails the
|
|
176
|
-
* round's routes with `SyncFailed` without a retry
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
198
|
+
* round's routes with `SyncFailed` without a retry. A mutation that throws is
|
|
199
|
+
* rolled back and reported as an {@link UnknownError} to the tab that made it,
|
|
200
|
+
* or to every tab when its Evolu instance was disposed first. Other unexpected
|
|
201
|
+
* SQLite exceptions remain unsupported and can panic the database worker. A
|
|
202
|
+
* frame the relay silently drops, such as invalid data, leaves the count above
|
|
203
|
+
* zero until the liveness rule below replaces the socket.
|
|
204
|
+
*
|
|
205
|
+
* A result that skipped a received message the database could not decrypt,
|
|
206
|
+
* verify, or decode records the error on the route as its `skippedError`,
|
|
207
|
+
* without ending its chain, so it requests no round, and the relay offers the
|
|
208
|
+
* message again in every round through the route until the database stores a
|
|
209
|
+
* message with that timestamp. The messages received elsewhere that the last
|
|
210
|
+
* condition names request no round through such a route, even while a requested
|
|
211
|
+
* round checks it again, because each round would download every skipped
|
|
212
|
+
* message again. The next round that a first use of the transport for the
|
|
213
|
+
* owner, an explicit request, a reopen, a replacement leader, or a failure
|
|
214
|
+
* sends through the route reconciles them.
|
|
180
215
|
*
|
|
181
216
|
* ### Liveness
|
|
182
217
|
*
|
|
@@ -196,11 +231,13 @@
|
|
|
196
231
|
* The timeout belongs to the transport, not to an owner, because a small reply
|
|
197
232
|
* for one owner can wait behind another owner's large frame on the socket. A
|
|
198
233
|
* grown timeout lasts while a request is outstanding on the socket or a
|
|
199
|
-
* database that has not refused startup has an
|
|
200
|
-
* including one waiting after a failure, and ends with the transport. A
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
234
|
+
* database that has not refused startup has an unsettled route through it,
|
|
235
|
+
* including one waiting after a failure, and ends with the transport. A settled
|
|
236
|
+
* route that is incomplete does not count, because its reconciliation has
|
|
237
|
+
* ended. A reply's speed proves nothing, because a recovery on a slow link
|
|
238
|
+
* starts with small replies that arrive quickly. The reopen resets the counts
|
|
239
|
+
* and starts the open rounds, so a dropped frame delays a route instead of
|
|
240
|
+
* stranding it.
|
|
204
241
|
*
|
|
205
242
|
* The shared worker sends nothing while idle, so a path that dies then may go
|
|
206
243
|
* unnoticed until its next request; until then, changes from other devices stop
|
|
@@ -226,7 +263,11 @@ import {
|
|
|
226
263
|
} from "../Assert.ts";
|
|
227
264
|
import type { Brand } from "../Brand.ts";
|
|
228
265
|
import type { ConsoleEntry, ConsoleLevel } from "../Console.ts";
|
|
229
|
-
import type {
|
|
266
|
+
import type {
|
|
267
|
+
DecryptWithXChaCha20Poly1305Error,
|
|
268
|
+
EncryptionKey,
|
|
269
|
+
} from "../Crypto.ts";
|
|
270
|
+
import { createUnknownError, type UnknownError } from "../Error.ts";
|
|
230
271
|
import { disposable, exhaustiveCheck } from "../Function.ts";
|
|
231
272
|
import { acquireLeaderLock, type LockManagerDep } from "../LockManager.ts";
|
|
232
273
|
import {
|
|
@@ -281,7 +322,6 @@ import type {
|
|
|
281
322
|
CreateWebSocketDep,
|
|
282
323
|
WebSocket,
|
|
283
324
|
WebSocketError,
|
|
284
|
-
WebSocketReadyState,
|
|
285
325
|
} from "../WebSocket.ts";
|
|
286
326
|
import type {
|
|
287
327
|
SharedWorker as CommonSharedWorker,
|
|
@@ -293,7 +333,7 @@ import type {
|
|
|
293
333
|
WorkerDeps,
|
|
294
334
|
} from "../Worker.ts";
|
|
295
335
|
import type { DbWorkerInit, UnsupportedDbVersionError } from "./Db.ts";
|
|
296
|
-
import type {
|
|
336
|
+
import type { Evolu, SyncStateDep } from "./Evolu.ts";
|
|
297
337
|
import type { Owner, OwnerId, OwnerTransport, SyncOwner } from "./Owner.ts";
|
|
298
338
|
import {
|
|
299
339
|
createProtocolBroadcastMessagesFromCrdtMessages,
|
|
@@ -303,7 +343,10 @@ import {
|
|
|
303
343
|
parseProtocolHeader,
|
|
304
344
|
type ApplyProtocolMessageAsClientResult,
|
|
305
345
|
type ProtocolError,
|
|
346
|
+
type ProtocolInvalidDataError,
|
|
306
347
|
type ProtocolMessage,
|
|
348
|
+
type ProtocolQuotaError,
|
|
349
|
+
type ProtocolTimestampMismatchError,
|
|
307
350
|
} from "./Protocol.ts";
|
|
308
351
|
import {
|
|
309
352
|
makePatches,
|
|
@@ -352,11 +395,12 @@ export type SharedWorkerOutput =
|
|
|
352
395
|
| DbWorkerInit
|
|
353
396
|
| {
|
|
354
397
|
/**
|
|
355
|
-
* Sent to one tab only: its database refused startup,
|
|
356
|
-
* keeps this worker waiting.
|
|
398
|
+
* Sent to one tab only: its database refused startup, a mutation it made
|
|
399
|
+
* could not be stored, or another build keeps this worker waiting.
|
|
357
400
|
*/
|
|
358
401
|
readonly type: "Error";
|
|
359
|
-
readonly error:
|
|
402
|
+
readonly error:
|
|
403
|
+
OtherBuildRunningError | UnknownError | UnsupportedDbVersionError;
|
|
360
404
|
}
|
|
361
405
|
| {
|
|
362
406
|
/**
|
|
@@ -393,7 +437,7 @@ export type ConsoleEntryOrError =
|
|
|
393
437
|
}
|
|
394
438
|
| {
|
|
395
439
|
readonly type: "Error";
|
|
396
|
-
readonly error:
|
|
440
|
+
readonly error: UnknownError;
|
|
397
441
|
};
|
|
398
442
|
|
|
399
443
|
export const consoleEntryOrErrorBroadcastChannelName =
|
|
@@ -451,31 +495,31 @@ export interface BuildWaitingRequest extends InferType<
|
|
|
451
495
|
export interface OtherBuildRunningError extends Typed<"OtherBuildRunningError"> {}
|
|
452
496
|
|
|
453
497
|
/**
|
|
454
|
-
* A snapshot of the transports and databases the shared worker manages.
|
|
455
|
-
*
|
|
456
|
-
*
|
|
498
|
+
* A snapshot of the transports and databases the shared worker manages. Each
|
|
499
|
+
* part is a union of the states the worker keeps for it, so each part holds
|
|
500
|
+
* only the fields valid for its state. Apps derive what to show with
|
|
501
|
+
* {@link syncStateToOwnerSyncStatus}; see Sync state in this module's
|
|
502
|
+
* documentation.
|
|
457
503
|
*/
|
|
458
504
|
export interface SyncState {
|
|
459
505
|
readonly transports: ReadonlyArray<SyncTransport>;
|
|
460
506
|
readonly tenants: ReadonlyArray<SyncTenant>;
|
|
461
507
|
}
|
|
462
508
|
|
|
463
|
-
/**
|
|
464
|
-
|
|
465
|
-
|
|
509
|
+
/**
|
|
510
|
+
* One transport, shared by every owner and database claiming it. Its `type` is
|
|
511
|
+
* the kind of transport, and its {@link SyncConnection} tells whether it is
|
|
512
|
+
* connected.
|
|
513
|
+
*/
|
|
514
|
+
export type SyncTransport = WebSocketSyncTransport;
|
|
515
|
+
|
|
516
|
+
/** A WebSocket {@link SyncTransport}. */
|
|
517
|
+
export interface WebSocketSyncTransport extends Typed<"WebSocket"> {
|
|
518
|
+
/** Opaque and stable for the transport's lifetime, across reconnects. */
|
|
466
519
|
readonly id: SyncTransportId;
|
|
467
|
-
/** The URL without
|
|
520
|
+
/** The relay URL without the owner-specific query. */
|
|
468
521
|
readonly label: string;
|
|
469
|
-
readonly
|
|
470
|
-
/** When the connection last opened, or null before its first open. */
|
|
471
|
-
readonly openedAt: Millis | null;
|
|
472
|
-
/** When the connection last closed, or null before its first close. */
|
|
473
|
-
readonly closedAt: Millis | null;
|
|
474
|
-
/**
|
|
475
|
-
* The last error, retained after a successful reconnect; null if none. Errors
|
|
476
|
-
* while reconnecting are routine.
|
|
477
|
-
*/
|
|
478
|
-
readonly error: SyncTransportError | null;
|
|
522
|
+
readonly connection: SyncConnection;
|
|
479
523
|
}
|
|
480
524
|
|
|
481
525
|
export type SyncTransportId = Id & Brand<"SyncTransport">;
|
|
@@ -485,109 +529,518 @@ export interface SyncTransportError {
|
|
|
485
529
|
readonly at: Millis;
|
|
486
530
|
}
|
|
487
531
|
|
|
488
|
-
/**
|
|
489
|
-
|
|
532
|
+
/**
|
|
533
|
+
* The connection of a {@link SyncTransport}: `Connecting` until its first
|
|
534
|
+
* connection opens or fails, `Open` while a connection is open, and
|
|
535
|
+
* `Disconnected` from a close, a failure, or an unanswered request that
|
|
536
|
+
* replaced the connection, until a connection opens again. It never returns to
|
|
537
|
+
* `Connecting`. The transport's events drive these states.
|
|
538
|
+
*/
|
|
539
|
+
export type SyncConnection =
|
|
540
|
+
ConnectingSyncConnection | OpenSyncConnection | DisconnectedSyncConnection;
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* A first connection, which has neither opened nor failed. A connection to a
|
|
544
|
+
* host that drops packets stays here until the operating system or browser
|
|
545
|
+
* gives up on it, which can take a minute or more.
|
|
546
|
+
*/
|
|
547
|
+
export interface ConnectingSyncConnection extends Typed<"Connecting"> {}
|
|
548
|
+
|
|
549
|
+
/** An open connection. */
|
|
550
|
+
export interface OpenSyncConnection extends Typed<"Open"> {
|
|
551
|
+
/** When this connection opened. */
|
|
552
|
+
readonly openedAt: Millis;
|
|
553
|
+
/** The last error of an earlier connection or attempt, or null. */
|
|
554
|
+
readonly error: SyncTransportError | null;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* No connection: a connection closed or failed, or the relay did not answer a
|
|
559
|
+
* request in time. The transport reconnects by itself.
|
|
560
|
+
*/
|
|
561
|
+
export interface DisconnectedSyncConnection extends Typed<"Disconnected"> {
|
|
562
|
+
/** When it disconnected. Failed reconnect attempts leave it unchanged. */
|
|
563
|
+
readonly disconnectedAt: Millis;
|
|
564
|
+
/** When the last connection opened, or null when none has. */
|
|
565
|
+
readonly openedAt: Millis | null;
|
|
566
|
+
/**
|
|
567
|
+
* The last error, or null. A close or an unanswered request records none, so
|
|
568
|
+
* it can come from an earlier connection. Errors while reconnecting are
|
|
569
|
+
* routine.
|
|
570
|
+
*/
|
|
571
|
+
readonly error: SyncTransportError | null;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** One named local database: `Active`, or `Refused` when it refused startup. */
|
|
575
|
+
export type SyncTenant = ActiveSyncTenant | RefusedSyncTenant;
|
|
576
|
+
|
|
577
|
+
/**
|
|
578
|
+
* A database that has not refused startup, including one still starting, with
|
|
579
|
+
* the owners its instances registered.
|
|
580
|
+
*/
|
|
581
|
+
export interface ActiveSyncTenant extends Typed<"Active"> {
|
|
490
582
|
readonly name: Name;
|
|
491
|
-
/** The database refused startup, so nothing it holds synchronizes. */
|
|
492
|
-
readonly refused: boolean;
|
|
493
583
|
readonly owners: ReadonlyArray<SyncTenantOwner>;
|
|
494
584
|
}
|
|
495
585
|
|
|
496
|
-
|
|
586
|
+
/**
|
|
587
|
+
* A database that refused startup, so nothing it holds syncs. Its tabs also
|
|
588
|
+
* receive the error through `evoluError`.
|
|
589
|
+
*/
|
|
590
|
+
export interface RefusedSyncTenant extends Typed<"Refused"> {
|
|
591
|
+
readonly name: Name;
|
|
592
|
+
readonly error: UnsupportedDbVersionError;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* An owner registered by any instance of an {@link ActiveSyncTenant}: `Writable`
|
|
597
|
+
* when any registration holds its write key, otherwise `Readonly`.
|
|
598
|
+
*/
|
|
599
|
+
export type SyncTenantOwner = WritableSyncTenantOwner | ReadonlySyncTenantOwner;
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* An owner the database syncs, with one route per transport claimed for it by
|
|
603
|
+
* this or any other database. Routes are briefly empty while the owner's
|
|
604
|
+
* transports are being claimed.
|
|
605
|
+
*/
|
|
606
|
+
export interface WritableSyncTenantOwner extends Typed<"Writable"> {
|
|
607
|
+
readonly ownerId: OwnerId;
|
|
608
|
+
readonly routes: ReadonlyArray<SyncRoute>;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* An owner registered only without its write key. Its registrations hold
|
|
613
|
+
* transports, which other databases' routes for the owner use, but this
|
|
614
|
+
* database does not sync it.
|
|
615
|
+
*/
|
|
616
|
+
export interface ReadonlySyncTenantOwner extends Typed<"Readonly"> {
|
|
497
617
|
readonly ownerId: OwnerId;
|
|
498
|
-
/** A readonly registration holds transports but never synchronizes. */
|
|
499
|
-
readonly writable: boolean;
|
|
500
618
|
/** Every transport claimed for the owner, by any database. */
|
|
501
619
|
readonly transportIds: ReadonlyArray<SyncTransportId>;
|
|
502
|
-
/** One route per transport for a writable owner; none for a readonly one. */
|
|
503
|
-
readonly routes: ReadonlyArray<SyncRoute>;
|
|
504
620
|
}
|
|
505
621
|
|
|
506
622
|
/**
|
|
507
|
-
* One database's use of one owner through one transport
|
|
508
|
-
*
|
|
623
|
+
* One database's use of one owner through one transport: `Pending` until its
|
|
624
|
+
* reconciliation ends, then `Complete`, or `Settled` while its relay offers a
|
|
625
|
+
* change the database skipped. See Synchronization completion in this module's
|
|
626
|
+
* documentation.
|
|
509
627
|
*/
|
|
510
|
-
export
|
|
628
|
+
export type SyncRoute = PendingSyncRoute | SettledSyncRoute | CompleteSyncRoute;
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* A route whose reconciliation has not ended: its transport is not open, a
|
|
632
|
+
* request or a received frame is outstanding, a replicated write is queued, or
|
|
633
|
+
* a round must still be sent through it.
|
|
634
|
+
*/
|
|
635
|
+
export interface PendingSyncRoute extends Typed<"Pending"> {
|
|
511
636
|
readonly transportId: SyncTransportId;
|
|
512
|
-
/**
|
|
513
|
-
|
|
637
|
+
/**
|
|
638
|
+
* The failure since the route last settled, or null. The first one requests a
|
|
639
|
+
* round; a further one waits for {@link Evolu.requestSync} or a reopen.
|
|
640
|
+
*/
|
|
641
|
+
readonly failure: SyncRouteError | null;
|
|
642
|
+
/**
|
|
643
|
+
* The first change skipped in the latest reply that skipped one, or null. The
|
|
644
|
+
* relay offers it again in every round through the route until this database
|
|
645
|
+
* stores a change with that timestamp.
|
|
646
|
+
*/
|
|
647
|
+
readonly skippedError: SyncRouteError | null;
|
|
514
648
|
/** When the route last became complete, or null. */
|
|
515
649
|
readonly completeAt: Millis | null;
|
|
516
650
|
/** When this database last sent a request through the route, or null. */
|
|
517
651
|
readonly lastSentAt: Millis | null;
|
|
518
652
|
/**
|
|
519
653
|
* When processing a frame from the route last finished, successfully or with
|
|
520
|
-
* a failure, or null. Aborted processing does not update
|
|
654
|
+
* a failure, or null. Aborted processing does not update it.
|
|
521
655
|
*/
|
|
522
656
|
readonly lastReceivedAt: Millis | null;
|
|
523
|
-
/** The last failed result on the route; cleared when the route completes. */
|
|
524
|
-
readonly error: SyncRouteError | null;
|
|
525
657
|
}
|
|
526
658
|
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
659
|
+
/**
|
|
660
|
+
* A route whose reconciliation ended while its relay offers a change the
|
|
661
|
+
* database skipped, so it is incomplete. Changes stored from other relays
|
|
662
|
+
* request no round through it; the next round requested through it, such as by
|
|
663
|
+
* {@link Evolu.requestSync} or a reopen, checks it again.
|
|
664
|
+
*/
|
|
665
|
+
export interface SettledSyncRoute extends Typed<"Settled"> {
|
|
666
|
+
readonly transportId: SyncTransportId;
|
|
667
|
+
/** The first change skipped in the latest reply that skipped one. */
|
|
668
|
+
readonly skippedError: SyncRouteError;
|
|
669
|
+
/** When the route last became complete, or null. */
|
|
670
|
+
readonly completeAt: Millis | null;
|
|
671
|
+
/** When this database last sent a request through the route. */
|
|
672
|
+
readonly lastSentAt: Millis;
|
|
673
|
+
/** When processing a frame from the route last finished, or null. */
|
|
674
|
+
readonly lastReceivedAt: Millis | null;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/** A route on which the database is reconciled with the relay for the owner. */
|
|
678
|
+
export interface CompleteSyncRoute extends Typed<"Complete"> {
|
|
679
|
+
readonly transportId: SyncTransportId;
|
|
680
|
+
/** When the route became complete. */
|
|
681
|
+
readonly completeAt: Millis;
|
|
682
|
+
/** When this database last sent a request through the route. */
|
|
683
|
+
readonly lastSentAt: Millis;
|
|
684
|
+
/** When processing a frame from the route last finished, or null. */
|
|
685
|
+
readonly lastReceivedAt: Millis | null;
|
|
530
686
|
}
|
|
531
687
|
|
|
532
688
|
/**
|
|
533
|
-
* A
|
|
534
|
-
*
|
|
535
|
-
*
|
|
536
|
-
*
|
|
689
|
+
* A failure or a skipped change of a {@link SyncRoute}, with the time it
|
|
690
|
+
* arrived.
|
|
691
|
+
*
|
|
692
|
+
* It is the error itself, so it carries its details, such as the expected and
|
|
693
|
+
* actual timestamps of a {@link ProtocolTimestampMismatchError}. A skipped
|
|
694
|
+
* message adds {@link DecryptWithXChaCha20Poly1305Error}. A
|
|
695
|
+
* {@link ProtocolInvalidDataError} leaves out its data, which can be a whole
|
|
696
|
+
* frame. The caught value in the `error` of either is an {@link UnknownError}.
|
|
697
|
+
* `WriteFailed` means a `writeMessages` call that threw, logged by the
|
|
698
|
+
* protocol, and `SyncFailed` means a logged failure while creating a round or
|
|
699
|
+
* reconciling ranges.
|
|
537
700
|
*/
|
|
538
|
-
export type
|
|
539
|
-
| ProtocolError
|
|
540
|
-
|
|
|
541
|
-
|
|
|
542
|
-
|
|
|
701
|
+
export type SyncRouteError = (
|
|
702
|
+
| Exclude<ProtocolError, ProtocolInvalidDataError>
|
|
703
|
+
| Omit<ProtocolInvalidDataError, "data">
|
|
704
|
+
| StorageWriteMessagesError
|
|
705
|
+
| DecryptWithXChaCha20Poly1305Error
|
|
706
|
+
| Typed<"WriteFailed">
|
|
707
|
+
| Typed<"SyncFailed">
|
|
708
|
+
) & { readonly at: Millis };
|
|
709
|
+
|
|
710
|
+
/** The type of a {@link SyncRouteError}. */
|
|
711
|
+
export type SyncRouteErrorType = SyncRouteError["type"];
|
|
543
712
|
|
|
544
713
|
/**
|
|
545
|
-
* One
|
|
546
|
-
*
|
|
714
|
+
* One relay of an owner in one database: a transport and the database's route
|
|
715
|
+
* through it, from {@link syncStateToRelaySyncStates}. Its status comes from
|
|
716
|
+
* {@link relaySyncStateToStatus} and is not stored beside them.
|
|
547
717
|
*/
|
|
548
|
-
export interface
|
|
549
|
-
readonly
|
|
550
|
-
readonly
|
|
551
|
-
readonly status: OwnerSyncStatus;
|
|
552
|
-
/** When a route of the owner last became complete, or null. */
|
|
553
|
-
readonly syncedAt: Millis | null;
|
|
554
|
-
/** The newest route error of the owner, or null. */
|
|
555
|
-
readonly error: SyncRouteError | null;
|
|
556
|
-
/** Each relay of the owner, in the order of its routes. */
|
|
557
|
-
readonly relays: ReadonlyArray<RelaySyncState>;
|
|
718
|
+
export interface RelaySyncState {
|
|
719
|
+
readonly transport: SyncTransport;
|
|
720
|
+
readonly route: SyncRoute;
|
|
558
721
|
}
|
|
559
722
|
|
|
560
723
|
/**
|
|
561
|
-
*
|
|
562
|
-
*
|
|
563
|
-
*
|
|
564
|
-
*
|
|
724
|
+
* What an app shows users about syncing one owner of one database, from
|
|
725
|
+
* {@link syncStateToOwnerSyncStatus} or the React and Vue `useOwnerSyncStatus`.
|
|
726
|
+
*
|
|
727
|
+
* Its variants:
|
|
728
|
+
*
|
|
729
|
+
* - `NoRelays`: nothing syncs the owner here. There is no snapshot yet, the
|
|
730
|
+
* owner's transports are still being set up, the app uses no relays for it,
|
|
731
|
+
* it is registered only as readonly, or the database refused startup, which
|
|
732
|
+
* `evoluError` reports.
|
|
733
|
+
* - `Syncing`: a relay connects for the first time or reconciles. A first
|
|
734
|
+
* connection to a host that drops packets stays `Syncing` until the platform
|
|
735
|
+
* gives up on it, which can take a minute.
|
|
736
|
+
* - `Synced`: a relay is up to date, and none syncs.
|
|
737
|
+
* - `Offline`: every relay is disconnected. Evolu keeps reconnecting, up to 30
|
|
738
|
+
* seconds apart, so `Offline` can briefly outlast the outage.
|
|
739
|
+
* - `Error`: a relay failed or offers a change this database skipped. `error` is
|
|
740
|
+
* the newest failure, or without one, the newest skipped change: a failure
|
|
741
|
+
* stops syncing through its relay, while a skipped change leaves out only
|
|
742
|
+
* that change.
|
|
743
|
+
*
|
|
744
|
+
* Evolu saves changes on the device before they sync, so sync needs no UI while
|
|
745
|
+
* it works: show nothing for `NoRelays`, `Syncing`, and `Synced`. An indicator
|
|
746
|
+
* that changes with every edit distracts, and screen readers announce each
|
|
747
|
+
* change.
|
|
748
|
+
*
|
|
749
|
+
* For `Offline` and `Error`, show one quiet line that lasts as long as the
|
|
750
|
+
* status, not a dialog, which interrupts, or a toast, which disappears while
|
|
751
|
+
* the problem lasts. Say that changes are saved on this device. Evolu reports
|
|
752
|
+
* `Offline` at once; an app may wait a few seconds before showing it, because
|
|
753
|
+
* brief disconnections, such as waking from sleep, reconnect quickly. For
|
|
754
|
+
* `Error`, write actionable text for the error types the app can act on, such
|
|
755
|
+
* as a {@link ProtocolQuotaError}: the relay stores no more data for the owner,
|
|
756
|
+
* so offer more quota, such as a plan upgrade, then call
|
|
757
|
+
* {@link Evolu.requestSync} with the owner's ID. For any other error, show
|
|
758
|
+
* generic text that names the error type, which helps when the user reports
|
|
759
|
+
* it.
|
|
760
|
+
*
|
|
761
|
+
* Render the line inside one element with `role="status"` that stays mounted:
|
|
762
|
+
* screen readers announce changes only in a live region that already exists,
|
|
763
|
+
* and a polite announcement fits a status that loses nothing. Do not use
|
|
764
|
+
* `role="alert"`.
|
|
765
|
+
*
|
|
766
|
+
* Show the app owner's status once for the whole app, such as below the header,
|
|
767
|
+
* and another owner's status where the app shows that owner's data. Each Evolu
|
|
768
|
+
* instance finds its own status by {@link Evolu.name}, so an app with several
|
|
769
|
+
* databases shows the status of the one the user works in.
|
|
770
|
+
*
|
|
771
|
+
* ### Example
|
|
772
|
+
*
|
|
773
|
+
* ```ts
|
|
774
|
+
* import { assertEqual, Millis } from "@evolu/common";
|
|
775
|
+
* import {
|
|
776
|
+
* testAppOwner,
|
|
777
|
+
* type OwnerSyncStatus,
|
|
778
|
+
* } from "@evolu/common/local-first";
|
|
779
|
+
*
|
|
780
|
+
* // What to tell the user, or null while sync works or is not used.
|
|
781
|
+
* const syncStatusToMessage = (status: OwnerSyncStatus): string | null => {
|
|
782
|
+
* switch (status.type) {
|
|
783
|
+
* case "NoRelays":
|
|
784
|
+
* case "Syncing":
|
|
785
|
+
* case "Synced":
|
|
786
|
+
* return null;
|
|
787
|
+
* case "Offline":
|
|
788
|
+
* return "Offline. Your changes are saved on this device.";
|
|
789
|
+
* case "Error":
|
|
790
|
+
* return status.error.type === "ProtocolQuotaError"
|
|
791
|
+
* ? "Sync is paused because the sync server is full. Your changes are saved on this device."
|
|
792
|
+
* : `Sync error: ${status.error.type}. Your changes are saved on this device.`;
|
|
793
|
+
* }
|
|
794
|
+
* };
|
|
795
|
+
*
|
|
796
|
+
* assertEqual(syncStatusToMessage({ type: "Synced" }), null);
|
|
797
|
+
* assertEqual(
|
|
798
|
+
* syncStatusToMessage({ type: "Offline" }),
|
|
799
|
+
* "Offline. Your changes are saved on this device.",
|
|
800
|
+
* );
|
|
801
|
+
* assertEqual(
|
|
802
|
+
* syncStatusToMessage({
|
|
803
|
+
* type: "Error",
|
|
804
|
+
* error: {
|
|
805
|
+
* type: "ProtocolQuotaError",
|
|
806
|
+
* ownerId: testAppOwner.id,
|
|
807
|
+
* at: Millis.orThrow(1000),
|
|
808
|
+
* },
|
|
809
|
+
* }),
|
|
810
|
+
* "Sync is paused because the sync server is full. Your changes are saved on this device.",
|
|
811
|
+
* );
|
|
812
|
+
* assertEqual(
|
|
813
|
+
* syncStatusToMessage({
|
|
814
|
+
* type: "Error",
|
|
815
|
+
* error: { type: "SyncFailed", at: Millis.orThrow(1000) },
|
|
816
|
+
* }),
|
|
817
|
+
* "Sync error: SyncFailed. Your changes are saved on this device.",
|
|
818
|
+
* );
|
|
819
|
+
* ```
|
|
565
820
|
*/
|
|
566
|
-
export type OwnerSyncStatus =
|
|
821
|
+
export type OwnerSyncStatus = NoRelaysSyncStatus | RelaySyncStatus;
|
|
567
822
|
|
|
568
823
|
/**
|
|
569
|
-
*
|
|
570
|
-
*
|
|
824
|
+
* The status of a {@link RelaySyncState}, from {@link relaySyncStateToStatus}:
|
|
825
|
+
* `Error` when its route has a failure or a skipped change, the failure first;
|
|
826
|
+
* otherwise `Synced` when the route is complete, `Syncing` while the
|
|
827
|
+
* transport's connection is `Connecting` or `Open`, and `Offline` while it is
|
|
828
|
+
* `Disconnected`.
|
|
571
829
|
*/
|
|
572
|
-
export
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
830
|
+
export type RelaySyncStatus =
|
|
831
|
+
SyncingSyncStatus | SyncedSyncStatus | OfflineSyncStatus | ErrorSyncStatus;
|
|
832
|
+
|
|
833
|
+
/** Evolu reports no relay syncing the owner in the database. */
|
|
834
|
+
export interface NoRelaysSyncStatus extends Typed<"NoRelays"> {}
|
|
835
|
+
|
|
836
|
+
/** A relay makes its first connection or reconciles. */
|
|
837
|
+
export interface SyncingSyncStatus extends Typed<"Syncing"> {}
|
|
838
|
+
|
|
839
|
+
/** A relay is reconciled with the database for the owner. */
|
|
840
|
+
export interface SyncedSyncStatus extends Typed<"Synced"> {}
|
|
841
|
+
|
|
842
|
+
/** Relays are disconnected and reconnecting, so changes wait on this device. */
|
|
843
|
+
export interface OfflineSyncStatus extends Typed<"Offline"> {}
|
|
844
|
+
|
|
845
|
+
/** A route failed or holds a change the database skipped. */
|
|
846
|
+
export interface ErrorSyncStatus extends Typed<"Error"> {
|
|
847
|
+
/**
|
|
848
|
+
* The failure, or without one, the skipped change. For an owner, the newest
|
|
849
|
+
* failure of any relay, or without one, the newest skipped change.
|
|
850
|
+
*/
|
|
851
|
+
readonly error: SyncRouteError;
|
|
576
852
|
}
|
|
577
853
|
|
|
578
854
|
/**
|
|
579
|
-
*
|
|
580
|
-
*
|
|
581
|
-
*
|
|
855
|
+
* Tells what an app shows about syncing an owner in a database: `Error` when a
|
|
856
|
+
* relay has one, holding the newest failure of any relay or, without one, the
|
|
857
|
+
* newest skipped change; otherwise the first of `Syncing`, `Synced`, and
|
|
858
|
+
* `Offline` that a relay has, or `NoRelays`. Accepts null, the store's value
|
|
859
|
+
* before the first snapshot. See {@link OwnerSyncStatus} for what to show.
|
|
860
|
+
*
|
|
861
|
+
* The same status is the same object: statuses other than `Error` are shared
|
|
862
|
+
* constants, and an `Error` status is the same object for the same error
|
|
863
|
+
* object, which {@link SyncStateDep.syncState} keeps between snapshots while it
|
|
864
|
+
* is unchanged. So compare statuses with `===`.
|
|
865
|
+
*
|
|
866
|
+
* ### Example
|
|
867
|
+
*
|
|
868
|
+
* ```ts
|
|
869
|
+
* import {
|
|
870
|
+
* assertEqual,
|
|
871
|
+
* assertSame,
|
|
872
|
+
* createId,
|
|
873
|
+
* createUnknownError,
|
|
874
|
+
* Millis,
|
|
875
|
+
* testCreateDeps,
|
|
876
|
+
* testName,
|
|
877
|
+
* } from "@evolu/common";
|
|
878
|
+
* import {
|
|
879
|
+
* syncStateToOwnerSyncStatus,
|
|
880
|
+
* testAppOwner,
|
|
881
|
+
* type PendingSyncRoute,
|
|
882
|
+
* type SyncConnection,
|
|
883
|
+
* type SyncRoute,
|
|
884
|
+
* type SyncState,
|
|
885
|
+
* type SyncTransportId,
|
|
886
|
+
* } from "@evolu/common/local-first";
|
|
887
|
+
*
|
|
888
|
+
* const deps = testCreateDeps();
|
|
889
|
+
* const primaryId = createId<"SyncTransport">(deps);
|
|
890
|
+
* const backupId = createId<"SyncTransport">(deps);
|
|
891
|
+
*
|
|
892
|
+
* const stateOf = (
|
|
893
|
+
* connections: ReadonlyArray<SyncConnection>,
|
|
894
|
+
* routes: ReadonlyArray<SyncRoute>,
|
|
895
|
+
* ): SyncState => ({
|
|
896
|
+
* transports: connections.map((connection, index) => ({
|
|
897
|
+
* type: "WebSocket",
|
|
898
|
+
* id: index === 0 ? primaryId : backupId,
|
|
899
|
+
* label: "wss://relay.example",
|
|
900
|
+
* connection,
|
|
901
|
+
* })),
|
|
902
|
+
* tenants: [
|
|
903
|
+
* {
|
|
904
|
+
* type: "Active",
|
|
905
|
+
* name: testName,
|
|
906
|
+
* owners: [{ type: "Writable", ownerId: testAppOwner.id, routes }],
|
|
907
|
+
* },
|
|
908
|
+
* ],
|
|
909
|
+
* });
|
|
910
|
+
* const pending = (transportId: SyncTransportId): PendingSyncRoute => ({
|
|
911
|
+
* type: "Pending",
|
|
912
|
+
* transportId,
|
|
913
|
+
* failure: null,
|
|
914
|
+
* skippedError: null,
|
|
915
|
+
* completeAt: null,
|
|
916
|
+
* lastSentAt: null,
|
|
917
|
+
* lastReceivedAt: null,
|
|
918
|
+
* });
|
|
919
|
+
* const statusOf = (state: SyncState | null) =>
|
|
920
|
+
* syncStateToOwnerSyncStatus(state, testName, testAppOwner.id);
|
|
921
|
+
*
|
|
922
|
+
* // Before the first snapshot, nothing syncs the owner.
|
|
923
|
+
* assertEqual(statusOf(null), { type: "NoRelays" });
|
|
924
|
+
*
|
|
925
|
+
* // A first connection is syncing; a lost one is offline.
|
|
926
|
+
* const connecting: SyncConnection = { type: "Connecting" };
|
|
927
|
+
* assertEqual(statusOf(stateOf([connecting], [pending(primaryId)])), {
|
|
928
|
+
* type: "Syncing",
|
|
929
|
+
* });
|
|
930
|
+
* const disconnected: SyncConnection = {
|
|
931
|
+
* type: "Disconnected",
|
|
932
|
+
* disconnectedAt: Millis.orThrow(2000),
|
|
933
|
+
* openedAt: Millis.orThrow(1000),
|
|
934
|
+
* error: null,
|
|
935
|
+
* };
|
|
936
|
+
* assertEqual(statusOf(stateOf([disconnected], [pending(primaryId)])), {
|
|
937
|
+
* type: "Offline",
|
|
938
|
+
* });
|
|
939
|
+
*
|
|
940
|
+
* // A quota failure on one relay shows before a newer skipped change on
|
|
941
|
+
* // another, because the app can act on it.
|
|
942
|
+
* const quotaError = {
|
|
943
|
+
* type: "ProtocolQuotaError",
|
|
944
|
+
* ownerId: testAppOwner.id,
|
|
945
|
+
* at: Millis.orThrow(3000),
|
|
946
|
+
* } as const;
|
|
947
|
+
* const open: SyncConnection = {
|
|
948
|
+
* type: "Open",
|
|
949
|
+
* openedAt: Millis.orThrow(3400),
|
|
950
|
+
* error: null,
|
|
951
|
+
* };
|
|
952
|
+
* assertEqual(
|
|
953
|
+
* statusOf(
|
|
954
|
+
* stateOf(
|
|
955
|
+
* [disconnected, open],
|
|
956
|
+
* [
|
|
957
|
+
* { ...pending(primaryId), failure: quotaError },
|
|
958
|
+
* {
|
|
959
|
+
* type: "Settled",
|
|
960
|
+
* transportId: backupId,
|
|
961
|
+
* skippedError: {
|
|
962
|
+
* type: "DecryptWithXChaCha20Poly1305Error",
|
|
963
|
+
* error: createUnknownError(new Error("invalid tag")),
|
|
964
|
+
* at: Millis.orThrow(4000),
|
|
965
|
+
* },
|
|
966
|
+
* completeAt: null,
|
|
967
|
+
* lastSentAt: Millis.orThrow(3500),
|
|
968
|
+
* lastReceivedAt: Millis.orThrow(4000),
|
|
969
|
+
* },
|
|
970
|
+
* ],
|
|
971
|
+
* ),
|
|
972
|
+
* ),
|
|
973
|
+
* { type: "Error", error: quotaError },
|
|
974
|
+
* );
|
|
975
|
+
*
|
|
976
|
+
* // The same error gives the same status object.
|
|
977
|
+
* const quotaState = stateOf(
|
|
978
|
+
* [disconnected],
|
|
979
|
+
* [{ ...pending(primaryId), failure: quotaError }],
|
|
980
|
+
* );
|
|
981
|
+
* assertSame(statusOf(quotaState), statusOf(quotaState));
|
|
982
|
+
* ```
|
|
582
983
|
*/
|
|
583
|
-
export
|
|
984
|
+
export const syncStateToOwnerSyncStatus = (
|
|
985
|
+
state: SyncState | null,
|
|
986
|
+
name: Name,
|
|
987
|
+
ownerId: OwnerId,
|
|
988
|
+
): OwnerSyncStatus => {
|
|
989
|
+
let failure: SyncRouteError | null = null;
|
|
990
|
+
let skippedError: SyncRouteError | null = null;
|
|
991
|
+
let isSyncing = false;
|
|
992
|
+
let isSynced = false;
|
|
993
|
+
let isOffline = false;
|
|
994
|
+
for (const relay of syncStateToRelaySyncStates(state, name, ownerId)) {
|
|
995
|
+
const { route } = relay;
|
|
996
|
+
// A failure stops syncing through its relay, and a skipped change is
|
|
997
|
+
// stamped again by every reply that skips it, so a newer skipped change
|
|
998
|
+
// must not hide an older failure the app can act on.
|
|
999
|
+
if (
|
|
1000
|
+
route.type === "Pending" &&
|
|
1001
|
+
route.failure &&
|
|
1002
|
+
(!failure || route.failure.at > failure.at)
|
|
1003
|
+
)
|
|
1004
|
+
failure = route.failure;
|
|
1005
|
+
if (
|
|
1006
|
+
route.type !== "Complete" &&
|
|
1007
|
+
route.skippedError &&
|
|
1008
|
+
(!skippedError || route.skippedError.at > skippedError.at)
|
|
1009
|
+
)
|
|
1010
|
+
skippedError = route.skippedError;
|
|
1011
|
+
const status = relaySyncStateToStatus(relay);
|
|
1012
|
+
switch (status.type) {
|
|
1013
|
+
case "Error":
|
|
1014
|
+
break;
|
|
1015
|
+
case "Syncing":
|
|
1016
|
+
isSyncing = true;
|
|
1017
|
+
break;
|
|
1018
|
+
case "Synced":
|
|
1019
|
+
isSynced = true;
|
|
1020
|
+
break;
|
|
1021
|
+
case "Offline":
|
|
1022
|
+
isOffline = true;
|
|
1023
|
+
break;
|
|
1024
|
+
default:
|
|
1025
|
+
exhaustiveCheck(status);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
const error = failure ?? skippedError;
|
|
1029
|
+
return error
|
|
1030
|
+
? syncRouteErrorToSyncStatus(error)
|
|
1031
|
+
: isSyncing
|
|
1032
|
+
? syncingSyncStatus
|
|
1033
|
+
: isSynced
|
|
1034
|
+
? syncedSyncStatus
|
|
1035
|
+
: isOffline
|
|
1036
|
+
? offlineSyncStatus
|
|
1037
|
+
: noRelaysSyncStatus;
|
|
1038
|
+
};
|
|
584
1039
|
|
|
585
1040
|
/**
|
|
586
|
-
*
|
|
587
|
-
*
|
|
588
|
-
*
|
|
589
|
-
* {@link RelaySyncState}. A database that refused startup and a readonly
|
|
590
|
-
* registration synchronize nothing, so they are left out.
|
|
1041
|
+
* Pairs each route of an owner in a database with its transport, in route
|
|
1042
|
+
* order. Returns none for a null snapshot, a missing or refused database, and a
|
|
1043
|
+
* missing or readonly owner.
|
|
591
1044
|
*
|
|
592
1045
|
* ### Example
|
|
593
1046
|
*
|
|
@@ -600,7 +1053,8 @@ export type RelaySyncStatus = "syncing" | "synced" | "offline" | "error";
|
|
|
600
1053
|
* testName,
|
|
601
1054
|
* } from "@evolu/common";
|
|
602
1055
|
* import {
|
|
603
|
-
*
|
|
1056
|
+
* relaySyncStateToStatus,
|
|
1057
|
+
* syncStateToRelaySyncStates,
|
|
604
1058
|
* testAppOwner,
|
|
605
1059
|
* type SyncRoute,
|
|
606
1060
|
* type SyncState,
|
|
@@ -609,99 +1063,106 @@ export type RelaySyncStatus = "syncing" | "synced" | "offline" | "error";
|
|
|
609
1063
|
*
|
|
610
1064
|
* const deps = testCreateDeps();
|
|
611
1065
|
* const transport: SyncTransport = {
|
|
1066
|
+
* type: "WebSocket",
|
|
612
1067
|
* id: createId<"SyncTransport">(deps),
|
|
613
1068
|
* label: "wss://relay.example",
|
|
614
|
-
*
|
|
615
|
-
*
|
|
616
|
-
*
|
|
617
|
-
*
|
|
1069
|
+
* connection: {
|
|
1070
|
+
* type: "Open",
|
|
1071
|
+
* openedAt: Millis.orThrow(800),
|
|
1072
|
+
* error: null,
|
|
1073
|
+
* },
|
|
618
1074
|
* };
|
|
619
1075
|
* const route: SyncRoute = {
|
|
1076
|
+
* type: "Complete",
|
|
620
1077
|
* transportId: transport.id,
|
|
621
|
-
* complete: true,
|
|
622
1078
|
* completeAt: Millis.orThrow(1000),
|
|
623
1079
|
* lastSentAt: Millis.orThrow(900),
|
|
624
1080
|
* lastReceivedAt: Millis.orThrow(1000),
|
|
625
|
-
* error: null,
|
|
626
1081
|
* };
|
|
627
1082
|
* const state: SyncState = {
|
|
628
1083
|
* transports: [transport],
|
|
629
1084
|
* tenants: [
|
|
630
1085
|
* {
|
|
1086
|
+
* type: "Active",
|
|
631
1087
|
* name: testName,
|
|
632
|
-
* refused: false,
|
|
633
1088
|
* owners: [
|
|
634
|
-
* {
|
|
635
|
-
* ownerId: testAppOwner.id,
|
|
636
|
-
* writable: true,
|
|
637
|
-
* transportIds: [transport.id],
|
|
638
|
-
* routes: [route],
|
|
639
|
-
* },
|
|
1089
|
+
* { type: "Writable", ownerId: testAppOwner.id, routes: [route] },
|
|
640
1090
|
* ],
|
|
641
1091
|
* },
|
|
642
1092
|
* ],
|
|
643
1093
|
* };
|
|
644
1094
|
*
|
|
645
|
-
*
|
|
646
|
-
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
649
|
-
*
|
|
650
|
-
*
|
|
651
|
-
*
|
|
652
|
-
*
|
|
653
|
-
*
|
|
654
|
-
*
|
|
1095
|
+
* const relays = syncStateToRelaySyncStates(
|
|
1096
|
+
* state,
|
|
1097
|
+
* testName,
|
|
1098
|
+
* testAppOwner.id,
|
|
1099
|
+
* );
|
|
1100
|
+
* assertEqual(relays, [{ transport, route }]);
|
|
1101
|
+
* assertEqual(relays.map(relaySyncStateToStatus), [{ type: "Synced" }]);
|
|
1102
|
+
* assertEqual(
|
|
1103
|
+
* syncStateToRelaySyncStates(null, testName, testAppOwner.id),
|
|
1104
|
+
* [],
|
|
1105
|
+
* );
|
|
655
1106
|
* ```
|
|
656
1107
|
*/
|
|
657
|
-
export const
|
|
658
|
-
state: SyncState,
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
);
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
1108
|
+
export const syncStateToRelaySyncStates = (
|
|
1109
|
+
state: SyncState | null,
|
|
1110
|
+
name: Name,
|
|
1111
|
+
ownerId: OwnerId,
|
|
1112
|
+
): ReadonlyArray<RelaySyncState> => {
|
|
1113
|
+
const tenant = state?.tenants.find((tenant) => tenant.name === name);
|
|
1114
|
+
if (!state || tenant?.type !== "Active") return emptyArray;
|
|
1115
|
+
const owner = tenant.owners.find((owner) => owner.ownerId === ownerId);
|
|
1116
|
+
if (owner?.type !== "Writable") return emptyArray;
|
|
1117
|
+
return owner.routes.map((route) => {
|
|
1118
|
+
// A snapshot lists the transport of every route.
|
|
1119
|
+
const transport = state.transports.find(
|
|
1120
|
+
({ id }) => id === route.transportId,
|
|
1121
|
+
);
|
|
1122
|
+
assertNonNullable(transport);
|
|
1123
|
+
return { transport, route };
|
|
1124
|
+
});
|
|
1125
|
+
};
|
|
1126
|
+
|
|
1127
|
+
/** Tells the status of one relay; a failure shows before a skipped change. */
|
|
1128
|
+
export const relaySyncStateToStatus = ({
|
|
1129
|
+
transport,
|
|
1130
|
+
route,
|
|
1131
|
+
}: RelaySyncState): RelaySyncStatus => {
|
|
1132
|
+
switch (route.type) {
|
|
1133
|
+
case "Complete":
|
|
1134
|
+
return syncedSyncStatus;
|
|
1135
|
+
case "Settled":
|
|
1136
|
+
return syncRouteErrorToSyncStatus(route.skippedError);
|
|
1137
|
+
case "Pending": {
|
|
1138
|
+
const error = route.failure ?? route.skippedError;
|
|
1139
|
+
if (error) return syncRouteErrorToSyncStatus(error);
|
|
1140
|
+
return transport.connection.type === "Disconnected"
|
|
1141
|
+
? offlineSyncStatus
|
|
1142
|
+
: syncingSyncStatus;
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
};
|
|
1146
|
+
|
|
1147
|
+
// Every status keeps its reference while it is unchanged, so bindings compare
|
|
1148
|
+
// statuses with `===`. The store shares an unchanged error object between
|
|
1149
|
+
// snapshots, and an error gives the same Error status object.
|
|
1150
|
+
const noRelaysSyncStatus: NoRelaysSyncStatus = { type: "NoRelays" };
|
|
1151
|
+
const syncingSyncStatus: SyncingSyncStatus = { type: "Syncing" };
|
|
1152
|
+
const syncedSyncStatus: SyncedSyncStatus = { type: "Synced" };
|
|
1153
|
+
const offlineSyncStatus: OfflineSyncStatus = { type: "Offline" };
|
|
1154
|
+
const errorSyncStatusByError = /*#__PURE__*/ new WeakMap<
|
|
1155
|
+
SyncRouteError,
|
|
1156
|
+
ErrorSyncStatus
|
|
1157
|
+
>();
|
|
1158
|
+
|
|
1159
|
+
const syncRouteErrorToSyncStatus = (error: SyncRouteError): ErrorSyncStatus => {
|
|
1160
|
+
let status = errorSyncStatusByError.get(error);
|
|
1161
|
+
if (!status) {
|
|
1162
|
+
status = { type: "Error", error };
|
|
1163
|
+
errorSyncStatusByError.set(error, status);
|
|
1164
|
+
}
|
|
1165
|
+
return status;
|
|
705
1166
|
};
|
|
706
1167
|
|
|
707
1168
|
export type EvoluInput =
|
|
@@ -744,6 +1205,11 @@ export type EvoluOutput =
|
|
|
744
1205
|
| {
|
|
745
1206
|
readonly type: "OnExport";
|
|
746
1207
|
readonly file: Uint8Array<ArrayBuffer>;
|
|
1208
|
+
}
|
|
1209
|
+
| {
|
|
1210
|
+
/** The mutation with these onComplete callbacks could not be stored. */
|
|
1211
|
+
readonly type: "OnMutateFailed";
|
|
1212
|
+
readonly onCompleteIds: ReadonlyArray<Id>;
|
|
747
1213
|
};
|
|
748
1214
|
|
|
749
1215
|
export type DbWorkerInput =
|
|
@@ -820,6 +1286,11 @@ export type DbWorkerQueuedResponse =
|
|
|
820
1286
|
>;
|
|
821
1287
|
readonly rowsByQuery: RowsByQueryMap;
|
|
822
1288
|
}
|
|
1289
|
+
| {
|
|
1290
|
+
/** The mutation threw, so it rolled back and nothing was stored. */
|
|
1291
|
+
readonly type: "MutateFailed";
|
|
1292
|
+
readonly error: UnknownError;
|
|
1293
|
+
}
|
|
823
1294
|
| {
|
|
824
1295
|
readonly type: "Query";
|
|
825
1296
|
readonly rowsByQuery: RowsByQueryMap;
|
|
@@ -846,6 +1317,15 @@ export type DbWorkerQueuedResponse =
|
|
|
846
1317
|
readonly clock: Timestamp;
|
|
847
1318
|
readonly ownerId: OwnerId;
|
|
848
1319
|
readonly didWriteMessages: boolean;
|
|
1320
|
+
/**
|
|
1321
|
+
* The first error of a received message the DbWorker skipped while
|
|
1322
|
+
* storing the rest, or null. It does not end the round.
|
|
1323
|
+
*/
|
|
1324
|
+
readonly skippedError:
|
|
1325
|
+
| DecryptWithXChaCha20Poly1305Error
|
|
1326
|
+
| ProtocolInvalidDataError
|
|
1327
|
+
| ProtocolTimestampMismatchError
|
|
1328
|
+
| null;
|
|
849
1329
|
readonly result: Result<
|
|
850
1330
|
ApplyProtocolMessageAsClientResult,
|
|
851
1331
|
ProtocolError | StorageWriteMessagesError | AbortError
|
|
@@ -898,19 +1378,31 @@ interface EvoluTenant extends AsyncDisposable {
|
|
|
898
1378
|
}
|
|
899
1379
|
|
|
900
1380
|
/** A tenant's part of {@link SyncState}, with its transports still keyed. */
|
|
901
|
-
|
|
1381
|
+
type TenantSyncState = ActiveTenantSyncState | RefusedSyncTenant;
|
|
1382
|
+
|
|
1383
|
+
interface ActiveTenantSyncState extends Typed<"Active"> {
|
|
902
1384
|
readonly name: Name;
|
|
903
|
-
readonly
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
1385
|
+
readonly owners: ReadonlyArray<TenantSyncOwner>;
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
type TenantSyncOwner = WritableTenantSyncOwner | ReadonlyTenantSyncOwner;
|
|
1389
|
+
|
|
1390
|
+
interface WritableTenantSyncOwner extends Typed<"Writable"> {
|
|
1391
|
+
readonly ownerId: OwnerId;
|
|
1392
|
+
readonly routes: ReadonlyArray<TenantSyncRoute>;
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
interface ReadonlyTenantSyncOwner extends Typed<"Readonly"> {
|
|
1396
|
+
readonly ownerId: OwnerId;
|
|
1397
|
+
readonly transportKeys: ReadonlyArray<StructuralLookupKey>;
|
|
910
1398
|
}
|
|
911
1399
|
|
|
912
|
-
|
|
1400
|
+
/** A tenant's route, with its transport still keyed. */
|
|
1401
|
+
interface TenantSyncRoute {
|
|
913
1402
|
readonly transportKey: StructuralLookupKey;
|
|
1403
|
+
readonly progress: RouteProgress;
|
|
1404
|
+
readonly lastSentAt: Millis | null;
|
|
1405
|
+
readonly lastReceivedAt: Millis | null;
|
|
914
1406
|
}
|
|
915
1407
|
|
|
916
1408
|
/**
|
|
@@ -929,6 +1421,39 @@ const syncRequestTimeout = PositiveMillis.orThrow(90_000);
|
|
|
929
1421
|
*/
|
|
930
1422
|
const maxSyncRequestTimeout = PositiveMillis.orThrow(16 * syncRequestTimeout);
|
|
931
1423
|
|
|
1424
|
+
const connectingSyncConnection: ConnectingSyncConnection = {
|
|
1425
|
+
type: "Connecting",
|
|
1426
|
+
};
|
|
1427
|
+
|
|
1428
|
+
/**
|
|
1429
|
+
* A connection already disconnected keeps when it disconnected, so failed
|
|
1430
|
+
* reconnect attempts only record their error.
|
|
1431
|
+
*/
|
|
1432
|
+
const disconnectSyncConnection = (
|
|
1433
|
+
connection: SyncConnection,
|
|
1434
|
+
at: Millis,
|
|
1435
|
+
error: SyncTransportError | null,
|
|
1436
|
+
): DisconnectedSyncConnection => {
|
|
1437
|
+
switch (connection.type) {
|
|
1438
|
+
case "Connecting":
|
|
1439
|
+
return {
|
|
1440
|
+
type: "Disconnected",
|
|
1441
|
+
disconnectedAt: at,
|
|
1442
|
+
openedAt: null,
|
|
1443
|
+
error,
|
|
1444
|
+
};
|
|
1445
|
+
case "Open":
|
|
1446
|
+
return {
|
|
1447
|
+
type: "Disconnected",
|
|
1448
|
+
disconnectedAt: at,
|
|
1449
|
+
openedAt: connection.openedAt,
|
|
1450
|
+
error: error ?? connection.error,
|
|
1451
|
+
};
|
|
1452
|
+
case "Disconnected":
|
|
1453
|
+
return error ? { ...connection, error } : connection;
|
|
1454
|
+
}
|
|
1455
|
+
};
|
|
1456
|
+
|
|
932
1457
|
/** Where the protocol messages produced by a queued sync request are sent. */
|
|
933
1458
|
type SyncTarget =
|
|
934
1459
|
| { readonly type: "AllTransports" }
|
|
@@ -955,6 +1480,118 @@ const isTargetTransport = (
|
|
|
955
1480
|
): boolean =>
|
|
956
1481
|
target.type === "AllTransports" || target.key === structuralLookup(transport);
|
|
957
1482
|
|
|
1483
|
+
/**
|
|
1484
|
+
* A tenant's progress toward completing a route, per the Synchronization
|
|
1485
|
+
* completion rules. Events move the route to `Pending`, which keeps what the
|
|
1486
|
+
* route must remember until it settles, except that messages stored elsewhere
|
|
1487
|
+
* leave a `Settled` route settled. Refreshing the routes settles a `Pending`
|
|
1488
|
+
* route once the tenant's reconciliation conditions hold, and returns a settled
|
|
1489
|
+
* or complete route to `Pending` when they no longer hold.
|
|
1490
|
+
*/
|
|
1491
|
+
type RouteProgress = PendingRoute | SettledRoute | CompleteRoute;
|
|
1492
|
+
|
|
1493
|
+
/**
|
|
1494
|
+
* A route whose reconciliation has not ended, or has not been evaluated since
|
|
1495
|
+
* an event.
|
|
1496
|
+
*/
|
|
1497
|
+
interface PendingRoute extends Typed<"Pending"> {
|
|
1498
|
+
/** A round must be sent through the route before it can settle. */
|
|
1499
|
+
readonly roundRequired: boolean;
|
|
1500
|
+
/**
|
|
1501
|
+
* The failure since the route settled, or null. A further failure requests no
|
|
1502
|
+
* round until the route settles.
|
|
1503
|
+
*/
|
|
1504
|
+
readonly failure: SyncRouteError | null;
|
|
1505
|
+
/** The route's skipped message, or null. */
|
|
1506
|
+
readonly skip: RouteSkip | null;
|
|
1507
|
+
/** When the route last became complete, or null. */
|
|
1508
|
+
readonly completeAt: Millis | null;
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
/**
|
|
1512
|
+
* A message a route skipped. Its relay offers the message again in every round,
|
|
1513
|
+
* so messages received elsewhere request no round through the route.
|
|
1514
|
+
*/
|
|
1515
|
+
interface RouteSkip {
|
|
1516
|
+
readonly error: SyncRouteError;
|
|
1517
|
+
/**
|
|
1518
|
+
* A round was requested through the route since the skip, and no messages
|
|
1519
|
+
* received elsewhere have been stored since that request, so the route
|
|
1520
|
+
* completes if it settles first.
|
|
1521
|
+
*/
|
|
1522
|
+
readonly isRechecking: boolean;
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
/**
|
|
1526
|
+
* A route whose reconciliation has ended, but whose relay may offer a skipped
|
|
1527
|
+
* message or lack messages stored elsewhere, so it is incomplete.
|
|
1528
|
+
*/
|
|
1529
|
+
interface SettledRoute extends Typed<"Settled"> {
|
|
1530
|
+
readonly error: SyncRouteError;
|
|
1531
|
+
readonly completeAt: Millis | null;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
interface CompleteRoute extends Typed<"Complete"> {
|
|
1535
|
+
readonly completeAt: Millis;
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
const routeToPending = (progress: RouteProgress): PendingRoute => {
|
|
1539
|
+
switch (progress.type) {
|
|
1540
|
+
case "Pending":
|
|
1541
|
+
return progress;
|
|
1542
|
+
case "Settled":
|
|
1543
|
+
return {
|
|
1544
|
+
type: "Pending",
|
|
1545
|
+
roundRequired: false,
|
|
1546
|
+
failure: null,
|
|
1547
|
+
skip: { error: progress.error, isRechecking: false },
|
|
1548
|
+
completeAt: progress.completeAt,
|
|
1549
|
+
};
|
|
1550
|
+
case "Complete":
|
|
1551
|
+
return {
|
|
1552
|
+
type: "Pending",
|
|
1553
|
+
roundRequired: false,
|
|
1554
|
+
failure: null,
|
|
1555
|
+
skip: null,
|
|
1556
|
+
completeAt: progress.completeAt,
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
};
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Converts an error to a {@link SyncRouteError}: a caught value becomes an
|
|
1563
|
+
* {@link UnknownError}, and a {@link ProtocolInvalidDataError} leaves out its
|
|
1564
|
+
* data.
|
|
1565
|
+
*/
|
|
1566
|
+
const errorToSyncRouteError = (
|
|
1567
|
+
error:
|
|
1568
|
+
| ProtocolError
|
|
1569
|
+
| StorageWriteMessagesError
|
|
1570
|
+
| DecryptWithXChaCha20Poly1305Error
|
|
1571
|
+
| Typed<"WriteFailed">
|
|
1572
|
+
| Typed<"SyncFailed">,
|
|
1573
|
+
at: Millis,
|
|
1574
|
+
): SyncRouteError => {
|
|
1575
|
+
// A caught value inside an error becomes an UnknownError, and a
|
|
1576
|
+
// ProtocolInvalidDataError leaves out its data, which can be a whole frame.
|
|
1577
|
+
if (error.type === "ProtocolInvalidDataError") {
|
|
1578
|
+
const { data: _data, ...rest } = error;
|
|
1579
|
+
return { ...rest, error: createUnknownError(rest.error), at };
|
|
1580
|
+
}
|
|
1581
|
+
if (error.type === "DecryptWithXChaCha20Poly1305Error")
|
|
1582
|
+
return { ...error, error: createUnknownError(error.error), at };
|
|
1583
|
+
return { ...error, at };
|
|
1584
|
+
};
|
|
1585
|
+
|
|
1586
|
+
const failRoute = (
|
|
1587
|
+
progress: RouteProgress,
|
|
1588
|
+
failure: SyncRouteError,
|
|
1589
|
+
): PendingRoute => ({
|
|
1590
|
+
...routeToPending(progress),
|
|
1591
|
+
roundRequired: true,
|
|
1592
|
+
failure,
|
|
1593
|
+
});
|
|
1594
|
+
|
|
958
1595
|
type EvoluTenantDeps = SharedWorkerDeps &
|
|
959
1596
|
PostConsoleEntryOrErrorDep &
|
|
960
1597
|
PublishSyncStateDep &
|
|
@@ -1117,8 +1754,15 @@ export const initSharedWorker =
|
|
|
1117
1754
|
});
|
|
1118
1755
|
};
|
|
1119
1756
|
|
|
1120
|
-
//
|
|
1121
|
-
//
|
|
1757
|
+
// Held while this worker runs. Its DbWorkers stop once they can take it,
|
|
1758
|
+
// because a Dispose posted right before this worker closes can be lost, as
|
|
1759
|
+
// in Firefox. Taken before the build lock, so nothing delays the end of
|
|
1760
|
+
// starting once that lock is held.
|
|
1761
|
+
disposer.use(await run.ok(acquireLeaderLock(workerId)));
|
|
1762
|
+
|
|
1763
|
+
// Released after every tenant is disposed and has told its DbWorker to
|
|
1764
|
+
// stop. Earlier releases take the same lock in their leader tab; see
|
|
1765
|
+
// Builds.
|
|
1122
1766
|
disposer.use(await run.ok(acquireLeaderLock("tab")));
|
|
1123
1767
|
starting.dispose();
|
|
1124
1768
|
|
|
@@ -1156,13 +1800,12 @@ export const initSharedWorker =
|
|
|
1156
1800
|
* another, so a reply can wait behind another owner's large frame. It
|
|
1157
1801
|
* doubles after each timeout and survives reconnects until nothing is
|
|
1158
1802
|
* outstanding and every route through the transport of a database that
|
|
1159
|
-
* has not refused startup
|
|
1803
|
+
* has not refused startup has settled.
|
|
1160
1804
|
*/
|
|
1161
1805
|
timeout: PositiveMillis;
|
|
1162
1806
|
socket: WebSocket | null;
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
error: SyncTransportError | null;
|
|
1807
|
+
/** Socket events drive it, so publishing never reads the socket. */
|
|
1808
|
+
connection: SyncConnection;
|
|
1166
1809
|
/** Armed while a request is outstanding on an open socket. */
|
|
1167
1810
|
timeoutId: TimeoutId | null;
|
|
1168
1811
|
}
|
|
@@ -1181,62 +1824,96 @@ export const initSharedWorker =
|
|
|
1181
1824
|
isPublishScheduled = false;
|
|
1182
1825
|
if (isDisposed) return;
|
|
1183
1826
|
const transports = [...transportsByKey.values()].map(
|
|
1184
|
-
({
|
|
1185
|
-
|
|
1186
|
-
label,
|
|
1187
|
-
socket,
|
|
1188
|
-
openedAt,
|
|
1189
|
-
closedAt,
|
|
1190
|
-
error,
|
|
1191
|
-
}): SyncTransport => ({
|
|
1827
|
+
({ id, label, connection }): SyncTransport => ({
|
|
1828
|
+
type: "WebSocket",
|
|
1192
1829
|
id,
|
|
1193
1830
|
label,
|
|
1194
|
-
|
|
1195
|
-
// reads a disposed one.
|
|
1196
|
-
readyState: socket?.getReadyState() ?? "connecting",
|
|
1197
|
-
openedAt,
|
|
1198
|
-
closedAt,
|
|
1199
|
-
error,
|
|
1831
|
+
connection,
|
|
1200
1832
|
}),
|
|
1201
1833
|
);
|
|
1834
|
+
// A grown timeout lasts while a request is outstanding on the socket
|
|
1835
|
+
// or a database that has not refused startup has an unsettled route
|
|
1836
|
+
// through it. Every change to either publishes, including a route that
|
|
1837
|
+
// goes away without settling.
|
|
1838
|
+
const unsettledTransportIds = new Set<SyncTransportId>();
|
|
1202
1839
|
const tenants = [...currentTenantsByName.values()].map(
|
|
1203
1840
|
(tenant): SyncTenant => {
|
|
1204
|
-
const
|
|
1841
|
+
const state = tenant.getSyncTenant();
|
|
1842
|
+
if (state.type === "Refused") return state;
|
|
1205
1843
|
return {
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
owners: owners.map(
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1844
|
+
type: "Active",
|
|
1845
|
+
name: state.name,
|
|
1846
|
+
owners: state.owners.map((owner): SyncTenantOwner =>
|
|
1847
|
+
owner.type === "Readonly"
|
|
1848
|
+
? {
|
|
1849
|
+
type: "Readonly",
|
|
1850
|
+
ownerId: owner.ownerId,
|
|
1851
|
+
transportIds: owner.transportKeys.flatMap((key) => {
|
|
1852
|
+
const entry = transportsByKey.get(key);
|
|
1853
|
+
return entry ? [entry.id] : [];
|
|
1854
|
+
}),
|
|
1855
|
+
}
|
|
1856
|
+
: {
|
|
1857
|
+
type: "Writable",
|
|
1858
|
+
ownerId: owner.ownerId,
|
|
1859
|
+
routes: owner.routes.flatMap(
|
|
1860
|
+
({
|
|
1861
|
+
transportKey,
|
|
1862
|
+
progress,
|
|
1863
|
+
lastSentAt,
|
|
1864
|
+
lastReceivedAt,
|
|
1865
|
+
}): Array<SyncRoute> => {
|
|
1866
|
+
const entry = transportsByKey.get(transportKey);
|
|
1867
|
+
if (!entry) return [];
|
|
1868
|
+
const transportId = entry.id;
|
|
1869
|
+
if (progress.type !== "Pending") {
|
|
1870
|
+
// Only a sent round settles a route or completes
|
|
1871
|
+
// it.
|
|
1872
|
+
assertNonNullable(lastSentAt);
|
|
1873
|
+
return [
|
|
1874
|
+
progress.type === "Complete"
|
|
1875
|
+
? {
|
|
1876
|
+
type: "Complete",
|
|
1877
|
+
transportId,
|
|
1878
|
+
completeAt: progress.completeAt,
|
|
1879
|
+
lastSentAt,
|
|
1880
|
+
lastReceivedAt,
|
|
1881
|
+
}
|
|
1882
|
+
: {
|
|
1883
|
+
type: "Settled",
|
|
1884
|
+
transportId,
|
|
1885
|
+
skippedError: progress.error,
|
|
1886
|
+
completeAt: progress.completeAt,
|
|
1887
|
+
lastSentAt,
|
|
1888
|
+
lastReceivedAt,
|
|
1889
|
+
},
|
|
1890
|
+
];
|
|
1891
|
+
}
|
|
1892
|
+
// Only a database that has not refused startup
|
|
1893
|
+
// publishes routes.
|
|
1894
|
+
unsettledTransportIds.add(transportId);
|
|
1895
|
+
return [
|
|
1896
|
+
{
|
|
1897
|
+
type: "Pending",
|
|
1898
|
+
transportId,
|
|
1899
|
+
failure: progress.failure,
|
|
1900
|
+
skippedError: progress.skip?.error ?? null,
|
|
1901
|
+
completeAt: progress.completeAt,
|
|
1902
|
+
lastSentAt,
|
|
1903
|
+
lastReceivedAt,
|
|
1904
|
+
},
|
|
1905
|
+
];
|
|
1906
|
+
},
|
|
1907
|
+
),
|
|
1908
|
+
},
|
|
1221
1909
|
),
|
|
1222
1910
|
};
|
|
1223
1911
|
},
|
|
1224
1912
|
);
|
|
1225
|
-
// A grown timeout lasts while a request is outstanding on the socket
|
|
1226
|
-
// or a database that has not refused startup has an incomplete route
|
|
1227
|
-
// through it. Every change to either publishes, including a route that
|
|
1228
|
-
// goes away without completing.
|
|
1229
|
-
const incompleteTransportIds = new Set<SyncTransportId>();
|
|
1230
|
-
for (const { refused, owners } of tenants) {
|
|
1231
|
-
if (refused) continue;
|
|
1232
|
-
for (const { routes } of owners)
|
|
1233
|
-
for (const { transportId, complete } of routes)
|
|
1234
|
-
if (!complete) incompleteTransportIds.add(transportId);
|
|
1235
|
-
}
|
|
1236
1913
|
for (const entry of transportsByKey.values())
|
|
1237
1914
|
if (
|
|
1238
1915
|
entry.outstandingByOwnerId.size === 0 &&
|
|
1239
|
-
!
|
|
1916
|
+
!unsettledTransportIds.has(entry.id)
|
|
1240
1917
|
)
|
|
1241
1918
|
entry.timeout = syncRequestTimeout;
|
|
1242
1919
|
syncStateBroadcastChannel.postMessage({ transports, tenants });
|
|
@@ -1295,10 +1972,14 @@ export const initSharedWorker =
|
|
|
1295
1972
|
);
|
|
1296
1973
|
// Reconnecting abandons the connection and starts a fresh retry
|
|
1297
1974
|
// schedule. The socket reports no close for it, so the transport
|
|
1298
|
-
//
|
|
1975
|
+
// disconnects here.
|
|
1299
1976
|
entry.socket?.reconnect();
|
|
1300
|
-
// `now` is monotonic; the reported
|
|
1301
|
-
entry.
|
|
1977
|
+
// `now` is monotonic; the reported time is wall clock.
|
|
1978
|
+
entry.connection = disconnectSyncConnection(
|
|
1979
|
+
entry.connection,
|
|
1980
|
+
deps.time.now(),
|
|
1981
|
+
null,
|
|
1982
|
+
);
|
|
1302
1983
|
refreshAllSyncRoutes();
|
|
1303
1984
|
publishSyncState();
|
|
1304
1985
|
}, delay);
|
|
@@ -1346,9 +2027,7 @@ export const initSharedWorker =
|
|
|
1346
2027
|
outstandingByOwnerId: new Map(),
|
|
1347
2028
|
timeout: syncRequestTimeout,
|
|
1348
2029
|
socket: null,
|
|
1349
|
-
|
|
1350
|
-
closedAt: null,
|
|
1351
|
-
error: null,
|
|
2030
|
+
connection: connectingSyncConnection,
|
|
1352
2031
|
timeoutId: null,
|
|
1353
2032
|
};
|
|
1354
2033
|
await using disposer = new AsyncDisposableStack();
|
|
@@ -1371,7 +2050,16 @@ export const initSharedWorker =
|
|
|
1371
2050
|
// answered.
|
|
1372
2051
|
entry.outstandingByOwnerId.clear();
|
|
1373
2052
|
clearSyncRequestTimeout(entry);
|
|
1374
|
-
|
|
2053
|
+
// A connection that opens keeps the last error of an earlier
|
|
2054
|
+
// one.
|
|
2055
|
+
entry.connection = {
|
|
2056
|
+
type: "Open",
|
|
2057
|
+
openedAt: run.deps.time.now(),
|
|
2058
|
+
error:
|
|
2059
|
+
entry.connection.type === "Connecting"
|
|
2060
|
+
? null
|
|
2061
|
+
: entry.connection.error,
|
|
2062
|
+
};
|
|
1375
2063
|
publishSyncState();
|
|
1376
2064
|
const ownerIds = transports.getClaimsForResource(transport);
|
|
1377
2065
|
console.debug("transportOpen", {
|
|
@@ -1395,7 +2083,11 @@ export const initSharedWorker =
|
|
|
1395
2083
|
code: event.code,
|
|
1396
2084
|
wasClean: event.wasClean,
|
|
1397
2085
|
});
|
|
1398
|
-
entry.
|
|
2086
|
+
entry.connection = disconnectSyncConnection(
|
|
2087
|
+
entry.connection,
|
|
2088
|
+
run.deps.time.now(),
|
|
2089
|
+
null,
|
|
2090
|
+
);
|
|
1399
2091
|
clearSyncRequestTimeout(entry);
|
|
1400
2092
|
refreshAllSyncRoutes();
|
|
1401
2093
|
publishSyncState();
|
|
@@ -1406,10 +2098,17 @@ export const initSharedWorker =
|
|
|
1406
2098
|
url: transport.url,
|
|
1407
2099
|
type: error.type,
|
|
1408
2100
|
});
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
2101
|
+
// Every error disconnects. A browser reports a failed
|
|
2102
|
+
// attempt only as an error, because the socket stops
|
|
2103
|
+
// listening before its close arrives. A connection error
|
|
2104
|
+
// arrives once the socket is closed, before its close, and
|
|
2105
|
+
// exhausted retries end reconnecting.
|
|
2106
|
+
const now = run.deps.time.now();
|
|
2107
|
+
entry.connection = disconnectSyncConnection(
|
|
2108
|
+
entry.connection,
|
|
2109
|
+
now,
|
|
2110
|
+
{ type: error.type, at: now },
|
|
2111
|
+
);
|
|
1413
2112
|
refreshAllSyncRoutes();
|
|
1414
2113
|
publishSyncState();
|
|
1415
2114
|
},
|
|
@@ -1474,9 +2173,8 @@ export const initSharedWorker =
|
|
|
1474
2173
|
// LIFO: the transport drops its timer and its socket reference
|
|
1475
2174
|
// before the socket is disposed. `disposable` guards every method
|
|
1476
2175
|
// of the socket the claims lease, and disposing the socket awaits
|
|
1477
|
-
// its retry, so
|
|
1478
|
-
//
|
|
1479
|
-
// read a disposed one.
|
|
2176
|
+
// its retry, so this entry stays registered meanwhile. It keeps its
|
|
2177
|
+
// last connection and holds no socket to reconnect.
|
|
1480
2178
|
disposer.defer(() => {
|
|
1481
2179
|
clearSyncRequestTimeout(entry);
|
|
1482
2180
|
entry.socket = null;
|
|
@@ -1541,6 +2239,7 @@ export const initSharedWorker =
|
|
|
1541
2239
|
? { ...message, memoryOnly: true }
|
|
1542
2240
|
: message,
|
|
1543
2241
|
currentTenantsByName,
|
|
2242
|
+
workerId,
|
|
1544
2243
|
),
|
|
1545
2244
|
{
|
|
1546
2245
|
idleDisposeAfter: "3s",
|
|
@@ -1571,6 +2270,7 @@ const createEvoluTenant =
|
|
|
1571
2270
|
memoryOnly,
|
|
1572
2271
|
}: ExtractTyped<SharedWorkerInput, "CreateEvolu">,
|
|
1573
2272
|
currentTenantsByName: Map<Name, BorrowedResource<EvoluTenant>>,
|
|
2273
|
+
workerId: SharedWorkerId,
|
|
1574
2274
|
): Task<EvoluTenant, never, EvoluTenantDeps> =>
|
|
1575
2275
|
async (run) => {
|
|
1576
2276
|
await using disposer = new AsyncDisposableStack();
|
|
@@ -1741,6 +2441,7 @@ const createEvoluTenant =
|
|
|
1741
2441
|
sqliteSchema,
|
|
1742
2442
|
encryptionKey,
|
|
1743
2443
|
memoryOnly,
|
|
2444
|
+
sharedWorkerId: workerId,
|
|
1744
2445
|
port: dbWorkerChannel.port1.native,
|
|
1745
2446
|
},
|
|
1746
2447
|
[dbWorkerChannel.port1.native],
|
|
@@ -1896,20 +2597,16 @@ const createEvoluTenant =
|
|
|
1896
2597
|
}
|
|
1897
2598
|
};
|
|
1898
2599
|
|
|
1899
|
-
|
|
2600
|
+
// Disposal does not wait for the DbWorker. It holds the database lock
|
|
2601
|
+
// until Dispose arrives, its tab closes, or this worker ends, and the next
|
|
2602
|
+
// DbWorker for this database, of this worker or another build, waits for
|
|
2603
|
+
// that lock before it reads the clock. A requested DbWorker that reports
|
|
2604
|
+
// in later gets Dispose from the isDisposing check.
|
|
2605
|
+
disposer.defer(() => {
|
|
1900
2606
|
isDisposing = true;
|
|
1901
2607
|
dbWorkerPort?.postMessage({ type: "Dispose" });
|
|
1902
2608
|
dbWorkerPort = null;
|
|
1903
2609
|
activeDispatch = null;
|
|
1904
|
-
|
|
1905
|
-
// The DbWorker holds this tenant leader lock while it is alive. Tenant
|
|
1906
|
-
// disposal sends Dispose, then acquires the same lock to wait until the
|
|
1907
|
-
// DbWorker releases it: either because Dispose was delivered or because
|
|
1908
|
-
// the hosting tab closed. A worker requested from a later tab leader may
|
|
1909
|
-
// be queued for the lock first; it is told to stop when it reports in.
|
|
1910
|
-
// The wait is unabortable because tenant disposal must finish even after
|
|
1911
|
-
// tenantRun receives an abort request.
|
|
1912
|
-
await using _ = await tenantRun.ok(acquireLeaderLock(name));
|
|
1913
2610
|
});
|
|
1914
2611
|
|
|
1915
2612
|
const handleResponseForEvolu = (
|
|
@@ -1996,23 +2693,47 @@ const createEvoluTenant =
|
|
|
1996
2693
|
break;
|
|
1997
2694
|
}
|
|
1998
2695
|
|
|
2696
|
+
case "MutateFailed": {
|
|
2697
|
+
// The tab shows it, and the instance releases the mutation's
|
|
2698
|
+
// onComplete callbacks without running them. A write outlives its
|
|
2699
|
+
// instance, so without one, every tab is told.
|
|
2700
|
+
//
|
|
2701
|
+
// A replay after leader replacement can fail although the earlier
|
|
2702
|
+
// leader committed the write and only its answer was lost. The tab
|
|
2703
|
+
// then shows an error for a stored write, and its onComplete
|
|
2704
|
+
// callbacks never run. Nothing is lost or reused: the replacement
|
|
2705
|
+
// adopted the stored clock, refreshed every instance's queries, and
|
|
2706
|
+
// reconciles every used owner.
|
|
2707
|
+
const { error } = response.message;
|
|
2708
|
+
if (instance) {
|
|
2709
|
+
assertSame(first.message.type, "Mutate");
|
|
2710
|
+
instance.tabPort.postMessage({ type: "Error", error });
|
|
2711
|
+
instance.port.postMessage({
|
|
2712
|
+
type: "OnMutateFailed",
|
|
2713
|
+
onCompleteIds: first.message.onCompleteIds,
|
|
2714
|
+
});
|
|
2715
|
+
} else {
|
|
2716
|
+
deps.postConsoleEntryOrError({ type: "Error", error });
|
|
2717
|
+
}
|
|
2718
|
+
break;
|
|
2719
|
+
}
|
|
2720
|
+
|
|
1999
2721
|
case "Export":
|
|
2000
2722
|
instance?.port.postMessage(
|
|
2001
2723
|
{ type: "OnExport", file: response.message.file },
|
|
2002
2724
|
[response.message.file.buffer],
|
|
2003
2725
|
);
|
|
2004
2726
|
break;
|
|
2727
|
+
|
|
2728
|
+
default:
|
|
2729
|
+
exhaustiveCheck(response.message);
|
|
2005
2730
|
}
|
|
2006
2731
|
};
|
|
2007
2732
|
|
|
2008
2733
|
interface RouteState {
|
|
2009
|
-
|
|
2010
|
-
roundRequired: boolean;
|
|
2011
|
-
complete: boolean;
|
|
2012
|
-
completeAt: Millis | null;
|
|
2734
|
+
progress: RouteProgress;
|
|
2013
2735
|
lastSentAt: Millis | null;
|
|
2014
2736
|
lastReceivedAt: Millis | null;
|
|
2015
|
-
error: SyncRouteError | null;
|
|
2016
2737
|
}
|
|
2017
2738
|
const routesByOwnerIdByKey = new Map<
|
|
2018
2739
|
StructuralLookupKey,
|
|
@@ -2031,12 +2752,15 @@ const createEvoluTenant =
|
|
|
2031
2752
|
let route = routesByOwnerId.get(ownerId);
|
|
2032
2753
|
if (!route) {
|
|
2033
2754
|
route = {
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2755
|
+
progress: {
|
|
2756
|
+
type: "Pending",
|
|
2757
|
+
roundRequired: true,
|
|
2758
|
+
failure: null,
|
|
2759
|
+
skip: null,
|
|
2760
|
+
completeAt: null,
|
|
2761
|
+
},
|
|
2037
2762
|
lastSentAt: null,
|
|
2038
2763
|
lastReceivedAt: null,
|
|
2039
|
-
error: null,
|
|
2040
2764
|
};
|
|
2041
2765
|
routesByOwnerId.set(ownerId, route);
|
|
2042
2766
|
}
|
|
@@ -2099,20 +2823,28 @@ const createEvoluTenant =
|
|
|
2099
2823
|
// worker answers it. Local-only changes create no synchronization
|
|
2100
2824
|
// work.
|
|
2101
2825
|
const hasQueuedWrite = pendingWriteCountByOwnerId.has(ownerId);
|
|
2102
|
-
|
|
2103
|
-
// queued writes without uploading them.
|
|
2104
|
-
const complete =
|
|
2105
|
-
startupError === null &&
|
|
2826
|
+
const isReconciled =
|
|
2106
2827
|
isOpen &&
|
|
2107
2828
|
deps.syncRequests.getOutstanding(ownerId, key) === 0 &&
|
|
2108
2829
|
!hasQueuedApply &&
|
|
2109
|
-
!hasQueuedWrite
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
route.
|
|
2830
|
+
!hasQueuedWrite;
|
|
2831
|
+
// Settling drops the failure, so the next one requests a round
|
|
2832
|
+
// again. A route with a skip it is not rechecking settles without
|
|
2833
|
+
// completing, because its relay may offer the skipped message or
|
|
2834
|
+
// lack messages stored elsewhere.
|
|
2835
|
+
const pending = routeToPending(route.progress);
|
|
2836
|
+
route.progress =
|
|
2837
|
+
!isReconciled || pending.roundRequired
|
|
2838
|
+
? pending
|
|
2839
|
+
: pending.skip && !pending.skip.isRechecking
|
|
2840
|
+
? {
|
|
2841
|
+
type: "Settled",
|
|
2842
|
+
error: pending.skip.error,
|
|
2843
|
+
completeAt: pending.completeAt,
|
|
2844
|
+
}
|
|
2845
|
+
: route.progress.type === "Complete"
|
|
2846
|
+
? route.progress
|
|
2847
|
+
: { type: "Complete", completeAt: deps.time.now() };
|
|
2116
2848
|
}
|
|
2117
2849
|
}
|
|
2118
2850
|
};
|
|
@@ -2144,8 +2876,10 @@ const createEvoluTenant =
|
|
|
2144
2876
|
.get(structuralLookup(transport))
|
|
2145
2877
|
?.get(ownerId);
|
|
2146
2878
|
if (!route) return;
|
|
2147
|
-
route.
|
|
2148
|
-
|
|
2879
|
+
route.progress = failRoute(route.progress, {
|
|
2880
|
+
type: "SyncFailed",
|
|
2881
|
+
at: now,
|
|
2882
|
+
});
|
|
2149
2883
|
});
|
|
2150
2884
|
}
|
|
2151
2885
|
deps.publishSyncState();
|
|
@@ -2159,7 +2893,14 @@ const createEvoluTenant =
|
|
|
2159
2893
|
const { ownerId, result } = response.message;
|
|
2160
2894
|
const error = result.ok ? null : result.error;
|
|
2161
2895
|
const isAborted = error?.type === "AbortError";
|
|
2162
|
-
|
|
2896
|
+
// An aborted apply reports no skipped message.
|
|
2897
|
+
const skippedError = isAborted ? null : response.message.skippedError;
|
|
2898
|
+
let failure:
|
|
2899
|
+
| ProtocolError
|
|
2900
|
+
| StorageWriteMessagesError
|
|
2901
|
+
| Typed<"WriteFailed">
|
|
2902
|
+
| Typed<"SyncFailed">
|
|
2903
|
+
| null = null;
|
|
2163
2904
|
if (isAborted) {
|
|
2164
2905
|
// An abort proves no convergence. A local sibling copy affects
|
|
2165
2906
|
// every route; a relay frame affects only its source route.
|
|
@@ -2168,17 +2909,22 @@ const createEvoluTenant =
|
|
|
2168
2909
|
const key = structuralLookup(transport);
|
|
2169
2910
|
if (source.type === "Transport" && source.key !== key) return;
|
|
2170
2911
|
const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
|
|
2171
|
-
if (route)
|
|
2912
|
+
if (route)
|
|
2913
|
+
route.progress = {
|
|
2914
|
+
...routeToPending(route.progress),
|
|
2915
|
+
roundRequired: true,
|
|
2916
|
+
};
|
|
2172
2917
|
});
|
|
2173
2918
|
} else if (error !== null) {
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
});
|
|
2919
|
+
// A relay's error shows on its route, not as an EvoluError,
|
|
2920
|
+
// because it belongs to one relay and often repeats in every
|
|
2921
|
+
// round. A sibling copy's error is reported below.
|
|
2922
|
+
failure = error;
|
|
2179
2923
|
} else if (result.ok && result.value.type === "Failed") {
|
|
2180
|
-
failure =
|
|
2181
|
-
|
|
2924
|
+
failure = {
|
|
2925
|
+
type:
|
|
2926
|
+
result.value.cause === "Write" ? "WriteFailed" : "SyncFailed",
|
|
2927
|
+
};
|
|
2182
2928
|
}
|
|
2183
2929
|
if (source.type === "Transport") {
|
|
2184
2930
|
// A registration or claim may have been removed while this apply
|
|
@@ -2188,39 +2934,69 @@ const createEvoluTenant =
|
|
|
2188
2934
|
const now = deps.time.now();
|
|
2189
2935
|
// An aborted apply applied nothing.
|
|
2190
2936
|
if (!isAborted) route.lastReceivedAt = now;
|
|
2937
|
+
// A skipped message is recorded as the route's skip, not a
|
|
2938
|
+
// failure, and does not end the round, whose response below is
|
|
2939
|
+
// still sent, so it requests no round. The relay offers the
|
|
2940
|
+
// message again in every later round, so the route stays
|
|
2941
|
+
// incomplete until a round requested through it settles without
|
|
2942
|
+
// skipping a message and without messages stored elsewhere since
|
|
2943
|
+
// that request.
|
|
2944
|
+
if (skippedError !== null)
|
|
2945
|
+
route.progress = {
|
|
2946
|
+
...routeToPending(route.progress),
|
|
2947
|
+
skip: {
|
|
2948
|
+
error: errorToSyncRouteError(skippedError, now),
|
|
2949
|
+
isRechecking: false,
|
|
2950
|
+
},
|
|
2951
|
+
};
|
|
2191
2952
|
if (failure !== null) {
|
|
2192
|
-
// The first failure since the route
|
|
2953
|
+
// The first failure since the route settled requests one
|
|
2193
2954
|
// round. Further failures wait for an explicit request or a
|
|
2194
2955
|
// reopen, even after a converged reply, which may answer
|
|
2195
2956
|
// another request.
|
|
2196
|
-
|
|
2957
|
+
const requestsRetry =
|
|
2958
|
+
route.progress.type !== "Pending" ||
|
|
2959
|
+
route.progress.failure === null;
|
|
2960
|
+
route.progress = failRoute(
|
|
2961
|
+
route.progress,
|
|
2962
|
+
errorToSyncRouteError(failure, now),
|
|
2963
|
+
);
|
|
2964
|
+
if (requestsRetry)
|
|
2197
2965
|
requestCreateSyncMessages(new Set([ownerId]), source);
|
|
2198
|
-
route.roundRequired = true;
|
|
2199
|
-
route.error = { type: failure, at: now };
|
|
2200
2966
|
}
|
|
2201
2967
|
}
|
|
2202
|
-
} else if (failure !== null) {
|
|
2203
|
-
// A sibling's
|
|
2204
|
-
//
|
|
2205
|
-
|
|
2968
|
+
} else if (failure !== null || skippedError !== null) {
|
|
2969
|
+
// A sibling's copy comes from this worker, not from a relay, so no
|
|
2970
|
+
// route shows its failure. It is a Broadcast, which carries no
|
|
2971
|
+
// relay error, so an error or a skip means a bug, such as
|
|
2972
|
+
// databases holding different keys for the owner, and is reported
|
|
2973
|
+
// as an unexpected failure. A Failed result was logged, which
|
|
2974
|
+
// reports it already. An error is the failure, which is never an
|
|
2975
|
+
// abort, and like a route, the report leaves out its frame.
|
|
2976
|
+
const unexpected = error !== null ? failure : skippedError;
|
|
2977
|
+
if (unexpected !== null)
|
|
2978
|
+
deps.postConsoleEntryOrError({
|
|
2979
|
+
type: "Error",
|
|
2980
|
+
error: createUnknownError(
|
|
2981
|
+
errorToSyncRouteError(unexpected, deps.time.now()),
|
|
2982
|
+
),
|
|
2983
|
+
});
|
|
2984
|
+
// Some of a sibling's messages were not stored; rounds fetch them
|
|
2985
|
+
// from the relays, whose routes then record a skip for any message
|
|
2986
|
+
// this database skips.
|
|
2987
|
+
requestRoundsForReceivedMessages(ownerId, {
|
|
2988
|
+
except: null,
|
|
2989
|
+
afterQueuedWrites: true,
|
|
2990
|
+
});
|
|
2206
2991
|
}
|
|
2207
2992
|
|
|
2208
2993
|
if (response.message.didWriteMessages) {
|
|
2209
2994
|
refreshQueries();
|
|
2210
2995
|
// Reconcile newly stored messages through each other transport.
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
if (isTargetTransport(target, transport)) return;
|
|
2215
|
-
keys.push(structuralLookup(transport));
|
|
2996
|
+
requestRoundsForReceivedMessages(ownerId, {
|
|
2997
|
+
except: target,
|
|
2998
|
+
afterQueuedWrites: false,
|
|
2216
2999
|
});
|
|
2217
|
-
for (const key of keys) {
|
|
2218
|
-
requestCreateSyncMessages(
|
|
2219
|
-
new Set([ownerId]),
|
|
2220
|
-
{ type: "Transport", key },
|
|
2221
|
-
{ afterQueuedWrites: false },
|
|
2222
|
-
);
|
|
2223
|
-
}
|
|
2224
3000
|
}
|
|
2225
3001
|
|
|
2226
3002
|
if (result.ok) {
|
|
@@ -2299,7 +3075,11 @@ const createEvoluTenant =
|
|
|
2299
3075
|
deps.syncRequests.noteSent(ownerId, key);
|
|
2300
3076
|
const route = getRoute(ownerId, key);
|
|
2301
3077
|
route.lastSentAt = deps.time.now();
|
|
2302
|
-
if (isRound)
|
|
3078
|
+
if (isRound)
|
|
3079
|
+
route.progress = {
|
|
3080
|
+
...routeToPending(route.progress),
|
|
3081
|
+
roundRequired: false,
|
|
3082
|
+
};
|
|
2303
3083
|
},
|
|
2304
3084
|
);
|
|
2305
3085
|
}
|
|
@@ -2307,6 +3087,48 @@ const createEvoluTenant =
|
|
|
2307
3087
|
if (protocolMessagesByOwnerId.size > 0) deps.refreshAllSyncRoutes();
|
|
2308
3088
|
};
|
|
2309
3089
|
|
|
3090
|
+
/**
|
|
3091
|
+
* Requests a round through each transport claimed for the owner outside
|
|
3092
|
+
* `except`, after messages from elsewhere were stored or a sibling copy was
|
|
3093
|
+
* not fully stored. Rounds toward the same transport coalesce whatever
|
|
3094
|
+
* their source. A route that skipped a message gets none, because its relay
|
|
3095
|
+
* would offer that message again, and a route rechecking one stops
|
|
3096
|
+
* rechecking, because its round may have read the database before this
|
|
3097
|
+
* event. A later requested round through such a route reconciles it.
|
|
3098
|
+
*/
|
|
3099
|
+
const requestRoundsForReceivedMessages = (
|
|
3100
|
+
ownerId: OwnerId,
|
|
3101
|
+
{
|
|
3102
|
+
except,
|
|
3103
|
+
afterQueuedWrites,
|
|
3104
|
+
}: { except: SyncTarget | null; afterQueuedWrites: boolean },
|
|
3105
|
+
): void => {
|
|
3106
|
+
const keys: Array<StructuralLookupKey> = [];
|
|
3107
|
+
deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
|
|
3108
|
+
if (except && isTargetTransport(except, transport)) return;
|
|
3109
|
+
const key = structuralLookup(transport);
|
|
3110
|
+
const route = routesByOwnerIdByKey.get(key)?.get(ownerId);
|
|
3111
|
+
if (route?.progress.type === "Settled") return;
|
|
3112
|
+
if (route?.progress.type === "Pending" && route.progress.skip) {
|
|
3113
|
+
// A round already requested may have read the database before these
|
|
3114
|
+
// messages were stored, so it can no longer complete the route.
|
|
3115
|
+
route.progress = {
|
|
3116
|
+
...route.progress,
|
|
3117
|
+
skip: { ...route.progress.skip, isRechecking: false },
|
|
3118
|
+
};
|
|
3119
|
+
return;
|
|
3120
|
+
}
|
|
3121
|
+
keys.push(key);
|
|
3122
|
+
});
|
|
3123
|
+
for (const key of keys) {
|
|
3124
|
+
requestCreateSyncMessages(
|
|
3125
|
+
new Set([ownerId]),
|
|
3126
|
+
{ type: "Transport", key },
|
|
3127
|
+
{ afterQueuedWrites },
|
|
3128
|
+
);
|
|
3129
|
+
}
|
|
3130
|
+
};
|
|
3131
|
+
|
|
2310
3132
|
/** The keys of every transport claimed for the owner, by any database. */
|
|
2311
3133
|
const getClaimedKeys = (
|
|
2312
3134
|
ownerId: OwnerId,
|
|
@@ -2341,11 +3163,18 @@ const createEvoluTenant =
|
|
|
2341
3163
|
if (startupError) return;
|
|
2342
3164
|
const usedOwnersById = getUsedOwnersById(ownerIds);
|
|
2343
3165
|
// Opening, storing messages from another transport, a failure, and an
|
|
2344
|
-
// explicit request each require a new round before completion.
|
|
3166
|
+
// explicit request each require a new round before completion. The
|
|
3167
|
+
// round also checks again whether the relay holds a skipped message.
|
|
2345
3168
|
for (const ownerId of usedOwnersById.keys()) {
|
|
2346
3169
|
deps.transports.forEachResourceForClaim(ownerId, (_, transport) => {
|
|
2347
3170
|
if (!isTargetTransport(target, transport)) return;
|
|
2348
|
-
getRoute(ownerId, structuralLookup(transport))
|
|
3171
|
+
const route = getRoute(ownerId, structuralLookup(transport));
|
|
3172
|
+
const pending = routeToPending(route.progress);
|
|
3173
|
+
route.progress = {
|
|
3174
|
+
...pending,
|
|
3175
|
+
roundRequired: true,
|
|
3176
|
+
skip: pending.skip && { ...pending.skip, isRechecking: true },
|
|
3177
|
+
};
|
|
2349
3178
|
});
|
|
2350
3179
|
}
|
|
2351
3180
|
refreshSyncRoutes();
|
|
@@ -2493,33 +3322,32 @@ const createEvoluTenant =
|
|
|
2493
3322
|
});
|
|
2494
3323
|
const tenant = disposable<EvoluTenant>(
|
|
2495
3324
|
{
|
|
2496
|
-
getSyncTenant: () =>
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
}),
|
|
3325
|
+
getSyncTenant: () =>
|
|
3326
|
+
startupError
|
|
3327
|
+
? { type: "Refused", name, error: startupError }
|
|
3328
|
+
: {
|
|
3329
|
+
type: "Active",
|
|
3330
|
+
name,
|
|
3331
|
+
owners: getSyncOwners().map(
|
|
3332
|
+
({ ownerId, writable, transportKeys }): TenantSyncOwner =>
|
|
3333
|
+
writable
|
|
3334
|
+
? {
|
|
3335
|
+
type: "Writable",
|
|
3336
|
+
ownerId,
|
|
3337
|
+
routes: transportKeys.map((key): TenantSyncRoute => {
|
|
3338
|
+
const route = routesByOwnerIdByKey
|
|
3339
|
+
.get(key)
|
|
3340
|
+
?.get(ownerId);
|
|
3341
|
+
// Registration and claim changes refresh routes
|
|
3342
|
+
// before yielding. Snapshot reads must not create
|
|
3343
|
+
// missing routes.
|
|
3344
|
+
assertNotUndefined(route);
|
|
3345
|
+
return { transportKey: key, ...route };
|
|
3346
|
+
}),
|
|
3347
|
+
}
|
|
3348
|
+
: { type: "Readonly", ownerId, transportKeys },
|
|
3349
|
+
),
|
|
3350
|
+
},
|
|
2523
3351
|
|
|
2524
3352
|
refreshSyncRoutes,
|
|
2525
3353
|
|
|
@@ -2553,19 +3381,19 @@ const createEvoluTenant =
|
|
|
2553
3381
|
|
|
2554
3382
|
disposer.defer(instance.onDisposed);
|
|
2555
3383
|
|
|
2556
|
-
disposer.defer(
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
deps.refreshAllSyncRoutes();
|
|
2564
|
-
deps.publishSyncState();
|
|
2565
|
-
return ok();
|
|
2566
|
-
}),
|
|
2567
|
-
);
|
|
3384
|
+
disposer.defer(() => {
|
|
3385
|
+
for (const leases of instance.ownerRegistrations.values()) {
|
|
3386
|
+
for (const lease of leases) lease?.release();
|
|
3387
|
+
}
|
|
3388
|
+
instance.ownerRegistrations.clear();
|
|
3389
|
+
deps.refreshAllSyncRoutes();
|
|
3390
|
+
deps.publishSyncState();
|
|
2568
3391
|
});
|
|
3392
|
+
// Cleanup starts no Task, because a root abort may dispose every Run
|
|
3393
|
+
// first. Disposed before the claims above are released, this Run
|
|
3394
|
+
// aborts queued UseOwner batches and waits for the running one, whose
|
|
3395
|
+
// transport claim cannot be aborted, so the release sees every lease.
|
|
3396
|
+
const instanceRun = disposer.use(tenantRun.create());
|
|
2569
3397
|
|
|
2570
3398
|
disposer.defer(() => {
|
|
2571
3399
|
instancesById.delete(instance.id);
|
|
@@ -2581,7 +3409,7 @@ const createEvoluTenant =
|
|
|
2581
3409
|
// while it is alive. Acquiring the same lock here means the main
|
|
2582
3410
|
// thread instance was disposed or its tab closed, so the tenant-side
|
|
2583
3411
|
// instance must dispose itself.
|
|
2584
|
-
void
|
|
3412
|
+
void instanceRun
|
|
2585
3413
|
.abortable(acquireLeaderLock(message.id))
|
|
2586
3414
|
.then((lock) => {
|
|
2587
3415
|
if (!lock.ok) return;
|
|
@@ -2622,7 +3450,7 @@ const createEvoluTenant =
|
|
|
2622
3450
|
break;
|
|
2623
3451
|
}
|
|
2624
3452
|
case "UseOwner": {
|
|
2625
|
-
void
|
|
3453
|
+
void instanceRun(
|
|
2626
3454
|
instance.useOwnerMutex.withLock(async (run) => {
|
|
2627
3455
|
for (const action of message.actions) {
|
|
2628
3456
|
switch (action.action) {
|
|
@@ -2729,14 +3557,14 @@ const createEvoluTenant =
|
|
|
2729
3557
|
// })
|
|
2730
3558
|
|
|
2731
3559
|
// TODO: SharedWorker follow-ups.
|
|
2732
|
-
// - Complete the queue head when a DbWorker mutation returns an error.
|
|
2733
3560
|
// - Rotate the node ID when a copied database is detected; see the Duplicate
|
|
2734
3561
|
// node IDs section in the Timestamp module.
|
|
2735
3562
|
// - Detect DbWorker and port liveness so a worker-only crash resumes the queue.
|
|
2736
3563
|
// Defer panicked-worker restart until failure detection and recovery are
|
|
2737
|
-
// defined, accounting for SQLite WASM's detection limits.
|
|
2738
|
-
// operations are expected not to throw;
|
|
2739
|
-
// can make replicated writes fail, are
|
|
3564
|
+
// defined, accounting for SQLite WASM's detection limits. A mutation that
|
|
3565
|
+
// throws is answered, but other SQLite operations are expected not to throw;
|
|
3566
|
+
// user-defined UNIQUE indexes, which can make replicated writes fail, are
|
|
3567
|
+
// planned to be forbidden.
|
|
2740
3568
|
// - Split worker protocol types and the EvoluTenant implementation into focused
|
|
2741
3569
|
// modules.
|
|
2742
3570
|
// - Remove the obsolete commented protocol block above.
|