@procedures/store 0.18.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +22 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/browser.d.ts +1 -1
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js.map +1 -1
- package/dist/storage/index.d.ts +1 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/tab-coordinator.d.ts +131 -15
- package/dist/storage/tab-coordinator.d.ts.map +1 -1
- package/dist/storage/tab-coordinator.js +374 -32
- package/dist/storage/tab-coordinator.js.map +1 -1
- package/package.json +4 -4
- package/src/browser.ts +1 -0
- package/src/storage/index.ts +1 -0
- package/src/storage/tab-coordinator.ts +455 -37
|
@@ -18,6 +18,12 @@
|
|
|
18
18
|
* acknowledged.
|
|
19
19
|
* - Without the Web Locks API or `BroadcastChannel`, the coordinator is in the single mode: the tab
|
|
20
20
|
* is the leader at once, and it opens no channel.
|
|
21
|
+
* - A tab becomes the leader only after its `HybridStorage` loaded the queue. When the setup fails
|
|
22
|
+
* (the function `remote` throws, or the load of the queue fails), the tab reports the error and
|
|
23
|
+
* releases the lock. It stays a follower, and it asks for the lock again after a backoff (roadmap
|
|
24
|
+
* CLI-67). Thus a failure that goes away does not leave the browser without a sync.
|
|
25
|
+
* - The leader sends the status of its sync to the followers: at each change, and in answer to a
|
|
26
|
+
* hello. `getStatus()` and `onStatus()` give the status in each tab (roadmap CLI-65).
|
|
21
27
|
*
|
|
22
28
|
* The limits:
|
|
23
29
|
* - When the leader stops during a push, the next leader sends the operation again: the delivery to
|
|
@@ -39,7 +45,10 @@ import type {
|
|
|
39
45
|
StorageWriteOptions,
|
|
40
46
|
} from "@procedures/collections";
|
|
41
47
|
import { StorageListeners, reportListenerError, type ListenerErrorHandler } from "@procedures/collections";
|
|
48
|
+
import { optionalMs, realClock, type Cancel, type Clock, type Duration } from "@procedures/client/browser";
|
|
49
|
+
import { backoffDelay, type Backoff } from "@mark1russell7/service/backoff";
|
|
42
50
|
import { HybridStorage, type HybridStorageOptions, type LocalCopyStorage } from "./hybrid.js";
|
|
51
|
+
import type { SyncError, SyncStatus } from "./types.js";
|
|
43
52
|
|
|
44
53
|
/** The signal that stops a lock request that waits. The `AbortSignal` of a browser fits it. */
|
|
45
54
|
export interface TabAbortSignalLike {
|
|
@@ -74,7 +83,7 @@ export type TabChannelFactory = (name: string) => TabChannelLike;
|
|
|
74
83
|
* The role of a tab.
|
|
75
84
|
*
|
|
76
85
|
* - `follower`: the tab waits for the lock. It copies the changes of the other tabs, and it sends
|
|
77
|
-
* its own changes to the leader.
|
|
86
|
+
* its own changes to the leader. A tab whose setup failed is a follower until its next try.
|
|
78
87
|
* - `leader`: the tab holds the lock, and its `HybridStorage` runs the sync.
|
|
79
88
|
* - `single`: the browser has no Web Locks API or no `BroadcastChannel`. The tab runs the sync
|
|
80
89
|
* alone, and it opens no channel.
|
|
@@ -103,7 +112,10 @@ export interface TabCoordinatorOptions<T> {
|
|
|
103
112
|
*/
|
|
104
113
|
copy: "shared" | "tab";
|
|
105
114
|
|
|
106
|
-
/**
|
|
115
|
+
/**
|
|
116
|
+
* The function that makes the remote of the `HybridStorage`. Only a tab that got the lock calls
|
|
117
|
+
* it. When it throws, the tab releases the lock and tries again after the backoff of `retry`.
|
|
118
|
+
*/
|
|
107
119
|
remote: () => CollectionStorage<T>;
|
|
108
120
|
|
|
109
121
|
/**
|
|
@@ -125,20 +137,92 @@ export interface TabCoordinatorOptions<T> {
|
|
|
125
137
|
*/
|
|
126
138
|
channel?: TabChannelFactory | null;
|
|
127
139
|
|
|
140
|
+
/**
|
|
141
|
+
* The wait before a new try after a failed setup of the sync (roadmap CLI-67). A setup fails when
|
|
142
|
+
* the function `remote` throws, when the `HybridStorage` cannot start, or when it cannot load its
|
|
143
|
+
* queue. The tab releases the lock, stays a follower, and asks for the lock again after the wait.
|
|
144
|
+
* In the single mode, the tab tries the setup again after the wait.
|
|
145
|
+
*
|
|
146
|
+
* After `n` failed setups in a row, the wait is `delay × factor^(n − 1)`, at most `maxDelay`. The
|
|
147
|
+
* random part `jitter` makes the wait shorter by up to this fraction. A setup that succeeds sets
|
|
148
|
+
* the wait back to `delay`.
|
|
149
|
+
*/
|
|
150
|
+
retry?: TabRetryOptions;
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The clock of the wait of `retry`. A test gives `fakeClock` of `@mark1russell7/service/testing`.
|
|
154
|
+
* @default the `clock` of `hybrid`, or else the clock of the platform (`realClock` of `@procedures/client`)
|
|
155
|
+
*/
|
|
156
|
+
clock?: Clock;
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* The random numbers of the jitter of `retry`, from 0 up to 1. A test gives a fixed generator.
|
|
160
|
+
* @default the `random` of `hybrid`, or else `Math.random`
|
|
161
|
+
*/
|
|
162
|
+
random?: () => number;
|
|
163
|
+
|
|
128
164
|
/**
|
|
129
165
|
* The function that gets the errors: of the lock, of the channel, of a write to the local copy,
|
|
130
|
-
* of the `HybridStorage
|
|
166
|
+
* of the `HybridStorage`, of a failed setup and of the observers. The default throws the error
|
|
167
|
+
* again in a microtask.
|
|
131
168
|
*/
|
|
132
169
|
onError?: ListenerErrorHandler;
|
|
133
170
|
}
|
|
134
171
|
|
|
172
|
+
/** The backoff of a new try after a failed setup of the sync (roadmap CLI-67). */
|
|
173
|
+
export interface TabRetryOptions {
|
|
174
|
+
/** The wait after the first failure, for example `seconds(1)`. It must be more than zero. @default 1 second */
|
|
175
|
+
delay?: Duration;
|
|
176
|
+
/** The number that multiplies the wait after each failure, 1 or more. @default 2 */
|
|
177
|
+
factor?: number;
|
|
178
|
+
/** The longest wait, for example `minutes(1)`. A `delay` that is longer is the longest wait. @default 1 minute */
|
|
179
|
+
maxDelay?: Duration;
|
|
180
|
+
/** The random part of each wait, from 0 to 1. The wait is from `(1 − jitter) ×` the backoff up to the backoff. @default 0.5 */
|
|
181
|
+
jitter?: number;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** The storage of a tab that got the lock, before the tab becomes the leader. */
|
|
185
|
+
interface Candidate<T> {
|
|
186
|
+
readonly hybrid: HybridStorage<T>;
|
|
187
|
+
/** False until the tab leads: the capture of the storage ignores the local copy */
|
|
188
|
+
readonly gate: { open: boolean };
|
|
189
|
+
/** The stop of the observer of the storage */
|
|
190
|
+
readonly stop: () => void;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The status of the sync in a message: only values that a structured clone keeps. The error of the
|
|
195
|
+
* leader becomes its name, its message and its code. The follower makes an `Error` of them again.
|
|
196
|
+
*/
|
|
197
|
+
interface PortableStatus {
|
|
198
|
+
readonly connected: boolean;
|
|
199
|
+
readonly pendingChanges: number;
|
|
200
|
+
readonly syncing: boolean;
|
|
201
|
+
readonly endpoint?: string;
|
|
202
|
+
readonly lastSync?: number;
|
|
203
|
+
readonly mode?: NonNullable<SyncStatus["mode"]>;
|
|
204
|
+
readonly error?: PortableSyncError;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
interface PortableSyncError {
|
|
208
|
+
readonly operation: SyncError["operation"];
|
|
209
|
+
readonly name: string;
|
|
210
|
+
readonly message: string;
|
|
211
|
+
readonly code?: string | number;
|
|
212
|
+
}
|
|
213
|
+
|
|
135
214
|
/** A change batch of a tab, with its number. */
|
|
136
215
|
interface SentBatch<T> {
|
|
137
216
|
readonly seq: number;
|
|
138
217
|
readonly changes: StorageChangeBatch<T>;
|
|
139
218
|
}
|
|
140
219
|
|
|
141
|
-
/**
|
|
220
|
+
/**
|
|
221
|
+
* The version of the messages. A tab ignores a message of another version, and a message of a type
|
|
222
|
+
* that it does not know. Thus a new type of message (`status`, roadmap CLI-65) keeps the version 1:
|
|
223
|
+
* a tab of an older release ignores it, and it still copies and sends the batches. A new version
|
|
224
|
+
* stops the batches between the tabs of two releases, so only a change of a known message needs it.
|
|
225
|
+
*/
|
|
142
226
|
const PROTOCOL = 1;
|
|
143
227
|
|
|
144
228
|
/**
|
|
@@ -150,12 +234,16 @@ const PROTOCOL = 1;
|
|
|
150
234
|
* acknowledged. A tab sends it when it starts, and when a new leader starts.
|
|
151
235
|
* - `leader`: the tab `from` is the new leader.
|
|
152
236
|
* - `ack`: the queue store of the leader holds the batches of the tab `to`, up to `seq`.
|
|
237
|
+
* - `status`: the status of the sync of the leader `from` (roadmap CLI-65). The leader sends it at
|
|
238
|
+
* each change, and in answer to a hello. The numbers of the statuses of a leader increase by one,
|
|
239
|
+
* and the answer to a hello sends the last number again: a tab ignores a number that it has.
|
|
153
240
|
*/
|
|
154
241
|
type TabMessage<T> =
|
|
155
242
|
| { readonly v: typeof PROTOCOL; readonly type: "changes"; readonly from: string; readonly seq: number; readonly changes: StorageChangeBatch<T> }
|
|
156
243
|
| { readonly v: typeof PROTOCOL; readonly type: "hello"; readonly from: string; readonly seq: number; readonly pending: ReadonlyArray<SentBatch<T>> }
|
|
157
244
|
| { readonly v: typeof PROTOCOL; readonly type: "leader"; readonly from: string }
|
|
158
|
-
| { readonly v: typeof PROTOCOL; readonly type: "ack"; readonly from: string; readonly to: string; readonly seq: number }
|
|
245
|
+
| { readonly v: typeof PROTOCOL; readonly type: "ack"; readonly from: string; readonly to: string; readonly seq: number }
|
|
246
|
+
| { readonly v: typeof PROTOCOL; readonly type: "status"; readonly from: string; readonly seq: number; readonly status: PortableStatus };
|
|
159
247
|
|
|
160
248
|
/**
|
|
161
249
|
* One leader tab runs the sync of a `HybridStorage`, and the other tabs copy its changes.
|
|
@@ -178,6 +266,7 @@ type TabMessage<T> =
|
|
|
178
266
|
* });
|
|
179
267
|
*
|
|
180
268
|
* users.observe((batch) => render(batch)); // The changes of this tab and of the other tabs
|
|
269
|
+
* users.onStatus((status) => showSync(status)); // The status of the sync of the leader, in each tab
|
|
181
270
|
* await local.set("1", { name: "Ada" }); // The leader pushes the write
|
|
182
271
|
* ```
|
|
183
272
|
*/
|
|
@@ -197,11 +286,31 @@ export class TabCoordinator<T> {
|
|
|
197
286
|
private readonly onError: ListenerErrorHandler;
|
|
198
287
|
private readonly observers: StorageListeners<StorageChangeBatch<T>>;
|
|
199
288
|
private readonly roleListeners: StorageListeners<TabRole>;
|
|
289
|
+
private readonly statusListeners: StorageListeners<SyncStatus>;
|
|
200
290
|
/** The observers that the `HybridStorage` of the leader adds to its local copy: the capture */
|
|
201
291
|
private readonly captures: StorageListeners<StorageChangeBatch<T>>;
|
|
292
|
+
/** The backoff of a new try after a failed setup, its clock and its random numbers */
|
|
293
|
+
private readonly backoff: Backoff;
|
|
294
|
+
private readonly clock: Clock;
|
|
295
|
+
private readonly random: () => number;
|
|
202
296
|
private currentRole: TabRole = "follower";
|
|
297
|
+
/** The lock manager, or undefined in the single mode */
|
|
298
|
+
private locks: TabLockManagerLike | undefined;
|
|
203
299
|
private channel: TabChannelLike | undefined;
|
|
204
300
|
private hybrid: HybridStorage<T> | undefined;
|
|
301
|
+
/** The storage of this tab while it loads its queue, before the tab leads */
|
|
302
|
+
private candidate: Candidate<T> | undefined;
|
|
303
|
+
/** The number of the failed setups in a row */
|
|
304
|
+
private failures = 0;
|
|
305
|
+
/** The stop of the wait before the next try */
|
|
306
|
+
private cancelRetry: Cancel | undefined;
|
|
307
|
+
/** The status of the sync: of the `HybridStorage` of this tab, or of the leader */
|
|
308
|
+
private status: SyncStatus | undefined;
|
|
309
|
+
/** In the leader: the number of its last status. In a follower: the tab and the number of the last status that it got. */
|
|
310
|
+
private statusSeq = 0;
|
|
311
|
+
private statusFrom: string | undefined;
|
|
312
|
+
/** The stop of the status of the `HybridStorage` of this tab */
|
|
313
|
+
private stopStatus: (() => void) | undefined;
|
|
205
314
|
/** The stop of the observer of the changes of this tab: on the local copy, or on the `HybridStorage` of the leader */
|
|
206
315
|
private stopTap: (() => void) | undefined;
|
|
207
316
|
/** True while the leader gives a batch of another tab to the capture of its `HybridStorage` */
|
|
@@ -230,6 +339,7 @@ export class TabCoordinator<T> {
|
|
|
230
339
|
throw new TypeError(`Unknown copy ${JSON.stringify(options.copy)}: use shared or tab`);
|
|
231
340
|
}
|
|
232
341
|
if (typeof options.remote !== "function") throw new TypeError("TabCoordinator needs the function remote");
|
|
342
|
+
this.backoff = retryBackoff(options.retry);
|
|
233
343
|
this.id = randomId();
|
|
234
344
|
this.lockName = `@procedures/store/tab:${options.name}`;
|
|
235
345
|
this.local = options.local;
|
|
@@ -241,23 +351,28 @@ export class TabCoordinator<T> {
|
|
|
241
351
|
this.onError = options.onError ?? reportListenerError;
|
|
242
352
|
this.observers = new StorageListeners<StorageChangeBatch<T>>(this.onError);
|
|
243
353
|
this.roleListeners = new StorageListeners<TabRole>(this.onError);
|
|
354
|
+
this.statusListeners = new StorageListeners<SyncStatus>(this.onError);
|
|
244
355
|
this.captures = new StorageListeners<StorageChangeBatch<T>>(this.onError);
|
|
356
|
+
this.clock = options.clock ?? this.hybridOptions.clock ?? realClock;
|
|
357
|
+
this.random = options.random ?? this.hybridOptions.random ?? Math.random;
|
|
245
358
|
|
|
359
|
+
// The observer of a follower. A tab that leads stops it. A single tab whose setup failed keeps its batches with it.
|
|
360
|
+
this.stopTap = this.local.observe!((batch) => this.onFollowerBatch(batch));
|
|
246
361
|
const locks = options.locks === undefined ? globalLocks() : (options.locks ?? undefined);
|
|
247
362
|
const openChannel = options.channel === undefined ? globalChannelFactory() : (options.channel ?? undefined);
|
|
248
363
|
if (!locks || !openChannel) {
|
|
249
|
-
this.
|
|
364
|
+
this.startSingle();
|
|
250
365
|
return;
|
|
251
366
|
}
|
|
252
367
|
try {
|
|
253
368
|
this.channel = openChannel(this.lockName);
|
|
254
369
|
} catch (error) {
|
|
255
370
|
this.onError(error);
|
|
256
|
-
this.
|
|
371
|
+
this.startSingle();
|
|
257
372
|
return;
|
|
258
373
|
}
|
|
374
|
+
this.locks = locks;
|
|
259
375
|
this.channel.onmessage = (event: { data?: unknown } | undefined) => this.receive(event?.data);
|
|
260
|
-
this.stopTap = this.local.observe!((batch) => this.onFollowerBatch(batch));
|
|
261
376
|
this.hello();
|
|
262
377
|
this.elect(locks);
|
|
263
378
|
}
|
|
@@ -273,9 +388,9 @@ export class TabCoordinator<T> {
|
|
|
273
388
|
}
|
|
274
389
|
|
|
275
390
|
/**
|
|
276
|
-
* The `HybridStorage` of the sync while this tab runs it, or else undefined.
|
|
277
|
-
* gives the state of the sync. Write to the local copy, not to this
|
|
278
|
-
* in the same way.
|
|
391
|
+
* The `HybridStorage` of the sync while this tab runs it, or else undefined. `getStatus()` of the
|
|
392
|
+
* coordinator gives the state of the sync in each tab. Write to the local copy, not to this
|
|
393
|
+
* storage: each tab then writes in the same way.
|
|
279
394
|
*/
|
|
280
395
|
get syncStorage(): HybridStorage<T> | undefined {
|
|
281
396
|
return this.hybrid;
|
|
@@ -306,10 +421,40 @@ export class TabCoordinator<T> {
|
|
|
306
421
|
return this.roleListeners.add(listener);
|
|
307
422
|
}
|
|
308
423
|
|
|
424
|
+
/**
|
|
425
|
+
* The status of the sync (roadmap CLI-65), or undefined before the first status and after
|
|
426
|
+
* `close()`.
|
|
427
|
+
*
|
|
428
|
+
* - In the leader and in the single mode, it is the status of the `HybridStorage` of this tab.
|
|
429
|
+
* - In a follower, it is the last status that the leader sent. A new tab gets it in the answer to
|
|
430
|
+
* its hello. After a change of the leader, a follower keeps the last status until the new
|
|
431
|
+
* leader sends its status.
|
|
432
|
+
* - In a follower, `error.error` is a copy of the error of the leader: an `Error` with its `name`,
|
|
433
|
+
* its `message` and its `code`. The copy has no `cause`.
|
|
434
|
+
*/
|
|
435
|
+
getStatus(): SyncStatus | undefined {
|
|
436
|
+
return this.status;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* This method starts to send the status of the sync to a listener at each change (roadmap
|
|
441
|
+
* CLI-65): in the leader, each status of its `HybridStorage`, and in a follower, each status that
|
|
442
|
+
* the leader sends. The listener gets each status one time. When the tab becomes the leader, the
|
|
443
|
+
* listener gets the status of its own `HybridStorage`.
|
|
444
|
+
*
|
|
445
|
+
* @returns The function that stops the statuses. A second use of it does nothing.
|
|
446
|
+
* @throws Error after `close()`
|
|
447
|
+
*/
|
|
448
|
+
onStatus(listener: (status: SyncStatus) => void): () => void {
|
|
449
|
+
if (this.closed) throw new Error("The TabCoordinator is closed");
|
|
450
|
+
return this.statusListeners.add(listener);
|
|
451
|
+
}
|
|
452
|
+
|
|
309
453
|
/**
|
|
310
454
|
* This method closes the coordinator.
|
|
311
455
|
*
|
|
312
|
-
* - A follower stops its lock request. It does not become the leader.
|
|
456
|
+
* - A follower stops its lock request, or its wait before a new try. It does not become the leader.
|
|
457
|
+
* - A tab that loads its queue to become the leader closes its `HybridStorage`, and releases the lock.
|
|
313
458
|
* - The leader closes its `HybridStorage`: it sends the queue, and the queue store keeps the
|
|
314
459
|
* operations that the remote did not take. Then the leader releases the lock, and the next
|
|
315
460
|
* tab becomes the leader.
|
|
@@ -328,9 +473,17 @@ export class TabCoordinator<T> {
|
|
|
328
473
|
this.stopTap = undefined;
|
|
329
474
|
this.abortRequest?.();
|
|
330
475
|
this.abortRequest = undefined;
|
|
476
|
+
this.cancelRetry?.();
|
|
477
|
+
this.cancelRetry = undefined;
|
|
478
|
+
this.stopStatus?.();
|
|
479
|
+
this.stopStatus = undefined;
|
|
331
480
|
this.setRole("closed");
|
|
332
|
-
const
|
|
481
|
+
const candidate = this.candidate;
|
|
482
|
+
this.candidate = undefined;
|
|
483
|
+
candidate?.stop();
|
|
484
|
+
const hybrid = this.hybrid ?? candidate?.hybrid;
|
|
333
485
|
this.hybrid = undefined;
|
|
486
|
+
this.status = undefined;
|
|
334
487
|
try {
|
|
335
488
|
// The leader holds the lock until its queue reached the remote or its queue store
|
|
336
489
|
if (hybrid) await hybrid.close();
|
|
@@ -348,6 +501,7 @@ export class TabCoordinator<T> {
|
|
|
348
501
|
this.channel = undefined;
|
|
349
502
|
this.observers.clear();
|
|
350
503
|
this.roleListeners.clear();
|
|
504
|
+
this.statusListeners.clear();
|
|
351
505
|
this.captures.clear();
|
|
352
506
|
}
|
|
353
507
|
}
|
|
@@ -368,9 +522,12 @@ export class TabCoordinator<T> {
|
|
|
368
522
|
// A lock after close(): the tab releases it at once
|
|
369
523
|
if (this.closed) return;
|
|
370
524
|
this.abortRequest = undefined;
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
525
|
+
if (await this.takeLead()) {
|
|
526
|
+
await held;
|
|
527
|
+
return;
|
|
528
|
+
}
|
|
529
|
+
// A failed setup releases the lock: another tab tries, and this tab tries again after the backoff
|
|
530
|
+
this.retryLater();
|
|
374
531
|
});
|
|
375
532
|
} catch (error) {
|
|
376
533
|
request = Promise.reject(error);
|
|
@@ -381,50 +538,143 @@ export class TabCoordinator<T> {
|
|
|
381
538
|
// The lock manager refused the request: this tab runs the sync alone
|
|
382
539
|
this.onError(error);
|
|
383
540
|
this.stopChannel();
|
|
384
|
-
this.
|
|
541
|
+
this.locks = undefined;
|
|
542
|
+
this.startSingle();
|
|
385
543
|
});
|
|
386
544
|
}
|
|
387
545
|
|
|
388
546
|
/**
|
|
389
|
-
* This method makes the `HybridStorage` of
|
|
390
|
-
*
|
|
547
|
+
* This method makes the `HybridStorage` of a tab that got the lock, and waits for the load of its
|
|
548
|
+
* queue. Then the tab becomes the leader. Until then, the tab is a follower: its own batches go to
|
|
549
|
+
* the other tabs and to its outbox, and the capture of the storage ignores the local copy.
|
|
550
|
+
*
|
|
551
|
+
* @returns False when the setup failed (the error went to `onError`), or after `close()`
|
|
552
|
+
*/
|
|
553
|
+
private async takeLead(): Promise<boolean> {
|
|
554
|
+
const candidate = this.makeCandidate();
|
|
555
|
+
if (!candidate) return false;
|
|
556
|
+
this.candidate = candidate;
|
|
557
|
+
let failed = false;
|
|
558
|
+
let failure: unknown;
|
|
559
|
+
try {
|
|
560
|
+
// A storage whose load failed refuses each operation: it cannot lead
|
|
561
|
+
await candidate.hybrid.whenQueued();
|
|
562
|
+
} catch (error) {
|
|
563
|
+
failed = true;
|
|
564
|
+
failure = error;
|
|
565
|
+
}
|
|
566
|
+
// close() took the storage
|
|
567
|
+
if (this.candidate !== candidate) return false;
|
|
568
|
+
this.candidate = undefined;
|
|
569
|
+
if (!failed) {
|
|
570
|
+
this.promote(candidate, "leader");
|
|
571
|
+
return true;
|
|
572
|
+
}
|
|
573
|
+
this.onError(failure);
|
|
574
|
+
candidate.stop();
|
|
575
|
+
try {
|
|
576
|
+
await candidate.hybrid.close();
|
|
577
|
+
} catch (error) {
|
|
578
|
+
this.onError(error);
|
|
579
|
+
}
|
|
580
|
+
return false;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/** This method makes the `HybridStorage` of the single mode. The tab tries again after the backoff when it fails. */
|
|
584
|
+
private startSingle(): void {
|
|
585
|
+
const candidate = this.makeCandidate();
|
|
586
|
+
if (candidate) this.promote(candidate, "single");
|
|
587
|
+
else this.retryLater();
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* This method makes the `HybridStorage` of the sync, with its observer. Its capture ignores the
|
|
592
|
+
* local copy until the tab leads.
|
|
391
593
|
*
|
|
392
|
-
* @returns
|
|
594
|
+
* @returns Undefined when the remote or the storage could not start (the error went to `onError`), or after `close()`
|
|
393
595
|
*/
|
|
394
|
-
private
|
|
395
|
-
if (this.closed) return
|
|
596
|
+
private makeCandidate(): Candidate<T> | undefined {
|
|
597
|
+
if (this.closed) return undefined;
|
|
598
|
+
let remote: CollectionStorage<T>;
|
|
599
|
+
try {
|
|
600
|
+
remote = this.makeRemote();
|
|
601
|
+
} catch (error) {
|
|
602
|
+
this.onError(error);
|
|
603
|
+
return undefined;
|
|
604
|
+
}
|
|
605
|
+
const gate = { open: false };
|
|
396
606
|
let hybrid: HybridStorage<T>;
|
|
397
607
|
try {
|
|
398
|
-
hybrid = new HybridStorage<T>(
|
|
608
|
+
hybrid = new HybridStorage<T>(remote, {
|
|
399
609
|
...this.hybridOptions,
|
|
400
|
-
local: this.captureCopy(),
|
|
610
|
+
local: this.captureCopy(gate),
|
|
401
611
|
capture: "observe",
|
|
402
612
|
origin: this.tag,
|
|
403
613
|
});
|
|
404
614
|
} catch (error) {
|
|
405
615
|
this.onError(error);
|
|
406
|
-
|
|
616
|
+
// The remote of a storage that did not start stays open otherwise
|
|
617
|
+
Promise.resolve()
|
|
618
|
+
.then(() => remote.close())
|
|
619
|
+
.catch((closeError: unknown) => this.onError(closeError));
|
|
620
|
+
return undefined;
|
|
407
621
|
}
|
|
622
|
+
// The observer starts at once: the other tabs get the changes of the load and of the first sync
|
|
623
|
+
const stop = hybrid.observe((batch) => this.onLeaderBatch(batch));
|
|
624
|
+
return { hybrid, gate, stop };
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* This method makes this tab the leader, or the single tab, with the storage of a candidate. The
|
|
629
|
+
* capture of the storage gets the batches of this tab that no leader acknowledged.
|
|
630
|
+
*/
|
|
631
|
+
private promote(candidate: Candidate<T>, role: "leader" | "single"): void {
|
|
632
|
+
const { hybrid, gate, stop } = candidate;
|
|
408
633
|
this.hybrid = hybrid;
|
|
409
|
-
|
|
634
|
+
this.failures = 0;
|
|
635
|
+
// The capture starts and the observer of the follower stops in one step: no change of the local copy goes between them
|
|
636
|
+
gate.open = true;
|
|
410
637
|
const stopFollower = this.stopTap;
|
|
411
|
-
this.stopTap =
|
|
638
|
+
this.stopTap = stop;
|
|
412
639
|
stopFollower?.();
|
|
413
640
|
for (const batch of this.outbox.splice(0)) this.capture(batch.changes);
|
|
414
641
|
this.setRole(role);
|
|
415
642
|
if (role === "leader") this.post({ v: PROTOCOL, type: "leader", from: this.id });
|
|
416
|
-
|
|
643
|
+
// The status of this tab is now the status of its storage, also when it has the values of the last status of the old leader. The followers get it.
|
|
644
|
+
this.status = undefined;
|
|
645
|
+
this.statusFrom = undefined;
|
|
646
|
+
this.stopStatus = hybrid.onStatus((status) => this.onOwnStatus(hybrid, status));
|
|
647
|
+
this.onOwnStatus(hybrid, hybrid.getStatus());
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/** This method asks for the lock again, or tries the single mode again, after the backoff (roadmap CLI-67). */
|
|
651
|
+
private retryLater(): void {
|
|
652
|
+
if (this.closed) return;
|
|
653
|
+
this.failures++;
|
|
654
|
+
const wait = backoffDelay(this.backoff, this.failures, this.random);
|
|
655
|
+
const cancel = this.clock.after(wait, () => {
|
|
656
|
+
this.cancelRetry = undefined;
|
|
657
|
+
if (this.closed) return;
|
|
658
|
+
if (this.locks) this.elect(this.locks);
|
|
659
|
+
else this.startSingle();
|
|
660
|
+
});
|
|
661
|
+
// The wait must not keep a Node process alive
|
|
662
|
+
cancel.unref?.();
|
|
663
|
+
this.cancelRetry = cancel;
|
|
417
664
|
}
|
|
418
665
|
|
|
419
666
|
/**
|
|
420
667
|
* The local copy for the `HybridStorage` of the leader. Its `observe()` gives the changes of the
|
|
421
|
-
* local copy and the batches of the other tabs, so the capture queues both.
|
|
422
|
-
* nothing: the application owns
|
|
668
|
+
* local copy and the batches of the other tabs, so the capture queues both. It gives the changes
|
|
669
|
+
* of the local copy only when `gate` is open. Its `close()` does nothing: the application owns
|
|
670
|
+
* the local copy.
|
|
423
671
|
*/
|
|
424
|
-
private captureCopy(): LocalCopyStorage<T> {
|
|
672
|
+
private captureCopy(gate: { readonly open: boolean }): LocalCopyStorage<T> {
|
|
425
673
|
const captures = this.captures;
|
|
426
674
|
const observe = (observer: StorageObserver<T>): (() => void) => {
|
|
427
|
-
const stopLocal = this.local.observe!(
|
|
675
|
+
const stopLocal = this.local.observe!((batch) => {
|
|
676
|
+
if (gate.open) observer(batch);
|
|
677
|
+
});
|
|
428
678
|
const stopCapture = captures.add(observer);
|
|
429
679
|
return () => {
|
|
430
680
|
stopLocal();
|
|
@@ -462,6 +712,37 @@ export class TabCoordinator<T> {
|
|
|
462
712
|
this.roleListeners.emit(role);
|
|
463
713
|
}
|
|
464
714
|
|
|
715
|
+
//
|
|
716
|
+
// ═══ The status of the sync (roadmap CLI-65) ═══
|
|
717
|
+
//
|
|
718
|
+
|
|
719
|
+
/** A status of the `HybridStorage` of this tab. The leader sends each new status to the followers. */
|
|
720
|
+
private onOwnStatus(hybrid: HybridStorage<T>, status: SyncStatus): void {
|
|
721
|
+
if (this.closed || this.hybrid !== hybrid) return;
|
|
722
|
+
if (this.status !== undefined && sameStatus(this.status, status)) return;
|
|
723
|
+
this.status = status;
|
|
724
|
+
this.statusSeq++;
|
|
725
|
+
if (this.currentRole === "leader") this.postStatus();
|
|
726
|
+
this.statusListeners.emit(status);
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/** This method sends the last status of the leader, with its number. */
|
|
730
|
+
private postStatus(): void {
|
|
731
|
+
if (this.status === undefined) return;
|
|
732
|
+
this.post({ v: PROTOCOL, type: "status", from: this.id, seq: this.statusSeq, status: portableStatus(this.status) });
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/** A status of the leader. A tab that runs the sync keeps its own status. */
|
|
736
|
+
private onLeaderStatus(from: string, seq: number, status: PortableStatus): void {
|
|
737
|
+
if (this.hybrid) return;
|
|
738
|
+
// The answer to a hello sends the last status again
|
|
739
|
+
if (from === this.statusFrom && seq <= this.statusSeq) return;
|
|
740
|
+
this.statusFrom = from;
|
|
741
|
+
this.statusSeq = seq;
|
|
742
|
+
this.status = statusOf(status);
|
|
743
|
+
this.statusListeners.emit(this.status);
|
|
744
|
+
}
|
|
745
|
+
|
|
465
746
|
//
|
|
466
747
|
// ═══ The batches of this tab ═══
|
|
467
748
|
//
|
|
@@ -485,12 +766,15 @@ export class TabCoordinator<T> {
|
|
|
485
766
|
this.share(batch, false);
|
|
486
767
|
}
|
|
487
768
|
|
|
488
|
-
/**
|
|
769
|
+
/**
|
|
770
|
+
* This method gives a batch of this tab to its observers and to the other tabs. A follower keeps
|
|
771
|
+
* it for the leader. A single tab whose setup failed keeps it for its next try.
|
|
772
|
+
*/
|
|
489
773
|
private share(changes: StorageChangeBatch<T>, keep: boolean): void {
|
|
490
774
|
const seq = ++this.seq;
|
|
491
775
|
this.observers.emit(changes);
|
|
492
776
|
const sent = this.post({ v: PROTOCOL, type: "changes", from: this.id, seq, changes });
|
|
493
|
-
if (
|
|
777
|
+
if (keep && (sent || !this.channel)) this.outbox.push({ seq, changes });
|
|
494
778
|
}
|
|
495
779
|
|
|
496
780
|
/** This method tells the leader the batches of this tab that no leader acknowledged. */
|
|
@@ -543,6 +827,9 @@ export class TabCoordinator<T> {
|
|
|
543
827
|
case "ack":
|
|
544
828
|
if (data.to === this.id) this.outbox = this.outbox.filter((batch) => batch.seq > data.seq);
|
|
545
829
|
break;
|
|
830
|
+
case "status":
|
|
831
|
+
this.onLeaderStatus(data.from, data.seq, data.status);
|
|
832
|
+
break;
|
|
546
833
|
}
|
|
547
834
|
}
|
|
548
835
|
|
|
@@ -562,7 +849,8 @@ export class TabCoordinator<T> {
|
|
|
562
849
|
|
|
563
850
|
/**
|
|
564
851
|
* The hello of another tab. The leader captures the batches that it did not capture, and then each
|
|
565
|
-
* batch of the tab after `seq`. Each tab copies the batches that it
|
|
852
|
+
* batch of the tab after `seq`. It answers with its status. Each tab copies the batches that it
|
|
853
|
+
* did not see.
|
|
566
854
|
*/
|
|
567
855
|
private onHello(from: string, seq: number, pending: ReadonlyArray<SentBatch<T>>): void {
|
|
568
856
|
if (this.hybrid && this.currentRole === "leader") {
|
|
@@ -570,6 +858,8 @@ export class TabCoordinator<T> {
|
|
|
570
858
|
for (const batch of pending) if (known === undefined || batch.seq > known) this.capture(batch.changes);
|
|
571
859
|
this.captured.set(from, Math.max(known ?? 0, seq));
|
|
572
860
|
if (pending.length > 0) this.acknowledge(from, seq);
|
|
861
|
+
// A new tab gets the status now. The other tabs have its number already.
|
|
862
|
+
this.postStatus();
|
|
573
863
|
}
|
|
574
864
|
for (const batch of pending) this.copy(from, batch);
|
|
575
865
|
}
|
|
@@ -625,6 +915,120 @@ export class TabCoordinator<T> {
|
|
|
625
915
|
}
|
|
626
916
|
}
|
|
627
917
|
|
|
918
|
+
/**
|
|
919
|
+
* The backoff of the option `retry`.
|
|
920
|
+
*
|
|
921
|
+
* @throws RangeError for a `delay` of zero or less, a `factor` below 1, or a `jitter` outside 0 to 1
|
|
922
|
+
* @throws TypeError for a time that is not a `Duration`
|
|
923
|
+
*/
|
|
924
|
+
function retryBackoff(retry: TabRetryOptions | undefined): Backoff {
|
|
925
|
+
const delay = optionalMs(retry?.delay, "retry.delay") ?? 1000;
|
|
926
|
+
const maxDelay = optionalMs(retry?.maxDelay, "retry.maxDelay") ?? 60_000;
|
|
927
|
+
const factor = retry?.factor ?? 2;
|
|
928
|
+
const jitter = retry?.jitter ?? 0.5;
|
|
929
|
+
if (!(delay > 0)) throw new RangeError(`retry.delay must be a time of more than zero, not ${delay} ms`);
|
|
930
|
+
if (!(Number.isFinite(factor) && factor >= 1)) throw new RangeError(`retry.factor must be a number of 1 or more, not ${factor}`);
|
|
931
|
+
if (!(jitter >= 0 && jitter <= 1)) throw new RangeError(`retry.jitter must be from 0 to 1, not ${jitter}`);
|
|
932
|
+
return {
|
|
933
|
+
base: delay,
|
|
934
|
+
factor,
|
|
935
|
+
cap: Math.max(maxDelay, delay),
|
|
936
|
+
jitter: jitter > 0 ? { kind: "subtract", fraction: jitter } : { kind: "none" },
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/** True when two statuses have the same values. The errors are the same object. */
|
|
941
|
+
function sameStatus(a: SyncStatus, b: SyncStatus): boolean {
|
|
942
|
+
return (
|
|
943
|
+
a.connected === b.connected &&
|
|
944
|
+
a.endpoint === b.endpoint &&
|
|
945
|
+
a.lastSync === b.lastSync &&
|
|
946
|
+
a.pendingChanges === b.pendingChanges &&
|
|
947
|
+
a.syncing === b.syncing &&
|
|
948
|
+
a.mode === b.mode &&
|
|
949
|
+
a.error === b.error
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
/**
|
|
954
|
+
* The status in a form that a structured clone keeps. The status has numbers, strings and booleans,
|
|
955
|
+
* and they clone. Its error can be any value, for example an error with a function in a property,
|
|
956
|
+
* so the message has only the name, the message and the code of the error.
|
|
957
|
+
*/
|
|
958
|
+
function portableStatus(status: SyncStatus): PortableStatus {
|
|
959
|
+
return {
|
|
960
|
+
connected: status.connected,
|
|
961
|
+
pendingChanges: status.pendingChanges,
|
|
962
|
+
syncing: status.syncing,
|
|
963
|
+
...(status.endpoint !== undefined && { endpoint: status.endpoint }),
|
|
964
|
+
...(status.lastSync !== undefined && { lastSync: status.lastSync }),
|
|
965
|
+
...(status.mode !== undefined && { mode: status.mode }),
|
|
966
|
+
...(status.error !== undefined && { error: { operation: status.error.operation, ...portableError(status.error.error) } }),
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
/** The name, the message and the code of an error. A value that is not an error becomes its text. */
|
|
971
|
+
function portableError(value: unknown): Omit<PortableSyncError, "operation"> {
|
|
972
|
+
try {
|
|
973
|
+
if (typeof value === "object" && value !== null) {
|
|
974
|
+
const { name, message, code } = value as { name?: unknown; message?: unknown; code?: unknown };
|
|
975
|
+
if (typeof message === "string") {
|
|
976
|
+
return {
|
|
977
|
+
name: typeof name === "string" ? name : "Error",
|
|
978
|
+
message,
|
|
979
|
+
...((typeof code === "string" || typeof code === "number") && { code }),
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
return { name: "Error", message: String(value) };
|
|
984
|
+
} catch {
|
|
985
|
+
// A value without a text, for example an object without a prototype
|
|
986
|
+
return { name: "Error", message: "An error without a text" };
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/** The status of a message of the leader. Its error becomes an `Error` again. */
|
|
991
|
+
function statusOf(status: PortableStatus): SyncStatus {
|
|
992
|
+
const error = status.error;
|
|
993
|
+
return {
|
|
994
|
+
connected: status.connected,
|
|
995
|
+
pendingChanges: status.pendingChanges,
|
|
996
|
+
syncing: status.syncing,
|
|
997
|
+
...(status.endpoint !== undefined && { endpoint: status.endpoint }),
|
|
998
|
+
...(status.lastSync !== undefined && { lastSync: status.lastSync }),
|
|
999
|
+
...(status.mode !== undefined && { mode: status.mode }),
|
|
1000
|
+
...(error !== undefined && {
|
|
1001
|
+
error: {
|
|
1002
|
+
operation: error.operation,
|
|
1003
|
+
error: Object.assign(new Error(error.message), { name: error.name }, error.code !== undefined ? { code: error.code } : {}),
|
|
1004
|
+
},
|
|
1005
|
+
}),
|
|
1006
|
+
};
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
const MODES: ReadonlySet<string> = new Set<NonNullable<SyncStatus["mode"]>>(["local-only", "write-through", "write-back", "remote-only"]);
|
|
1010
|
+
const OPERATIONS: ReadonlySet<string> = new Set<SyncError["operation"]>(["load", "push", "pull", "sync", "store"]);
|
|
1011
|
+
|
|
1012
|
+
/** True when a value is the status of a message. */
|
|
1013
|
+
function isPortableStatus(value: unknown): value is PortableStatus {
|
|
1014
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1015
|
+
const status = value as { connected?: unknown; pendingChanges?: unknown; syncing?: unknown; endpoint?: unknown; lastSync?: unknown; mode?: unknown; error?: unknown };
|
|
1016
|
+
if (typeof status.connected !== "boolean" || typeof status.syncing !== "boolean" || !isSeq(status.pendingChanges)) return false;
|
|
1017
|
+
if (status.endpoint !== undefined && typeof status.endpoint !== "string") return false;
|
|
1018
|
+
if (status.lastSync !== undefined && typeof status.lastSync !== "number") return false;
|
|
1019
|
+
if (status.mode !== undefined && !(typeof status.mode === "string" && MODES.has(status.mode))) return false;
|
|
1020
|
+
if (status.error === undefined) return true;
|
|
1021
|
+
if (typeof status.error !== "object" || status.error === null) return false;
|
|
1022
|
+
const { operation, name, message, code } = status.error as { operation?: unknown; name?: unknown; message?: unknown; code?: unknown };
|
|
1023
|
+
return (
|
|
1024
|
+
typeof operation === "string" &&
|
|
1025
|
+
OPERATIONS.has(operation) &&
|
|
1026
|
+
typeof name === "string" &&
|
|
1027
|
+
typeof message === "string" &&
|
|
1028
|
+
(code === undefined || typeof code === "string" || typeof code === "number")
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
628
1032
|
/** A new random id. */
|
|
629
1033
|
function randomId(): string {
|
|
630
1034
|
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
@@ -664,10 +1068,22 @@ function isBatch(value: unknown): value is StorageChangeEvent<unknown>[] {
|
|
|
664
1068
|
});
|
|
665
1069
|
}
|
|
666
1070
|
|
|
667
|
-
/**
|
|
1071
|
+
/**
|
|
1072
|
+
* True when a value is a message of the channel, of this version. A message of a type that this
|
|
1073
|
+
* release does not know is not a message: a tab of an older release ignores a new type.
|
|
1074
|
+
*/
|
|
668
1075
|
function isTabMessage<T>(value: unknown): value is TabMessage<T> {
|
|
669
1076
|
if (typeof value !== "object" || value === null) return false;
|
|
670
|
-
const message = value as {
|
|
1077
|
+
const message = value as {
|
|
1078
|
+
v?: unknown;
|
|
1079
|
+
type?: unknown;
|
|
1080
|
+
from?: unknown;
|
|
1081
|
+
to?: unknown;
|
|
1082
|
+
seq?: unknown;
|
|
1083
|
+
changes?: unknown;
|
|
1084
|
+
pending?: unknown;
|
|
1085
|
+
status?: unknown;
|
|
1086
|
+
};
|
|
671
1087
|
if (message.v !== PROTOCOL || typeof message.from !== "string") return false;
|
|
672
1088
|
switch (message.type) {
|
|
673
1089
|
case "changes":
|
|
@@ -686,6 +1102,8 @@ function isTabMessage<T>(value: unknown): value is TabMessage<T> {
|
|
|
686
1102
|
return true;
|
|
687
1103
|
case "ack":
|
|
688
1104
|
return typeof message.to === "string" && isSeq(message.seq);
|
|
1105
|
+
case "status":
|
|
1106
|
+
return isSeq(message.seq) && isPortableStatus(message.status);
|
|
689
1107
|
default:
|
|
690
1108
|
return false;
|
|
691
1109
|
}
|