@procedures/store 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +59 -1
- package/dist/.tsbuildinfo +1 -1
- package/dist/browser.d.ts +3 -2
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +2 -1
- package/dist/browser.js.map +1 -1
- package/dist/storage/index.d.ts +2 -2
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +1 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/tab-coordinator.d.ts +194 -145
- package/dist/storage/tab-coordinator.d.ts.map +1 -1
- package/dist/storage/tab-coordinator.js +1018 -251
- package/dist/storage/tab-coordinator.js.map +1 -1
- package/package.json +4 -4
- package/src/browser.ts +6 -1
- package/src/storage/index.ts +5 -1
- package/src/storage/tab-coordinator.ts +1199 -256
|
@@ -25,6 +25,39 @@
|
|
|
25
25
|
* - The leader sends the status of its sync to the followers: at each change, and in answer to a
|
|
26
26
|
* hello. `getStatus()` and `onStatus()` give the status in each tab (roadmap CLI-65).
|
|
27
27
|
*
|
|
28
|
+
* A `TabGroup` runs the sync of many collections with one lock and one channel (roadmap CLI-64),
|
|
29
|
+
* for example all the collections of a tenant.
|
|
30
|
+
*
|
|
31
|
+
* - The leader of the group runs one `HybridStorage` for each collection. Each message has the
|
|
32
|
+
* field `collection`, and each collection has its own batches, acknowledgements and status.
|
|
33
|
+
* - Each collection leads when its own queue loaded, and it tells the other tabs at once. Thus a
|
|
34
|
+
* slow collection does not stop the batches of the other collections.
|
|
35
|
+
* - A collection whose setup fails tries again alone after its backoff, and the other collections
|
|
36
|
+
* keep their sync. The tab sends a status with the error of the setup to the other tabs.
|
|
37
|
+
* - When no collection of the group leads or loads its queue, the tab releases the lock. It asks for
|
|
38
|
+
* the lock again after the backoff of the group, as a `TabCoordinator` does. This is also true for
|
|
39
|
+
* a collection that the application adds after the tab got the lock.
|
|
40
|
+
* - The lock and the channel of a group have the prefix `@procedures/store/tabs:`. A
|
|
41
|
+
* `TabCoordinator` has the prefix `@procedures/store/tab:`. Thus a group and a coordinator never
|
|
42
|
+
* share a lock or a channel, also with the same name.
|
|
43
|
+
* - With the option `open`, the leader makes a collection that another tab has and that this tab
|
|
44
|
+
* does not have. Thus the tabs can open different collections.
|
|
45
|
+
*
|
|
46
|
+
* A snapshot for a new tab (roadmap CLI-66): with `copy: "tab"` and the option `snapshot`, a new
|
|
47
|
+
* tab asks the leader for the entries of its local copy.
|
|
48
|
+
*
|
|
49
|
+
* - The leader reads the entries of its copy, and it sends them in chunks of `snapshot.chunk`
|
|
50
|
+
* entries. Each chunk has the numbers of the last batches that the copy holds, one for each tab.
|
|
51
|
+
* - The new tab keeps the batches of the other tabs until the last chunk comes. It writes the
|
|
52
|
+
* entries to its copy, and then it writes the batches with a number after the snapshot.
|
|
53
|
+
* - The new tab keeps the values of its own writes after the snapshot.
|
|
54
|
+
* - A set in the entries can come again in a batch after the snapshot, for example a write of the
|
|
55
|
+
* leader during its read. The second write of the set gives the same state.
|
|
56
|
+
* - When the leader changes before the last chunk, the tab asks the new leader. When no chunk comes
|
|
57
|
+
* in the time `snapshot.timeout`, the tab reports an error and continues without the snapshot.
|
|
58
|
+
* - In the tab that holds the lock, the wait renews itself while the collection loads its queue, at
|
|
59
|
+
* most `SNAPSHOT_RENEWALS` times. Thus the batches of the other tabs wait for a limited time.
|
|
60
|
+
*
|
|
28
61
|
* The limits:
|
|
29
62
|
* - When the leader stops during a push, the next leader sends the operation again: the delivery to
|
|
30
63
|
* the remote is at least once.
|
|
@@ -32,6 +65,12 @@
|
|
|
32
65
|
* capture. The tabs agree again after the push and the next pull.
|
|
33
66
|
* - The load of the queue in a new leader writes the queued operations to the local copy again, and
|
|
34
67
|
* the observers get them again with the origin `syncOrigin`.
|
|
68
|
+
* - The leader reads all the entries of a snapshot at one time, and the new tab keeps all the chunks
|
|
69
|
+
* until the last one. Thus a snapshot is for a local copy that fits in the memory of a tab.
|
|
70
|
+
* - `snapshot.chunk` counts the entries of a message, not its bytes: large items give large messages.
|
|
71
|
+
* - A `TabCoordinator` and a `TabGroup` of one collection do not see each other. When tabs of both
|
|
72
|
+
* releases are open, two tabs lead the collection. Give the group a new queue store, or close
|
|
73
|
+
* the tabs of the older release first.
|
|
35
74
|
*
|
|
36
75
|
* Nothing in this module reads a global value when it loads: Node can import it. The coordinator
|
|
37
76
|
* reads `navigator.locks` and `BroadcastChannel` when it starts, and a test gives its own.
|
|
@@ -42,11 +81,20 @@ import { backoffDelay } from "@mark1russell7/service/backoff";
|
|
|
42
81
|
import { HybridStorage } from "./hybrid.js";
|
|
43
82
|
/**
|
|
44
83
|
* The version of the messages. A tab ignores a message of another version, and a message of a type
|
|
45
|
-
* that it does not know. Thus a new type of message (`status`, roadmap CLI-65
|
|
46
|
-
* a tab of an older release ignores it, and it still copies
|
|
47
|
-
* stops the batches between the tabs of two releases, so only
|
|
84
|
+
* that it does not know. Thus a new type of message (`status`, roadmap CLI-65, and `snapshot`,
|
|
85
|
+
* roadmap CLI-66) keeps the version 1: a tab of an older release ignores it, and it still copies
|
|
86
|
+
* and sends the batches. A new version stops the batches between the tabs of two releases, so only
|
|
87
|
+
* a change of a known message needs it.
|
|
48
88
|
*/
|
|
49
89
|
const PROTOCOL = 1;
|
|
90
|
+
/** The prefixes of the lock and of the channel: of a `TabCoordinator`, and of a `TabGroup`. */
|
|
91
|
+
const COORDINATOR_PREFIX = "@procedures/store/tab:";
|
|
92
|
+
const GROUP_PREFIX = "@procedures/store/tabs:";
|
|
93
|
+
/** The defaults of the snapshot (roadmap CLI-66). */
|
|
94
|
+
const SNAPSHOT_CHUNK = 500;
|
|
95
|
+
const SNAPSHOT_TIMEOUT = 10_000;
|
|
96
|
+
/** The largest number of renewals of the wait for a snapshot in the tab that holds the lock. */
|
|
97
|
+
const SNAPSHOT_RENEWALS = 5;
|
|
50
98
|
/**
|
|
51
99
|
* One leader tab runs the sync of a `HybridStorage`, and the other tabs copy its changes.
|
|
52
100
|
*
|
|
@@ -77,31 +125,571 @@ export class TabCoordinator {
|
|
|
77
125
|
id;
|
|
78
126
|
/** The name of the lock and of the channel. */
|
|
79
127
|
lockName;
|
|
128
|
+
group;
|
|
129
|
+
collection;
|
|
130
|
+
constructor(options) {
|
|
131
|
+
if (typeof options?.name !== "string" || options.name === "")
|
|
132
|
+
throw new TypeError("TabCoordinator needs a name");
|
|
133
|
+
const settings = collectionSettings(options, "TabCoordinator");
|
|
134
|
+
const hybrid = options.hybrid ?? {};
|
|
135
|
+
this.group = new GroupEngine({
|
|
136
|
+
prefix: COORDINATOR_PREFIX,
|
|
137
|
+
name: options.name,
|
|
138
|
+
single: true,
|
|
139
|
+
open: undefined,
|
|
140
|
+
backoff: retryBackoff(options.retry),
|
|
141
|
+
clock: options.clock ?? hybrid.clock ?? realClock,
|
|
142
|
+
random: options.random ?? hybrid.random ?? Math.random,
|
|
143
|
+
onError: options.onError ?? reportListenerError,
|
|
144
|
+
});
|
|
145
|
+
this.id = this.group.id;
|
|
146
|
+
this.lockName = this.group.lockName;
|
|
147
|
+
this.collection = this.group.add(options.name, options, settings);
|
|
148
|
+
this.group.start(options.locks, options.channel);
|
|
149
|
+
}
|
|
150
|
+
/** The role of this tab now. */
|
|
151
|
+
get role() {
|
|
152
|
+
return this.collection.role;
|
|
153
|
+
}
|
|
154
|
+
/** True when this tab runs the sync: the leader, or a tab in the single mode. */
|
|
155
|
+
get isLeader() {
|
|
156
|
+
return this.collection.isLeader;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* The `HybridStorage` of the sync while this tab runs it, or else undefined. `getStatus()` of the
|
|
160
|
+
* coordinator gives the state of the sync in each tab. Write to the local copy, not to this
|
|
161
|
+
* storage: each tab then writes in the same way.
|
|
162
|
+
*/
|
|
163
|
+
get syncStorage() {
|
|
164
|
+
return this.collection.syncStorage;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* This method starts to send the change batches to an observer: the batches of the local copy of
|
|
168
|
+
* this tab, and the batches of the other tabs. Each batch comes one time, with the origin of each
|
|
169
|
+
* change. In the leader, the batches of the sync have the origin `syncOrigin` of the
|
|
170
|
+
* `HybridStorage`. A batch of another tab comes after the local copy of this tab has it. The
|
|
171
|
+
* entries of a snapshot come in one batch of sets, with the origin `syncOrigin`. A set of the
|
|
172
|
+
* snapshot can come again in a later batch, with the same value.
|
|
173
|
+
*
|
|
174
|
+
* @returns The function that stops the batches. A second use of it does nothing.
|
|
175
|
+
* @throws Error after `close()`
|
|
176
|
+
*/
|
|
177
|
+
observe(observer) {
|
|
178
|
+
return this.collection.observe(observer);
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* This method starts to send the role of this tab to a listener at each change.
|
|
182
|
+
*
|
|
183
|
+
* @returns The function that stops the roles. A second use of it does nothing.
|
|
184
|
+
* @throws Error after `close()`
|
|
185
|
+
*/
|
|
186
|
+
onRole(listener) {
|
|
187
|
+
return this.collection.onRole(listener);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The status of the sync (roadmap CLI-65), or undefined before the first status and after
|
|
191
|
+
* `close()`.
|
|
192
|
+
*
|
|
193
|
+
* - In the leader and in the single mode, it is the status of the `HybridStorage` of this tab.
|
|
194
|
+
* - In a follower, it is the last status that the leader sent. A new tab gets it in the answer to
|
|
195
|
+
* its hello. After a change of the leader, a follower keeps the last status until the new
|
|
196
|
+
* leader sends its status.
|
|
197
|
+
* - In a follower, `error.error` is a copy of the error of the leader: an `Error` with its `name`,
|
|
198
|
+
* its `message` and its `code`. The copy has no `cause`.
|
|
199
|
+
*/
|
|
200
|
+
getStatus() {
|
|
201
|
+
return this.collection.getStatus();
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* This method starts to send the status of the sync to a listener at each change (roadmap
|
|
205
|
+
* CLI-65): in the leader, each status of its `HybridStorage`, and in a follower, each status that
|
|
206
|
+
* the leader sends. The listener gets each status one time. When the tab becomes the leader, the
|
|
207
|
+
* listener gets the status of its own `HybridStorage`.
|
|
208
|
+
*
|
|
209
|
+
* @returns The function that stops the statuses. A second use of it does nothing.
|
|
210
|
+
* @throws Error after `close()`
|
|
211
|
+
*/
|
|
212
|
+
onStatus(listener) {
|
|
213
|
+
return this.collection.onStatus(listener);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* This method closes the coordinator.
|
|
217
|
+
*
|
|
218
|
+
* - A follower stops its lock request, or its wait before a new try. It does not become the leader.
|
|
219
|
+
* - A tab that loads its queue to become the leader closes its `HybridStorage`, and releases the lock.
|
|
220
|
+
* - The leader closes its `HybridStorage`: it sends the queue, and the queue store keeps the
|
|
221
|
+
* operations that the remote did not take. Then the leader releases the lock, and the next
|
|
222
|
+
* tab becomes the leader.
|
|
223
|
+
* - The coordinator closes its channel. It does not close the local copy: the application owns it.
|
|
224
|
+
* - After it, the coordinator sends and copies nothing. A second `close()` gives the promise of
|
|
225
|
+
* the first one.
|
|
226
|
+
*/
|
|
227
|
+
close() {
|
|
228
|
+
return this.group.close();
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* One leader tab runs the sync of many collections, with one lock and one channel (roadmap CLI-64).
|
|
233
|
+
* The other tabs copy the changes of each collection.
|
|
234
|
+
*
|
|
235
|
+
* @example
|
|
236
|
+
* ```typescript
|
|
237
|
+
* import { ApiStorage, IndexedDbQueueStore, TabGroup } from "@procedures/store/browser";
|
|
238
|
+
*
|
|
239
|
+
* const tenant = new TabGroup({
|
|
240
|
+
* name: tenantId,
|
|
241
|
+
* open: (collection) => ({
|
|
242
|
+
* local: localCopyOf(collection),
|
|
243
|
+
* copy: "tab",
|
|
244
|
+
* remote: () => new ApiStorage(client, { collection }),
|
|
245
|
+
* hybrid: { queueStore: new IndexedDbQueueStore(`${tenantId}-${collection}-queue`) },
|
|
246
|
+
* snapshot: true,
|
|
247
|
+
* }),
|
|
248
|
+
* });
|
|
249
|
+
*
|
|
250
|
+
* const notes = tenant.collection<Note>("notes");
|
|
251
|
+
* notes.observe((batch) => render(batch));
|
|
252
|
+
* notes.onStatus((status) => showSync("notes", status));
|
|
253
|
+
* ```
|
|
254
|
+
*/
|
|
255
|
+
export class TabGroup {
|
|
256
|
+
/** The id of this tab: a new random id for each group. */
|
|
257
|
+
id;
|
|
258
|
+
/** The name of the lock and of the channel. */
|
|
259
|
+
lockName;
|
|
260
|
+
group;
|
|
261
|
+
constructor(options) {
|
|
262
|
+
if (typeof options?.name !== "string" || options.name === "")
|
|
263
|
+
throw new TypeError("TabGroup needs a name");
|
|
264
|
+
if (options.open !== undefined && typeof options.open !== "function")
|
|
265
|
+
throw new TypeError("The option open of a TabGroup must be a function");
|
|
266
|
+
this.group = new GroupEngine({
|
|
267
|
+
prefix: GROUP_PREFIX,
|
|
268
|
+
name: options.name,
|
|
269
|
+
single: false,
|
|
270
|
+
open: options.open,
|
|
271
|
+
backoff: retryBackoff(options.retry),
|
|
272
|
+
clock: options.clock ?? realClock,
|
|
273
|
+
random: options.random ?? Math.random,
|
|
274
|
+
onError: options.onError ?? reportListenerError,
|
|
275
|
+
});
|
|
276
|
+
this.id = this.group.id;
|
|
277
|
+
this.lockName = this.group.lockName;
|
|
278
|
+
this.group.start(options.locks, options.channel);
|
|
279
|
+
}
|
|
280
|
+
/** The role of this tab in the group. A collection has its own role too. */
|
|
281
|
+
get role() {
|
|
282
|
+
return this.group.role;
|
|
283
|
+
}
|
|
284
|
+
/** True when this tab holds the lock of the group, or runs in the single mode. */
|
|
285
|
+
get isLeader() {
|
|
286
|
+
return this.group.role === "leader" || this.group.role === "single";
|
|
287
|
+
}
|
|
288
|
+
/** The names of the collections of the group in this tab. */
|
|
289
|
+
get collections() {
|
|
290
|
+
return [...this.group.members.keys()];
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* This method gives a collection of the group. It adds the collection at its first use. When the
|
|
294
|
+
* group has the collection, the method gives it and ignores `options`.
|
|
295
|
+
*
|
|
296
|
+
* @param options The options of a new collection. The default is the result of `open(name)`.
|
|
297
|
+
* @throws TypeError for a new collection without options, or with options of a wrong shape
|
|
298
|
+
* @throws Error after `close()`
|
|
299
|
+
*/
|
|
300
|
+
collection(name, options) {
|
|
301
|
+
return this.group.collection(name, options);
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* This method starts to send the role of this tab in the group to a listener at each change.
|
|
305
|
+
*
|
|
306
|
+
* @returns The function that stops the roles. A second use of it does nothing.
|
|
307
|
+
* @throws Error after `close()`
|
|
308
|
+
*/
|
|
309
|
+
onRole(listener) {
|
|
310
|
+
if (this.group.closed)
|
|
311
|
+
throw new Error("The TabGroup is closed");
|
|
312
|
+
return this.group.roleListeners.add(listener);
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* This method closes the group and each of its collections, as `TabCoordinator.close()` does.
|
|
316
|
+
* The leader closes each `HybridStorage`, and then it releases the lock.
|
|
317
|
+
*/
|
|
318
|
+
close() {
|
|
319
|
+
return this.group.close();
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* The lock, the channel and the collections of a tab. A `TabCoordinator` is a group with one
|
|
324
|
+
* collection, whose messages have no field `collection`: thus it talks with a tab of an older release.
|
|
325
|
+
*/
|
|
326
|
+
class GroupEngine {
|
|
327
|
+
id;
|
|
328
|
+
lockName;
|
|
329
|
+
/** True for a `TabCoordinator` */
|
|
330
|
+
legacy;
|
|
331
|
+
backoff;
|
|
332
|
+
clock;
|
|
333
|
+
random;
|
|
334
|
+
onError;
|
|
335
|
+
roleListeners;
|
|
336
|
+
/** The collections of this tab, by name */
|
|
337
|
+
members = new Map();
|
|
338
|
+
/** An object for each time that this tab holds the lock. A collection leads only with the lease of now. */
|
|
339
|
+
lease;
|
|
340
|
+
/** True in the single mode */
|
|
341
|
+
alone = false;
|
|
342
|
+
closed = false;
|
|
343
|
+
open;
|
|
344
|
+
/** The names for which `open` gave no options */
|
|
345
|
+
refused = new Set();
|
|
346
|
+
currentRole = "follower";
|
|
347
|
+
started = false;
|
|
348
|
+
/** The lock manager, or undefined in the single mode */
|
|
349
|
+
locks;
|
|
350
|
+
channel;
|
|
351
|
+
/** The number of the failed setups of the group in a row: each setup where no collection could lead */
|
|
352
|
+
failures = 0;
|
|
353
|
+
/** The stop of the wait before the next try */
|
|
354
|
+
cancelRetry;
|
|
355
|
+
closing;
|
|
356
|
+
/** The abort of the lock request that waits */
|
|
357
|
+
abortRequest;
|
|
358
|
+
/** The release of the lock */
|
|
359
|
+
releaseLock;
|
|
360
|
+
constructor(options) {
|
|
361
|
+
this.id = randomId();
|
|
362
|
+
this.lockName = `${options.prefix}${options.name}`;
|
|
363
|
+
this.legacy = options.single;
|
|
364
|
+
this.open = options.open;
|
|
365
|
+
this.backoff = options.backoff;
|
|
366
|
+
this.clock = options.clock;
|
|
367
|
+
this.random = options.random;
|
|
368
|
+
this.onError = options.onError;
|
|
369
|
+
this.roleListeners = new StorageListeners(this.onError);
|
|
370
|
+
}
|
|
371
|
+
get role() {
|
|
372
|
+
return this.currentRole;
|
|
373
|
+
}
|
|
374
|
+
/** True while the tab has a channel. */
|
|
375
|
+
get hasChannel() {
|
|
376
|
+
return this.channel !== undefined;
|
|
377
|
+
}
|
|
378
|
+
/** The name of the coordinator or of the group, for the messages of the errors. */
|
|
379
|
+
get kind() {
|
|
380
|
+
return this.legacy ? "TabCoordinator" : "TabGroup";
|
|
381
|
+
}
|
|
382
|
+
/** This method opens the channel and asks for the lock, or starts the single mode. */
|
|
383
|
+
start(locksOption, channelOption) {
|
|
384
|
+
this.started = true;
|
|
385
|
+
const locks = locksOption === undefined ? globalLocks() : (locksOption ?? undefined);
|
|
386
|
+
const openChannel = channelOption === undefined ? globalChannelFactory() : (channelOption ?? undefined);
|
|
387
|
+
if (!locks || !openChannel) {
|
|
388
|
+
this.startSingle();
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
try {
|
|
392
|
+
this.channel = openChannel(this.lockName);
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
this.onError(error);
|
|
396
|
+
this.startSingle();
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
this.locks = locks;
|
|
400
|
+
this.channel.onmessage = (event) => this.receive(event?.data);
|
|
401
|
+
for (const member of this.members.values())
|
|
402
|
+
member.begin();
|
|
403
|
+
this.elect(locks);
|
|
404
|
+
}
|
|
405
|
+
/** This method gives a collection of a `TabGroup`, and adds it at its first use. */
|
|
406
|
+
collection(name, options) {
|
|
407
|
+
if (this.closed)
|
|
408
|
+
throw new Error("The TabGroup is closed");
|
|
409
|
+
if (typeof name !== "string" || name === "")
|
|
410
|
+
throw new TypeError("A collection of a TabGroup needs a name");
|
|
411
|
+
const known = this.members.get(name);
|
|
412
|
+
if (known)
|
|
413
|
+
return known;
|
|
414
|
+
const resolved = options ?? this.open?.(name);
|
|
415
|
+
if (!resolved)
|
|
416
|
+
throw new TypeError(`The TabGroup has no options for the collection ${JSON.stringify(name)}: give options or the function open`);
|
|
417
|
+
return this.add(name, resolved, collectionSettings(resolved, "A collection of a TabGroup"));
|
|
418
|
+
}
|
|
419
|
+
/** This method adds a collection. After the start, the collection says hello, and it leads when the tab holds the lock. */
|
|
420
|
+
add(name, options, settings) {
|
|
421
|
+
const member = new CollectionSync(this, name, options, settings);
|
|
422
|
+
this.members.set(name, member);
|
|
423
|
+
this.refused.delete(name);
|
|
424
|
+
if (!this.started)
|
|
425
|
+
return member;
|
|
426
|
+
if (this.alone) {
|
|
427
|
+
member.startSingle();
|
|
428
|
+
return member;
|
|
429
|
+
}
|
|
430
|
+
member.begin();
|
|
431
|
+
if (this.lease)
|
|
432
|
+
member.startLead(this.lease, true);
|
|
433
|
+
return member;
|
|
434
|
+
}
|
|
435
|
+
/** @returns False when the message did not go out: no channel, or a value that the channel cannot clone */
|
|
436
|
+
post(collection, message) {
|
|
437
|
+
const channel = this.channel;
|
|
438
|
+
if (!channel)
|
|
439
|
+
return false;
|
|
440
|
+
try {
|
|
441
|
+
channel.postMessage(this.legacy || collection === undefined ? message : { ...message, collection });
|
|
442
|
+
return true;
|
|
443
|
+
}
|
|
444
|
+
catch (error) {
|
|
445
|
+
this.onError(error);
|
|
446
|
+
return false;
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
close() {
|
|
450
|
+
if (!this.closing)
|
|
451
|
+
this.closing = this.shutDown();
|
|
452
|
+
return this.closing;
|
|
453
|
+
}
|
|
454
|
+
async shutDown() {
|
|
455
|
+
this.closed = true;
|
|
456
|
+
this.abortRequest?.();
|
|
457
|
+
this.abortRequest = undefined;
|
|
458
|
+
this.cancelRetry?.();
|
|
459
|
+
this.cancelRetry = undefined;
|
|
460
|
+
this.lease = undefined;
|
|
461
|
+
this.setRole("closed");
|
|
462
|
+
const members = [...this.members.values()];
|
|
463
|
+
try {
|
|
464
|
+
// The leader holds the lock until each queue reached the remote or its queue store
|
|
465
|
+
await Promise.all(members.map((member) => member.shutDown()));
|
|
466
|
+
}
|
|
467
|
+
finally {
|
|
468
|
+
this.releaseLock?.();
|
|
469
|
+
this.releaseLock = undefined;
|
|
470
|
+
// The acknowledgements of the close of the storages went out before
|
|
471
|
+
try {
|
|
472
|
+
this.channel?.close();
|
|
473
|
+
}
|
|
474
|
+
catch {
|
|
475
|
+
// The channel is closed already
|
|
476
|
+
}
|
|
477
|
+
this.channel = undefined;
|
|
478
|
+
for (const member of members)
|
|
479
|
+
member.clearListeners();
|
|
480
|
+
this.roleListeners.clear();
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
//
|
|
484
|
+
// ═══ The election ═══
|
|
485
|
+
//
|
|
486
|
+
/** This method asks for the lock. The tab becomes the leader when it gets the lock. */
|
|
487
|
+
elect(locks) {
|
|
488
|
+
const Controller = globalAbortController();
|
|
489
|
+
const controller = Controller ? new Controller() : undefined;
|
|
490
|
+
this.abortRequest = controller ? () => controller.abort() : undefined;
|
|
491
|
+
const held = new Promise((resolve) => (this.releaseLock = resolve));
|
|
492
|
+
let request;
|
|
493
|
+
try {
|
|
494
|
+
request = locks.request(this.lockName, controller ? { signal: controller.signal } : {}, async () => {
|
|
495
|
+
// A lock after close(): the tab releases it at once
|
|
496
|
+
if (this.closed)
|
|
497
|
+
return;
|
|
498
|
+
this.abortRequest = undefined;
|
|
499
|
+
if (await this.lead()) {
|
|
500
|
+
// close() ends the hold. A group whose collections all failed after the start ends it too.
|
|
501
|
+
await held;
|
|
502
|
+
if (this.closed)
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
// A failed setup releases the lock: another tab tries, and this tab tries again after the backoff
|
|
506
|
+
this.retryLater();
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
request = Promise.reject(error);
|
|
511
|
+
}
|
|
512
|
+
request.then(undefined, (error) => {
|
|
513
|
+
// The abort of close(), or a tab that leads already
|
|
514
|
+
if (this.closed || this.lease)
|
|
515
|
+
return;
|
|
516
|
+
// The lock manager refused the request: this tab runs the sync alone
|
|
517
|
+
this.onError(error);
|
|
518
|
+
this.stopChannel();
|
|
519
|
+
this.locks = undefined;
|
|
520
|
+
this.startSingle();
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
/**
|
|
524
|
+
* This method runs in a tab that got the lock. Each collection makes its `HybridStorage` and
|
|
525
|
+
* loads its queue, and it leads at the end of its load. When no collection can lead, the tab
|
|
526
|
+
* releases the lock. Else the collections whose setup failed try again alone after their backoff.
|
|
527
|
+
*
|
|
528
|
+
* @returns False when the tab must release the lock: no collection could lead, or `close()`
|
|
529
|
+
*/
|
|
530
|
+
async lead() {
|
|
531
|
+
const lease = {};
|
|
532
|
+
this.lease = lease;
|
|
533
|
+
const members = [...this.members.values()];
|
|
534
|
+
// Each collection tells the other tabs at once when it leads: a slow collection does not stop the batches of the others
|
|
535
|
+
const results = await Promise.all(members.map((member) => member.takeLead(lease, true)));
|
|
536
|
+
if (this.lease !== lease)
|
|
537
|
+
return false;
|
|
538
|
+
// A group without collections keeps the lock: its first collection leads, or releases the lock when its setup fails
|
|
539
|
+
if (!this.canLead()) {
|
|
540
|
+
this.stepDown();
|
|
541
|
+
return false;
|
|
542
|
+
}
|
|
543
|
+
this.failures = 0;
|
|
544
|
+
this.setRole("leader");
|
|
545
|
+
// Each follower says hello for each collection that it has and that the leader did not announce: the leader makes the collections that it does not have
|
|
546
|
+
if (!this.legacy)
|
|
547
|
+
this.post(undefined, { v: PROTOCOL, type: "leader", from: this.id });
|
|
548
|
+
members.forEach((member, at) => {
|
|
549
|
+
if (!results[at])
|
|
550
|
+
member.retryInLeader();
|
|
551
|
+
});
|
|
552
|
+
return true;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* This method runs after a failed setup of a collection in a tab that holds the lock, and keeps
|
|
556
|
+
* it after `lead()`. When no collection of the group leads or loads its queue, the tab releases
|
|
557
|
+
* the lock and asks for it again after the backoff of the group. Else the collection tries again
|
|
558
|
+
* alone after its own backoff.
|
|
559
|
+
*/
|
|
560
|
+
afterFailedSetup(member, lease) {
|
|
561
|
+
if (this.closed || this.lease !== lease)
|
|
562
|
+
return;
|
|
563
|
+
if (this.canLead()) {
|
|
564
|
+
member.retryInLeader();
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
this.stepDown();
|
|
568
|
+
this.setRole("follower");
|
|
569
|
+
// The lock callback then asks for the lock again after the backoff
|
|
570
|
+
const release = this.releaseLock;
|
|
571
|
+
this.releaseLock = undefined;
|
|
572
|
+
release?.();
|
|
573
|
+
}
|
|
574
|
+
/** True when the group has no collections yet, or when one of its collections leads or loads its queue. */
|
|
575
|
+
canLead() {
|
|
576
|
+
const members = [...this.members.values()];
|
|
577
|
+
return members.length === 0 || members.some((member) => member.leads || member.settingUp);
|
|
578
|
+
}
|
|
579
|
+
/** This method ends the lease of the lock. A collection that waits for its next try stops its wait: the next lease tries it. */
|
|
580
|
+
stepDown() {
|
|
581
|
+
this.lease = undefined;
|
|
582
|
+
for (const member of this.members.values())
|
|
583
|
+
member.stopRetry();
|
|
584
|
+
}
|
|
585
|
+
/** This method starts the single mode: each collection runs its sync alone. */
|
|
586
|
+
startSingle() {
|
|
587
|
+
this.alone = true;
|
|
588
|
+
this.setRole("single");
|
|
589
|
+
for (const member of [...this.members.values()])
|
|
590
|
+
member.startSingle();
|
|
591
|
+
}
|
|
592
|
+
/** This method asks for the lock again after the backoff (roadmap CLI-67). */
|
|
593
|
+
retryLater() {
|
|
594
|
+
if (this.closed)
|
|
595
|
+
return;
|
|
596
|
+
this.failures++;
|
|
597
|
+
const wait = backoffDelay(this.backoff, this.failures, this.random);
|
|
598
|
+
const cancel = this.clock.after(wait, () => {
|
|
599
|
+
this.cancelRetry = undefined;
|
|
600
|
+
if (this.closed || !this.locks)
|
|
601
|
+
return;
|
|
602
|
+
this.elect(this.locks);
|
|
603
|
+
});
|
|
604
|
+
// The wait must not keep a Node process alive
|
|
605
|
+
cancel.unref?.();
|
|
606
|
+
this.cancelRetry = cancel;
|
|
607
|
+
}
|
|
608
|
+
setRole(role) {
|
|
609
|
+
if (this.currentRole === role)
|
|
610
|
+
return;
|
|
611
|
+
this.currentRole = role;
|
|
612
|
+
this.roleListeners.emit(role);
|
|
613
|
+
}
|
|
614
|
+
stopChannel() {
|
|
615
|
+
const channel = this.channel;
|
|
616
|
+
this.channel = undefined;
|
|
617
|
+
if (!channel)
|
|
618
|
+
return;
|
|
619
|
+
channel.onmessage = null;
|
|
620
|
+
try {
|
|
621
|
+
channel.close();
|
|
622
|
+
}
|
|
623
|
+
catch {
|
|
624
|
+
// The channel is closed already
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
//
|
|
628
|
+
// ═══ The messages of the other tabs ═══
|
|
629
|
+
//
|
|
630
|
+
receive(data) {
|
|
631
|
+
if (this.closed || !isTabMessage(data) || data.from === this.id)
|
|
632
|
+
return;
|
|
633
|
+
if (this.legacy) {
|
|
634
|
+
for (const member of this.members.values())
|
|
635
|
+
member.receive(data);
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
638
|
+
const name = data.collection;
|
|
639
|
+
if (name === undefined) {
|
|
640
|
+
// A new leader of the group: each collection of this tab that the leader did not announce says hello
|
|
641
|
+
if (data.type === "leader")
|
|
642
|
+
for (const member of [...this.members.values()])
|
|
643
|
+
member.onLeader(data.from, false);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
const member = this.members.get(name) ?? this.adopt(name, data.type);
|
|
647
|
+
member?.receive(data);
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* The leader makes a collection that another tab has, with the function `open`. Thus the leader
|
|
651
|
+
* runs the sync of each collection of each tab.
|
|
652
|
+
*/
|
|
653
|
+
adopt(name, type) {
|
|
654
|
+
if ((type !== "hello" && type !== "changes") || this.currentRole !== "leader" || !this.open || this.refused.has(name))
|
|
655
|
+
return undefined;
|
|
656
|
+
try {
|
|
657
|
+
const options = this.open(name);
|
|
658
|
+
if (options)
|
|
659
|
+
return this.add(name, options, collectionSettings(options, "A collection of a TabGroup"));
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
this.onError(error);
|
|
663
|
+
}
|
|
664
|
+
this.refused.add(name);
|
|
665
|
+
return undefined;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* The sync of one collection in a tab: the leader runs its `HybridStorage`, and a follower copies
|
|
670
|
+
* the batches of the other tabs. The group gives the lock and the channel.
|
|
671
|
+
*/
|
|
672
|
+
class CollectionSync {
|
|
673
|
+
name;
|
|
674
|
+
group;
|
|
80
675
|
local;
|
|
81
676
|
shared;
|
|
82
677
|
makeRemote;
|
|
83
678
|
hybridOptions;
|
|
679
|
+
settings;
|
|
84
680
|
/** The origin of the `HybridStorage` and of the writes of the coordinator to the local copy. */
|
|
85
681
|
tag;
|
|
86
682
|
writeOptions;
|
|
87
|
-
onError;
|
|
88
683
|
observers;
|
|
89
684
|
roleListeners;
|
|
90
685
|
statusListeners;
|
|
91
686
|
/** The observers that the `HybridStorage` of the leader adds to its local copy: the capture */
|
|
92
687
|
captures;
|
|
93
|
-
/** The backoff of a new try after a failed setup, its clock and its random numbers */
|
|
94
|
-
backoff;
|
|
95
|
-
clock;
|
|
96
|
-
random;
|
|
97
688
|
currentRole = "follower";
|
|
98
|
-
/** The lock manager, or undefined in the single mode */
|
|
99
|
-
locks;
|
|
100
|
-
channel;
|
|
101
689
|
hybrid;
|
|
102
690
|
/** The storage of this tab while it loads its queue, before the tab leads */
|
|
103
691
|
candidate;
|
|
104
|
-
/** The number of the failed setups in a row */
|
|
692
|
+
/** The number of the failed setups of this collection in a row, when the group leads or is single */
|
|
105
693
|
failures = 0;
|
|
106
694
|
/** The stop of the wait before the next try */
|
|
107
695
|
cancelRetry;
|
|
@@ -122,157 +710,97 @@ export class TabCoordinator {
|
|
|
122
710
|
outbox = [];
|
|
123
711
|
/** The number of the last batch of each other tab that this tab copied */
|
|
124
712
|
seen = new Map();
|
|
713
|
+
/** The number of the last batch of each other tab that the local copy of this tab holds (roadmap CLI-66) */
|
|
714
|
+
applied = new Map();
|
|
125
715
|
/** In the leader: the number of the last batch of each tab that the leader captured. A tab without a hello is not here. */
|
|
126
716
|
captured = new Map();
|
|
127
717
|
/** The batches of the other tabs: they reach the local copy and the observers in their order */
|
|
128
718
|
incoming = Promise.resolve();
|
|
129
719
|
closed = false;
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
/** The
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
this.
|
|
146
|
-
this.
|
|
147
|
-
this.lockName = `@procedures/store/tab:${options.name}`;
|
|
720
|
+
/** The snapshot that this tab waits for, and the number of its last request (roadmap CLI-66) */
|
|
721
|
+
snapshot;
|
|
722
|
+
requests = 0;
|
|
723
|
+
/** The batches of the other tabs that wait for the snapshot, in their order */
|
|
724
|
+
held = [];
|
|
725
|
+
/** The own writes while the snapshot waits: the number of the last own batch of each id, and of the last clear */
|
|
726
|
+
ownWrites = new Map();
|
|
727
|
+
ownClear = 0;
|
|
728
|
+
/** The stop of the wait for the next chunk */
|
|
729
|
+
cancelSnapshotWait;
|
|
730
|
+
/** The last tab that told this tab that it leads this collection: the `leader` message of its group does not start a second hello */
|
|
731
|
+
announcedBy;
|
|
732
|
+
/** The error of the last failed setup of the sync */
|
|
733
|
+
setupError;
|
|
734
|
+
constructor(group, name, options, settings) {
|
|
735
|
+
this.group = group;
|
|
736
|
+
this.name = name;
|
|
148
737
|
this.local = options.local;
|
|
149
738
|
this.shared = options.copy === "shared";
|
|
150
739
|
this.makeRemote = options.remote;
|
|
151
740
|
this.hybridOptions = options.hybrid ?? {};
|
|
152
|
-
this.
|
|
741
|
+
this.settings = settings;
|
|
742
|
+
this.tag = this.hybridOptions.origin ?? `tab-${group.id}`;
|
|
153
743
|
this.writeOptions = { origin: this.tag };
|
|
154
|
-
this.
|
|
155
|
-
this.
|
|
156
|
-
this.
|
|
157
|
-
this.
|
|
158
|
-
this.captures = new StorageListeners(this.onError);
|
|
159
|
-
this.clock = options.clock ?? this.hybridOptions.clock ?? realClock;
|
|
160
|
-
this.random = options.random ?? this.hybridOptions.random ?? Math.random;
|
|
744
|
+
this.observers = new StorageListeners(group.onError);
|
|
745
|
+
this.roleListeners = new StorageListeners(group.onError);
|
|
746
|
+
this.statusListeners = new StorageListeners(group.onError);
|
|
747
|
+
this.captures = new StorageListeners(group.onError);
|
|
161
748
|
// The observer of a follower. A tab that leads stops it. A single tab whose setup failed keeps its batches with it.
|
|
162
749
|
this.stopTap = this.local.observe((batch) => this.onFollowerBatch(batch));
|
|
163
|
-
const locks = options.locks === undefined ? globalLocks() : (options.locks ?? undefined);
|
|
164
|
-
const openChannel = options.channel === undefined ? globalChannelFactory() : (options.channel ?? undefined);
|
|
165
|
-
if (!locks || !openChannel) {
|
|
166
|
-
this.startSingle();
|
|
167
|
-
return;
|
|
168
|
-
}
|
|
169
|
-
try {
|
|
170
|
-
this.channel = openChannel(this.lockName);
|
|
171
|
-
}
|
|
172
|
-
catch (error) {
|
|
173
|
-
this.onError(error);
|
|
174
|
-
this.startSingle();
|
|
175
|
-
return;
|
|
176
|
-
}
|
|
177
|
-
this.locks = locks;
|
|
178
|
-
this.channel.onmessage = (event) => this.receive(event?.data);
|
|
179
|
-
this.hello();
|
|
180
|
-
this.elect(locks);
|
|
181
750
|
}
|
|
182
|
-
/** The role of this tab now. */
|
|
183
751
|
get role() {
|
|
184
752
|
return this.currentRole;
|
|
185
753
|
}
|
|
186
|
-
/** True when this tab runs the sync: the leader, or a tab in the single mode. */
|
|
187
754
|
get isLeader() {
|
|
188
755
|
return this.currentRole === "leader" || this.currentRole === "single";
|
|
189
756
|
}
|
|
190
|
-
/**
|
|
191
|
-
* The `HybridStorage` of the sync while this tab runs it, or else undefined. `getStatus()` of the
|
|
192
|
-
* coordinator gives the state of the sync in each tab. Write to the local copy, not to this
|
|
193
|
-
* storage: each tab then writes in the same way.
|
|
194
|
-
*/
|
|
195
757
|
get syncStorage() {
|
|
196
758
|
return this.hybrid;
|
|
197
759
|
}
|
|
198
|
-
/**
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
*/
|
|
760
|
+
/** True while the `HybridStorage` of this tab runs the sync of the collection. */
|
|
761
|
+
get leads() {
|
|
762
|
+
return this.hybrid !== undefined;
|
|
763
|
+
}
|
|
764
|
+
/** True while the `HybridStorage` of this tab loads its queue, before the collection leads. */
|
|
765
|
+
get settingUp() {
|
|
766
|
+
return this.candidate !== undefined;
|
|
767
|
+
}
|
|
207
768
|
observe(observer) {
|
|
208
769
|
if (this.closed)
|
|
209
|
-
throw new Error(
|
|
770
|
+
throw new Error(`The ${this.group.kind} is closed`);
|
|
210
771
|
return this.observers.add(observer);
|
|
211
772
|
}
|
|
212
|
-
/**
|
|
213
|
-
* This method starts to send the role of this tab to a listener at each change.
|
|
214
|
-
*
|
|
215
|
-
* @returns The function that stops the roles. A second use of it does nothing.
|
|
216
|
-
* @throws Error after `close()`
|
|
217
|
-
*/
|
|
218
773
|
onRole(listener) {
|
|
219
774
|
if (this.closed)
|
|
220
|
-
throw new Error(
|
|
775
|
+
throw new Error(`The ${this.group.kind} is closed`);
|
|
221
776
|
return this.roleListeners.add(listener);
|
|
222
777
|
}
|
|
223
|
-
/**
|
|
224
|
-
* The status of the sync (roadmap CLI-65), or undefined before the first status and after
|
|
225
|
-
* `close()`.
|
|
226
|
-
*
|
|
227
|
-
* - In the leader and in the single mode, it is the status of the `HybridStorage` of this tab.
|
|
228
|
-
* - In a follower, it is the last status that the leader sent. A new tab gets it in the answer to
|
|
229
|
-
* its hello. After a change of the leader, a follower keeps the last status until the new
|
|
230
|
-
* leader sends its status.
|
|
231
|
-
* - In a follower, `error.error` is a copy of the error of the leader: an `Error` with its `name`,
|
|
232
|
-
* its `message` and its `code`. The copy has no `cause`.
|
|
233
|
-
*/
|
|
234
778
|
getStatus() {
|
|
235
779
|
return this.status;
|
|
236
780
|
}
|
|
237
|
-
/**
|
|
238
|
-
* This method starts to send the status of the sync to a listener at each change (roadmap
|
|
239
|
-
* CLI-65): in the leader, each status of its `HybridStorage`, and in a follower, each status that
|
|
240
|
-
* the leader sends. The listener gets each status one time. When the tab becomes the leader, the
|
|
241
|
-
* listener gets the status of its own `HybridStorage`.
|
|
242
|
-
*
|
|
243
|
-
* @returns The function that stops the statuses. A second use of it does nothing.
|
|
244
|
-
* @throws Error after `close()`
|
|
245
|
-
*/
|
|
246
781
|
onStatus(listener) {
|
|
247
782
|
if (this.closed)
|
|
248
|
-
throw new Error(
|
|
783
|
+
throw new Error(`The ${this.group.kind} is closed`);
|
|
249
784
|
return this.statusListeners.add(listener);
|
|
250
785
|
}
|
|
251
|
-
/**
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
* operations that the remote did not take. Then the leader releases the lock, and the next
|
|
258
|
-
* tab becomes the leader.
|
|
259
|
-
* - The coordinator closes its channel. It does not close the local copy: the application owns it.
|
|
260
|
-
* - After it, the coordinator sends and copies nothing. A second `close()` gives the promise of
|
|
261
|
-
* the first one.
|
|
262
|
-
*/
|
|
263
|
-
close() {
|
|
264
|
-
if (!this.closing)
|
|
265
|
-
this.closing = this.shutDown();
|
|
266
|
-
return this.closing;
|
|
786
|
+
/** This method starts the collection on the channel: it says hello, and it asks for a snapshot. */
|
|
787
|
+
begin() {
|
|
788
|
+
if (this.settings.snapshot.want)
|
|
789
|
+
this.snapshot = { request: 0, chunks: [], received: 0 };
|
|
790
|
+
this.hello();
|
|
791
|
+
this.requestSnapshot();
|
|
267
792
|
}
|
|
793
|
+
/** This method closes the storage of the collection. The group then releases the lock and closes the channel. */
|
|
268
794
|
async shutDown() {
|
|
269
795
|
this.closed = true;
|
|
270
796
|
this.stopTap?.();
|
|
271
797
|
this.stopTap = undefined;
|
|
272
|
-
this.abortRequest?.();
|
|
273
|
-
this.abortRequest = undefined;
|
|
274
798
|
this.cancelRetry?.();
|
|
275
799
|
this.cancelRetry = undefined;
|
|
800
|
+
this.cancelSnapshotWait?.();
|
|
801
|
+
this.cancelSnapshotWait = undefined;
|
|
802
|
+
this.snapshot = undefined;
|
|
803
|
+
this.held = [];
|
|
276
804
|
this.stopStatus?.();
|
|
277
805
|
this.stopStatus = undefined;
|
|
278
806
|
this.setRole("closed");
|
|
@@ -283,79 +811,61 @@ export class TabCoordinator {
|
|
|
283
811
|
this.hybrid = undefined;
|
|
284
812
|
this.status = undefined;
|
|
285
813
|
try {
|
|
286
|
-
// The leader holds the lock until its queue reached the remote or its queue store
|
|
287
814
|
if (hybrid)
|
|
288
815
|
await hybrid.close();
|
|
289
816
|
}
|
|
290
817
|
catch (error) {
|
|
291
|
-
this.onError(error);
|
|
292
|
-
}
|
|
293
|
-
finally {
|
|
294
|
-
this.releaseLock?.();
|
|
295
|
-
this.releaseLock = undefined;
|
|
296
|
-
// The acknowledgements of the close of the HybridStorage went out before
|
|
297
|
-
try {
|
|
298
|
-
this.channel?.close();
|
|
299
|
-
}
|
|
300
|
-
catch {
|
|
301
|
-
// The channel is closed already
|
|
302
|
-
}
|
|
303
|
-
this.channel = undefined;
|
|
304
|
-
this.observers.clear();
|
|
305
|
-
this.roleListeners.clear();
|
|
306
|
-
this.statusListeners.clear();
|
|
307
|
-
this.captures.clear();
|
|
818
|
+
this.group.onError(error);
|
|
308
819
|
}
|
|
309
820
|
}
|
|
821
|
+
clearListeners() {
|
|
822
|
+
this.observers.clear();
|
|
823
|
+
this.roleListeners.clear();
|
|
824
|
+
this.statusListeners.clear();
|
|
825
|
+
this.captures.clear();
|
|
826
|
+
}
|
|
310
827
|
//
|
|
311
|
-
// ═══ The
|
|
828
|
+
// ═══ The setup of the sync ═══
|
|
312
829
|
//
|
|
313
|
-
/**
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
request = locks.request(this.lockName, controller ? { signal: controller.signal } : {}, async () => {
|
|
322
|
-
// A lock after close(): the tab releases it at once
|
|
323
|
-
if (this.closed)
|
|
324
|
-
return;
|
|
325
|
-
this.abortRequest = undefined;
|
|
326
|
-
if (await this.takeLead()) {
|
|
327
|
-
await held;
|
|
328
|
-
return;
|
|
329
|
-
}
|
|
330
|
-
// A failed setup releases the lock: another tab tries, and this tab tries again after the backoff
|
|
331
|
-
this.retryLater();
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
catch (error) {
|
|
335
|
-
request = Promise.reject(error);
|
|
336
|
-
}
|
|
337
|
-
request.then(undefined, (error) => {
|
|
338
|
-
// The abort of close(), or a tab that leads already
|
|
339
|
-
if (this.closed || this.hybrid)
|
|
340
|
-
return;
|
|
341
|
-
// The lock manager refused the request: this tab runs the sync alone
|
|
342
|
-
this.onError(error);
|
|
343
|
-
this.stopChannel();
|
|
344
|
-
this.locks = undefined;
|
|
345
|
-
this.startSingle();
|
|
830
|
+
/**
|
|
831
|
+
* This method sets up the sync in a tab that holds the lock. After a failure, the collection tries
|
|
832
|
+
* again after its backoff. When no collection of the group can lead, the tab releases the lock.
|
|
833
|
+
*/
|
|
834
|
+
startLead(lease, announce) {
|
|
835
|
+
void this.takeLead(lease, announce).then((led) => {
|
|
836
|
+
if (!led && !this.closed && this.group.lease === lease)
|
|
837
|
+
this.group.afterFailedSetup(this, lease);
|
|
346
838
|
});
|
|
347
839
|
}
|
|
840
|
+
/** This method stops the wait before the next try of the setup. */
|
|
841
|
+
stopRetry() {
|
|
842
|
+
this.cancelRetry?.();
|
|
843
|
+
this.cancelRetry = undefined;
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* This method runs after a failed setup in a tab that keeps the lock. No other tab leads the
|
|
847
|
+
* collection, so no snapshot comes: the batches that waited reach the local copy now.
|
|
848
|
+
*/
|
|
849
|
+
retryInLeader() {
|
|
850
|
+
this.endSnapshot();
|
|
851
|
+
this.retryLater();
|
|
852
|
+
}
|
|
348
853
|
/**
|
|
349
854
|
* This method makes the `HybridStorage` of a tab that got the lock, and waits for the load of its
|
|
350
|
-
* queue. Then the
|
|
351
|
-
*
|
|
855
|
+
* queue. Then the collection leads. Until then, it is a follower: its own batches go to the other
|
|
856
|
+
* tabs and to its outbox, and the capture of the storage ignores the local copy.
|
|
352
857
|
*
|
|
353
|
-
* @
|
|
858
|
+
* @param announce True: the collection tells the other tabs that it leads. Else the group tells it.
|
|
859
|
+
* @returns False when the setup failed (the error went to `onError`), after `close()`, or when the tab lost the lease
|
|
354
860
|
*/
|
|
355
|
-
async takeLead() {
|
|
861
|
+
async takeLead(lease, announce) {
|
|
862
|
+
if (this.hybrid)
|
|
863
|
+
return true;
|
|
356
864
|
const candidate = this.makeCandidate();
|
|
357
|
-
if (!candidate)
|
|
865
|
+
if (!candidate) {
|
|
866
|
+
this.reportFailure(lease);
|
|
358
867
|
return false;
|
|
868
|
+
}
|
|
359
869
|
this.candidate = candidate;
|
|
360
870
|
let failed = false;
|
|
361
871
|
let failure;
|
|
@@ -371,28 +881,55 @@ export class TabCoordinator {
|
|
|
371
881
|
if (this.candidate !== candidate)
|
|
372
882
|
return false;
|
|
373
883
|
this.candidate = undefined;
|
|
374
|
-
if (!failed) {
|
|
375
|
-
this.promote(candidate, "leader");
|
|
884
|
+
if (!failed && this.group.lease === lease) {
|
|
885
|
+
this.promote(candidate, "leader", announce);
|
|
376
886
|
return true;
|
|
377
887
|
}
|
|
378
|
-
|
|
888
|
+
if (failed) {
|
|
889
|
+
this.setupFailed(failure);
|
|
890
|
+
this.reportFailure(lease);
|
|
891
|
+
}
|
|
379
892
|
candidate.stop();
|
|
380
893
|
try {
|
|
381
894
|
await candidate.hybrid.close();
|
|
382
895
|
}
|
|
383
896
|
catch (error) {
|
|
384
|
-
this.onError(error);
|
|
897
|
+
this.group.onError(error);
|
|
385
898
|
}
|
|
386
899
|
return false;
|
|
387
900
|
}
|
|
388
|
-
/** This method makes the `HybridStorage` of the single mode. The
|
|
901
|
+
/** This method makes the `HybridStorage` of the single mode. The collection tries again after the backoff when it fails. */
|
|
389
902
|
startSingle() {
|
|
903
|
+
if (this.closed || this.hybrid)
|
|
904
|
+
return;
|
|
905
|
+
// No other tab sends a snapshot
|
|
906
|
+
this.endSnapshot();
|
|
390
907
|
const candidate = this.makeCandidate();
|
|
391
908
|
if (candidate)
|
|
392
|
-
this.promote(candidate, "single");
|
|
909
|
+
this.promote(candidate, "single", false);
|
|
393
910
|
else
|
|
394
911
|
this.retryLater();
|
|
395
912
|
}
|
|
913
|
+
/** This method tries the setup again after the backoff of the collection (roadmap CLI-67). */
|
|
914
|
+
retryLater() {
|
|
915
|
+
if (this.closed)
|
|
916
|
+
return;
|
|
917
|
+
this.cancelRetry?.();
|
|
918
|
+
this.failures++;
|
|
919
|
+
const wait = backoffDelay(this.group.backoff, this.failures, this.group.random);
|
|
920
|
+
const cancel = this.group.clock.after(wait, () => {
|
|
921
|
+
this.cancelRetry = undefined;
|
|
922
|
+
if (this.closed || this.hybrid)
|
|
923
|
+
return;
|
|
924
|
+
if (this.group.alone)
|
|
925
|
+
this.startSingle();
|
|
926
|
+
else if (this.group.lease)
|
|
927
|
+
this.startLead(this.group.lease, true);
|
|
928
|
+
});
|
|
929
|
+
// The wait must not keep a Node process alive
|
|
930
|
+
cancel.unref?.();
|
|
931
|
+
this.cancelRetry = cancel;
|
|
932
|
+
}
|
|
396
933
|
/**
|
|
397
934
|
* This method makes the `HybridStorage` of the sync, with its observer. Its capture ignores the
|
|
398
935
|
* local copy until the tab leads.
|
|
@@ -407,7 +944,7 @@ export class TabCoordinator {
|
|
|
407
944
|
remote = this.makeRemote();
|
|
408
945
|
}
|
|
409
946
|
catch (error) {
|
|
410
|
-
this.
|
|
947
|
+
this.setupFailed(error);
|
|
411
948
|
return undefined;
|
|
412
949
|
}
|
|
413
950
|
const gate = { open: false };
|
|
@@ -421,24 +958,57 @@ export class TabCoordinator {
|
|
|
421
958
|
});
|
|
422
959
|
}
|
|
423
960
|
catch (error) {
|
|
424
|
-
this.
|
|
961
|
+
this.setupFailed(error);
|
|
425
962
|
// The remote of a storage that did not start stays open otherwise
|
|
426
963
|
Promise.resolve()
|
|
427
964
|
.then(() => remote.close())
|
|
428
|
-
.catch((closeError) => this.onError(closeError));
|
|
965
|
+
.catch((closeError) => this.group.onError(closeError));
|
|
429
966
|
return undefined;
|
|
430
967
|
}
|
|
431
968
|
// The observer starts at once: the other tabs get the changes of the load and of the first sync
|
|
432
969
|
const stop = hybrid.observe((batch) => this.onLeaderBatch(batch));
|
|
433
970
|
return { hybrid, gate, stop };
|
|
434
971
|
}
|
|
972
|
+
/** This method gives the error of a failed setup to `onError`, and keeps it for the status of the failure. */
|
|
973
|
+
setupFailed(error) {
|
|
974
|
+
this.setupError = error;
|
|
975
|
+
this.group.onError(error);
|
|
976
|
+
}
|
|
977
|
+
/**
|
|
978
|
+
* In a group, the tab that holds the lock tells the other tabs that the setup of the collection
|
|
979
|
+
* failed (roadmap CLI-65). Its status has the error, with the operation `load`, and the other
|
|
980
|
+
* values of the last status. Thus a follower does not keep the status of an old leader. A
|
|
981
|
+
* `TabCoordinator` releases the lock after a failed setup, and sends no status.
|
|
982
|
+
*/
|
|
983
|
+
reportFailure(lease) {
|
|
984
|
+
if (this.group.legacy || this.closed || this.group.lease !== lease)
|
|
985
|
+
return;
|
|
986
|
+
const last = this.status;
|
|
987
|
+
const status = {
|
|
988
|
+
connected: false,
|
|
989
|
+
pendingChanges: last?.pendingChanges ?? 0,
|
|
990
|
+
syncing: false,
|
|
991
|
+
...(last?.endpoint !== undefined && { endpoint: last.endpoint }),
|
|
992
|
+
...(last?.lastSync !== undefined && { lastSync: last.lastSync }),
|
|
993
|
+
...(last?.mode !== undefined && { mode: last.mode }),
|
|
994
|
+
error: { operation: "load", error: this.setupError },
|
|
995
|
+
};
|
|
996
|
+
this.status = status;
|
|
997
|
+
// The status is the status of this tab now: its number follows the last number
|
|
998
|
+
this.statusFrom = undefined;
|
|
999
|
+
this.statusSeq++;
|
|
1000
|
+
this.postStatus();
|
|
1001
|
+
this.statusListeners.emit(status);
|
|
1002
|
+
}
|
|
435
1003
|
/**
|
|
436
|
-
* This method makes this tab the leader, or the single tab, with the storage of
|
|
437
|
-
* capture of the storage gets the batches of this tab that no leader acknowledged.
|
|
1004
|
+
* This method makes this tab the leader of the collection, or the single tab, with the storage of
|
|
1005
|
+
* a candidate. The capture of the storage gets the batches of this tab that no leader acknowledged.
|
|
438
1006
|
*/
|
|
439
|
-
promote(candidate, role) {
|
|
1007
|
+
promote(candidate, role, announce) {
|
|
440
1008
|
const { hybrid, gate, stop } = candidate;
|
|
441
1009
|
this.hybrid = hybrid;
|
|
1010
|
+
this.cancelRetry?.();
|
|
1011
|
+
this.cancelRetry = undefined;
|
|
442
1012
|
this.failures = 0;
|
|
443
1013
|
// The capture starts and the observer of the follower stops in one step: no change of the local copy goes between them
|
|
444
1014
|
gate.open = true;
|
|
@@ -447,34 +1017,17 @@ export class TabCoordinator {
|
|
|
447
1017
|
stopFollower?.();
|
|
448
1018
|
for (const batch of this.outbox.splice(0))
|
|
449
1019
|
this.capture(batch.changes);
|
|
1020
|
+
// The copy of this tab is now the copy that the other tabs get: a snapshot that waits ends without entries
|
|
1021
|
+
this.endSnapshot();
|
|
450
1022
|
this.setRole(role);
|
|
451
|
-
if (
|
|
452
|
-
this.post({ v: PROTOCOL, type: "leader", from: this.id });
|
|
1023
|
+
if (announce)
|
|
1024
|
+
this.post({ v: PROTOCOL, type: "leader", from: this.group.id });
|
|
453
1025
|
// The status of this tab is now the status of its storage, also when it has the values of the last status of the old leader. The followers get it.
|
|
454
1026
|
this.status = undefined;
|
|
455
1027
|
this.statusFrom = undefined;
|
|
456
1028
|
this.stopStatus = hybrid.onStatus((status) => this.onOwnStatus(hybrid, status));
|
|
457
1029
|
this.onOwnStatus(hybrid, hybrid.getStatus());
|
|
458
1030
|
}
|
|
459
|
-
/** This method asks for the lock again, or tries the single mode again, after the backoff (roadmap CLI-67). */
|
|
460
|
-
retryLater() {
|
|
461
|
-
if (this.closed)
|
|
462
|
-
return;
|
|
463
|
-
this.failures++;
|
|
464
|
-
const wait = backoffDelay(this.backoff, this.failures, this.random);
|
|
465
|
-
const cancel = this.clock.after(wait, () => {
|
|
466
|
-
this.cancelRetry = undefined;
|
|
467
|
-
if (this.closed)
|
|
468
|
-
return;
|
|
469
|
-
if (this.locks)
|
|
470
|
-
this.elect(this.locks);
|
|
471
|
-
else
|
|
472
|
-
this.startSingle();
|
|
473
|
-
});
|
|
474
|
-
// The wait must not keep a Node process alive
|
|
475
|
-
cancel.unref?.();
|
|
476
|
-
this.cancelRetry = cancel;
|
|
477
|
-
}
|
|
478
1031
|
/**
|
|
479
1032
|
* The local copy for the `HybridStorage` of the leader. Its `observe()` gives the changes of the
|
|
480
1033
|
* local copy and the batches of the other tabs, so the capture queues both. It gives the changes
|
|
@@ -546,7 +1099,7 @@ export class TabCoordinator {
|
|
|
546
1099
|
postStatus() {
|
|
547
1100
|
if (this.status === undefined)
|
|
548
1101
|
return;
|
|
549
|
-
this.post({ v: PROTOCOL, type: "status", from: this.id, seq: this.statusSeq, status: portableStatus(this.status) });
|
|
1102
|
+
this.post({ v: PROTOCOL, type: "status", from: this.group.id, seq: this.statusSeq, status: portableStatus(this.status) });
|
|
550
1103
|
}
|
|
551
1104
|
/** A status of the leader. A tab that runs the sync keeps its own status. */
|
|
552
1105
|
onLeaderStatus(from, seq, status) {
|
|
@@ -589,69 +1142,71 @@ export class TabCoordinator {
|
|
|
589
1142
|
*/
|
|
590
1143
|
share(changes, keep) {
|
|
591
1144
|
const seq = ++this.seq;
|
|
1145
|
+
if (this.snapshot)
|
|
1146
|
+
this.noteOwn(changes, seq);
|
|
592
1147
|
this.observers.emit(changes);
|
|
593
|
-
const sent = this.post({ v: PROTOCOL, type: "changes", from: this.id, seq, changes });
|
|
594
|
-
if (keep && (sent || !this.
|
|
1148
|
+
const sent = this.post({ v: PROTOCOL, type: "changes", from: this.group.id, seq, changes });
|
|
1149
|
+
if (keep && (sent || !this.group.hasChannel))
|
|
595
1150
|
this.outbox.push({ seq, changes });
|
|
596
1151
|
}
|
|
597
1152
|
/** This method tells the leader the batches of this tab that no leader acknowledged. */
|
|
598
1153
|
hello() {
|
|
599
|
-
this.post({ v: PROTOCOL, type: "hello", from: this.id, seq: this.seq, pending: this.outbox });
|
|
1154
|
+
this.post({ v: PROTOCOL, type: "hello", from: this.group.id, seq: this.seq, pending: this.outbox });
|
|
600
1155
|
}
|
|
601
|
-
/** @returns False when the message did not go out
|
|
1156
|
+
/** @returns False when the message did not go out */
|
|
602
1157
|
post(message) {
|
|
603
|
-
|
|
604
|
-
if (!channel)
|
|
605
|
-
return false;
|
|
606
|
-
try {
|
|
607
|
-
channel.postMessage(message);
|
|
608
|
-
return true;
|
|
609
|
-
}
|
|
610
|
-
catch (error) {
|
|
611
|
-
this.onError(error);
|
|
612
|
-
return false;
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
stopChannel() {
|
|
616
|
-
const channel = this.channel;
|
|
617
|
-
this.channel = undefined;
|
|
618
|
-
if (!channel)
|
|
619
|
-
return;
|
|
620
|
-
channel.onmessage = null;
|
|
621
|
-
try {
|
|
622
|
-
channel.close();
|
|
623
|
-
}
|
|
624
|
-
catch {
|
|
625
|
-
// The channel is closed already
|
|
626
|
-
}
|
|
1158
|
+
return this.group.post(this.name, message);
|
|
627
1159
|
}
|
|
628
1160
|
//
|
|
629
1161
|
// ═══ The batches of the other tabs ═══
|
|
630
1162
|
//
|
|
1163
|
+
/** A message of another tab for this collection. */
|
|
631
1164
|
receive(data) {
|
|
632
|
-
if (this.closed
|
|
1165
|
+
if (this.closed)
|
|
633
1166
|
return;
|
|
634
|
-
|
|
1167
|
+
const message = data;
|
|
1168
|
+
switch (message.type) {
|
|
635
1169
|
case "changes":
|
|
636
|
-
this.onChanges(
|
|
1170
|
+
this.onChanges(message.from, { seq: message.seq, changes: message.changes });
|
|
637
1171
|
break;
|
|
638
1172
|
case "hello":
|
|
639
|
-
this.onHello(
|
|
1173
|
+
this.onHello(message.from, message.seq, message.pending);
|
|
640
1174
|
break;
|
|
641
1175
|
case "leader":
|
|
642
|
-
|
|
643
|
-
if (this.currentRole === "follower")
|
|
644
|
-
this.hello();
|
|
1176
|
+
this.onLeader(message.from, true);
|
|
645
1177
|
break;
|
|
646
1178
|
case "ack":
|
|
647
|
-
if (
|
|
648
|
-
this.outbox = this.outbox.filter((batch) => batch.seq >
|
|
1179
|
+
if (message.to === this.group.id)
|
|
1180
|
+
this.outbox = this.outbox.filter((batch) => batch.seq > message.seq);
|
|
649
1181
|
break;
|
|
650
1182
|
case "status":
|
|
651
|
-
this.onLeaderStatus(
|
|
1183
|
+
this.onLeaderStatus(message.from, message.seq, message.status);
|
|
1184
|
+
break;
|
|
1185
|
+
case "snapshot-request":
|
|
1186
|
+
this.sendSnapshot(message.from, message.request);
|
|
1187
|
+
break;
|
|
1188
|
+
case "snapshot":
|
|
1189
|
+
this.onSnapshot(message);
|
|
652
1190
|
break;
|
|
653
1191
|
}
|
|
654
1192
|
}
|
|
1193
|
+
/**
|
|
1194
|
+
* A new leader. A follower tells it the batches that no leader acknowledged, and asks it for the
|
|
1195
|
+
* snapshot that waits.
|
|
1196
|
+
*
|
|
1197
|
+
* @param own True when the tab `from` leads this collection. False for the message of a group for
|
|
1198
|
+
* all its collections: a collection that the tab `from` announced already ignores it.
|
|
1199
|
+
*/
|
|
1200
|
+
onLeader(from, own) {
|
|
1201
|
+
if (this.closed || this.currentRole !== "follower")
|
|
1202
|
+
return;
|
|
1203
|
+
if (own)
|
|
1204
|
+
this.announcedBy = from;
|
|
1205
|
+
else if (this.announcedBy === from)
|
|
1206
|
+
return;
|
|
1207
|
+
this.hello();
|
|
1208
|
+
this.requestSnapshot();
|
|
1209
|
+
}
|
|
655
1210
|
/**
|
|
656
1211
|
* A batch of another tab. The leader captures it when the tab said hello to it. Each tab copies
|
|
657
1212
|
* it one time.
|
|
@@ -682,27 +1237,40 @@ export class TabCoordinator {
|
|
|
682
1237
|
// A new tab gets the status now. The other tabs have its number already.
|
|
683
1238
|
this.postStatus();
|
|
684
1239
|
}
|
|
1240
|
+
else if (!this.group.legacy && !this.hybrid && this.group.lease && this.status !== undefined && this.statusFrom === undefined) {
|
|
1241
|
+
// The status of a failed setup in the tab that holds the lock (roadmap CLI-65)
|
|
1242
|
+
this.postStatus();
|
|
1243
|
+
}
|
|
685
1244
|
for (const batch of pending)
|
|
686
1245
|
this.copy(from, batch);
|
|
687
1246
|
}
|
|
688
1247
|
/**
|
|
689
1248
|
* This method copies a batch of another tab that this tab did not see: to the local copy of a
|
|
690
|
-
* tab, and then to the observers. The batches go in their order.
|
|
1249
|
+
* tab, and then to the observers. The batches go in their order. While a snapshot waits, the
|
|
1250
|
+
* batch waits for it.
|
|
691
1251
|
*/
|
|
692
1252
|
copy(from, batch) {
|
|
693
1253
|
if (batch.seq <= (this.seen.get(from) ?? 0))
|
|
694
1254
|
return;
|
|
695
1255
|
this.seen.set(from, batch.seq);
|
|
1256
|
+
if (this.snapshot)
|
|
1257
|
+
this.held.push({ from, batch });
|
|
1258
|
+
else
|
|
1259
|
+
this.apply(from, batch);
|
|
1260
|
+
}
|
|
1261
|
+
/** This method writes a batch of another tab to the local copy, and then gives it to the observers. */
|
|
1262
|
+
apply(from, batch) {
|
|
696
1263
|
this.incoming = this.incoming
|
|
697
1264
|
.then(async () => {
|
|
698
1265
|
if (this.closed)
|
|
699
1266
|
return;
|
|
700
1267
|
if (!this.shared)
|
|
701
1268
|
await this.write(batch.changes);
|
|
1269
|
+
this.applied.set(from, Math.max(this.applied.get(from) ?? 0, batch.seq));
|
|
702
1270
|
if (!this.closed)
|
|
703
1271
|
this.observers.emit(batch.changes);
|
|
704
1272
|
})
|
|
705
|
-
.catch((error) => this.onError(error));
|
|
1273
|
+
.catch((error) => this.group.onError(error));
|
|
706
1274
|
}
|
|
707
1275
|
/**
|
|
708
1276
|
* This method writes a batch of another tab to the local copy of this tab, with the tag of the
|
|
@@ -736,8 +1304,190 @@ export class TabCoordinator {
|
|
|
736
1304
|
const hybrid = this.hybrid;
|
|
737
1305
|
if (!hybrid)
|
|
738
1306
|
return;
|
|
739
|
-
hybrid.whenQueued().then(() => void this.post({ v: PROTOCOL, type: "ack", from: this.id, to, seq }), (error) => this.onError(error));
|
|
1307
|
+
hybrid.whenQueued().then(() => void this.post({ v: PROTOCOL, type: "ack", from: this.group.id, to, seq }), (error) => this.group.onError(error));
|
|
1308
|
+
}
|
|
1309
|
+
//
|
|
1310
|
+
// ═══ The snapshot for a new tab (roadmap CLI-66) ═══
|
|
1311
|
+
//
|
|
1312
|
+
/** This method asks the leader for a snapshot, while the snapshot waits. A new request replaces the last one. */
|
|
1313
|
+
requestSnapshot() {
|
|
1314
|
+
if (!this.snapshot)
|
|
1315
|
+
return;
|
|
1316
|
+
const request = ++this.requests;
|
|
1317
|
+
this.snapshot = { request, chunks: [], received: 0 };
|
|
1318
|
+
this.post({ v: PROTOCOL, type: "snapshot-request", from: this.group.id, request });
|
|
1319
|
+
this.waitSnapshot();
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* This method starts the wait for the next chunk again. At its end, the tab continues without the
|
|
1323
|
+
* snapshot. In the tab that holds the lock, the wait renews itself while the collection loads its
|
|
1324
|
+
* queue, at most `SNAPSHOT_RENEWALS` times: a load that hangs does not hold the batches forever.
|
|
1325
|
+
*
|
|
1326
|
+
* @param renewals The number of the renewals of the wait in the tab that holds the lock
|
|
1327
|
+
*/
|
|
1328
|
+
waitSnapshot(renewals = 0) {
|
|
1329
|
+
this.cancelSnapshotWait?.();
|
|
1330
|
+
const cancel = this.group.clock.after(this.settings.snapshot.timeout, () => {
|
|
1331
|
+
this.cancelSnapshotWait = undefined;
|
|
1332
|
+
if (this.closed || !this.snapshot)
|
|
1333
|
+
return;
|
|
1334
|
+
if (this.group.lease) {
|
|
1335
|
+
// This tab holds the lock and loads its queue: the snapshot ends when the collection leads
|
|
1336
|
+
if (renewals < SNAPSHOT_RENEWALS) {
|
|
1337
|
+
this.waitSnapshot(renewals + 1);
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
this.group.onError(new Error(`The collection ${JSON.stringify(this.name)} did not lead in time: it continues without the snapshot`));
|
|
1341
|
+
}
|
|
1342
|
+
else {
|
|
1343
|
+
this.group.onError(new Error(`No leader sent the snapshot of the collection ${JSON.stringify(this.name)} in time`));
|
|
1344
|
+
}
|
|
1345
|
+
this.endSnapshot();
|
|
1346
|
+
});
|
|
1347
|
+
// The wait must not keep a Node process alive
|
|
1348
|
+
cancel.unref?.();
|
|
1349
|
+
this.cancelSnapshotWait = cancel;
|
|
1350
|
+
}
|
|
1351
|
+
/**
|
|
1352
|
+
* The leader sends the entries of its local copy to the tab `to`, in chunks. The numbers come
|
|
1353
|
+
* before the read of the entries: thus the entries hold each batch up to the numbers. A batch
|
|
1354
|
+
* after them can be in the entries too, and the new tab writes it again with the same values.
|
|
1355
|
+
*/
|
|
1356
|
+
sendSnapshot(to, request) {
|
|
1357
|
+
if (!this.hybrid || this.currentRole !== "leader")
|
|
1358
|
+
return;
|
|
1359
|
+
const chunk = this.settings.snapshot.chunk;
|
|
1360
|
+
// The batches of the other tabs that this tab got before the request reach the local copy first
|
|
1361
|
+
this.incoming = this.incoming
|
|
1362
|
+
.then(async () => {
|
|
1363
|
+
if (this.closed || this.currentRole !== "leader")
|
|
1364
|
+
return;
|
|
1365
|
+
const seqs = [[this.group.id, this.seq], ...this.applied];
|
|
1366
|
+
const entries = await this.local.getEntries();
|
|
1367
|
+
if (this.closed)
|
|
1368
|
+
return;
|
|
1369
|
+
const parts = Math.max(1, Math.ceil(entries.length / chunk));
|
|
1370
|
+
for (let part = 0; part < parts; part++) {
|
|
1371
|
+
const slice = entries.slice(part * chunk, (part + 1) * chunk);
|
|
1372
|
+
if (!this.post({ v: PROTOCOL, type: "snapshot", from: this.group.id, to, request, part, parts, seqs, entries: slice }))
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
})
|
|
1376
|
+
.catch((error) => this.group.onError(error));
|
|
1377
|
+
}
|
|
1378
|
+
/** A chunk of a snapshot. The tab keeps the chunks of its last request from one leader, and applies them after the last one. */
|
|
1379
|
+
onSnapshot(message) {
|
|
1380
|
+
const pending = this.snapshot;
|
|
1381
|
+
if (!pending || message.to !== this.group.id || message.request !== pending.request)
|
|
1382
|
+
return;
|
|
1383
|
+
if (pending.from === undefined) {
|
|
1384
|
+
pending.from = message.from;
|
|
1385
|
+
pending.parts = message.parts;
|
|
1386
|
+
pending.seqs = message.seqs;
|
|
1387
|
+
}
|
|
1388
|
+
else if (message.from !== pending.from || message.parts !== pending.parts) {
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
1391
|
+
if (pending.chunks[message.part] !== undefined)
|
|
1392
|
+
return;
|
|
1393
|
+
pending.chunks[message.part] = message.entries;
|
|
1394
|
+
pending.received++;
|
|
1395
|
+
if (pending.received < message.parts)
|
|
1396
|
+
this.waitSnapshot();
|
|
1397
|
+
else
|
|
1398
|
+
this.applySnapshot(pending);
|
|
1399
|
+
}
|
|
1400
|
+
/**
|
|
1401
|
+
* This method writes the entries of a snapshot to the local copy, and then the batches that
|
|
1402
|
+
* waited. It skips the batches that the snapshot holds, and the entries of the ids that this tab
|
|
1403
|
+
* wrote after the snapshot.
|
|
1404
|
+
*/
|
|
1405
|
+
applySnapshot(pending) {
|
|
1406
|
+
this.snapshot = undefined;
|
|
1407
|
+
this.cancelSnapshotWait?.();
|
|
1408
|
+
this.cancelSnapshotWait = undefined;
|
|
1409
|
+
const seqs = new Map(pending.seqs ?? []);
|
|
1410
|
+
const mine = seqs.get(this.group.id) ?? 0;
|
|
1411
|
+
const entries = [];
|
|
1412
|
+
if (this.ownClear <= mine) {
|
|
1413
|
+
for (const chunk of pending.chunks) {
|
|
1414
|
+
for (const [id, value] of chunk ?? [])
|
|
1415
|
+
if ((this.ownWrites.get(id) ?? 0) <= mine)
|
|
1416
|
+
entries.push([id, value]);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
this.ownWrites.clear();
|
|
1420
|
+
this.ownClear = 0;
|
|
1421
|
+
seqs.delete(this.group.id);
|
|
1422
|
+
// A batch that the snapshot holds and that comes later is not new
|
|
1423
|
+
for (const [from, seq] of seqs)
|
|
1424
|
+
if (seq > (this.seen.get(from) ?? 0))
|
|
1425
|
+
this.seen.set(from, seq);
|
|
1426
|
+
const origin = this.hybridOptions.syncOrigin ?? "sync";
|
|
1427
|
+
this.incoming = this.incoming
|
|
1428
|
+
.then(async () => {
|
|
1429
|
+
if (this.closed)
|
|
1430
|
+
return;
|
|
1431
|
+
if (entries.length > 0)
|
|
1432
|
+
await this.local.setBatch(entries, this.writeOptions);
|
|
1433
|
+
for (const [from, seq] of seqs)
|
|
1434
|
+
this.applied.set(from, Math.max(this.applied.get(from) ?? 0, seq));
|
|
1435
|
+
if (!this.closed && entries.length > 0)
|
|
1436
|
+
this.observers.emit(entries.map(([id, value]) => ({ type: "set", id, value, origin })));
|
|
1437
|
+
})
|
|
1438
|
+
.catch((error) => this.group.onError(error));
|
|
1439
|
+
for (const { from, batch } of this.held.splice(0))
|
|
1440
|
+
if (batch.seq > (seqs.get(from) ?? 0))
|
|
1441
|
+
this.apply(from, batch);
|
|
1442
|
+
}
|
|
1443
|
+
/** This method ends a snapshot that waits, without entries: the batches that waited reach the local copy. */
|
|
1444
|
+
endSnapshot() {
|
|
1445
|
+
if (!this.snapshot)
|
|
1446
|
+
return;
|
|
1447
|
+
this.snapshot = undefined;
|
|
1448
|
+
this.cancelSnapshotWait?.();
|
|
1449
|
+
this.cancelSnapshotWait = undefined;
|
|
1450
|
+
this.ownWrites.clear();
|
|
1451
|
+
this.ownClear = 0;
|
|
1452
|
+
for (const { from, batch } of this.held.splice(0))
|
|
1453
|
+
this.apply(from, batch);
|
|
1454
|
+
}
|
|
1455
|
+
/** This method records the ids of an own batch while the snapshot waits. */
|
|
1456
|
+
noteOwn(changes, seq) {
|
|
1457
|
+
for (const change of changes) {
|
|
1458
|
+
if (change.type === "clear")
|
|
1459
|
+
this.ownClear = seq;
|
|
1460
|
+
else
|
|
1461
|
+
this.ownWrites.set(change.id, seq);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
/**
|
|
1466
|
+
* The settings of the options of a collection.
|
|
1467
|
+
*
|
|
1468
|
+
* @throws TypeError for a local copy without `observe()`, an unknown `copy`, no `remote`, or a snapshot of a shared copy
|
|
1469
|
+
* @throws RangeError for a `snapshot.chunk` that is not a whole number of 1 or more
|
|
1470
|
+
*/
|
|
1471
|
+
function collectionSettings(options, owner) {
|
|
1472
|
+
if (typeof options?.local?.observe !== "function")
|
|
1473
|
+
throw new TypeError(`${owner} needs a local copy with observe()`);
|
|
1474
|
+
if (options.copy !== "shared" && options.copy !== "tab") {
|
|
1475
|
+
throw new TypeError(`Unknown copy ${JSON.stringify(options.copy)}: use shared or tab`);
|
|
740
1476
|
}
|
|
1477
|
+
if (typeof options.remote !== "function")
|
|
1478
|
+
throw new TypeError(`${owner} needs the function remote`);
|
|
1479
|
+
const snapshot = options.snapshot ?? false;
|
|
1480
|
+
const want = snapshot !== false;
|
|
1481
|
+
if (want && options.copy !== "tab")
|
|
1482
|
+
throw new TypeError('The option snapshot needs copy: "tab": the tabs of a shared copy have the entries');
|
|
1483
|
+
const details = typeof snapshot === "object" && snapshot !== null ? snapshot : {};
|
|
1484
|
+
const chunk = details.chunk ?? SNAPSHOT_CHUNK;
|
|
1485
|
+
if (!(Number.isSafeInteger(chunk) && chunk >= 1))
|
|
1486
|
+
throw new RangeError(`snapshot.chunk must be a whole number of 1 or more, not ${chunk}`);
|
|
1487
|
+
const timeout = optionalMs(details.timeout, "snapshot.timeout") ?? SNAPSHOT_TIMEOUT;
|
|
1488
|
+
if (!(timeout > 0))
|
|
1489
|
+
throw new RangeError(`snapshot.timeout must be a time of more than zero, not ${timeout} ms`);
|
|
1490
|
+
return { snapshot: { want, chunk, timeout } };
|
|
741
1491
|
}
|
|
742
1492
|
/**
|
|
743
1493
|
* The backoff of the option `retry`.
|
|
@@ -891,6 +1641,10 @@ function isBatch(value) {
|
|
|
891
1641
|
return type === "delete" || (type === "set" && "value" in change);
|
|
892
1642
|
});
|
|
893
1643
|
}
|
|
1644
|
+
/** True when a value is a list of pairs whose first value is a string, and whose second value passes `second`. */
|
|
1645
|
+
function isPairs(value, second) {
|
|
1646
|
+
return Array.isArray(value) && value.every((pair) => Array.isArray(pair) && pair.length === 2 && typeof pair[0] === "string" && second(pair[1]));
|
|
1647
|
+
}
|
|
894
1648
|
/**
|
|
895
1649
|
* True when a value is a message of the channel, of this version. A message of a type that this
|
|
896
1650
|
* release does not know is not a message: a tab of an older release ignores a new type.
|
|
@@ -901,6 +1655,8 @@ function isTabMessage(value) {
|
|
|
901
1655
|
const message = value;
|
|
902
1656
|
if (message.v !== PROTOCOL || typeof message.from !== "string")
|
|
903
1657
|
return false;
|
|
1658
|
+
if (message.collection !== undefined && typeof message.collection !== "string")
|
|
1659
|
+
return false;
|
|
904
1660
|
switch (message.type) {
|
|
905
1661
|
case "changes":
|
|
906
1662
|
return isSeq(message.seq) && isBatch(message.changes);
|
|
@@ -919,6 +1675,17 @@ function isTabMessage(value) {
|
|
|
919
1675
|
return typeof message.to === "string" && isSeq(message.seq);
|
|
920
1676
|
case "status":
|
|
921
1677
|
return isSeq(message.seq) && isPortableStatus(message.status);
|
|
1678
|
+
case "snapshot-request":
|
|
1679
|
+
return isSeq(message.request);
|
|
1680
|
+
case "snapshot":
|
|
1681
|
+
return (typeof message.to === "string" &&
|
|
1682
|
+
isSeq(message.request) &&
|
|
1683
|
+
isSeq(message.parts) &&
|
|
1684
|
+
message.parts >= 1 &&
|
|
1685
|
+
isSeq(message.part) &&
|
|
1686
|
+
message.part < message.parts &&
|
|
1687
|
+
isPairs(message.seqs, isSeq) &&
|
|
1688
|
+
isPairs(message.entries, () => true));
|
|
922
1689
|
default:
|
|
923
1690
|
return false;
|
|
924
1691
|
}
|