@procedures/store 0.18.0 → 0.19.0

package/dist/storage/tab-coordinator.js

@@ -18,6 +18,12 @@
  *   acknowledged.
  * - Without the Web Locks API or `BroadcastChannel`, the coordinator is in the single mode: the tab
  *   is the leader at once, and it opens no channel.
+ * - A tab becomes the leader only after its `HybridStorage` loaded the queue. When the setup fails
+ *   (the function `remote` throws, or the load of the queue fails), the tab reports the error and
+ *   releases the lock. It stays a follower, and it asks for the lock again after a backoff (roadmap
+ *   CLI-67). Thus a failure that goes away does not leave the browser without a sync.
+ * - The leader sends the status of its sync to the followers: at each change, and in answer to a
+ *   hello. `getStatus()` and `onStatus()` give the status in each tab (roadmap CLI-65).
  *
  * The limits:
  * - When the leader stops during a push, the next leader sends the operation again: the delivery to
@@ -31,8 +37,15 @@
  * reads `navigator.locks` and `BroadcastChannel` when it starts, and a test gives its own.
  */
 import { StorageListeners, reportListenerError } from "@procedures/collections";
+import { optionalMs, realClock } from "@procedures/client/browser";
+import { backoffDelay } from "@mark1russell7/service/backoff";
 import { HybridStorage } from "./hybrid.js";
-/** The version of the messages. A tab ignores a message of another version. */
+/**
+ * The version of the messages. A tab ignores a message of another version, and a message of a type
+ * that it does not know. Thus a new type of message (`status`, roadmap CLI-65) keeps the version 1:
+ * a tab of an older release ignores it, and it still copies and sends the batches. A new version
+ * stops the batches between the tabs of two releases, so only a change of a known message needs it.
+ */
 const PROTOCOL = 1;
 /**
  * One leader tab runs the sync of a `HybridStorage`, and the other tabs copy its changes.
@@ -55,6 +68,7 @@ const PROTOCOL = 1;
  * });
  *
  * users.observe((batch) => render(batch)); // The changes of this tab and of the other tabs
+ * users.onStatus((status) => showSync(status)); // The status of the sync of the leader, in each tab
  * await local.set("1", { name: "Ada" }); // The leader pushes the write
  * ```
  */
@@ -73,11 +87,31 @@ export class TabCoordinator {
     onError;
     observers;
     roleListeners;
+    statusListeners;
     /** The observers that the `HybridStorage` of the leader adds to its local copy: the capture */
     captures;
+    /** The backoff of a new try after a failed setup, its clock and its random numbers */
+    backoff;
+    clock;
+    random;
     currentRole = "follower";
+    /** The lock manager, or undefined in the single mode */
+    locks;
     channel;
     hybrid;
+    /** The storage of this tab while it loads its queue, before the tab leads */
+    candidate;
+    /** The number of the failed setups in a row */
+    failures = 0;
+    /** The stop of the wait before the next try */
+    cancelRetry;
+    /** The status of the sync: of the `HybridStorage` of this tab, or of the leader */
+    status;
+    /** In the leader: the number of its last status. In a follower: the tab and the number of the last status that it got. */
+    statusSeq = 0;
+    statusFrom;
+    /** The stop of the status of the `HybridStorage` of this tab */
+    stopStatus;
     /** The stop of the observer of the changes of this tab: on the local copy, or on the `HybridStorage` of the leader */
     stopTap;
     /** True while the leader gives a batch of another tab to the capture of its `HybridStorage` */
@@ -108,6 +142,7 @@ export class TabCoordinator {
         }
         if (typeof options.remote !== "function")
             throw new TypeError("TabCoordinator needs the function remote");
+        this.backoff = retryBackoff(options.retry);
         this.id = randomId();
         this.lockName = `@procedures/store/tab:${options.name}`;
         this.local = options.local;
@@ -119,11 +154,16 @@ export class TabCoordinator {
         this.onError = options.onError ?? reportListenerError;
         this.observers = new StorageListeners(this.onError);
         this.roleListeners = new StorageListeners(this.onError);
+        this.statusListeners = new StorageListeners(this.onError);
         this.captures = new StorageListeners(this.onError);
+        this.clock = options.clock ?? this.hybridOptions.clock ?? realClock;
+        this.random = options.random ?? this.hybridOptions.random ?? Math.random;
+        // The observer of a follower. A tab that leads stops it. A single tab whose setup failed keeps its batches with it.
+        this.stopTap = this.local.observe((batch) => this.onFollowerBatch(batch));
         const locks = options.locks === undefined ? globalLocks() : (options.locks ?? undefined);
         const openChannel = options.channel === undefined ? globalChannelFactory() : (options.channel ?? undefined);
         if (!locks || !openChannel) {
-            this.lead("single");
+            this.startSingle();
             return;
         }
         try {
@@ -131,11 +171,11 @@ export class TabCoordinator {
         }
         catch (error) {
             this.onError(error);
-            this.lead("single");
+            this.startSingle();
             return;
         }
+        this.locks = locks;
         this.channel.onmessage = (event) => this.receive(event?.data);
-        this.stopTap = this.local.observe((batch) => this.onFollowerBatch(batch));
         this.hello();
         this.elect(locks);
     }
@@ -148,9 +188,9 @@ export class TabCoordinator {
         return this.currentRole === "leader" || this.currentRole === "single";
     }
     /**
-     * The `HybridStorage` of the sync while this tab runs it, or else undefined. Its `getStatus()`
-     * gives the state of the sync. Write to the local copy, not to this storage: each tab then writes
-     * in the same way.
+     * The `HybridStorage` of the sync while this tab runs it, or else undefined. `getStatus()` of the
+     * coordinator gives the state of the sync in each tab. Write to the local copy, not to this
+     * storage: each tab then writes in the same way.
      */
     get syncStorage() {
         return this.hybrid;
@@ -180,10 +220,39 @@ export class TabCoordinator {
             throw new Error("The TabCoordinator is closed");
         return this.roleListeners.add(listener);
     }
+    /**
+     * The status of the sync (roadmap CLI-65), or undefined before the first status and after
+     * `close()`.
+     *
+     * - In the leader and in the single mode, it is the status of the `HybridStorage` of this tab.
+     * - In a follower, it is the last status that the leader sent. A new tab gets it in the answer to
+     *   its hello. After a change of the leader, a follower keeps the last status until the new
+     *   leader sends its status.
+     * - In a follower, `error.error` is a copy of the error of the leader: an `Error` with its `name`,
+     *   its `message` and its `code`. The copy has no `cause`.
+     */
+    getStatus() {
+        return this.status;
+    }
+    /**
+     * This method starts to send the status of the sync to a listener at each change (roadmap
+     * CLI-65): in the leader, each status of its `HybridStorage`, and in a follower, each status that
+     * the leader sends. The listener gets each status one time. When the tab becomes the leader, the
+     * listener gets the status of its own `HybridStorage`.
+     *
+     * @returns The function that stops the statuses. A second use of it does nothing.
+     * @throws Error after `close()`
+     */
+    onStatus(listener) {
+        if (this.closed)
+            throw new Error("The TabCoordinator is closed");
+        return this.statusListeners.add(listener);
+    }
     /**
      * This method closes the coordinator.
      *
-     * - A follower stops its lock request. It does not become the leader.
+     * - A follower stops its lock request, or its wait before a new try. It does not become the leader.
+     * - A tab that loads its queue to become the leader closes its `HybridStorage`, and releases the lock.
      * - The leader closes its `HybridStorage`: it sends the queue, and the queue store keeps the
      *   operations that the remote did not take. Then the leader releases the lock, and the next
      *   tab becomes the leader.
@@ -202,9 +271,17 @@ export class TabCoordinator {
         this.stopTap = undefined;
         this.abortRequest?.();
         this.abortRequest = undefined;
+        this.cancelRetry?.();
+        this.cancelRetry = undefined;
+        this.stopStatus?.();
+        this.stopStatus = undefined;
         this.setRole("closed");
-        const hybrid = this.hybrid;
+        const candidate = this.candidate;
+        this.candidate = undefined;
+        candidate?.stop();
+        const hybrid = this.hybrid ?? candidate?.hybrid;
         this.hybrid = undefined;
+        this.status = undefined;
         try {
             // The leader holds the lock until its queue reached the remote or its queue store
             if (hybrid)
@@ -226,6 +303,7 @@ export class TabCoordinator {
             this.channel = undefined;
             this.observers.clear();
             this.roleListeners.clear();
+            this.statusListeners.clear();
             this.captures.clear();
         }
     }
@@ -245,10 +323,12 @@ export class TabCoordinator {
                 if (this.closed)
                     return;
                 this.abortRequest = undefined;
-                // A HybridStorage that cannot start releases the lock: another tab tries
-                if (!this.lead("leader"))
+                if (await this.takeLead()) {
+                    await held;
                     return;
-                await held;
+                }
+                // A failed setup releases the lock: another tab tries, and this tab tries again after the backoff
+                this.retryLater();
             });
         }
         catch (error) {
@@ -261,52 +341,153 @@ export class TabCoordinator {
             // The lock manager refused the request: this tab runs the sync alone
             this.onError(error);
             this.stopChannel();
-            this.lead("single");
+            this.locks = undefined;
+            this.startSingle();
         });
     }
     /**
-     * This method makes the `HybridStorage` of the sync: the tab becomes the leader, or the single
-     * tab. The capture of the storage gets the batches of this tab that no leader acknowledged.
+     * This method makes the `HybridStorage` of a tab that got the lock, and waits for the load of its
+     * queue. Then the tab becomes the leader. Until then, the tab is a follower: its own batches go to
+     * the other tabs and to its outbox, and the capture of the storage ignores the local copy.
      *
-     * @returns False when the storage could not start
+     * @returns False when the setup failed (the error went to `onError`), or after `close()`
      */
-    lead(role) {
-        if (this.closed)
+    async takeLead() {
+        const candidate = this.makeCandidate();
+        if (!candidate)
             return false;
+        this.candidate = candidate;
+        let failed = false;
+        let failure;
+        try {
+            // A storage whose load failed refuses each operation: it cannot lead
+            await candidate.hybrid.whenQueued();
+        }
+        catch (error) {
+            failed = true;
+            failure = error;
+        }
+        // close() took the storage
+        if (this.candidate !== candidate)
+            return false;
+        this.candidate = undefined;
+        if (!failed) {
+            this.promote(candidate, "leader");
+            return true;
+        }
+        this.onError(failure);
+        candidate.stop();
+        try {
+            await candidate.hybrid.close();
+        }
+        catch (error) {
+            this.onError(error);
+        }
+        return false;
+    }
+    /** This method makes the `HybridStorage` of the single mode. The tab tries again after the backoff when it fails. */
+    startSingle() {
+        const candidate = this.makeCandidate();
+        if (candidate)
+            this.promote(candidate, "single");
+        else
+            this.retryLater();
+    }
+    /**
+     * This method makes the `HybridStorage` of the sync, with its observer. Its capture ignores the
+     * local copy until the tab leads.
+     *
+     * @returns Undefined when the remote or the storage could not start (the error went to `onError`), or after `close()`
+     */
+    makeCandidate() {
+        if (this.closed)
+            return undefined;
+        let remote;
+        try {
+            remote = this.makeRemote();
+        }
+        catch (error) {
+            this.onError(error);
+            return undefined;
+        }
+        const gate = { open: false };
         let hybrid;
         try {
-            hybrid = new HybridStorage(this.makeRemote(), {
+            hybrid = new HybridStorage(remote, {
                 ...this.hybridOptions,
-                local: this.captureCopy(),
+                local: this.captureCopy(gate),
                 capture: "observe",
                 origin: this.tag,
             });
         }
         catch (error) {
             this.onError(error);
-            return false;
+            // The remote of a storage that did not start stays open otherwise
+            Promise.resolve()
+                .then(() => remote.close())
+                .catch((closeError) => this.onError(closeError));
+            return undefined;
         }
+        // The observer starts at once: the other tabs get the changes of the load and of the first sync
+        const stop = hybrid.observe((batch) => this.onLeaderBatch(batch));
+        return { hybrid, gate, stop };
+    }
+    /**
+     * This method makes this tab the leader, or the single tab, with the storage of a candidate. The
+     * capture of the storage gets the batches of this tab that no leader acknowledged.
+     */
+    promote(candidate, role) {
+        const { hybrid, gate, stop } = candidate;
         this.hybrid = hybrid;
-        // The new observer starts before the old one stops: no change of the local copy goes between them
+        this.failures = 0;
+        // The capture starts and the observer of the follower stops in one step: no change of the local copy goes between them
+        gate.open = true;
         const stopFollower = this.stopTap;
-        this.stopTap = hybrid.observe((batch) => this.onLeaderBatch(batch));
+        this.stopTap = stop;
         stopFollower?.();
         for (const batch of this.outbox.splice(0))
             this.capture(batch.changes);
         this.setRole(role);
         if (role === "leader")
             this.post({ v: PROTOCOL, type: "leader", from: this.id });
-        return true;
+        // The status of this tab is now the status of its storage, also when it has the values of the last status of the old leader. The followers get it.
+        this.status = undefined;
+        this.statusFrom = undefined;
+        this.stopStatus = hybrid.onStatus((status) => this.onOwnStatus(hybrid, status));
+        this.onOwnStatus(hybrid, hybrid.getStatus());
+    }
+    /** This method asks for the lock again, or tries the single mode again, after the backoff (roadmap CLI-67). */
+    retryLater() {
+        if (this.closed)
+            return;
+        this.failures++;
+        const wait = backoffDelay(this.backoff, this.failures, this.random);
+        const cancel = this.clock.after(wait, () => {
+            this.cancelRetry = undefined;
+            if (this.closed)
+                return;
+            if (this.locks)
+                this.elect(this.locks);
+            else
+                this.startSingle();
+        });
+        // The wait must not keep a Node process alive
+        cancel.unref?.();
+        this.cancelRetry = cancel;
     }
     /**
      * The local copy for the `HybridStorage` of the leader. Its `observe()` gives the changes of the
-     * local copy and the batches of the other tabs, so the capture queues both. Its `close()` does
-     * nothing: the application owns the local copy.
+     * local copy and the batches of the other tabs, so the capture queues both. It gives the changes
+     * of the local copy only when `gate` is open. Its `close()` does nothing: the application owns
+     * the local copy.
      */
-    captureCopy() {
+    captureCopy(gate) {
         const captures = this.captures;
         const observe = (observer) => {
-            const stopLocal = this.local.observe(observer);
+            const stopLocal = this.local.observe((batch) => {
+                if (gate.open)
+                    observer(batch);
+            });
             const stopCapture = captures.add(observer);
             return () => {
                 stopLocal();
@@ -347,6 +528,39 @@ export class TabCoordinator {
         this.roleListeners.emit(role);
     }
     //
+    // ═══ The status of the sync (roadmap CLI-65) ═══
+    //
+    /** A status of the `HybridStorage` of this tab. The leader sends each new status to the followers. */
+    onOwnStatus(hybrid, status) {
+        if (this.closed || this.hybrid !== hybrid)
+            return;
+        if (this.status !== undefined && sameStatus(this.status, status))
+            return;
+        this.status = status;
+        this.statusSeq++;
+        if (this.currentRole === "leader")
+            this.postStatus();
+        this.statusListeners.emit(status);
+    }
+    /** This method sends the last status of the leader, with its number. */
+    postStatus() {
+        if (this.status === undefined)
+            return;
+        this.post({ v: PROTOCOL, type: "status", from: this.id, seq: this.statusSeq, status: portableStatus(this.status) });
+    }
+    /** A status of the leader. A tab that runs the sync keeps its own status. */
+    onLeaderStatus(from, seq, status) {
+        if (this.hybrid)
+            return;
+        // The answer to a hello sends the last status again
+        if (from === this.statusFrom && seq <= this.statusSeq)
+            return;
+        this.statusFrom = from;
+        this.statusSeq = seq;
+        this.status = statusOf(status);
+        this.statusListeners.emit(this.status);
+    }
+    //
     // ═══ The batches of this tab ═══
     //
     /**
@@ -369,12 +583,15 @@ export class TabCoordinator {
             return;
         this.share(batch, false);
     }
-    /** This method gives a batch of this tab to its observers and to the other tabs. A follower keeps it for the leader. */
+    /**
+     * This method gives a batch of this tab to its observers and to the other tabs. A follower keeps
+     * it for the leader. A single tab whose setup failed keeps it for its next try.
+     */
     share(changes, keep) {
         const seq = ++this.seq;
         this.observers.emit(changes);
         const sent = this.post({ v: PROTOCOL, type: "changes", from: this.id, seq, changes });
-        if (sent && keep)
+        if (keep && (sent || !this.channel))
             this.outbox.push({ seq, changes });
     }
     /** This method tells the leader the batches of this tab that no leader acknowledged. */
@@ -430,6 +647,9 @@ export class TabCoordinator {
                 if (data.to === this.id)
                     this.outbox = this.outbox.filter((batch) => batch.seq > data.seq);
                 break;
+            case "status":
+                this.onLeaderStatus(data.from, data.seq, data.status);
+                break;
         }
     }
     /**
@@ -447,7 +667,8 @@ export class TabCoordinator {
     }
     /**
      * The hello of another tab. The leader captures the batches that it did not capture, and then each
-     * batch of the tab after `seq`. Each tab copies the batches that it did not see.
+     * batch of the tab after `seq`. It answers with its status. Each tab copies the batches that it
+     * did not see.
      */
     onHello(from, seq, pending) {
         if (this.hybrid && this.currentRole === "leader") {
@@ -458,6 +679,8 @@ export class TabCoordinator {
             this.captured.set(from, Math.max(known ?? 0, seq));
             if (pending.length > 0)
                 this.acknowledge(from, seq);
+            // A new tab gets the status now. The other tabs have its number already.
+            this.postStatus();
         }
         for (const batch of pending)
             this.copy(from, batch);
@@ -516,6 +739,120 @@ export class TabCoordinator {
         hybrid.whenQueued().then(() => void this.post({ v: PROTOCOL, type: "ack", from: this.id, to, seq }), (error) => this.onError(error));
     }
 }
+/**
+ * The backoff of the option `retry`.
+ *
+ * @throws RangeError for a `delay` of zero or less, a `factor` below 1, or a `jitter` outside 0 to 1
+ * @throws TypeError for a time that is not a `Duration`
+ */
+function retryBackoff(retry) {
+    const delay = optionalMs(retry?.delay, "retry.delay") ?? 1000;
+    const maxDelay = optionalMs(retry?.maxDelay, "retry.maxDelay") ?? 60_000;
+    const factor = retry?.factor ?? 2;
+    const jitter = retry?.jitter ?? 0.5;
+    if (!(delay > 0))
+        throw new RangeError(`retry.delay must be a time of more than zero, not ${delay} ms`);
+    if (!(Number.isFinite(factor) && factor >= 1))
+        throw new RangeError(`retry.factor must be a number of 1 or more, not ${factor}`);
+    if (!(jitter >= 0 && jitter <= 1))
+        throw new RangeError(`retry.jitter must be from 0 to 1, not ${jitter}`);
+    return {
+        base: delay,
+        factor,
+        cap: Math.max(maxDelay, delay),
+        jitter: jitter > 0 ? { kind: "subtract", fraction: jitter } : { kind: "none" },
+    };
+}
+/** True when two statuses have the same values. The errors are the same object. */
+function sameStatus(a, b) {
+    return (a.connected === b.connected &&
+        a.endpoint === b.endpoint &&
+        a.lastSync === b.lastSync &&
+        a.pendingChanges === b.pendingChanges &&
+        a.syncing === b.syncing &&
+        a.mode === b.mode &&
+        a.error === b.error);
+}
+/**
+ * The status in a form that a structured clone keeps. The status has numbers, strings and booleans,
+ * and they clone. Its error can be any value, for example an error with a function in a property,
+ * so the message has only the name, the message and the code of the error.
+ */
+function portableStatus(status) {
+    return {
+        connected: status.connected,
+        pendingChanges: status.pendingChanges,
+        syncing: status.syncing,
+        ...(status.endpoint !== undefined && { endpoint: status.endpoint }),
+        ...(status.lastSync !== undefined && { lastSync: status.lastSync }),
+        ...(status.mode !== undefined && { mode: status.mode }),
+        ...(status.error !== undefined && { error: { operation: status.error.operation, ...portableError(status.error.error) } }),
+    };
+}
+/** The name, the message and the code of an error. A value that is not an error becomes its text. */
+function portableError(value) {
+    try {
+        if (typeof value === "object" && value !== null) {
+            const { name, message, code } = value;
+            if (typeof message === "string") {
+                return {
+                    name: typeof name === "string" ? name : "Error",
+                    message,
+                    ...((typeof code === "string" || typeof code === "number") && { code }),
+                };
+            }
+        }
+        return { name: "Error", message: String(value) };
+    }
+    catch {
+        // A value without a text, for example an object without a prototype
+        return { name: "Error", message: "An error without a text" };
+    }
+}
+/** The status of a message of the leader. Its error becomes an `Error` again. */
+function statusOf(status) {
+    const error = status.error;
+    return {
+        connected: status.connected,
+        pendingChanges: status.pendingChanges,
+        syncing: status.syncing,
+        ...(status.endpoint !== undefined && { endpoint: status.endpoint }),
+        ...(status.lastSync !== undefined && { lastSync: status.lastSync }),
+        ...(status.mode !== undefined && { mode: status.mode }),
+        ...(error !== undefined && {
+            error: {
+                operation: error.operation,
+                error: Object.assign(new Error(error.message), { name: error.name }, error.code !== undefined ? { code: error.code } : {}),
+            },
+        }),
+    };
+}
+const MODES = new Set(["local-only", "write-through", "write-back", "remote-only"]);
+const OPERATIONS = new Set(["load", "push", "pull", "sync", "store"]);
+/** True when a value is the status of a message. */
+function isPortableStatus(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const status = value;
+    if (typeof status.connected !== "boolean" || typeof status.syncing !== "boolean" || !isSeq(status.pendingChanges))
+        return false;
+    if (status.endpoint !== undefined && typeof status.endpoint !== "string")
+        return false;
+    if (status.lastSync !== undefined && typeof status.lastSync !== "number")
+        return false;
+    if (status.mode !== undefined && !(typeof status.mode === "string" && MODES.has(status.mode)))
+        return false;
+    if (status.error === undefined)
+        return true;
+    if (typeof status.error !== "object" || status.error === null)
+        return false;
+    const { operation, name, message, code } = status.error;
+    return (typeof operation === "string" &&
+        OPERATIONS.has(operation) &&
+        typeof name === "string" &&
+        typeof message === "string" &&
+        (code === undefined || typeof code === "string" || typeof code === "number"));
+}
 /** A new random id. */
 function randomId() {
     return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
@@ -554,7 +891,10 @@ function isBatch(value) {
         return type === "delete" || (type === "set" && "value" in change);
     });
 }
-/** True when a value is a message of the channel, of this version. */
+/**
+ * True when a value is a message of the channel, of this version. A message of a type that this
+ * release does not know is not a message: a tab of an older release ignores a new type.
+ */
 function isTabMessage(value) {
     if (typeof value !== "object" || value === null)
         return false;
@@ -577,6 +917,8 @@ function isTabMessage(value) {
             return true;
         case "ack":
             return typeof message.to === "string" && isSeq(message.seq);
+        case "status":
+            return isSeq(message.seq) && isPortableStatus(message.status);
         default:
             return false;
     }