@procedures/store 0.48.1 → 0.49.0

package/dist/storage/hybrid.js

@@ -49,7 +49,7 @@ import { InMemoryStorage, StorageListeners } from "@procedures/collections";
 import { queryEntries, queryOptionsProblems, selectorProblems, versionConflictsOf, } from "@procedures/client/query";
 import { optionalMs, realClock } from "@procedures/client/browser";
 import { backoffDelay } from "@mark1russell7/service/backoff";
-import { MemoryQueueStore, idsOf, withoutId } from "./queue.js";
+import { MemoryQueueStore, idsOf, singlesOf, withoutId } from "./queue.js";
 import { isSyncState } from "./sync-state.js";
 const MODES = new Set(["local-only", "write-through", "write-back", "remote-only", "read-only"]);
 /** True for the modes that keep a local copy and read the remote changes into it. */
@@ -283,6 +283,8 @@ const defaultIsRefusal = (error) => {
     const code = error?.code;
     return typeof code === "string" && REFUSAL_CODES.has(code);
 };
+/** The default of `isTooLarge`: an error with the code `PAYLOAD_TOO_LARGE` (roadmap CLI-108). */
+const defaultIsTooLarge = (error) => error?.code === "PAYLOAD_TOO_LARGE";
 /** True when a remote does not know a procedure: an older server without `changes`. */
 const isNotFound = (error) => typeof error === "object" && error !== null && error.code === "NOT_FOUND";
 const defaultKeyOf = (item) => {
@@ -469,16 +471,20 @@ class BaseVersions extends Map {
     changed = new Set();
     /** True when the storage has a sync state store */
     tracked = false;
+    /** This function gets each change of a version, after the change. */
+    onChange;
     set(id, version) {
         super.set(id, version);
         if (this.tracked)
             this.changed.add(id);
+        this.onChange?.(id, version);
         return this;
     }
     delete(id) {
         const had = super.delete(id);
         if (had && this.tracked)
             this.changed.add(id);
+        this.onChange?.(id, undefined);
         return had;
     }
 }
@@ -569,6 +575,13 @@ export class HybridStorage {
     changesUnsupported = false;
     /** With `versionOf`: the last known remote version of each id. `null`: the remote has no item. */
     bases = new BaseVersions();
+    /**
+     * With `versionOf`, the policy `merge` and `capture: "observe"`, this map holds the remote value at the known remote
+     * version of each id (roadmap CLI-109). A captured change has no value before it, so its base value comes from here.
+     * A write of the storage reads its base value from the local copy, and needs no map. A new version of an id drops
+     * its value. A sync or a send gives the value of the new version.
+     */
+    baseValues = new Map();
     /**
      * With `versionOf`: the version of each local item that the storage wrote or read. A pull does
      * not read an item whose version it knows. The version of an id that is not here is unknown.
@@ -600,6 +613,10 @@ export class HybridStorage {
     resetPull = false;
     /** The stop of the capture of the local copy */
     stopCapture;
+    /** True with `capture: "observe"`. */
+    capturing;
+    /** True when a queued write keeps its base value: with `versionOf` and the policy `merge` (roadmap CLI-109). */
+    keepsBaseValues;
     /** The captured changes that came before the end of the load. Undefined after the load. */
     captured = [];
     /** With `versionOf`: the versions of the own writes that wait for their echo */
@@ -630,6 +647,7 @@ export class HybridStorage {
         cacheMisses: 0,
         syncCount: 0,
         conflicts: 0,
+        dropped: 0,
     };
     constructor(remote, options = {}) {
         // The local copy tests the preconditions of the callers (roadmap CLI-16): it gets `versionOf`
@@ -652,7 +670,9 @@ export class HybridStorage {
             offlineQueue: options.offlineQueue ?? true,
             isUnavailable: options.isUnavailable ?? defaultIsUnavailable,
             isRefusal: options.isRefusal ?? defaultIsRefusal,
+            isTooLarge: options.isTooLarge ?? defaultIsTooLarge,
             ...(options.onRefused !== undefined && { onRefused: options.onRefused }),
+            ...(options.onDropped !== undefined && { onDropped: options.onDropped }),
         };
         this.statusListeners = new StorageListeners(options.onListenerError);
         this.queryPolicy = options.queryPolicy;
@@ -667,6 +687,10 @@ export class HybridStorage {
         this.aroundRemoteApply = options.aroundRemoteApply;
         this.stateStore = options.syncState;
         this.bases.tracked = this.stateStore !== undefined;
+        this.bases.onChange = (id, version) => {
+            if (this.baseValues.get(id)?.version !== version)
+                this.baseValues.delete(id);
+        };
         this.localGiven = options.local !== undefined;
         // The timers on the clock, with the backoff after a failure (roadmap CLI-13)
         const factor = options.backoff?.factor ?? 2;
@@ -703,6 +727,8 @@ export class HybridStorage {
         const capture = options.capture ?? "none";
         if (capture !== "none" && capture !== "observe")
             throw new TypeError(`Unknown capture ${JSON.stringify(capture)}: use none or observe`);
+        this.capturing = capture === "observe";
+        this.keepsBaseValues = this.options.conflictResolution === "merge" && this.options.versionOf !== undefined;
         if (capture === "observe") {
             if (!this.local.observe)
                 throw new TypeError('capture: "observe" needs a local copy with observe()');
@@ -855,6 +881,10 @@ export class HybridStorage {
                 op.base = previous.base;
             else
                 delete op.base;
+            if (previous.baseValues)
+                op.baseValues = previous.baseValues;
+            else
+                delete op.baseValues;
             this.offlineOps[index] = op;
         }
         else {
@@ -1579,6 +1609,7 @@ export class HybridStorage {
                 this.ensureQueueRoom(op);
                 this.attachBase(op);
                 const undo = await this.snapshot(ids, origin);
+                this.fillBaseValues(op, undo.before);
                 const result = await applyLocal();
                 this.noteOwnWrite(op);
                 await this.enqueue(op, undo);
@@ -1618,6 +1649,7 @@ export class HybridStorage {
                         // The status shows the queue and the connection. The caller gets PendingWriteError.
                         this.isOnline = false;
                         this.attachBase(op);
+                        this.fillBaseValues(op, undo.before);
                         await this.enqueue(op, undo);
                         this.scheduleDrain();
                         throw new PendingWriteError(type, error);
@@ -1765,7 +1797,7 @@ export class HybridStorage {
         // The versions after the write: this write adds one to each id (clear adds to clearVersion)
         const versions = new Map(touched.map((id) => [id, (this.writeVersions.get(id) ?? 0) + (ids === "*" ? 0 : 1)]));
         const clearVersion = this.clearVersion + (ids === "*" ? 1 : 0);
-        return async () => {
+        const undo = async () => {
             if (this.clearVersion !== clearVersion)
                 return;
             for (const id of touched) {
@@ -1778,6 +1810,7 @@ export class HybridStorage {
                     await this.localSet(id, value, origin);
             }
         };
+        return Object.assign(undo, { before });
     }
     //
     // ═══ Sync Operations ═══
@@ -1942,6 +1975,7 @@ export class HybridStorage {
                 cacheMisses: this.stats.cacheMisses,
                 syncCount: this.stats.syncCount,
                 conflicts: this.stats.conflicts,
+                dropped: this.stats.dropped,
                 pendingOps: this.offlineOps.length,
                 isOnline: this.isOnline,
                 mode: this.mode,
@@ -2294,6 +2328,7 @@ export class HybridStorage {
                 return false;
             if (knownVersion === remoteVersion) {
                 this.bases.set(id, remoteVersion);
+                this.noteBaseValue(id, remoteVersion, item);
                 return false;
             }
             const localItem = await this.readLocal(id);
@@ -2301,15 +2336,17 @@ export class HybridStorage {
                 return false;
             if (localItem !== undefined && this.versionOfItem(localItem) === remoteVersion) {
                 this.bases.set(id, remoteVersion);
+                this.noteBaseValue(id, remoteVersion, item);
                 return false;
             }
-            return this.resolvePending(id, localItem, item, remoteVersion, conflicts);
+            return this.resolvePending(id, localItem, item, remoteVersion, conflicts, "pull");
         }
         if (knownVersion !== undefined) {
             // Only the remote changed: it is not a conflict. The version of the local item is known.
             if (knownVersion !== remoteVersion)
                 await this.localSet(id, item, this.syncOrigin);
             this.bases.set(id, remoteVersion);
+            this.noteBaseValue(id, remoteVersion, item);
             return false;
         }
         const localItem = await this.readLocal(id);
@@ -2335,8 +2372,10 @@ export class HybridStorage {
             await this.localSet(id, resolved, this.syncOrigin);
             this.stats.conflicts++;
         }
-        if (remoteVersion !== undefined)
+        if (remoteVersion !== undefined) {
             this.bases.set(id, remoteVersion);
+            this.noteBaseValue(id, remoteVersion, item);
+        }
         return false;
     }
     /** This method applies one remote delete. It gives true when the queue changed. */
@@ -2375,6 +2414,7 @@ export class HybridStorage {
                     break;
                 await this.localDelete(id, this.syncOrigin);
                 this.bases.set(id, null);
+                await this.reportDropped({ id, value: localItem, reason: "remote-delete", during: "pull" });
                 return true;
             case "error":
                 conflicts.push(id);
@@ -2390,7 +2430,7 @@ export class HybridStorage {
      * A queued local write of an id whose remote version changed: a conflict, and the policy
      * decides. True when the queue changed.
      */
-    async resolvePending(id, localItem, remoteItem, remoteVersion, conflicts) {
+    async resolvePending(id, localItem, remoteItem, remoteVersion, conflicts, during) {
         this.stats.conflicts++;
         switch (this.options.conflictResolution) {
             case "remote":
@@ -2399,16 +2439,20 @@ export class HybridStorage {
                     break;
                 await this.localSet(id, remoteItem, this.syncOrigin);
                 this.bases.set(id, remoteVersion);
+                this.noteBaseValue(id, remoteVersion, remoteItem);
+                await this.reportDropped({ id, value: localItem, reason: "conflict", during, remote: remoteItem });
                 return true;
             case "merge": {
                 if (localItem === undefined)
                     break;
-                const merged = this.resolveConflict(localItem, remoteItem);
+                const merged = this.resolveConflict(localItem, remoteItem, this.queuedBaseValue(id));
                 if (!this.dropQueued(id))
                     break;
                 await this.localSet(id, merged, this.syncOrigin);
                 this.bases.set(id, remoteVersion);
-                this.offlineOps.push({ type: "set", data: { id, value: merged }, timestamp: Date.now(), base: [[id, remoteVersion]] });
+                this.noteBaseValue(id, remoteVersion, remoteItem);
+                // The merged write changes the remote item of this conflict: that item is its base
+                this.offlineOps.push({ type: "set", data: { id, value: merged }, timestamp: Date.now(), base: [[id, remoteVersion]], baseValues: [[id, remoteItem]] });
                 return true;
             }
             case "error":
@@ -2421,6 +2465,16 @@ export class HybridStorage {
         this.bases.set(id, remoteVersion);
         return false;
     }
+    /** This method counts a dropped write and gives it to the hook `onDropped`. An error of the hook does not stop the sync. */
+    async reportDropped(drop) {
+        this.stats.dropped++;
+        try {
+            await this.options.onDropped?.(drop);
+        }
+        catch {
+            // An error of the hook does not stop the sync
+        }
+    }
     /**
      * This method removes an id from the queued operations. It gives false when a queued clear touches it: nothing
      * changes then.
@@ -2479,6 +2533,8 @@ export class HybridStorage {
     async pushQueue() {
         this.pushing = true;
         let sent = 0;
+        // True when a batch went again as one write for each id: the queue store gets the new operations
+        let split = false;
         // The operations whose conflicts this push resolved once (roadmap CLI-16)
         const resolved = new Set();
         try {
@@ -2509,18 +2565,25 @@ export class HybridStorage {
                         }
                         continue;
                     }
-                    if (conflicts === undefined &&
-                        !(error instanceof QueueStoreError) &&
-                        !this.options.isUnavailable(error) &&
-                        this.options.isRefusal(error)) {
-                        // The remote refused the write: it leaves the queue, and the push goes on (roadmap O67)
+                    const tooLarge = conflicts === undefined && !(error instanceof QueueStoreError) && this.options.isTooLarge(error);
+                    const refused = tooLarge || (conflicts === undefined && !(error instanceof QueueStoreError) && !this.options.isUnavailable(error) && this.options.isRefusal(error));
+                    if (refused) {
                         this.isOnline = true;
+                        const singles = singlesOf(op);
+                        if (singles.length > 1 && this.offlineOps[0] === op) {
+                            // A batch goes again as one write for each id, so only the bad write leaves the queue (roadmap CLI-108)
+                            this.offlineOps.splice(0, 1, ...singles);
+                            split = true;
+                            this.storeDirty = true;
+                            continue;
+                        }
+                        // The remote refused the write: it leaves the queue, and the push goes on (roadmap O67)
                         if (this.offlineOps[0] === op) {
                             this.offlineOps.shift();
                             sent++;
                             this.storeDirty = true;
                         }
-                        await this.refuseQueued(op, error);
+                        await this.refuseQueued(op, error, tooLarge ? "too-large" : "refused");
                         continue;
                     }
                     // This operation and the later ones stay queued
@@ -2540,7 +2603,7 @@ export class HybridStorage {
         }
         finally {
             this.pushing = false;
-            if (sent > 0) {
+            if (sent > 0 || split) {
                 // The store keeps sent operations, and the status gets the failure. A later direct write
                 // repairs the store first.
                 await this.persistAll().catch(() => undefined);
@@ -2552,7 +2615,7 @@ export class HybridStorage {
      * gets the remote values, and the hook `onRefused` gets the refusal. A refused clear asks for a
      * full pull.
      */
-    async refuseQueued(op, error) {
+    async refuseQueued(op, error, reason) {
         const ids = idsOf(op);
         const all = ids.includes("*");
         let restored = false;
@@ -2568,6 +2631,10 @@ export class HybridStorage {
         catch {
             // An error of the hook does not stop the queue
         }
+        for (const id of all ? ["*"] : ids) {
+            const written = this.writtenValue(op, id);
+            await this.reportDropped({ id, value: written ? written.value : undefined, reason, during: "push", error });
+        }
     }
     /**
      * This method writes the remote values of these ids into the local copy. It skips an id that a
@@ -2630,10 +2697,57 @@ export class HybridStorage {
             return;
         }
         const version = this.versionOfItem(item);
-        if (version === undefined)
+        if (version === undefined) {
             this.bases.delete(id);
-        else
-            this.bases.set(id, version);
+            return;
+        }
+        this.bases.set(id, version);
+        this.noteBaseValue(id, version, item);
+    }
+    /** This method keeps the remote value of the remote version of an id, for a captured change (roadmap CLI-109). */
+    noteBaseValue(id, version, value) {
+        if (!this.keepsBaseValues || !this.capturing)
+            return;
+        if (this.bases.get(id) === version)
+            this.baseValues.set(id, { version, value });
+    }
+    /**
+     * This method gives a write of the storage the base value of each id that has no queued write (roadmap CLI-109).
+     * The value is the local item before the write, when that item has the base version. The local copy then holds the
+     * remote value of a sync or a send.
+     */
+    fillBaseValues(op, before) {
+        if (!this.keepsBaseValues || !op.base)
+            return;
+        const values = op.baseValues ? [...op.baseValues] : [];
+        const known = new Set(values.map(([id]) => id));
+        for (const [id, version] of op.base) {
+            if (known.has(id) || version === null)
+                continue;
+            const value = before.get(id);
+            if (value !== undefined && this.versionOfItem(value) === version)
+                values.push([id, value]);
+        }
+        if (values.length > 0)
+            op.baseValues = values;
+    }
+    /** The remote value at a remote version of an id: `null` for no item, and undefined when the storage does not know it. */
+    baseValueAt(id, version) {
+        if (version === null)
+            return null;
+        const known = this.baseValues.get(id);
+        return known !== undefined && known.version === version ? known.value : undefined;
+    }
+    /**
+     * The remote value that the queued writes of an id changed: the base value of the first queued operation of the id.
+     * The value is undefined when that operation has none, or when it is a clear.
+     */
+    queuedBaseValue(id) {
+        const op = this.offlineOps.find((queued) => {
+            const ids = idsOf(queued);
+            return ids.includes(id) || ids.includes("*");
+        });
+        return op?.baseValues?.find(([valueId]) => valueId === id)?.[1];
     }
     /** This method records the remote versions after a write reached the remote. */
     noteRemoteWrite(op) {
@@ -2670,15 +2784,56 @@ export class HybridStorage {
         if (!this.options.versionOf)
             return;
         const base = [];
+        const values = [];
         for (const id of idsOf(op)) {
             if (id === "*")
                 continue;
             const version = this.expectedBase(id);
             if (version !== undefined)
                 base.push([id, version]);
+            const value = this.expectedBaseValue(id, version);
+            if (value !== undefined)
+                values.push([id, value]);
         }
         if (base.length > 0)
             op.base = base;
+        if (values.length > 0)
+            op.baseValues = values;
+    }
+    /**
+     * This method gives the remote value at the version of `expectedBase`, for the policy `merge` (roadmap CLI-109). It
+     * is the value of an older queued operation of the id. Without one, it is the kept value of the known remote version.
+     */
+    expectedBaseValue(id, version) {
+        if (!this.keepsBaseValues || version === undefined)
+            return undefined;
+        for (let index = this.offlineOps.length - 1; index >= 0; index--) {
+            const op = this.offlineOps[index];
+            const data = op.data;
+            switch (op.type) {
+                case "set":
+                    if (data.id === id)
+                        return data.value;
+                    break;
+                case "delete":
+                    if (data.id === id)
+                        return null;
+                    break;
+                case "setBatch": {
+                    const item = (data.items ?? []).findLast(([itemId]) => itemId === id);
+                    if (item)
+                        return item[1];
+                    break;
+                }
+                case "deleteBatch":
+                    if ((data.ids ?? []).includes(id))
+                        return null;
+                    break;
+                case "clear":
+                    return null;
+            }
+        }
+        return this.baseValueAt(id, version);
     }
     /** The remote version of an id after the queued operations, or undefined when it is not known. */
     expectedBase(id) {
@@ -2719,7 +2874,7 @@ export class HybridStorage {
     /**
      * Resolve conflict between local and remote versions.
      */
-    resolveConflict(local, remote) {
+    resolveConflict(local, remote, base) {
         switch (this.options.conflictResolution) {
             case "local":
                 return local;
@@ -2729,7 +2884,7 @@ export class HybridStorage {
                 if (!this.options.mergeFn) {
                     throw new Error("Merge function required for conflict resolution strategy 'merge'");
                 }
-                return this.options.mergeFn(local, remote);
+                return this.options.mergeFn(local, remote, base);
             case "error":
                 throw new Error(`Conflict detected between local and remote versions`);
             default:
@@ -2757,6 +2912,10 @@ export class HybridStorage {
                 op.base = previous.base;
             else
                 delete op.base;
+            if (previous.baseValues)
+                op.baseValues = previous.baseValues;
+            else
+                delete op.baseValues;
             this.offlineOps[index] = op;
             rollback = () => {
                 const at = this.offlineOps.indexOf(op);
@@ -3002,14 +3161,16 @@ export class HybridStorage {
                 if (remoteVersion === undefined || echo) {
                     if (remoteVersion === undefined)
                         this.bases.delete(id);
-                    else
+                    else {
                         this.bases.set(id, remoteVersion);
+                        this.noteBaseValue(id, remoteVersion, remoteItem);
+                    }
                     if (echo)
                         echoes.add(id);
                     continue;
                 }
                 const localItem = await this.readLocal(id);
-                if (await this.resolvePending(id, localItem, remoteItem, remoteVersion, ids))
+                if (await this.resolvePending(id, localItem, remoteItem, remoteVersion, ids, "push"))
                     queueChanged = true;
                 continue;
             }
@@ -3040,12 +3201,15 @@ export class HybridStorage {
     async resolveRemoteDelete(id, conflicts) {
         this.stats.conflicts++;
         switch (this.options.conflictResolution) {
-            case "remote":
+            case "remote": {
+                const value = await this.readLocal(id);
                 if (!this.dropQueued(id))
                     break;
                 await this.localDelete(id, this.syncOrigin);
                 this.bases.set(id, null);
+                await this.reportDropped({ id, value, reason: "remote-delete", during: "push" });
                 return true;
+            }
             case "error":
                 conflicts.push(id);
                 return false;