@freecorps/ember-offline 0.1.0 → 0.2.0

package/dist/store.js

@@ -6,10 +6,61 @@
 // row is deleted. So the replica is always the union of the last server results of the offline queries — never a
 // row the server did not return, never a row no stored result vouches for. Other queries only get their last value
 // cached (`source: "cache"`).
+//
+// Values by reference, changes as row diffs. A replicated document is stored once, as a row; the record keeps a
+// skeleton of its value where those documents are references `{ _id, $row: 1 }`. A new server value whose
+// skeleton is a doc array is persisted as a row diff against the previous skeleton (appended to the record's log,
+// usually a few ids) plus the rows that changed; the record is rewritten whole (compaction) when the log would
+// outgrow it. So a patch touching k rows of an n-row result writes O(k), not O(n).
+//
+// Reactive local results. `watchQueryLocally` keeps the read set of each local run (`runLocalQuery`) and re-runs
+// the query when replicated rows change inside it (this store's writes, another tab's through the leader, or a
+// backend's external writer), and re-sends a cached value when the record of its args changes.
 import { decodeJson, encodeJson, getFunctionName, stableStringify, } from "@freecorps/ember-client";
-import { documentsIn, localQueryRegistry, localTableOfId, runLocalQuery, } from "@freecorps/ember-server";
-import { recordKey } from "./backend.js";
+import { documentsIn, localQueryRegistry, localReadSetMatches, localTableOfId, runLocalQuery, } from "@freecorps/ember-server";
+import { applyRowDiff, isPlainObject, rowDiff } from "@freecorps/ember-values";
+import { NeedData, recordKey } from "./backend.js";
 const DAY = 86_400_000;
+/** Log entries per record before it is rewritten whole. */
+const MAX_LOG = 64;
+const ROW = "$row";
+const isRef = (x) => isPlainObject(x) && x[ROW] === 1 && typeof x._id === "string" && Object.keys(x).length === 2;
+/** `value` with the documents of `docs` (by identity) replaced by row references. */
+function toSkeleton(value, docs) {
+    if (Array.isArray(value))
+        return value.map((x) => toSkeleton(x, docs));
+    if (!isPlainObject(value))
+        return value;
+    if (docs.has(value))
+        return { _id: value._id, [ROW]: 1 };
+    const out = {};
+    for (const [k, x] of Object.entries(value))
+        out[k] = toSkeleton(x, docs);
+    return out;
+}
+function refsIn(skel, out = []) {
+    if (Array.isArray(skel))
+        for (const x of skel)
+            refsIn(x, out);
+    else if (isRef(skel))
+        out.push(skel._id);
+    else if (isPlainObject(skel))
+        for (const x of Object.values(skel))
+            refsIn(x, out);
+    return out;
+}
+function resolveRefs(skel, row) {
+    if (Array.isArray(skel))
+        return skel.map((x) => resolveRefs(x, row));
+    if (isRef(skel))
+        return row(skel._id);
+    if (!isPlainObject(skel))
+        return skel;
+    const out = {};
+    for (const [k, x] of Object.entries(skel))
+        out[k] = resolveRefs(x, row);
+    return out;
+}
 export class OfflineStore {
     backend;
     options;
@@ -20,11 +71,15 @@ export class OfflineStore {
     records = new Map();
     /** `${shard}\0${id}` → number of records owning the row. */
     refs = new Map();
-    /** record key → id → stable JSON of the doc (lazy): unchanged rows are not rewritten. */
-    sigs = new Map();
+    /** `${shard}\0${id}` → the row as last written / read (unknown after a restart until needed). */
+    rowDoc = new Map();
+    rowSig = new Map();
+    watches = new Set();
+    rerunQueued = false;
     meta = {};
     chain = Promise.resolve();
     opened = null;
+    stopExternal = null;
     now;
     constructor(backend, options) {
         this.backend = backend;
@@ -37,8 +92,8 @@ export class OfflineStore {
         this.persistPending = {
             getClientId: () => this.enqueue(async () => this.meta.clientId ?? null),
             setClientId: (id) => this.enqueue(async () => {
-                this.meta.clientId = id;
                 await this.backend.setMeta("clientId", id);
+                this.meta.clientId = id;
             }),
             load: () => this.enqueue(async () => await this.backend.loadPending()),
             put: (r) => this.enqueue(async () => void (await this.backend.putPending(r))),
@@ -64,23 +119,81 @@ export class OfflineStore {
     open() {
         if (!this.opened) {
             this.opened = (async () => {
-                const { meta, queries } = await this.backend.open();
+                const { meta, queries, log = [] } = await this.backend.open();
                 this.meta = { ...meta };
+                const logs = new Map();
+                for (const e of log)
+                    logs.set(e.key, [...(logs.get(e.key) ?? []), e]);
+                const broken = [];
                 for (const r of queries) {
-                    this.records.set(r.key, r);
-                    for (const id of r.ids)
-                        this.ref(r.shard, id, +1);
+                    const rec = this.replay(r, (logs.get(r.key) ?? []).sort((a, b) => a.seq - b.seq));
+                    this.records.set(rec.key, rec);
+                    for (const id of rec.ids)
+                        this.ref(rec.shard, id, +1);
+                    if (rec.skel === BROKEN)
+                        broken.push(rec);
                 }
                 // Retention: what the server has not refreshed for too long goes, with the rows only it owned.
                 const cutoff = this.now() - (this.options.retentionMs ?? 30 * DAY);
-                const stale = [...this.records.values()].filter((r) => r.updatedAt < cutoff);
+                const stale = [...this.records.values()].filter((r) => r.updatedAt < cutoff || r.skel === BROKEN);
                 if (stale.length)
                     await this.dropRecords(stale);
+                this.stopExternal = this.backend.onExternalChange?.((c) => this.onExternal(c)) ?? null;
             })();
             this.chain = this.opened.catch(() => undefined);
         }
         return this.opened;
     }
+    /** A stored record with its log applied. A log that does not apply marks the record broken (dropped on open). */
+    replay(r, entries) {
+        let skel = r.value;
+        let { version, updatedAt } = r;
+        let epoch = r.epoch;
+        const ids = new Set(r.ids);
+        let seq = 0;
+        let logBytes = 0;
+        try {
+            for (const e of entries) {
+                skel = applyRowDiff(skel, e.diff);
+                version = e.version;
+                epoch = e.epoch;
+                updatedAt = e.updatedAt;
+                for (const id of e.add)
+                    ids.add(id);
+                for (const id of e.remove)
+                    ids.delete(id);
+                seq = e.seq;
+                logBytes += encodeJson(e.diff).length;
+            }
+        }
+        catch {
+            // Own every row any entry added, so dropping the record releases them all.
+            for (const e of entries)
+                for (const id of e.add)
+                    ids.add(id);
+            skel = BROKEN;
+        }
+        const refs = r.refs === true;
+        return {
+            key: r.key,
+            shard: r.shard,
+            fn: r.fn,
+            args: r.args,
+            argsJson: r.argsJson,
+            version,
+            epoch,
+            updatedAt,
+            ids: [...ids],
+            skel,
+            refs,
+            materialized: !refs,
+            value: refs ? undefined : skel,
+            seq,
+            logCount: entries.length,
+            logBytes,
+            baseBytes: r.size ?? 0,
+        };
+    }
     ref(shard, id, d) {
         const k = `${shard}\u0000${id}`;
         const n = (this.refs.get(k) ?? 0) + d;
@@ -102,57 +215,155 @@ export class OfflineStore {
     applyPatch(u) {
         return this.enqueue(() => this.apply(u));
     }
-    sigsOf(r) {
-        let m = this.sigs.get(r.key);
-        if (!m) {
-            m = new Map(documentsIn(this.options.schema.manifest, r.value).map(({ doc }) => [doc._id, stableStringify(doc)]));
-            this.sigs.set(r.key, m);
+    /** Reads rows by id from the backend (async readers: fetch and retry until nothing is missing). */
+    async readRows(shard, ids) {
+        const m = this.options.schema.manifest;
+        const reader = this.backend.reader(shard);
+        for (let attempt = 0;; attempt++) {
+            const out = new Map();
+            for (const id of ids) {
+                const table = localTableOfId(m, id);
+                if (!table) {
+                    out.set(id, null);
+                    continue;
+                }
+                try {
+                    out.set(id, reader.get(table, id));
+                }
+                catch (e) {
+                    if (!(e instanceof NeedData))
+                        throw e;
+                }
+            }
+            if (reader.misses && reader.fetch && attempt < 100) {
+                await reader.fetch();
+                continue;
+            }
+            // Rows still missing after the retries are reported as missing (null), never left out of the map.
+            for (const id of ids)
+                if (!out.has(id))
+                    out.set(id, null);
+            return out;
+        }
+    }
+    /** The full value of a record (resolving its row references from the replica). */
+    async valueOf(rec) {
+        if (rec.materialized)
+            return rec.value;
+        const missing = [...new Set(refsIn(rec.skel))].filter((id) => !this.rowDoc.has(`${rec.shard}\u0000${id}`));
+        if (missing.length) {
+            for (const [id, doc] of await this.readRows(rec.shard, missing)) {
+                if (!doc)
+                    throw new Error(`offline store: replicated row ${id} of ${rec.fn} is missing`);
+                this.know(rec.shard, doc);
+            }
         }
-        return m;
+        rec.value = resolveRefs(rec.skel, (id) => this.rowDoc.get(`${rec.shard}\u0000${id}`));
+        rec.materialized = true;
+        return rec.value;
+    }
+    know(shard, doc, sig) {
+        const k = `${shard}\u0000${doc._id}`;
+        this.rowDoc.set(k, doc);
+        this.rowSig.set(k, sig ?? stableStringify(doc));
     }
     async apply(u) {
         const m = this.options.schema.manifest;
         const key = recordKey(u.shard, u.fn, u.argsJson);
         const prev = this.records.get(key);
-        const docs = this.replicated.has(u.fn) ? documentsIn(m, u.value) : [];
-        const oldSigs = prev ? this.sigsOf(prev) : new Map();
-        const newSigs = new Map();
-        const c = { put: [], upsert: [], delete: [] };
+        // Older than what is stored (another tab persisted a newer value first, or a call resent after a leader
+        // handover): dropped, so the replica never goes back. Versions only compare within one shard epoch; a new
+        // epoch (a restore reuses versions) or an unknown one is taken as it comes.
+        if (prev && prev.epoch !== undefined && prev.epoch === u.epoch && u.version < prev.version)
+            return;
+        // The previous rows' content tells which rows are unchanged (not rewritten) and how changed rows moved.
+        if (prev && !prev.materialized)
+            await this.valueOf(prev).catch(() => undefined);
+        const found = this.replicated.has(u.fn) ? documentsIn(m, u.value) : [];
+        const prevIds = new Set(prev?.ids ?? []);
+        const newIds = new Set();
+        const c = { upsert: [], delete: [] };
+        const changes = [];
+        const written = [];
+        const removed = [];
         // Refcount changes are applied as computed and undone if the commit fails, so memory never runs ahead of storage.
         const deltas = [];
         const ref = (id, d) => {
             deltas.push([id, d]);
             return this.ref(u.shard, id, d);
         };
-        for (const { table, doc } of docs) {
+        const before = (k) => (this.rowDoc.has(k) ? this.rowDoc.get(k) : this.refs.has(k) ? undefined : null);
+        for (const { table, doc } of found) {
+            const k = `${u.shard}\u0000${doc._id}`;
             const sig = stableStringify(doc);
-            newSigs.set(doc._id, sig);
-            if (oldSigs.get(doc._id) !== sig)
+            newIds.add(doc._id);
+            if (this.rowSig.get(k) !== sig) {
                 c.upsert.push({ table, doc });
-            if (!oldSigs.has(doc._id))
+                changes.push({ table, id: doc._id, before: before(k), after: doc });
+                written.push({ doc, sig });
+            }
+            if (!prevIds.has(doc._id))
                 ref(doc._id, +1);
         }
-        for (const id of prev?.ids ?? []) {
-            if (newSigs.has(id))
+        for (const id of prevIds) {
+            if (newIds.has(id))
                 continue;
+            const k = `${u.shard}\u0000${id}`;
+            const was = before(k);
             if (ref(id, -1) === 0) {
                 const table = localTableOfId(m, id);
-                if (table)
+                if (table) {
                     c.delete.push({ table, id });
+                    changes.push({ table, id, before: was, after: null });
+                    removed.push(k);
+                }
             }
         }
-        const record = {
-            key,
-            shard: u.shard,
-            fn: u.fn,
-            args: u.args,
-            argsJson: u.argsJson,
-            value: u.value,
-            version: u.version,
-            updatedAt: this.now(),
-            ids: [...newSigs.keys()],
-        };
-        c.put.push(record);
+        const docSet = new Set(found.map((f) => f.doc));
+        const diffs = this.options.persistDiffs !== false;
+        const skel = docSet.size && diffs ? toSkeleton(u.value, docSet) : u.value;
+        const now = this.now();
+        let entry = null;
+        let entryBytes = 0;
+        if (prev && prev.skel !== BROKEN && diffs && prev.logCount < MAX_LOG && prev.baseBytes > 0) {
+            const diff = rowDiff(prev.skel, skel);
+            if (diff) {
+                entryBytes = encodeJson(diff).length;
+                if (prev.logBytes + entryBytes <= prev.baseBytes) {
+                    entry = {
+                        key,
+                        seq: prev.seq + 1,
+                        diff,
+                        version: u.version,
+                        ...(u.epoch !== undefined ? { epoch: u.epoch } : {}),
+                        updatedAt: now,
+                        add: [...newIds].filter((id) => !prevIds.has(id)),
+                        remove: [...prevIds].filter((id) => !newIds.has(id)),
+                    };
+                }
+            }
+        }
+        let baseBytes = prev?.baseBytes ?? 0;
+        if (entry)
+            c.append = [entry];
+        else {
+            const record = {
+                key,
+                shard: u.shard,
+                fn: u.fn,
+                args: u.args,
+                argsJson: u.argsJson,
+                value: skel,
+                version: u.version,
+                ...(u.epoch !== undefined ? { epoch: u.epoch } : {}),
+                updatedAt: now,
+                ids: [...newIds],
+                size: 0,
+                ...(docSet.size && diffs ? { refs: true } : {}),
+            };
+            record.size = baseBytes = encodeJson(skel).length;
+            c.put = [record];
+        }
         try {
             await this.backend.commit(u.shard, c);
         }
@@ -161,8 +372,32 @@ export class OfflineStore {
                 this.ref(u.shard, id, -d);
             throw e;
         }
-        this.records.set(key, record);
-        this.sigs.set(key, newSigs);
+        for (const w of written)
+            this.know(u.shard, w.doc, w.sig);
+        for (const k of removed) {
+            this.rowDoc.delete(k);
+            this.rowSig.delete(k);
+        }
+        this.records.set(key, {
+            key,
+            shard: u.shard,
+            fn: u.fn,
+            args: u.args,
+            argsJson: u.argsJson,
+            version: u.version,
+            epoch: u.epoch,
+            updatedAt: now,
+            ids: [...newIds],
+            skel,
+            refs: docSet.size > 0 && diffs,
+            materialized: true,
+            value: u.value,
+            seq: entry ? entry.seq : 0,
+            logCount: entry ? prev.logCount + 1 : 0,
+            logBytes: entry ? prev.logBytes + entryBytes : 0,
+            baseBytes,
+        });
+        this.notify(u.shard, changes, [key]);
         await this.evict(key);
     }
     /** Over `maxQueries`: drop the least recently refreshed records (never the one just written). */
@@ -187,38 +422,60 @@ export class OfflineStore {
             byShard.set(r.shard, [...(byShard.get(r.shard) ?? []), r]);
         for (const [shard, records] of byShard) {
             const c = { drop: records.map((r) => r.key), delete: [] };
+            const changes = [];
             const dec = new Map();
             for (const r of records)
                 for (const id of r.ids)
                     dec.set(id, (dec.get(id) ?? 0) + 1);
             for (const [id, n] of dec) {
-                if ((this.refs.get(`${shard}\u0000${id}`) ?? 0) - n > 0)
+                const k = `${shard}\u0000${id}`;
+                if ((this.refs.get(k) ?? 0) - n > 0)
                     continue;
                 const table = localTableOfId(m, id);
-                if (table)
+                if (table) {
                     c.delete.push({ table, id });
+                    changes.push({ table, id, before: this.rowDoc.has(k) ? this.rowDoc.get(k) : undefined, after: null });
+                }
             }
             await this.backend.commit(shard, c);
-            for (const r of records) {
+            for (const r of records)
                 this.records.delete(r.key);
-                this.sigs.delete(r.key);
+            for (const [id, n] of dec) {
+                if (this.ref(shard, id, -n) > 0)
+                    continue;
+                this.rowDoc.delete(`${shard}\u0000${id}`);
+                this.rowSig.delete(`${shard}\u0000${id}`);
             }
-            for (const [id, n] of dec)
-                this.ref(shard, id, -n);
+            this.notify(shard, changes, records.map((r) => r.key));
         }
     }
     async runQueryLocally(req) {
         await this.flush();
+        return await this.evaluate(req, this.registry.get(req.fn)).then((r) => r.result);
+    }
+    /** The local result of a request (registered: run over the replica; else / on failure: the cached value). */
+    async evaluate(req, fn) {
         const cached = this.records.get(recordKey(req.shard, req.fn, req.argsJson));
-        const fn = this.registry.get(req.fn);
+        const fromCache = async () => {
+            if (!cached)
+                return undefined;
+            try {
+                return { value: await this.valueOf(cached), source: "cache" };
+            }
+            catch {
+                return undefined;
+            }
+        };
         if (!fn)
-            return cached ? { value: cached.value, source: "cache" } : undefined;
+            return { result: await fromCache(), readSet: null };
         try {
-            return { value: await this.runLocal(fn, req), source: "local" };
+            const { value, readSet } = await this.runLocal(fn, req);
+            return { result: { value, source: "local" }, readSet };
         }
         catch (e) {
-            if (cached)
-                return { value: cached.value, source: "cache" };
+            const result = await fromCache();
+            if (result)
+                return { result, readSet: null };
             throw e;
         }
     }
@@ -226,14 +483,12 @@ export class OfflineStore {
     async runLocal(fn, req) {
         const reader = this.backend.reader(req.shard);
         for (let attempt = 0;; attempt++) {
-            let value;
+            let out;
             let error;
-            let failed = false;
             try {
-                value = runLocalQuery(fn, req.args, { schema: this.options.schema, reader, identity: req.identity, now: this.now() }).value;
+                out = runLocalQuery(fn, req.args, { schema: this.options.schema, reader, identity: req.identity, now: this.now() });
             }
             catch (e) {
-                failed = true;
                 error = e;
             }
             if (reader.misses && reader.fetch) {
@@ -242,19 +497,128 @@ export class OfflineStore {
                 await reader.fetch();
                 continue;
             }
-            if (failed)
+            if (!out)
                 throw error;
-            return value;
+            return out;
+        }
+    }
+    // ---------------------------------------------------------------------------------------------- reactive local results
+    /**
+     * The local result of `req` now, and again whenever it may have changed: a registered query re-runs when rows
+     * inside its read set change; any request re-reads when the cached record of its args changes. Results equal to
+     * the last one sent are not re-sent.
+     */
+    watchQueryLocally(req, onResult, onError) {
+        const w = {
+            req,
+            key: recordKey(req.shard, req.fn, req.argsJson),
+            fn: this.registry.get(req.fn),
+            readSet: null,
+            last: null,
+            dirty: false,
+            closed: false,
+            failed: false,
+            onResult,
+            ...(onError ? { onError } : {}),
+        };
+        this.watches.add(w);
+        this.schedule([w]);
+        return () => {
+            w.closed = true;
+            this.watches.delete(w);
+        };
+    }
+    schedule(ws) {
+        let any = false;
+        for (const w of ws) {
+            if (w.closed)
+                continue;
+            w.dirty = true;
+            any = true;
         }
+        if (!any || this.rerunQueued)
+            return;
+        this.rerunQueued = true;
+        // In the write queue: a re-run sees every write handed to the store before it, and none half-done.
+        void this.enqueue(async () => {
+            this.rerunQueued = false;
+            for (const w of [...this.watches])
+                if (w.dirty)
+                    await this.rerun(w);
+        }).catch(() => {
+            this.rerunQueued = false;
+        });
+    }
+    async rerun(w) {
+        w.dirty = false;
+        let result;
+        try {
+            const r = await this.evaluate(w.req, w.fn);
+            result = r.result;
+            w.readSet = r.readSet;
+            w.failed = false;
+        }
+        catch (e) {
+            w.readSet = null;
+            if (!w.failed && !w.closed)
+                w.onError?.(e);
+            w.failed = true;
+            return;
+        }
+        if (w.closed)
+            return;
+        const sig = result ? `${result.source}\u0000${encodeJson(result.value)}` : "";
+        if (sig === w.last)
+            return;
+        w.last = sig;
+        try {
+            w.onResult(result);
+        }
+        catch {
+            /* a listener's problem */
+        }
+    }
+    /** Rows of `shard` changed (and these records were rewritten): mark the watches they may affect. */
+    notify(shard, changes, keys) {
+        if (!this.watches.size || (!changes.length && !keys.length))
+            return;
+        const touched = new Set(keys);
+        const hit = [];
+        for (const w of this.watches) {
+            if (w.req.shard !== shard)
+                continue;
+            if (touched.has(w.key))
+                hit.push(w);
+            else if (w.fn && changes.length && (!w.readSet || localReadSetMatches(this.options.schema, w.readSet, changes)))
+                hit.push(w);
+        }
+        this.schedule(hit);
+    }
+    /** Someone else wrote rows (a shared TinyBase store): what we knew about them is stale; re-run what reads them. */
+    onExternal(c) {
+        for (const k of [...this.rowDoc.keys()]) {
+            if (!k.startsWith(`${c.shard}\u0000`))
+                continue;
+            this.rowDoc.delete(k);
+            this.rowSig.delete(k);
+        }
+        for (const r of this.records.values()) {
+            if (r.shard === c.shard && r.refs) {
+                r.materialized = false;
+                r.value = undefined;
+            }
+        }
+        this.notify(c.shard, c.tables.map((table) => ({ table, id: "", before: undefined, after: null })), []);
     }
     async setIdentity(identity) {
         await this.enqueue(async () => {
             const v = identity ? encodeJson(identity) : null;
+            // Storage first: a failed write leaves memory as it was.
+            await this.backend.setMeta("identity", v);
             if (v === null)
                 delete this.meta.identity;
             else
                 this.meta.identity = v;
-            await this.backend.setMeta("identity", v);
         });
     }
     /** Forget every cached result and replicated row (e.g. on sign-out on a shared device). Pending mutations stay. */
@@ -263,9 +627,16 @@ export class OfflineStore {
     }
     async close() {
         await this.flush().catch(() => undefined);
+        for (const w of this.watches)
+            w.closed = true;
+        this.watches.clear();
+        this.stopExternal?.();
+        this.stopExternal = null;
         await this.backend.close();
     }
 }
+/** Marks a record whose log does not apply (dropped on open). */
+const BROKEN = Object.freeze({ broken: true });
 /** A `StorageAdapter` over any backend. */
 export function createOfflineStorage(backend, options) {
     return new OfflineStore(backend, options);