@omgbase/sync 0.6.4 → 0.7.2

package/dist/src/coordinator.d.ts

@@ -1,5 +1,5 @@
 import { type SyncSource, type SourceWatch } from "@omgbase/core";
-import type { EngineClient } from "./engine-client.js";
+import { type EngineClient } from "./engine-client.js";
 export interface SyncInSummary {
     ingested: string[];
     suppressed: string[];
@@ -8,8 +8,12 @@ export interface SyncInSummary {
 }
 export interface SyncOutSummary {
     cursor: number;
+    /** the feed's epoch the cursor belongs to (spec/sync §6, 4.0); a consumer stores both */
+    epoch: string | null;
     written: string[];
     removed: string[];
+    /** set when the feed had expired the cursor and the rescan procedure (spec/sync §6) ran before resuming */
+    rescanned?: boolean;
 }
 export declare class Coordinator {
     private readonly engine;
@@ -24,12 +28,33 @@ export declare class Coordinator {
     reconcile(paths: string[]): Promise<SyncInSummary>;
     /**
      * engine → source: write engine-authored changes out to the source. Walks the
-     * commit feed from `cursor` (paging through `truncated`), skipping
-     * observed-origin commits (they originated from a source — §6), and writes
-     * each changed doc's current bytes to the source (or removes a tombstoned one).
-     * Requires a write-through source; a read-only source exports nothing.
+     * change feed from `(epoch, cursor)` (paging through `truncated`), skipping
+     * observed-origin commits (they originated from a source — §6), and applies
+     * each digest's documents by `kind`: `created`/`changed`/`moved` → read the
+     * doc by `_id` and `write` at `_path` (a `moved` also `remove`s `from_path`);
+     * `deleted` → `remove`. When the feed has **expired** the cursor (another
+     * epoch, past the head, pruned behind the oldest retained row — spec/surface
+     * §3 `expired_cursor`) the rescan procedure of spec/sync §6 runs — the
+     * server's live documents are listed and the source reconverged to them —
+     * and the walk resumes from the error's `head` under its `epoch`. Requires a
+     * write-through source; a read-only source exports nothing.
      */
-    syncOut(cursor?: number): Promise<SyncOutSummary>;
+    syncOut(cursor?: number, epoch?: string): Promise<SyncOutSummary>;
+    /** `remove` a source path once per `syncOut` call (a tombstoned document's own digest and its earlier digests all name it). */
+    private removeOnce;
+    /**
+     * The rescan procedure, steps 2–3 (spec/sync §6, 4.0): page the server's
+     * live documents into `L = { (id, path, hash) }`; for every document whose
+     * hash differs from the source's copy (hashed as the engine hashes a file —
+     * sha256 of the bytes), pull the bytes and write them at `path`; every
+     * source-side path not in `L` is removed (absence from `L` is deletion — the
+     * source has paths, not ids, so a moved document is a write at the new path
+     * and a removal of the old one). The caller resumes the feed (step 4).
+     */
+    rescan(): Promise<{
+        written: string[];
+        removed: string[];
+    }>;
     /**
      * Live source→engine sync: subscribe to the source's change batches and
      * reconcile each. Returns the subscription (`.ready` resolves once the source

package/dist/src/coordinator.js

@@ -1,4 +1,6 @@
+import { createHash } from "node:crypto";
 import { storagePath } from "@omgbase/core";
+import { FeedExpired } from "./engine-client.js";
 export class Coordinator {
     engine;
     source;
@@ -48,43 +50,126 @@ export class Coordinator {
     }
     /**
      * engine → source: write engine-authored changes out to the source. Walks the
-     * commit feed from `cursor` (paging through `truncated`), skipping
-     * observed-origin commits (they originated from a source — §6), and writes
-     * each changed doc's current bytes to the source (or removes a tombstoned one).
-     * Requires a write-through source; a read-only source exports nothing.
+     * change feed from `(epoch, cursor)` (paging through `truncated`), skipping
+     * observed-origin commits (they originated from a source — §6), and applies
+     * each digest's documents by `kind`: `created`/`changed`/`moved` → read the
+     * doc by `_id` and `write` at `_path` (a `moved` also `remove`s `from_path`);
+     * `deleted` → `remove`. When the feed has **expired** the cursor (another
+     * epoch, past the head, pruned behind the oldest retained row — spec/surface
+     * §3 `expired_cursor`) the rescan procedure of spec/sync §6 runs — the
+     * server's live documents are listed and the source reconverged to them —
+     * and the walk resumes from the error's `head` under its `epoch`. Requires a
+     * write-through source; a read-only source exports nothing.
      */
-    async syncOut(cursor) {
+    async syncOut(cursor, epoch) {
         const written = [];
         const removed = [];
         let cur = cursor ?? 0;
+        let ep = epoch ?? null;
         if (!this.source.write)
-            return { cursor: cur, written, removed }; // read-only source
+            return { cursor: cur, epoch: ep, written, removed }; // read-only source
+        let rescanned = false;
         for (;;) {
-            const page = await this.engine.changesSince(cur);
+            let page;
+            try {
+                page = await this.engine.changesSince(cur, ep !== null ? { epoch: ep } : {});
+            }
+            catch (e) {
+                if (!(e instanceof FeedExpired) || rescanned)
+                    throw e;
+                // The rescan (spec/sync §6): record S = head and E = epoch from the
+                // error, reconverge the source to the server's live documents, resume
+                // from (E, S). Once per call: a second expiry means the bootstrap lease
+                // (the retention floor) did not hold and the caller retries.
+                const r = await this.rescan();
+                written.push(...r.written);
+                removed.push(...r.removed);
+                cur = e.head;
+                ep = e.epoch;
+                rescanned = true;
+                continue;
+            }
             for (const digest of page.digests) {
                 if (digest.origin === "observed")
                     continue; // came from a source; don't echo back
                 for (const rev of digest.revisions) {
-                    // `changes_since` reports a revision under the row's engine names
+                    // `changes_since` reports a document under the row's engine names
                     // (`_id`, `_path`, `_hash` — spec/surface §1.2, 5.0) in the
                     // reference form; the source takes storage paths.
                     const path = storagePath(rev._path);
+                    if (rev.kind === "moved" && rev.from_path !== undefined) {
+                        const from = storagePath(rev.from_path);
+                        if (from !== path)
+                            await this.removeOnce(from, removed);
+                    }
+                    if (rev.kind === "deleted") {
+                        await this.removeOnce(path, removed);
+                        continue;
+                    }
                     const doc = await this.engine.readDoc(path);
                     if (doc) {
                         await this.source.write(path, doc.content);
                         written.push(path);
                     }
-                    else if (this.source.remove) {
-                        await this.source.remove(path);
-                        removed.push(path);
+                    else {
+                        // The document no longer reads (tombstoned later in the feed): its
+                        // `deleted` digest follows and would remove the same path again.
+                        await this.removeOnce(path, removed);
                     }
                 }
             }
             cur = page.cursor;
+            ep = page.epoch ?? ep;
             if (!page.truncated)
                 break;
         }
-        return { cursor: cur, written, removed };
+        return { cursor: cur, epoch: ep, written, removed, ...(rescanned ? { rescanned } : {}) };
+    }
+    /** `remove` a source path once per `syncOut` call (a tombstoned document's own digest and its earlier digests all name it). */
+    async removeOnce(path, removed) {
+        if (!this.source.remove || removed.includes(path))
+            return;
+        await this.source.remove(path);
+        removed.push(path);
+    }
+    /**
+     * The rescan procedure, steps 2–3 (spec/sync §6, 4.0): page the server's
+     * live documents into `L = { (id, path, hash) }`; for every document whose
+     * hash differs from the source's copy (hashed as the engine hashes a file —
+     * sha256 of the bytes), pull the bytes and write them at `path`; every
+     * source-side path not in `L` is removed (absence from `L` is deletion — the
+     * source has paths, not ids, so a moved document is a write at the new path
+     * and a removal of the old one). The caller resumes the feed (step 4).
+     */
+    async rescan() {
+        const written = [];
+        const removed = [];
+        if (!this.source.write)
+            return { written, removed };
+        const listed = await this.engine.listDocs();
+        const live = new Set();
+        for (const d of listed) {
+            const path = storagePath(d._path);
+            live.add(path);
+            const have = await this.source.fetch(path);
+            const haveHash = have ? createHash("sha256").update(have.content, "utf8").digest("hex") : null;
+            if (haveHash !== null && haveHash === d._hash)
+                continue;
+            const doc = await this.engine.readDoc(path);
+            if (!doc)
+                continue;
+            await this.source.write(path, doc.content);
+            written.push(path);
+        }
+        if (this.source.remove) {
+            for (const entry of await this.source.enumerate()) {
+                if (live.has(entry.path))
+                    continue;
+                await this.source.remove(entry.path);
+                removed.push(entry.path);
+            }
+        }
+        return { written, removed };
     }
     /**
      * Live source→engine sync: subscribe to the source's change batches and

package/dist/src/engine-client.d.ts

@@ -3,9 +3,40 @@ export interface ChangesPage {
     digests: CommitDigest[];
     cursor: number;
     truncated: boolean;
-    /** the repo's current max commit seq — cursor > head means the cursor came from another repo/server */
+    /** the feed's epoch (spec/store 17.0, D7): a cursor is the pair `(epoch, seq)` — a consumer stores and sends both */
+    epoch?: string;
+    /** the oldest `seq` the feed still holds (0 when empty); a gap between a consumer's cursor and `oldest − 1` is a rescan */
+    oldest?: number;
+    /** the repo's current max seq — `cursor == head` means nothing new */
     head?: number;
 }
+/** A live document as the rescan lists it (spec/sync §6, 4.0): `query docs { select _id, _path, _hash }`. */
+export interface DocListing {
+    _id: string;
+    /** reference form (`/a.md`) */
+    _path: string;
+    _hash: string | null;
+}
+/**
+ * `expired_cursor` (spec/surface §3, 6.0; spec/sync §6, 4.0): the feed no
+ * longer serves the consumer's `(epoch, cursor)` — the repository's current
+ * position is `{ epoch, oldest, head }` and the consumer runs the rescan
+ * procedure, then resumes from `head` under `epoch`. Raised by both engine
+ * clients alike (the in-process one from the core's `ExpiredCursor`, the MCP
+ * one from the error envelope), so the coordinator sees one shape.
+ */
+export declare class FeedExpired extends Error {
+    readonly epoch: string;
+    readonly oldest: number;
+    readonly head: number;
+    constructor(epoch: string, oldest: number, head: number, message?: string);
+}
+export interface ChangesSinceOpts {
+    /** the cursor's epoch (default: the repository's current one) */
+    epoch?: string;
+    origin?: "api" | "observed" | "import";
+    limit?: number;
+}
 export interface DocBytes {
     content: string;
     /** hex of the doc's current rendered/file hash (matches a changes_since revision's `_hash`). */
@@ -19,11 +50,10 @@ export interface EngineClient {
     }[]): Promise<ObserveResult[]>;
     /** Mirror a source-side deletion: tombstone the doc (observed). */
     observeDelete(path: string): Promise<ObserveDeleteResult>;
-    /** The repo-wide commit feed after a cursor (for the export direction). */
-    changesSince(cursor?: number, opts?: {
-        origin?: "api" | "observed" | "import";
-        limit?: number;
-    }): Promise<ChangesPage>;
+    /** The repo-wide change feed after a cursor (for the export direction); throws `FeedExpired` when the feed no longer serves the cursor. */
+    changesSince(cursor?: number, opts?: ChangesSinceOpts): Promise<ChangesPage>;
+    /** Every live document of the repo as `{ _id, _path, _hash }` (the rescan's step 2, spec/sync §6). */
+    listDocs(): Promise<DocListing[]>;
     /** Current whole-file bytes + content hash for a path, or null if absent. */
     readDoc(path: string): Promise<DocBytes | null>;
     close(): Promise<void>;
@@ -39,10 +69,8 @@ export declare class InProcessEngineClient implements EngineClient {
         content: string;
     }[]): Promise<ObserveResult[]>;
     observeDelete(path: string): Promise<ObserveDeleteResult>;
-    changesSince(cursor?: number, opts?: {
-        origin?: "api" | "observed" | "import";
-        limit?: number;
-    }): Promise<ChangesPage>;
+    changesSince(cursor?: number, opts?: ChangesSinceOpts): Promise<ChangesPage>;
+    listDocs(): Promise<DocListing[]>;
     readDoc(path: string): Promise<DocBytes | null>;
     close(): Promise<void>;
 }

package/dist/src/engine-client.js

@@ -1,4 +1,24 @@
-import { observeMany, observeDelete, changesSince, docsRead, storagePath, } from "@omgbase/core";
+import { observeMany, observeDelete, changesSince, ExpiredCursor, docsRead, storagePath, referencePath, } from "@omgbase/core";
+/**
+ * `expired_cursor` (spec/surface §3, 6.0; spec/sync §6, 4.0): the feed no
+ * longer serves the consumer's `(epoch, cursor)` — the repository's current
+ * position is `{ epoch, oldest, head }` and the consumer runs the rescan
+ * procedure, then resumes from `head` under `epoch`. Raised by both engine
+ * clients alike (the in-process one from the core's `ExpiredCursor`, the MCP
+ * one from the error envelope), so the coordinator sees one shape.
+ */
+export class FeedExpired extends Error {
+    epoch;
+    oldest;
+    head;
+    constructor(epoch, oldest, head, message) {
+        super(message ?? `expired_cursor — the feed now begins at (${epoch}, ${oldest}); rescan and resume from head ${head}`);
+        this.epoch = epoch;
+        this.oldest = oldest;
+        this.head = head;
+        this.name = "FeedExpired";
+    }
+}
 /** In-process EngineClient: calls @omgbase/core directly against an open Store.
  *  Used by the local watcher/sync (no self-MCP-loop) and by tests. */
 export class InProcessEngineClient {
@@ -15,15 +35,29 @@ export class InProcessEngineClient {
         return Promise.resolve(observeDelete(this.store, this.repoId, path));
     }
     changesSince(cursor, opts) {
-        return Promise.resolve(changesSince(this.store, this.repoId, {
-            ...(cursor !== undefined ? { cursor } : {}),
-            ...(opts?.origin ? { origin: opts.origin } : {}),
-            ...(opts?.limit !== undefined ? { limit: opts.limit } : {}),
-        }));
+        try {
+            return Promise.resolve(changesSince(this.store, this.repoId, {
+                ...(cursor !== undefined ? { cursor } : {}),
+                ...(opts?.epoch !== undefined ? { epoch: opts.epoch } : {}),
+                ...(opts?.origin ? { origin: opts.origin } : {}),
+                ...(opts?.limit !== undefined ? { limit: opts.limit } : {}),
+            }));
+        }
+        catch (e) {
+            if (e instanceof ExpiredCursor)
+                return Promise.reject(new FeedExpired(e.epoch, e.oldest, e.head, e.message));
+            return Promise.reject(e instanceof Error ? e : new Error(String(e)));
+        }
+    }
+    listDocs() {
+        const rows = this.store.db
+            .prepare("SELECT doc_id, path, file_hash FROM docs WHERE repo_id = ? AND deleted_at IS NULL ORDER BY path")
+            .all(this.repoId);
+        return Promise.resolve(rows.map((r) => ({ _id: r.doc_id, _path: referencePath(r.path), _hash: r.file_hash ? r.file_hash.toString("hex") : null })));
     }
     readDoc(path) {
         const row = this.store.db
-            .prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_commit IS NULL")
+            .prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_at IS NULL")
             .get(this.repoId, storagePath(path));
         if (!row)
             return Promise.resolve(null);

package/dist/src/index.d.ts

@@ -1,4 +1,4 @@
-export { type EngineClient, type ChangesPage, type DocBytes, InProcessEngineClient } from "./engine-client.js";
+export { type EngineClient, type ChangesPage, type ChangesSinceOpts, type DocBytes, type DocListing, FeedExpired, InProcessEngineClient } from "./engine-client.js";
 export { McpEngineClient, connectStdioEngine, connectHttpEngine, connectEngine, parseEngineSpec, type EngineSpec } from "./mcp-engine-client.js";
 export { Coordinator, type SyncInSummary, type SyncOutSummary } from "./coordinator.js";
 export { runFsMirror, type FsMirrorOptions } from "./mirror.js";

package/dist/src/index.js

@@ -2,7 +2,7 @@
 // The Coordinator reconciles a SyncSource (external store, via the adapter
 // protocol) with an omgbase repo reached through an EngineClient seam:
 // InProcessEngineClient (local, direct Store) or McpEngineClient (remote, MCP).
-export { InProcessEngineClient } from "./engine-client.js";
+export { FeedExpired, InProcessEngineClient } from "./engine-client.js";
 export { McpEngineClient, connectStdioEngine, connectHttpEngine, connectEngine, parseEngineSpec } from "./mcp-engine-client.js";
 export { Coordinator } from "./coordinator.js";
 export { runFsMirror } from "./mirror.js";

package/dist/src/mcp-engine-client.d.ts

@@ -1,6 +1,6 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { type ObserveResult, type ObserveDeleteResult, type SurfaceObserveResult } from "@omgbase/core";
-import type { EngineClient, ChangesPage, DocBytes } from "./engine-client.js";
+import { type EngineClient, type ChangesPage, type ChangesSinceOpts, type DocBytes, type DocListing } from "./engine-client.js";
 /** Wraps a connected MCP SDK Client, calling the engine's sync tools. */
 export declare class McpEngineClient implements EngineClient {
     private readonly client;
@@ -26,10 +26,9 @@ export declare class McpEngineClient implements EngineClient {
         content: string;
     }[]): Promise<ObserveResult[]>;
     observeDelete(path: string): Promise<ObserveDeleteResult>;
-    changesSince(cursor?: number, opts?: {
-        origin?: "api" | "observed" | "import";
-        limit?: number;
-    }): Promise<ChangesPage>;
+    changesSince(cursor?: number, opts?: ChangesSinceOpts): Promise<ChangesPage>;
+    /** The rescan's listing (spec/sync §6 step 2): `query docs { select _id, _path, _hash }` on the minimal profile, paged through its cursor. */
+    listDocs(): Promise<DocListing[]>;
     readDoc(path: string): Promise<DocBytes | null>;
     close(): Promise<void>;
 }

package/dist/src/mcp-engine-client.js

@@ -3,6 +3,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
 import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
 import { VERSION } from "@omgbase/core";
+import { FeedExpired } from "./engine-client.js";
 /** Wraps a connected MCP SDK Client, calling the engine's sync tools. */
 export class McpEngineClient {
     client;
@@ -44,6 +45,11 @@ export class McpEngineClient {
         }
         if (res.isError) {
             const err = body;
+            // spec/surface §3 (6.0): the one error a feed consumer acts on — the
+            // coordinator rescans (spec/sync §6) — surfaces with its position.
+            if (err.error === "expired_cursor" && err.data && typeof err.data.epoch === "string" && typeof err.data.oldest === "number" && typeof err.data.head === "number") {
+                throw new FeedExpired(err.data.epoch, err.data.oldest, err.data.head, err.message);
+            }
             throw new Error(`tool ${name} failed: ${err.error ?? "error"} — ${err.message ?? text.slice(0, 200)}`);
         }
         return body;
@@ -64,10 +70,29 @@ export class McpEngineClient {
     changesSince(cursor, opts) {
         return this.callTool("changes_since", {
             ...(cursor !== undefined ? { cursor } : {}),
+            ...(opts?.epoch !== undefined ? { epoch: opts.epoch } : {}),
             ...(opts?.origin ? { origin: opts.origin } : {}),
             ...(opts?.limit !== undefined ? { limit: opts.limit } : {}),
         });
     }
+    /** The rescan's listing (spec/sync §6 step 2): `query docs { select _id, _path, _hash }` on the minimal profile, paged through its cursor. */
+    async listDocs() {
+        const out = [];
+        let cursor;
+        for (;;) {
+            const page = await this.callTool("query", {
+                query: "docs { select _id, _path, _hash }",
+                bare: true,
+                limit: 500,
+                ...(cursor !== undefined ? { cursor } : {}),
+            });
+            out.push(...page.hits);
+            if (!page.truncated || !page.cursor)
+                break;
+            cursor = page.cursor;
+        }
+        return out;
+    }
     async readDoc(path) {
         try {
             // `docs_read { _path }` — the argument addresses the row's `_path`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@omgbase/sync",
-  "version": "0.6.4",
+  "version": "0.7.2",
   "description": "Open Markdown Graph Base — standalone store-to-store synchronizer: a coordinator over an EngineClient seam (in-process or MCP-client) and source adapters",
   "license": "MIT",
   "repository": {
@@ -53,7 +53,7 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
-    "@omgbase/core": "^0.29.3",
+    "@omgbase/core": "^0.32.0",
     "@omgbase/fs-adapter": "^0.2.0"
   },
   "scripts": {