@omgbase/core 0.1.0 → 0.3.0

package/dist/src/sync/reconciling-ingest.d.ts

@@ -1,12 +1,52 @@
 import type { Database } from "better-sqlite3";
 import type { Store } from "../core/store/store.js";
-import type { IdResolver } from "../core/ingest.js";
+import type { BlockTree } from "../core/parse/types.js";
+import { type IdResolver } from "../core/ingest.js";
+import { type ResurrectionCandidate, type DocReconcileResult } from "../reconcile/reconcile.js";
 import { type MatchBlock, type ReconcileConfig } from "../reconcile/types.js";
 export declare function loadOldMatchBlocks(db: Database, docId: string): MatchBlock[];
+/** The live resurrection pool for a repo (rows not yet expired at `ts`). A batch
+ * caller loads it ONCE per checkpoint and threads a shared consumed-set through
+ * `prepareReconcile`, so two documents reconciled before either commits cannot
+ * both resurrect the same pooled id. */
+export declare function loadPool(db: Database, repoId: string, ts: string): ResurrectionCandidate[];
+/** One document parsed + reconciled but NOT committed: the first pass of a
+ * checkpoint (reconciliation-spec §8). The cross-document phase reads
+ * `oldBlocks`/`newBlocks`/`result` across every prepared doc, rewrites `result`
+ * in place (`applyCrossDocMatches`) and records the ids it carried in; the
+ * second pass commits by handing this to `makeReconcilingResolver({ prepared })`
+ * + `ingestFile({ parsed: tree })`. */
+export interface PreparedReconcile {
+    path: string;
+    content: string;
+    /** the doc row at `path` (live or tombstoned), or null when the path is new —
+     * exactly the docId ingestFile will pass the resolver. */
+    docId: string | null;
+    tree: BlockTree;
+    oldBlocks: MatchBlock[];
+    newBlocks: MatchBlock[];
+    result: DocReconcileResult;
+    /** ids carried into this document by the cross-doc phase (filled by it). */
+    crossDocIds: string[];
+}
+/**
+ * Pass 1 of an observed ingest: parse `content`, load the document's current
+ * tree, and reconcile — without writing. `pool`/`consumed` let a batch share one
+ * pool snapshot: candidates already in `consumed` are hidden from this document,
+ * and the ids this document resurrects are added to it.
+ */
+export declare function prepareReconcile(store: Store, repoId: string, path: string, content: string, opts?: {
+    ts?: string;
+    config?: ReconcileConfig;
+    pool?: ResurrectionCandidate[];
+    consumed?: Set<string>;
+}): PreparedReconcile;
 /** Build an IdResolver bound to this store + repo, reconciling against the
- * document's current revision. */
+ * document's current revision. With `prepared` (from `prepareReconcile`, after
+ * the cross-doc phase) the resolver skips the reconcile and applies that result. */
 export declare function makeReconcilingResolver(store: Store, repoId: string, opts?: {
     ts?: string;
     config?: ReconcileConfig;
     path?: string;
+    prepared?: PreparedReconcile;
 }): IdResolver;

package/dist/src/sync/reconciling-ingest.js

@@ -1,7 +1,8 @@
+import { parseForIngest } from "../core/ingest.js";
 import { mintId } from "../core/ids.js";
 import { extractFromBlock, extractFromFrontmatter, resolveRelativePath } from "../graph/extract.js";
 import { resolveExternal, resolveDocPath } from "../core/store/edges.js";
-import { sha256, normalizeVisibleText } from "../core/hash.js";
+import { sha256, childQuoteDepth, visibleText } from "../core/hash.js";
 import { reconcileDocument } from "../reconcile/reconcile.js";
 import { flatten } from "../reconcile/flatten.js";
 import { DEFAULT_CONFIG } from "../reconcile/types.js";
@@ -35,11 +36,41 @@ export function loadOldMatchBlocks(db, docId) {
         parentKeyOf.set(r.block_id, parentKey);
         return key;
     };
+    // `text` follows spec/format §4.1, which needs the tree: containers compose
+    // from their children and nested raws lose up to <blockquote-depth> `> `
+    // prefixes. Rebuild the shape (children by parent, in ordinal order) and
+    // compute text top-down with the same helper ingest used to store it.
     const blobBytes = db.prepare("SELECT bytes FROM blobs WHERE hash = ?");
+    const rawOf = new Map();
+    for (const r of rows) {
+        rawOf.set(r.block_id, blobBytes.get(r.raw_hash)?.bytes.toString("utf8") ?? "");
+    }
+    const childrenOf = new Map();
+    for (const r of rows) {
+        const parent = r.parent_block && byId.has(r.parent_block) ? r.parent_block : null;
+        let list = childrenOf.get(parent);
+        if (!list)
+            childrenOf.set(parent, (list = []));
+        list.push(r);
+    }
+    for (const list of childrenOf.values())
+        list.sort((a, b) => a.ordinal - b.ordinal);
+    const toTree = (r) => ({
+        type: r.type,
+        raw: rawOf.get(r.block_id) ?? "",
+        children: (childrenOf.get(r.block_id) ?? []).map(toTree),
+    });
+    const textOf = new Map();
+    const walk = (list, quoteDepth) => {
+        for (const r of list) {
+            textOf.set(r.block_id, visibleText(toTree(r), quoteDepth));
+            walk(childrenOf.get(r.block_id) ?? [], childQuoteDepth(r.type, quoteDepth));
+        }
+    };
+    walk(childrenOf.get(null) ?? [], 0);
     return rows.map((r) => {
         const key = resolveKey(r);
-        const raw = blobBytes.get(r.raw_hash)?.bytes.toString("utf8") ?? "";
-        const text = normalizeVisibleText(raw, r.type);
+        const text = textOf.get(r.block_id) ?? "";
         return {
             blockId: r.block_id,
             type: r.type,
@@ -53,7 +84,11 @@ export function loadOldMatchBlocks(db, docId) {
         };
     });
 }
-function loadPool(db, repoId, ts) {
+/** The live resurrection pool for a repo (rows not yet expired at `ts`). A batch
+ * caller loads it ONCE per checkpoint and threads a shared consumed-set through
+ * `prepareReconcile`, so two documents reconciled before either commits cannot
+ * both resurrect the same pooled id. */
+export function loadPool(db, repoId, ts) {
     const rows = db
         .prepare("SELECT block_id, raw_hash, norm_hash, type FROM resurrection_pool WHERE repo_id = ? AND expires_ts > ?")
         .all(repoId, ts);
@@ -70,12 +105,41 @@ function assignFromMap(blocks, assignment) {
     const walk = (list, parentKey) => list.map((b, index) => {
         const key = `${parentKey ?? ""}/${index}`;
         const blockId = assignment.get(key) ?? mintId("b");
-        return { blockId, type: b.type, raw: b.raw, trivia: b.trivia, attrs: b.attrs, children: walk(b.children, key) };
+        return { blockId, type: b.type, raw: b.raw, trivia: b.trivia, attrs: b.attrs, children: walk(b.children, key), span: b.span };
     });
     return walk(blocks, null);
 }
+/**
+ * Pass 1 of an observed ingest: parse `content`, load the document's current
+ * tree, and reconcile — without writing. `pool`/`consumed` let a batch share one
+ * pool snapshot: candidates already in `consumed` are hidden from this document,
+ * and the ids this document resurrects are added to it.
+ */
+export function prepareReconcile(store, repoId, path, content, opts = {}) {
+    const db = store.db;
+    const ts = opts.ts ?? new Date().toISOString();
+    const config = opts.config ?? DEFAULT_CONFIG;
+    // Same lookup ingestFile does (a tombstoned row still owns the path's identity).
+    const doc = db.prepare("SELECT doc_id FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
+    const docId = doc?.doc_id ?? null;
+    const { tree, rest } = parseForIngest(path, content);
+    const oldBlocks = docId ? loadOldMatchBlocks(db, docId) : [];
+    const newBlocks = flatten(rest.map(toFlatSource));
+    let pool = [];
+    if (docId) {
+        pool = opts.pool ?? loadPool(db, repoId, ts);
+        if (opts.consumed)
+            pool = pool.filter((c) => !opts.consumed.has(c.blockId));
+    }
+    const result = reconcileDocument(oldBlocks, newBlocks, { config, pool });
+    if (opts.consumed)
+        for (const id of result.consumedPool)
+            opts.consumed.add(id);
+    return { path, content, docId, tree, oldBlocks, newBlocks, result, crossDocIds: [] };
+}
 /** Build an IdResolver bound to this store + repo, reconciling against the
- * document's current revision. */
+ * document's current revision. With `prepared` (from `prepareReconcile`, after
+ * the cross-doc phase) the resolver skips the reconcile and applies that result. */
 export function makeReconcilingResolver(store, repoId, opts = {}) {
     const ts = opts.ts ?? new Date().toISOString();
     const config = opts.config ?? DEFAULT_CONFIG;
@@ -83,10 +147,18 @@ export function makeReconcilingResolver(store, repoId, opts = {}) {
     const docDir = opts.path ? opts.path.replace(/[^/]*$/, "") : "";
     return (rest, docId) => {
         const db = store.db;
-        const oldBlocks = docId ? loadOldMatchBlocks(db, docId) : [];
-        const newBlocks = flatten(rest.map(toFlatSource));
-        const pool = docId ? loadPool(db, repoId, ts) : [];
-        const result = reconcileDocument(oldBlocks, newBlocks, { config, pool });
+        let result;
+        let crossDocIds = [];
+        if (opts.prepared) {
+            result = opts.prepared.result;
+            crossDocIds = opts.prepared.crossDocIds;
+        }
+        else {
+            const oldBlocks = docId ? loadOldMatchBlocks(db, docId) : [];
+            const newBlocks = flatten(rest.map(toFlatSource));
+            const pool = docId ? loadPool(db, repoId, ts) : [];
+            result = reconcileDocument(oldBlocks, newBlocks, { config, pool });
+        }
         const assigned = assignFromMap(rest, result.assignment);
         const dispositions = result.dispositions.map((d) => ({
             blockId: d.blockId,
@@ -124,7 +196,7 @@ export function makeReconcilingResolver(store, repoId, opts = {}) {
             }
             return out;
         };
-        return { assigned, dispositions, deleted: result.deleted, consumedPool: result.consumedPool, extractEdges };
+        return { assigned, dispositions, deleted: result.deleted, consumedPool: result.consumedPool, crossDocIds, extractEdges };
     };
 }
 // Resolve an extracted edge's target to a node id (external mint, doc path or
@@ -188,11 +260,13 @@ function resolveAdapterEdge(db, repoId, srcDoc, e, docDir) {
         provenance: e.provenance,
     };
 }
-// Collect RawBlock-shaped objects from the id-assigned tree for adapter edge extraction.
+// Collect RawBlock-shaped objects from the id-assigned tree for adapter edge
+// extraction. The parsed span rides along so the adapter can blank a
+// container's children (spec/graph §1 own text).
 function collectRawBlocks(blocks) {
     return blocks.map((b) => ({
         type: b.type,
-        span: { start: 0, end: 0 },
+        span: b.span ?? { start: 0, end: 0 },
         raw: b.raw,
         text: "",
         attrs: b.attrs,

package/dist/src/sync/recovery.d.ts

@@ -1,6 +1,10 @@
 import type { Store } from "../core/store/store.js";
+import { type SyncFs } from "./fs-util.js";
 export interface RecoveryResult {
     healed: string[];
     missing: string[];
 }
-export declare function recoverRepo(store: Store, repoId: string, rootPath: string): RecoveryResult;
+export declare function recoverRepo(store: Store, repoId: string, rootPath: string, opts?: {
+    ts?: string;
+    fs?: SyncFs;
+}): RecoveryResult;

package/dist/src/sync/recovery.js

@@ -1,9 +1,9 @@
-import { readFileSync, existsSync } from "node:fs";
-import { join } from "node:path";
 import { sha256 } from "../core/hash.js";
 import { ingestFile } from "../core/ingest.js";
 import { makeReconcilingResolver } from "./reconciling-ingest.js";
-export function recoverRepo(store, repoId, rootPath) {
+import { nodeFs } from "./fs-util.js";
+export function recoverRepo(store, repoId, rootPath, opts = {}) {
+    const fs = opts.fs ?? nodeFs;
     const docs = store.db
         .prepare(`SELECT d.doc_id, d.path, d.file_hash, r.rendered_hash
        FROM docs d LEFT JOIN revisions r ON r.rev_id = d.current_rev
@@ -11,18 +11,18 @@ export function recoverRepo(store, repoId, rootPath) {
         .all(repoId);
     const healed = [];
     const missing = [];
-    const ts = new Date().toISOString();
+    const ts = opts.ts ?? new Date().toISOString();
     for (const doc of docs) {
-        const abs = join(rootPath, doc.path);
-        if (!existsSync(abs)) {
+        const bytes = fs.read(rootPath, doc.path);
+        if (bytes === null) {
             missing.push(doc.path);
             continue;
         }
-        const onDisk = sha256(readFileSync(abs, "utf8"));
+        const onDisk = sha256(bytes);
         const recorded = doc.rendered_hash ?? doc.file_hash;
         if (!recorded || !onDisk.equals(recorded)) {
             // Divergence: the file moved ahead of (or behind) the store. Re-ingest.
-            ingestFile(store, repoId, doc.path, readFileSync(abs, "utf8"), {
+            ingestFile(store, repoId, doc.path, bytes, {
                 ts,
                 resolveIds: makeReconcilingResolver(store, repoId, { ts, path: doc.path }),
             });

package/dist/src/sync/workspace.d.ts

@@ -20,13 +20,21 @@ export declare class Workspace {
     repos(): RepoRow[];
     repoBySlug(slug: string): RepoRow | null;
     /**
-     * Select the active repo. With an explicit slug, that repo (or throws
-     * repo_not_found). Otherwise the repo whose root_path contains `cwd`; if none
-     * or several match, throws AmbiguousRepo with the candidate slugs.
+     * Select the active repo (spec/sync §1 "Repo selection"): {@link selectRepo}
+     * over this workspace's repos.
      */
     selectRepo(cwd: string, slug?: string): RepoRow;
     close(): void;
 }
+/**
+ * Repo selection (spec/sync §1) over a plain `repos` list. With an explicit
+ * slug, that repo (or `repo_not_found` with the candidate slugs). Else with
+ * exactly one repo, that repo. Else the repos whose root path contains `cwd`
+ * (path-prefix containment after resolving both; a sourceless repo never
+ * matches): exactly one → it; none → `repo_not_found`; several (nested roots)
+ * → the one with the longest resolved root path.
+ */
+export declare function selectRepo(repos: RepoRow[], cwd: string, slug?: string): RepoRow;
 export declare class RepoSelectionError extends Error {
     readonly code: "repo_not_found";
     readonly candidates: string[];

package/dist/src/sync/workspace.js

@@ -70,43 +70,52 @@ export class Workspace {
         return this.repos().find((r) => r.slug === slug) ?? null;
     }
     /**
-     * Select the active repo. With an explicit slug, that repo (or throws
-     * repo_not_found). Otherwise the repo whose root_path contains `cwd`; if none
-     * or several match, throws AmbiguousRepo with the candidate slugs.
+     * Select the active repo (spec/sync §1 "Repo selection"): {@link selectRepo}
+     * over this workspace's repos.
      */
     selectRepo(cwd, slug) {
-        const repos = this.repos();
-        if (slug) {
-            const found = repos.find((r) => r.slug === slug);
-            if (!found) {
-                throw new RepoSelectionError("repo_not_found", `no repo with slug '${slug}'`, repos.map((r) => r.slug));
-            }
-            return found;
-        }
-        if (repos.length === 1)
-            return repos[0];
-        const here = resolve(cwd);
-        // Only filesystem-backed repos (a derived rootPath) can contain the cwd; a
-        // sourceless/headless repo has no location, so it never matches by cwd.
-        const containing = repos.filter((r) => {
-            if (r.rootPath == null)
-                return false;
-            const rel = relative(resolve(r.rootPath), here);
-            return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
-        });
-        if (containing.length === 1)
-            return containing[0];
-        if (containing.length === 0) {
-            throw new RepoSelectionError("repo_not_found", `no repo contains ${here}; select one with --repo`, repos.map((r) => r.slug));
-        }
-        // Several contain cwd (nested roots): pick the deepest (longest root path).
-        containing.sort((a, b) => resolve(b.rootPath).length - resolve(a.rootPath).length);
-        return containing[0];
+        return selectRepo(this.repos(), cwd, slug);
     }
     close() {
         this.store.close();
     }
 }
+/**
+ * Repo selection (spec/sync §1) over a plain `repos` list. With an explicit
+ * slug, that repo (or `repo_not_found` with the candidate slugs). Else with
+ * exactly one repo, that repo. Else the repos whose root path contains `cwd`
+ * (path-prefix containment after resolving both; a sourceless repo never
+ * matches): exactly one → it; none → `repo_not_found`; several (nested roots)
+ * → the one with the longest resolved root path.
+ */
+export function selectRepo(repos, cwd, slug) {
+    if (slug) {
+        const found = repos.find((r) => r.slug === slug);
+        if (!found) {
+            throw new RepoSelectionError("repo_not_found", `no repo with slug '${slug}'`, repos.map((r) => r.slug));
+        }
+        return found;
+    }
+    if (repos.length === 1)
+        return repos[0];
+    const here = resolve(cwd);
+    // Only filesystem-backed repos (a derived rootPath) can contain the cwd; a
+    // sourceless/headless repo has no location, so it never matches by cwd.
+    const containing = repos.filter((r) => {
+        if (r.rootPath == null)
+            return false;
+        const rel = relative(resolve(r.rootPath), here);
+        return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
+    });
+    if (containing.length === 1)
+        return containing[0];
+    if (containing.length === 0) {
+        throw new RepoSelectionError("repo_not_found", `no repo contains ${here}; select one with --repo`, repos.map((r) => r.slug));
+    }
+    // Several contain cwd (nested roots): pick the deepest (longest root path).
+    containing.sort((a, b) => resolve(b.rootPath).length - resolve(a.rootPath).length);
+    return containing[0];
+}
 export class RepoSelectionError extends Error {
     code;
     candidates;

package/dist/src/sync/writer-lock.js

@@ -8,12 +8,12 @@ import { join } from "node:path";
 // Node core exposes no flock(2), and the design forbids new runtime deps (11
 // §8). We implement the same advisory contract with an O_EXCL lockfile: exclusive
 // creation is atomic on local filesystems, the holder writes its pid for
-// liveness, and a lock whose pid is dead is stolen (no daemon, no lease sweeper —
-// the same "probe by trying" spirit as the watch lease in §3.4). This is an
-// as-built substitution for flock; the file-CAS + ingest-and-replay behavior in
-// the write protocol is unchanged.
+// liveness, and a lock whose pid is dead (or whose body does not parse) is
+// stolen — never one whose holder is alive, however old the record (spec/sync
+// §7; no daemon, no lease sweeper — the same "probe by trying" spirit as the
+// watch lease in §3.4). This is an as-built substitution for flock; the
+// file-CAS + ingest-and-replay behavior in the write protocol is unchanged.
 const LOCK_NAME = "writer.lock";
-const STALE_MS = 30_000;
 function pidAlive(pid) {
     try {
         process.kill(pid, 0);
@@ -40,20 +40,9 @@ function tryAcquire(lockPath) {
     catch (err) {
         if (err.code !== "EEXIST")
             throw err;
-        // Lock exists: steal it if the holder is dead or the record is stale.
-        let holderPid = null;
-        let ts = 0;
-        try {
-            const rec = JSON.parse(readFileSync(lockPath, "utf8"));
-            holderPid = rec.pid ?? null;
-            ts = rec.ts ?? 0;
-        }
-        catch {
-            // Unreadable/partial lock record — treat as stale.
-        }
-        const dead = holderPid != null && !pidAlive(holderPid);
-        const stale = Date.now() - ts > STALE_MS;
-        if (dead || stale) {
+        // Lock exists: steal it only if the holder is dead or the record is unreadable.
+        const holderPid = readHolderPid(lockPath);
+        if (holderPid == null || !pidAlive(holderPid)) {
             try {
                 unlinkSync(lockPath);
             }
@@ -73,14 +62,15 @@ function readHolderPid(lockPath) {
         return null;
     }
 }
-// Busy-wait with a blocking sleep between polls. Writers hold the lock only for
-// the brief file-CAS + ingest window, so contention waits are short; a sync
-// spin keeps the lock API synchronous like the rest of the write path.
+// Blocking sleep between polls. Writers hold the lock only for the brief
+// file-CAS + ingest window, so contention waits are short, and the lock API
+// stays synchronous like the rest of the write path. `Atomics.wait` parks the
+// thread in the kernel for `ms` — unlike a `Date.now()` spin it burns no CPU,
+// which matters precisely when several writers contend on a loaded machine
+// (each spinning waiter would otherwise steal cycles from the lock holder).
+const SLEEP_CELL = new Int32Array(new SharedArrayBuffer(4));
 function sleepSync(ms) {
-    const until = Date.now() + ms;
-    while (Date.now() < until) {
-        /* spin */
-    }
+    Atomics.wait(SLEEP_CELL, 0, 0, ms);
 }
 /**
  * Run fn while holding the workspace writer lock. Blocks (polling) until the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@omgbase/core",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Open Markdown Graph Base — embedded engine",
   "license": "MIT",
   "repository": {
@@ -55,7 +55,7 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
-    "@omgbase/oqx": "^0.10.2",
+    "@omgbase/oqx": "^0.13.0",
     "better-sqlite3": "^13.0.3",
     "mdast-util-from-markdown": "^2.0.2",
     "mdast-util-frontmatter": "^2.0.1",
@@ -73,7 +73,7 @@
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run",
+    "test": "vitest run --exclude src/core/perf.test.ts && vitest run src/core/perf.test.ts",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix"
   }