@omgbase/core 0.2.0 → 0.4.0

package/dist/src/search/rrf.js

@@ -50,7 +50,8 @@ export function hybridSearch(store, input) {
     const vecRanks = new Map();
     const cosineByBlock = new Map();
     if (input.vector) {
-        // First occurrence wins, exactly as for FTS (spec/search §4).
+        // First occurrence wins, exactly as for FTS (spec/search §4). Since search
+        // 1.1 vectorSearch yields one candidate per block, so this is a no-op guard.
         vectorSearch(store, input.repoId, input.vector.model, input.vector.vec, { limit: 200 }).forEach((h, i) => {
             if (vecRanks.has(h.blockId))
                 return;

package/dist/src/search/tasks.d.ts

@@ -1,6 +1,39 @@
 import type { Store } from "../core/store/store.js";
 import { type EmbedTask, type DocEmbedTask } from "./embeddings.js";
-/** Build EmbedTask[] for every embeddable live block in the repo. */
+/** The columns a block's current context (spec/search §2.2) is a function of. */
+export interface BlockContextInput {
+    doc_id: string;
+    path: string;
+    ordinal: number;
+    type: string;
+}
+/**
+ * Resolves a block's *current* embedding context (spec/search §2.2) — the one
+ * rule every reader of the `embeddings` cache keys with: the drain (tasks), the
+ * pooled document vector, `vector_search` (§3) and `semantic()` (spec/surface
+ * §1.3). Built once per repo (or per doc) and reused across blocks: one scan of
+ * the frontmatter titles and one of the sections, then a per-block walk of the
+ * doc's sections for the heading chain.
+ */
+export interface BlockContextResolver {
+    ctx(block: BlockContextInput): string;
+}
+/**
+ * Load the title/section facts the context prefix depends on. `scope` narrows
+ * the load to one repo (every doc — the drain and vector search) or one doc
+ * (a single block's lookup).
+ */
+export declare function makeBlockContextResolver(store: Store, scope: {
+    repoId: string;
+} | {
+    docId: string;
+}): BlockContextResolver;
+/**
+ * Build EmbedTask[] for every embeddable live block in the repo: the live
+ * LEAF blocks (spec/search §2.1, 1.2 — a container's text is its children's
+ * text, so embedding it would pool the same words twice) whose text clears the
+ * `shouldEmbed` floor.
+ */
 export declare function buildEmbedTasks(store: Store, repoId: string): EmbedTask[];
 /**
  * Build DocEmbedTask[] for every live document in the repo (doc-grain semantic

package/dist/src/search/tasks.js

@@ -1,5 +1,6 @@
 import { reconstructContent } from "../core/read/document.js";
 import { docPropertiesMerged } from "../core/store/properties.js";
+import { LIVE_LEAF_SQL } from "../core/store/fts.js";
 import { contextPrefix, shouldEmbed, estimateTokens } from "./embeddings.js";
 function docTitle(titleByDoc, firstHeadingByDoc, docId, path) {
     const t = titleByDoc.get(docId);
@@ -7,31 +8,31 @@ function docTitle(titleByDoc, firstHeadingByDoc, docId, path) {
         return t;
     return firstHeadingByDoc.get(docId) ?? path;
 }
-/** Build EmbedTask[] for every embeddable live block in the repo. */
-export function buildEmbedTasks(store, repoId) {
-    const blocks = store.db
-        .prepare(`SELECT b.block_id, b.doc_id, d.path AS path,
-              b.ordinal, b.type, b.text, b.raw_hash
-       FROM blocks b JOIN docs d ON d.doc_id = b.doc_id
-       WHERE b.repo_id = ? AND b.deleted_commit IS NULL
-       ORDER BY d.path, b.ordinal`)
-        .all(repoId);
-    // Frontmatter title per doc (scalar), for the embed context prefix. One scan
-    // of the properties table instead of a per-block metadata parse.
+/**
+ * Load the title/section facts the context prefix depends on. `scope` narrows
+ * the load to one repo (every doc — the drain and vector search) or one doc
+ * (a single block's lookup).
+ */
+export function makeBlockContextResolver(store, scope) {
+    const byRepo = "repoId" in scope;
+    // Frontmatter title per doc (scalar string), for the embed context prefix.
+    // One scan of the properties table instead of a per-block metadata parse.
     const titleByDoc = new Map();
     for (const r of store.db.prepare(`SELECT p.doc_id AS doc_id, p.val_text AS title FROM properties p
-     WHERE p.repo_id = ? AND p.source = 'frontmatter' AND p.key = 'title'
-       AND p.card = 'scalar' AND p.type = 'string' AND p.deleted_commit IS NULL`).all(repoId)) {
+     WHERE ${byRepo ? "p.repo_id = ?" : "p.doc_id = ?"} AND p.source = 'frontmatter' AND p.key = 'title'
+       AND p.card = 'scalar' AND p.type = 'string' AND p.deleted_commit IS NULL`).all(byRepo ? scope.repoId : scope.docId)) {
         if (r.title)
             titleByDoc.set(r.doc_id, r.title);
     }
-    // All sections in the repo, with their heading text, for chain lookup.
+    // The sections in scope with their heading text, for chain lookup. Ordered by
+    // first_ordinal so the first per doc is the title fallback (§2.2).
     const sections = store.db
         .prepare(`SELECT s.doc_id AS doc_id, hb.text AS heading_text, s.level AS level,
               s.first_ordinal AS first_ordinal, s.last_ordinal AS last_ordinal
        FROM sections s JOIN blocks hb ON hb.block_id = s.heading_block
-       WHERE hb.repo_id = ?`)
-        .all(repoId);
+       WHERE ${byRepo ? "hb.repo_id = ?" : "s.doc_id = ?"}
+       ORDER BY s.doc_id, s.first_ordinal, s.rowid`)
+        .all(byRepo ? scope.repoId : scope.docId);
     const sectionsByDoc = new Map();
     for (const s of sections) {
         (sectionsByDoc.get(s.doc_id) ?? sectionsByDoc.set(s.doc_id, []).get(s.doc_id)).push(s);
@@ -52,17 +53,35 @@ export function buildEmbedTasks(store, repoId) {
             .sort((a, b) => a.level - b.level)
             .map((s) => s.heading_text);
     };
-    const tasks = [];
-    for (const b of blocks) {
-        if (!shouldEmbed(b.text))
-            continue;
-        const ctx = contextPrefix({
+    return {
+        ctx: (b) => contextPrefix({
             docTitle: docTitle(titleByDoc, firstHeadingByDoc, b.doc_id, b.path),
             path: b.path,
             headingChain: headingChain(b.doc_id, b.ordinal),
             blockType: b.type,
-        });
-        tasks.push({ blockId: b.block_id, contentHashHex: b.raw_hash.toString("hex"), ctx, text: b.text });
+        }),
+    };
+}
+/**
+ * Build EmbedTask[] for every embeddable live block in the repo: the live
+ * LEAF blocks (spec/search §2.1, 1.2 — a container's text is its children's
+ * text, so embedding it would pool the same words twice) whose text clears the
+ * `shouldEmbed` floor.
+ */
+export function buildEmbedTasks(store, repoId) {
+    const blocks = store.db
+        .prepare(`SELECT b.block_id, b.doc_id, d.path AS path,
+              b.ordinal, b.type, b.text, b.raw_hash
+       FROM blocks b JOIN docs d ON d.doc_id = b.doc_id
+       WHERE b.repo_id = ? AND ${LIVE_LEAF_SQL}
+       ORDER BY d.path, b.ordinal`)
+        .all(repoId);
+    const contexts = makeBlockContextResolver(store, { repoId });
+    const tasks = [];
+    for (const b of blocks) {
+        if (!shouldEmbed(b.text))
+            continue;
+        tasks.push({ blockId: b.block_id, contentHashHex: b.raw_hash.toString("hex"), ctx: contexts.ctx(b), text: b.text });
     }
     return tasks;
 }

package/dist/src/search/vector.js

@@ -1,3 +1,5 @@
+import { ctxHashHex } from "./embeddings.js";
+import { buildEmbedTasks } from "./tasks.js";
 function cosine(a, b) {
     let dot = 0, na = 0, nb = 0;
     const n = Math.min(a.length, b.length);
@@ -12,25 +14,24 @@ function cosine(a, b) {
 }
 export function vectorSearch(store, repoId, model, queryVec, opts = {}) {
     const limit = opts.limit ?? 50;
-    const rows = store.db
-        .prepare(`SELECT b.block_id AS blockId, b.doc_id AS docId, d.path AS path, e.vec AS vec
-       FROM embeddings e
-       JOIN blocks b ON b.raw_hash = e.content_hash AND b.deleted_commit IS NULL
-       JOIN docs d ON d.doc_id = b.doc_id
-       WHERE b.repo_id = ? AND e.model = ?`)
-        .all(repoId, model);
-    // One hit per live block (spec/search §3). Several cache rows can join to the
-    // same block — a stale ctx row survives a heading rename next to the fresh one
-    // — and the block keeps its best cosine, not one hit per row.
-    const best = new Map();
-    for (const r of rows) {
-        const v = new Float32Array(r.vec.buffer, r.vec.byteOffset, r.vec.byteLength / 4);
-        const c = cosine(queryVec, v);
-        const prev = best.get(r.blockId);
-        if (!prev || c > prev.cosine)
-            best.set(r.blockId, { blockId: r.blockId, docId: r.docId, path: r.path, cosine: c });
+    // The candidates are exactly the drain's tasks: every live embeddable block
+    // with its current (content_hash, ctx). Reusing buildEmbedTasks keeps the
+    // context rule in one place; its cost is one walk of the repo's blocks and
+    // sections per query (fine at v1 scale, see the ceiling above).
+    const tasks = buildEmbedTasks(store, repoId);
+    const docPath = store.db.prepare("SELECT doc_id AS docId, path FROM blocks JOIN docs USING (doc_id) WHERE block_id = ?");
+    const vecFor = store.db.prepare("SELECT vec FROM embeddings WHERE content_hash = ? AND ctx_hash = ? AND model = ?");
+    const scored = [];
+    for (const t of tasks) {
+        const row = vecFor.get(Buffer.from(t.contentHashHex, "hex"), Buffer.from(ctxHashHex(t.ctx), "hex"), model);
+        if (!row)
+            continue; // stale (§2.3): no vector until the next drain
+        const meta = docPath.get(t.blockId);
+        if (!meta)
+            continue;
+        const v = new Float32Array(row.vec.buffer, row.vec.byteOffset, row.vec.byteLength / 4);
+        scored.push({ blockId: t.blockId, docId: meta.docId, path: meta.path, cosine: cosine(queryVec, v) });
     }
-    const scored = [...best.values()];
     scored.sort((a, b) => b.cosine - a.cosine || (a.blockId < b.blockId ? -1 : 1));
     return scored.slice(0, limit);
 }

package/dist/src/sync/external-source.js

@@ -17,7 +17,7 @@ export async function createExternalSource(spec) {
     }
     const rl = createInterface({ input: child.stdout });
     let watchListener = null;
-    const router = messageRouter(rl, (paths) => watchListener?.(paths));
+    const router = messageRouter(rl, (ev) => watchListener?.(ev));
     const died = new Promise((_, reject) => {
         child.once("error", (err) => reject(new Error(`sync adapter '${spec.command}' failed to spawn: ${err.message}`)));
         child.once("exit", (code) => reject(new Error(`sync adapter '${spec.command}' exited early (code ${code ?? "?"})`)));
@@ -69,10 +69,19 @@ export async function createExternalSource(spec) {
         },
         ...(caps.watch
             ? {
-                async watch(onBatch) {
-                    watchListener = onBatch;
+                async watch(listener) {
+                    // `ready` resolves on the adapter's {"event":"ready"}; the listener sees
+                    // it as an event too, in wire order with the batches (spec/sync §5, §8).
+                    let markReady = () => { };
+                    const ready = new Promise((resolve) => { markReady = resolve; });
+                    watchListener = (ev) => {
+                        if (ev.event === "ready")
+                            markReady();
+                        listener(ev);
+                    };
                     await call("watch");
                     return {
+                        ready,
                         async stop() {
                             watchListener = null;
                             try {
@@ -105,9 +114,9 @@ export async function createExternalSource(spec) {
     };
     return source;
 }
-// Demultiplex adapter stdout: {"event":"batch"} lines feed the watch listener;
-// every other line is an id-addressed response buffered for nextResponse().
-function messageRouter(rl, onBatch) {
+// Demultiplex adapter stdout: {"event":…} lines feed the watch listener (ready,
+// batch); every other line is an id-addressed response buffered for nextResponse().
+function messageRouter(rl, onEvent) {
     const buffered = [];
     const waiters = [];
     rl.on("line", (line) => {
@@ -117,7 +126,11 @@ function messageRouter(rl, onBatch) {
         try {
             const peek = JSON.parse(trimmed);
             if (peek.event === "batch") {
-                onBatch(peek.paths ?? []);
+                onEvent({ event: "batch", paths: peek.paths ?? [] });
+                return;
+            }
+            if (peek.event === "ready") {
+                onEvent({ event: "ready" });
                 return;
             }
         }

package/dist/src/sync/freshness.d.ts

@@ -87,7 +87,14 @@ export interface DiskDrift {
 export declare function detectDiskDrift(store: Store, repoId: string, rootPath: string, opts?: {
     fs?: SyncFs;
 }): DiskDrift;
-/** Rebuild file_stats from scratch by re-statting + re-hashing every file. */
+/**
+ * Rebuild `file_stats` from scratch (spec/sync §4.3, since 1.1): delete the repo's rows, then
+ * walk the tree and record a row only for a file whose bytes' hash equals its live doc's
+ * `file_hash` — the cache is allowed to say "known" only about bytes the store already holds.
+ * A file with no live doc, or whose bytes differ from what was ingested, gets no row, so the
+ * next sweep still treats it as a candidate and drift still reports it (a cache rebuild must
+ * never change what the next sweep does). Returns the number of files walked, recorded or not.
+ */
 export declare function rebuildFileStats(store: Store, repoId: string, rootPath: string, opts?: {
     fs?: SyncFs;
 }): number;

package/dist/src/sync/freshness.js

@@ -147,16 +147,36 @@ export function detectDiskDrift(store, repoId, rootPath, opts = {}) {
             deleted += 1;
     return { changed, deleted, untracked };
 }
-/** Rebuild file_stats from scratch by re-statting + re-hashing every file. */
+/**
+ * Rebuild `file_stats` from scratch (spec/sync §4.3, since 1.1): delete the repo's rows, then
+ * walk the tree and record a row only for a file whose bytes' hash equals its live doc's
+ * `file_hash` — the cache is allowed to say "known" only about bytes the store already holds.
+ * A file with no live doc, or whose bytes differ from what was ingested, gets no row, so the
+ * next sweep still treats it as a candidate and drift still reports it (a cache rebuild must
+ * never change what the next sweep does). Returns the number of files walked, recorded or not.
+ */
 export function rebuildFileStats(store, repoId, rootPath, opts = {}) {
     const fs = opts.fs ?? nodeFs;
     store.db.prepare("DELETE FROM file_stats WHERE repo_id = ?").run(repoId);
+    // The live docs' durable convergence signal, keyed by path (one query).
+    const live = new Map();
+    for (const row of store.db
+        .prepare("SELECT path, file_hash FROM docs WHERE repo_id = ? AND deleted_commit IS NULL AND file_hash IS NOT NULL")
+        .all(repoId)) {
+        live.set(row.path, row.file_hash);
+    }
     const paths = fs.walk(rootPath);
     for (const path of paths) {
+        const want = live.get(path);
+        if (!want)
+            continue; // no live doc: nothing the cache may vouch for
         const bytes = fs.read(rootPath, path);
         if (bytes === null)
-            continue;
-        recordStat(store, repoId, path, fs.stat(rootPath, path), sha256(bytes));
+            continue; // vanished between the walk and the read
+        const hash = sha256(bytes);
+        if (!hash.equals(want))
+            continue; // a pending edit: leave it visible
+        recordStat(store, repoId, path, fs.stat(rootPath, path), hash);
     }
     return paths.length;
 }

package/dist/src/sync/plugin.d.ts

@@ -25,13 +25,38 @@ export interface SourceItem extends SourceEntry {
     /** The engine hashes this itself for the authoritative echo/convergence gate. */
     content: string;
 }
-/** A batch of changed storage-keys delivered by a live watch (13 §4.3). */
-export type WatchListener = (paths: string[]) => void;
+/**
+ * An unsolicited event of a live watch (spec/sync §5): `ready` once — the feed
+ * is primed and every change from now on will be reported — then `batch` lines
+ * of changed storage keys. An adapter built before sync 1.2 never sends `ready`.
+ */
+export type WatchEvent = {
+    event: "ready";
+} | {
+    event: "batch";
+    paths: string[];
+};
+/** Receives every event of a live watch, in wire order. */
+export type WatchListener = (event: WatchEvent) => void;
 /** A live subscription created by SyncSource.watch. */
 export interface SourceWatch {
-    /** Stop delivering batches and release resources. */
+    /**
+     * Resolves when the source reported `ready` (spec/sync §5). An in-process
+     * source resolves it at once; an adapter that never reports leaves it pending,
+     * so a host bounds the wait (`awaitReady`).
+     */
+    ready: Promise<void>;
+    /** Stop delivering events and release resources. */
     stop(): Promise<void>;
 }
+/** How long a host waits for `ready` before proceeding as if ready (spec/sync §5: unpinned; 30 s). */
+export declare const WATCH_READY_PATIENCE_MS = 30000;
+/**
+ * Wait for a watch's `ready` with bounded patience: `true` when it resolved in
+ * time, `false` on timeout (the host proceeds as if ready, with a warning — an
+ * adapter without `ready` still works, with the old window).
+ */
+export declare function awaitReady(ready: Promise<void>, patienceMs?: number): Promise<boolean>;
 /**
  * The in-engine handle to an external source process. Only capabilities/
  * enumerate/fetch are always meaningful; watch exists when capabilities().watch,
@@ -43,8 +68,8 @@ export interface SyncSource {
     enumerate(): Promise<SourceEntry[]>;
     /** Current state of one member, or null if it left the scope (a delete). */
     fetch(path: string): Promise<SourceItem | null>;
-    /** Subscribe to debounced change batches (push sources). */
-    watch?(onBatch: WatchListener): Promise<SourceWatch>;
+    /** Subscribe to the change feed (push sources): `ready` once, then debounced `batch` events. */
+    watch?(listener: WatchListener): Promise<SourceWatch>;
     /** Persist engine-authored bytes back to the source (writeThrough only). */
     write?(path: string, content: string): Promise<void>;
     /** Remove a member from the source (writeThrough only). */

package/dist/src/sync/plugin.js

@@ -8,5 +8,17 @@
 // reconciliation, commits, convergence, and durable revision/cursor state — so a
 // source only reports membership and transports bytes; it can never write a
 // commit, mint identity, or bypass the convergence check.
-export {};
+/** How long a host waits for `ready` before proceeding as if ready (spec/sync §5: unpinned; 30 s). */
+export const WATCH_READY_PATIENCE_MS = 30_000;
+/**
+ * Wait for a watch's `ready` with bounded patience: `true` when it resolved in
+ * time, `false` on timeout (the host proceeds as if ready, with a warning — an
+ * adapter without `ready` still works, with the old window).
+ */
+export function awaitReady(ready, patienceMs = WATCH_READY_PATIENCE_MS) {
+    return new Promise((resolve) => {
+        const timer = setTimeout(() => resolve(false), patienceMs);
+        ready.then(() => { clearTimeout(timer); resolve(true); }, () => { clearTimeout(timer); resolve(false); });
+    });
+}
 //# sourceMappingURL=plugin.js.map

package/dist/src/sync/reconciling-ingest.js

@@ -105,7 +105,7 @@ function assignFromMap(blocks, assignment) {
     const walk = (list, parentKey) => list.map((b, index) => {
         const key = `${parentKey ?? ""}/${index}`;
         const blockId = assignment.get(key) ?? mintId("b");
-        return { blockId, type: b.type, raw: b.raw, trivia: b.trivia, attrs: b.attrs, children: walk(b.children, key) };
+        return { blockId, type: b.type, raw: b.raw, trivia: b.trivia, attrs: b.attrs, children: walk(b.children, key), span: b.span };
     });
     return walk(blocks, null);
 }
@@ -260,11 +260,13 @@ function resolveAdapterEdge(db, repoId, srcDoc, e, docDir) {
         provenance: e.provenance,
     };
 }
-// Collect RawBlock-shaped objects from the id-assigned tree for adapter edge extraction.
+// Collect RawBlock-shaped objects from the id-assigned tree for adapter edge
+// extraction. The parsed span rides along so the adapter can blank a
+// container's children (spec/graph §1 own text).
 function collectRawBlocks(blocks) {
     return blocks.map((b) => ({
         type: b.type,
-        span: { start: 0, end: 0 },
+        span: b.span ?? { start: 0, end: 0 },
         raw: b.raw,
         text: "",
         attrs: b.attrs,

package/dist/src/sync/watcher.d.ts

@@ -14,6 +14,8 @@ export declare class Watcher {
     private sub;
     constructor(store: Store, repoId: string, source: SyncSource, opts?: WatcherOptions);
     start(): Promise<void>;
+    /** The live subscription's `ready` (spec/sync §5); throws before `start()`. */
+    ready(): Promise<void>;
     private ingest;
     stop(): Promise<void>;
 }

package/dist/src/sync/watcher.js

@@ -14,11 +14,19 @@ export class Watcher {
     async start() {
         if (!this.source.watch)
             throw new Error("source does not support watch()");
-        this.sub = await this.source.watch((paths) => {
+        this.sub = await this.source.watch((ev) => {
+            if (ev.event !== "batch")
+                return; // `ready` is surfaced through ready()
             // A batch arrives from the adapter's stream; reconcile it off the callback.
-            void this.ingest(paths).catch((err) => this.opts.onError?.(err));
+            void this.ingest(ev.paths).catch((err) => this.opts.onError?.(err));
         });
     }
+    /** The live subscription's `ready` (spec/sync §5); throws before `start()`. */
+    ready() {
+        if (!this.sub)
+            throw new Error("watcher not started");
+        return this.sub.ready;
+    }
     async ingest(paths) {
         if (paths.length === 0)
             return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@omgbase/core",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Open Markdown Graph Base — embedded engine",
   "license": "MIT",
   "repository": {
@@ -55,7 +55,7 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
-    "@omgbase/oqx": "^0.12.0",
+    "@omgbase/oqx": "^0.13.0",
     "better-sqlite3": "^13.0.3",
     "mdast-util-from-markdown": "^2.0.2",
     "mdast-util-frontmatter": "^2.0.1",