@omgbase/core 0.11.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +1 -1
  2. package/dist/src/core/ids.d.ts +4 -0
  3. package/dist/src/core/ids.js +17 -0
  4. package/dist/src/core/ingest.d.ts +39 -3
  5. package/dist/src/core/ingest.js +233 -202
  6. package/dist/src/core/store/schema.d.ts +3 -2
  7. package/dist/src/core/store/schema.js +28 -3
  8. package/dist/src/core/store/store.d.ts +9 -0
  9. package/dist/src/core/store/store.js +11 -0
  10. package/dist/src/graph/history.js +15 -12
  11. package/dist/src/index.d.ts +9 -5
  12. package/dist/src/index.js +14 -7
  13. package/dist/src/mcp/arguments.d.ts +9 -2
  14. package/dist/src/mcp/arguments.js +15 -2
  15. package/dist/src/mcp/errors.d.ts +1 -1
  16. package/dist/src/mcp/errors.js +3 -0
  17. package/dist/src/mcp/server.d.ts +22 -0
  18. package/dist/src/mcp/server.js +262 -185
  19. package/dist/src/mutate/apply.d.ts +93 -12
  20. package/dist/src/mutate/apply.js +203 -113
  21. package/dist/src/mutate/commit.d.ts +150 -0
  22. package/dist/src/mutate/commit.js +349 -0
  23. package/dist/src/mutate/doc-store.d.ts +3 -3
  24. package/dist/src/mutate/doc-store.js +21 -6
  25. package/dist/src/mutate/docs.d.ts +42 -87
  26. package/dist/src/mutate/docs.js +50 -204
  27. package/dist/src/mutate/effects.d.ts +331 -0
  28. package/dist/src/mutate/effects.js +981 -0
  29. package/dist/src/mutate/materialize.d.ts +49 -0
  30. package/dist/src/mutate/materialize.js +174 -0
  31. package/dist/src/mutate/opset.d.ts +27 -2
  32. package/dist/src/mutate/opset.js +12 -5
  33. package/dist/src/mutate/plan-update.d.ts +1 -46
  34. package/dist/src/mutate/plan-update.js +19 -67
  35. package/dist/src/mutate/plan.d.ts +200 -0
  36. package/dist/src/mutate/plan.js +472 -0
  37. package/dist/src/mutate/sugar.d.ts +135 -0
  38. package/dist/src/mutate/sugar.js +167 -0
  39. package/dist/src/mutate/tree.d.ts +4 -2
  40. package/dist/src/spec-versions.d.ts +5 -5
  41. package/dist/src/spec-versions.js +5 -5
  42. package/dist/src/surface-paths.d.ts +17 -26
  43. package/dist/src/surface-paths.js +27 -22
  44. package/dist/src/sync/checkpoint.d.ts +5 -2
  45. package/dist/src/sync/checkpoint.js +10 -6
  46. package/dist/src/sync/freshness.d.ts +10 -7
  47. package/dist/src/sync/freshness.js +15 -17
  48. package/dist/src/sync/observe.d.ts +4 -1
  49. package/dist/src/sync/observe.js +56 -5
  50. package/dist/src/sync/recovery.d.ts +4 -0
  51. package/dist/src/sync/recovery.js +4 -1
  52. package/dist/src/sync/stat-cache.d.ts +7 -0
  53. package/dist/src/sync/stat-cache.js +22 -0
  54. package/dist/src/sync/tombstone.js +23 -19
  55. package/dist/src/sync/watcher.d.ts +2 -0
  56. package/dist/src/sync/watcher.js +4 -1
  57. package/dist/src/sync/writer-lock.d.ts +15 -0
  58. package/dist/src/sync/writer-lock.js +104 -17
  59. package/package.json +1 -1
package/README.md CHANGED
@@ -111,7 +111,7 @@ the `Store` first and are synchronous unless noted.
111
111
  | Search | Ranked "what does the caller mean" resolution; RRF fusion of FTS + vectors; raw vector search | `resolve`, `hybridSearch`, `vectorSearch`, `docVectorSearch` |
112
112
  | Embeddings | Out-of-process embedder client, batching worker, post-mutation drainer, task builders | `createExternalProvider`, `EmbeddingWorker`, `EmbedDrainer`, `buildEmbedTasks`, `embeddingSettings`, `SemanticUnavailable` |
113
113
  | Format adapters | Register/lookup the parsers that turn a file into blocks + nodes + edges | `registerAdapter`, `adapterForPath`, `detectFormat`, `markdownAdapter`, `yamlAdapter`, `jsonAdapter` |
114
- | MCP server | Build the full tool surface (45 tools: `docs_read`, `query`, `apply`, `observe`, `changes_since`, …) as an `McpServer`; wire it to stdio | `buildServer({ store, repoId, rootPath?, embedQuery?, onMutation? })`, `serveStdio(ctx)` |
114
+ | MCP server | Build the full tool surface (47 tools: `docs_read`, `query`, `apply`, `observe`, `changes_since`, …) as an `McpServer`; wire it to stdio | `buildServer({ store, repoId, rootPath?, embedQuery?, onMutation? })`, `serveStdio(ctx)` |
115
115
  | Admin | Rebuild derived indexes; garbage-collect unreferenced blobs | `rebuildIndex`, `runGc`, `VERSION`, `EngineError` |
116
116
 
117
117
  `omg mcp` is just a host for `serveStdio`: it opens the workspace, primes with
@@ -38,6 +38,10 @@ export declare function sequentialMinter(): IdMinter;
38
38
  * again, so a case run under it projects exactly as under `sequentialMinter`.
39
39
  */
40
40
  export declare function repeatingMinter(): IdMinter;
41
+ /** A mint observer (spec/mutate §11.6, 2.3): told every id `mintId` issues, in order. */
42
+ export type MintObserver = (prefix: IdPrefix, id: string) => void;
43
+ /** Run `body` with `observer` told every id minted meanwhile (also under a nested `withIdMinter`). */
44
+ export declare function observeMints<T>(observer: MintObserver, body: () => T): T;
41
45
  /**
42
46
  * Mint an id with the given prefix (spec/store §2.1 "Uniqueness at mint"):
43
47
  * draw a candidate — from the installed minter, else the CSPRNG — and redraw
@@ -96,6 +96,21 @@ export function repeatingMinter() {
96
96
  return id;
97
97
  };
98
98
  }
99
+ // The installed observers. The effect planner's `execute` records the `b`/`d`
100
+ // ids a plan's replay mints so the receipt can zip them, in mint order, onto
101
+ // the plan's symbolic `$new.n` identities — it observes what `mintId` RETURNS
102
+ // (after the in-use check), never a rejected candidate.
103
+ const observers = new Set();
104
+ /** Run `body` with `observer` told every id minted meanwhile (also under a nested `withIdMinter`). */
105
+ export function observeMints(observer, body) {
106
+ observers.add(observer);
107
+ try {
108
+ return body();
109
+ }
110
+ finally {
111
+ observers.delete(observer);
112
+ }
113
+ }
99
114
  function inUse(prefix, id, issued) {
100
115
  if (issued.has(id))
101
116
  return true;
@@ -118,6 +133,8 @@ export function mintId(prefix) {
118
133
  if (inUse(prefix, candidate, issued))
119
134
  continue;
120
135
  issued.add(candidate);
136
+ for (const observer of observers)
137
+ observer(prefix, candidate);
121
138
  return candidate;
122
139
  }
123
140
  throw new Error(`mintId(${prefix}): ${MINT_GIVE_UP_AFTER} consecutive candidates were already in use — the ${minter ? "installed" : "production"} minter cannot produce a fresh id`);
@@ -1,6 +1,7 @@
1
1
  import { Store } from "./store/store.js";
2
2
  import { type TreeInputBlock } from "./store/writers.js";
3
3
  import type { RawBlock, BlockTree } from "./parse/types.js";
4
+ import type { Database } from "better-sqlite3";
4
5
  export interface DispositionRow {
5
6
  blockId: string;
6
7
  kind: string;
@@ -48,8 +49,8 @@ export interface IngestResult {
48
49
  /** convergence: sha256(file bytes) === revision.rendered_hash */
49
50
  converged: boolean;
50
51
  }
51
- /** Ingest a single file's content into the store as one observed commit. */
52
- export declare function ingestFile(store: Store, repoId: string, path: string, content: string, opts?: {
52
+ /** The options of `ingestFile` — one observed/api/import commit of one file. */
53
+ export interface IngestFileOptions {
53
54
  ts?: string;
54
55
  /** Commit provenance. `observed` = a file edit picked up by sync; `api` = an intent write (apply/docs_*), which should also carry `actor`/`reason`; `import` = bulk ingest with no retro history. */
55
56
  origin?: "observed" | "import" | "api";
@@ -60,4 +61,39 @@ export declare function ingestFile(store: Store, repoId: string, path: string, c
60
61
  /** A tree already parsed from `content` (via `parseForIngest`) — skips the
61
62
  * re-parse. The caller guarantees it is the parse of these exact bytes. */
62
63
  parsed?: BlockTree;
63
- }): IngestResult;
64
+ /** The document's `conflicted` flag (git conflict markers in the bytes),
65
+ * written in the same statement as the current-revision pointers so the
66
+ * flag and the revision it describes land in one transaction (spec/store
67
+ * §5.4 step 14). Absent ⇒ the column is left untouched — the observe path
68
+ * sets it on every commit; api writes never touch it. */
69
+ conflicted?: boolean;
70
+ }
71
+ /** The options of `ingestInto`: the commit the revision hangs off is the
72
+ * caller's (spec/store §5.4 step 5 ran already), and `ts` is that commit's
73
+ * timestamp (the pool expiry is computed from it). */
74
+ export interface IngestIntoOptions {
75
+ ts: string;
76
+ /** The `commits` row this ingest writes its revision, block rows, dispositions, properties and edges under. */
77
+ commitId: string;
78
+ resolveIds?: IdResolver | undefined;
79
+ format?: string | undefined;
80
+ parsed?: BlockTree | undefined;
81
+ conflicted?: boolean | undefined;
82
+ }
83
+ /**
84
+ * Ingest a single file's content into the store as ONE commit of its own:
85
+ * mint the commit row, then `ingestInto` it, in one transaction. The
86
+ * observe path (`sync/observe.ts`, attach, recovery, the driver) commits one
87
+ * member per call this way; the api path (`mutate/commit.ts`) instead opens
88
+ * one transaction and one commit for a whole request and calls `ingestInto`
89
+ * per document (spec/store §5.4 "The api path", spec/mutate §4 step 5).
90
+ */
91
+ export declare function ingestFile(store: Store, repoId: string, path: string, content: string, opts?: IngestFileOptions): IngestResult;
92
+ /**
93
+ * spec/store §5.4 steps 1–4 and 6–14 for one document under an EXISTING
94
+ * commit row, inside the caller's transaction (`db` is the connection the
95
+ * caller's `store.write` handed it; nothing here opens one). The revision,
96
+ * the block rows' `created_commit`, the pool snapshot, the dispositions and
97
+ * block changes, the properties and the edge intervals all name `commitId`.
98
+ */
99
+ export declare function ingestInto(db: Database, repoId: string, path: string, content: string, opts: IngestIntoOptions): IngestResult;
@@ -74,218 +74,249 @@ function flatten(blocks, parent, depth, ancestorPath, out, quoteDepth = 0) {
74
74
  }
75
75
  });
76
76
  }
77
- /** Ingest a single file's content into the store as one observed commit. */
77
+ /**
78
+ * Ingest a single file's content into the store as ONE commit of its own:
79
+ * mint the commit row, then `ingestInto` it, in one transaction. The
80
+ * observe path (`sync/observe.ts`, attach, recovery, the driver) commits one
81
+ * member per call this way; the api path (`mutate/commit.ts`) instead opens
82
+ * one transaction and one commit for a whole request and calls `ingestInto`
83
+ * per document (spec/store §5.4 "The api path", spec/mutate §4 step 5).
84
+ */
78
85
  export function ingestFile(store, repoId, path, content, opts = {}) {
79
86
  const ts = opts.ts ?? new Date().toISOString();
80
87
  const origin = opts.origin ?? "observed";
81
- const adapter = adapterForPath(path);
82
- const format = opts.format ?? adapter?.format ?? "markdown";
83
88
  return store.write((db) => {
84
- const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
85
- const { fmBlock, rest } = extractFrontmatter(tree.children);
86
- // Frontmatter blob (markdown-specific) preserved for revision history.
87
- const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
88
- // The exact separator bytes between the frontmatter block and the first body
89
- // block (the frontmatter block's trailing trivia). Persisted so the body
90
- // reconstructs byte-for-byte instead of assuming a canonical blank line.
91
- // NULL when there is no frontmatter.
92
- const fmTrivia = fmBlock ? fmBlock.trivia : null;
93
- // Document property bag: whole-document metadata for the formats whose
94
- // adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
95
- // `extractMetadata` — the YAML of the `frontmatter` block and nothing else
96
- // (spec/properties §3.1: a document whose first block is not `frontmatter`
97
- // has no frontmatter rows, whatever else its bytes resemble). Flattened into
98
- // the properties table below (source=frontmatter); also drives frontmatter
99
- // edge extraction.
100
- const metadata = adapter?.extractMetadata
101
- ? (adapter.extractMetadata(content) ?? {})
102
- : parseFrontmatter(fmBlock);
103
- // Upsert the document row.
104
- let doc = db.prepare("SELECT doc_id, deleted_commit FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
105
- const docId = doc?.doc_id ?? mintId("d");
106
- const isNew = !doc;
107
- if (!doc) {
108
- db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
109
- doc = { doc_id: docId, deleted_commit: null };
110
- // Adopt phantom edges that pointed at this path so backlinks re-point.
111
- adoptPhantoms(db, path, docId);
112
- }
113
- else {
114
- // A tombstoned row still owns the path's identity (spec/store §5.6
115
- // "Re-creation"): bytes observed again at the path revive it — the row is
116
- // live again, so reads that filter `deleted_commit IS NULL` see it and the
117
- // next identical observation echo-gates instead of re-ingesting.
118
- const revived = doc.deleted_commit !== null;
119
- db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ?, deleted_commit = NULL WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
120
- // spec/graph §3.5: adoption happens whenever a row BECOMES live, so a
121
- // revived tombstone also collects the phantom edges minted at its path
122
- // while it was gone (§8 "Fixed — a revived document adopted no phantoms").
123
- if (revived)
124
- adoptPhantoms(db, path, docId);
125
- }
126
- // Assign block ids via the resolver when supplied (it reconciles against
127
- // the prior revision — for a new doc that's an empty old tree, so all blocks
128
- // mint — and also drives edge extraction). Without a resolver, fresh-mint.
129
- const resolved = opts.resolveIds
130
- ? opts.resolveIds(rest, isNew ? null : docId)
131
- : { assigned: assignIds(rest), dispositions: [], deleted: [] };
132
- const assigned = resolved.assigned;
133
- const rootTreeHex = writeBlockTree(db, assigned);
134
89
  const commit = newCommit(db, { repoId, ts, origin, actor: opts.actor ?? null, reason: opts.reason ?? null });
135
- const renderedHash = sha256(content);
136
- const rev = writeRevision(db, {
137
- docId,
138
- rootTreeHex,
139
- frontmatterBlobHex: fmBlobHex,
140
- renderedHash,
141
- path,
90
+ return ingestInto(db, repoId, path, content, {
91
+ ts,
142
92
  commitId: commit.commitId,
93
+ resolveIds: opts.resolveIds,
94
+ format: opts.format,
95
+ parsed: opts.parsed,
96
+ conflicted: opts.conflicted,
143
97
  });
144
- // Snapshot deleted blocks into the resurrection pool BEFORE dropping the
145
- // doc's rows (the ingest rebuilds current-state blocks wholesale).
146
- if (resolved.deleted.length > 0) {
147
- const expires = new Date(Date.parse(ts) + 30 * 24 * 3600 * 1000).toISOString();
148
- const pool = db.prepare(`INSERT OR REPLACE INTO resurrection_pool (block_id, repo_id, doc_id, raw_hash, norm_hash, type, deleted_commit, expires_ts)
149
- SELECT block_id, repo_id, doc_id, raw_hash, norm_hash, type, ?, ? FROM blocks WHERE block_id = ? AND doc_id = ?`);
150
- for (const id of resolved.deleted)
151
- pool.run(commit.commitId, expires, id, docId);
152
- }
153
- // Ids arriving from ANOTHER document — a cross-document move whose source
154
- // commits later in this checkpoint (or was tombstoned in it), or a
155
- // resurrection out of a tombstoned doc whose rows still sit in `blocks` —
156
- // must not collide on the block_id primary key. Evict the foreign row (and
157
- // its FTS entry when live) and drop any pool row: the id is live here now.
158
- const incoming = [...(resolved.crossDocIds ?? []), ...(resolved.consumedPool ?? [])];
159
- if (incoming.length > 0)
160
- evictForeignBlockRows(db, docId, incoming);
161
- // Refresh current-state blocks (clear + repopulate with carried/minted ids).
162
- // FTS is external-content: delete old index rows before dropping blocks.
163
- ftsDeleteDoc(db, docId);
164
- db.prepare("DELETE FROM blocks WHERE doc_id = ?").run(docId);
165
- const rows = [];
166
- flatten(assigned, null, 0, "/", rows);
167
- const insert = db.prepare(`INSERT INTO blocks
168
- (block_id, repo_id, doc_id, parent_block, order_key, ordinal, depth,
169
- ancestor_path, type, attrs, text, raw_hash, norm_hash, trivia_hash, created_commit)
170
- VALUES
171
- (@blockId, @repoId, @docId, @parentBlock, @orderKey, @ordinal, @depth,
172
- @ancestorPath, @type, @attrs, @text, @rawHash, @normHash, @triviaHash, @createdCommit)`);
173
- for (const r of rows) {
174
- insert.run({ ...r, repoId, docId, createdCommit: commit.commitId });
175
- }
176
- ftsIndexDoc(db, docId);
177
- rebuildSections(db, docId);
178
- // Node projection: adapter-provided semantic features from parsed blocks,
179
- // plus engine-derived `md:section` nodes from the (just-rebuilt) sections
180
- // table. Written together in one pass (writeDocNodes clears + repopulates).
181
- const propertyRows = [];
182
- const sectionNodes = projectSectionNodes(db, docId);
183
- if (adapter?.projectNodes) {
184
- // Zip assigned block ids onto the parsed blocks so projectNodes can anchor
185
- // each node to its own block (the parsed RawBlocks are pre-identity).
186
- const rawBlocks = zipAssignedIds(rest, assigned);
187
- const projected = adapter.projectNodes(rawBlocks);
188
- // Keep only nodes anchored to a real assigned block id.
189
- const idSet = collectBlockIds(assigned);
190
- const withIds = projected.map((n) => ({
191
- ...n,
192
- blockId: n.blockId && idSet.has(n.blockId) ? n.blockId : "",
193
- }));
194
- // Adapters record spans as string indices; the store keeps byte offsets
195
- // into the block's UTF-8 raw (spec/graph §2.3), so convert here — the one
196
- // place that has both the nodes and the raws they index.
197
- const rawById = collectRaws(assigned);
198
- const byteSpanned = toByteSpans(withIds, (id) => rawById.get(id));
199
- writeDocNodes(db, repoId, docId, [...byteSpanned, ...sectionNodes]);
200
- // Inline properties (key:: value) → property rows (source=inline). The
201
- // authored shape within the inline source drives `card`: a key that occurs
202
- // exactly once in the document is `scalar` (so `element == "fire"` can
203
- // match a lone `element:: fire`); a key that repeats is `list`. Repeats
204
- // accumulate as ord-indexed rows in document order (block ordinal, then
205
- // in-block position — the order withIds already carries). Effective bare-
206
- // key cardinality (a frontmatter+inline collision) is decided at query
207
- // time by the single-value gate on scalar comparisons (see compile.ts),
208
- // not here — each source stays honest to its own authored shape.
209
- const inlineFields = withIds.filter((n) => n.kind === "md:inline_field" && n.name);
210
- const inlineCounts = new Map();
211
- for (const n of inlineFields)
212
- inlineCounts.set(n.name, (inlineCounts.get(n.name) ?? 0) + 1);
213
- const inlineOrd = new Map();
214
- for (const n of inlineFields) {
215
- const ord = inlineOrd.get(n.name) ?? 0;
216
- inlineOrd.set(n.name, ord + 1);
217
- propertyRows.push({
218
- source: "inline",
219
- blockId: n.blockId || null,
220
- key: n.name,
221
- card: (inlineCounts.get(n.name) ?? 0) > 1 ? "list" : "scalar",
222
- ord,
223
- ...typedInlineValue(n.value),
224
- });
225
- }
226
- }
227
- else if (sectionNodes.length > 0) {
228
- // No adapter node projection, but the document still has sections.
229
- writeDocNodes(db, repoId, docId, sectionNodes);
230
- }
231
- // Frontmatter → property rows (source=frontmatter). Flatten nested maps to
232
- // dotted keys, arrays to ord-indexed list rows, scalars to a scalar row.
233
- for (const r of flattenFrontmatter(metadata)) {
234
- propertyRows.push({ source: "frontmatter", blockId: null, ...r });
235
- }
236
- // Computed → property rows (source=computed). Engine-derived $-intrinsics
237
- // ($title, $tags) from the adapter; never claim authored keys.
238
- if (adapter?.computeProperties) {
239
- const computed = adapter.computeProperties(rest, metadata);
240
- for (const r of flattenComputed(computed)) {
241
- propertyRows.push({ source: "computed", blockId: null, ...r });
242
- }
243
- }
244
- writeDocProperties(db, repoId, docId, commit.commitId, propertyRows);
245
- // Edge extraction + interval maintenance (05 §2), if the resolver supplies
246
- // an extractor. Runs in this commit transaction.
247
- if (resolved.extractEdges) {
248
- const extracted = resolved.extractEdges(docId, metadata);
249
- maintainEdges(db, repoId, docId, commit.commitId, extracted);
250
- }
251
- // Persist dispositions + block-history projection. With a resolver, use the
252
- // real reconciliation dispositions; otherwise record every block as inserted.
253
- const dispositions = resolved.dispositions.length > 0 || opts.resolveIds
254
- ? resolved.dispositions
255
- : rows.map((r) => ({ blockId: r.blockId, kind: "inserted", confidence: null, reason: null, matcherV: null, detail: {} }));
256
- const insDisp = db.prepare(`INSERT OR IGNORE INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail)
257
- VALUES (?, ?, ?, ?, ?, ?, ?)`);
258
- const bc = db.prepare("INSERT OR IGNORE INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
259
- for (const d of dispositions) {
260
- insDisp.run(commit.commitId, d.blockId, d.kind, d.confidence, d.reason, d.matcherV, JSON.stringify(d.detail));
261
- bc.run(d.blockId, commit.commitId, d.kind);
98
+ });
99
+ }
100
+ /**
101
+ * spec/store §5.4 steps 1–4 and 6–14 for one document under an EXISTING
102
+ * commit row, inside the caller's transaction (`db` is the connection the
103
+ * caller's `store.write` handed it; nothing here opens one). The revision,
104
+ * the block rows' `created_commit`, the pool snapshot, the dispositions and
105
+ * block changes, the properties and the edge intervals all name `commitId`.
106
+ */
107
+ export function ingestInto(db, repoId, path, content, opts) {
108
+ const { ts, commitId } = opts;
109
+ const adapter = adapterForPath(path);
110
+ const format = opts.format ?? adapter?.format ?? "markdown";
111
+ const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
112
+ const { fmBlock, rest } = extractFrontmatter(tree.children);
113
+ // Frontmatter blob (markdown-specific) preserved for revision history.
114
+ const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
115
+ // The exact separator bytes between the frontmatter block and the first body
116
+ // block (the frontmatter block's trailing trivia). Persisted so the body
117
+ // reconstructs byte-for-byte instead of assuming a canonical blank line.
118
+ // NULL when there is no frontmatter.
119
+ const fmTrivia = fmBlock ? fmBlock.trivia : null;
120
+ // Document property bag: whole-document metadata for the formats whose
121
+ // adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
122
+ // `extractMetadata` — the YAML of the `frontmatter` block and nothing else
123
+ // (spec/properties §3.1: a document whose first block is not `frontmatter`
124
+ // has no frontmatter rows, whatever else its bytes resemble). Flattened into
125
+ // the properties table below (source=frontmatter); also drives frontmatter
126
+ // edge extraction.
127
+ const metadata = adapter?.extractMetadata
128
+ ? (adapter.extractMetadata(content) ?? {})
129
+ : parseFrontmatter(fmBlock);
130
+ // Upsert the document row.
131
+ let doc = db.prepare("SELECT doc_id, deleted_commit FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
132
+ const docId = doc?.doc_id ?? mintId("d");
133
+ const isNew = !doc;
134
+ if (!doc) {
135
+ db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
136
+ doc = { doc_id: docId, deleted_commit: null };
137
+ // Adopt phantom edges that pointed at this path so backlinks re-point.
138
+ adoptPhantoms(db, path, docId);
139
+ }
140
+ else {
141
+ // A tombstoned row still owns the path's identity (spec/store §5.6
142
+ // "Re-creation"): bytes observed again at the path revive it — the row is
143
+ // live again, so reads that filter `deleted_commit IS NULL` see it and the
144
+ // next identical observation echo-gates instead of re-ingesting.
145
+ const revived = doc.deleted_commit !== null;
146
+ db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ?, deleted_commit = NULL WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
147
+ // spec/graph §3.5: adoption happens whenever a row BECOMES live, so a
148
+ // revived tombstone also collects the phantom edges minted at its path
149
+ // while it was gone (§8 "Fixed — a revived document adopted no phantoms").
150
+ if (revived)
151
+ adoptPhantoms(db, path, docId);
152
+ }
153
+ // Assign block ids via the resolver when supplied (it reconciles against
154
+ // the prior revision — for a new doc that's an empty old tree, so all blocks
155
+ // mint — and also drives edge extraction). Without a resolver, fresh-mint.
156
+ const resolved = opts.resolveIds
157
+ ? opts.resolveIds(rest, isNew ? null : docId)
158
+ : { assigned: assignIds(rest), dispositions: [], deleted: [] };
159
+ const assigned = resolved.assigned;
160
+ const rootTreeHex = writeBlockTree(db, assigned);
161
+ const renderedHash = sha256(content);
162
+ const rev = writeRevision(db, {
163
+ docId,
164
+ rootTreeHex,
165
+ frontmatterBlobHex: fmBlobHex,
166
+ renderedHash,
167
+ path,
168
+ commitId,
169
+ });
170
+ // Snapshot deleted blocks into the resurrection pool BEFORE dropping the
171
+ // doc's rows (the ingest rebuilds current-state blocks wholesale).
172
+ if (resolved.deleted.length > 0) {
173
+ const expires = new Date(Date.parse(ts) + 30 * 24 * 3600 * 1000).toISOString();
174
+ const pool = db.prepare(`INSERT OR REPLACE INTO resurrection_pool (block_id, repo_id, doc_id, raw_hash, norm_hash, type, deleted_commit, expires_ts)
175
+ SELECT block_id, repo_id, doc_id, raw_hash, norm_hash, type, ?, ? FROM blocks WHERE block_id = ? AND doc_id = ?`);
176
+ for (const id of resolved.deleted)
177
+ pool.run(commitId, expires, id, docId);
178
+ }
179
+ // Ids arriving from ANOTHER document — a cross-document move whose source
180
+ // commits later in this checkpoint (or was tombstoned in it), or a
181
+ // resurrection out of a tombstoned doc whose rows still sit in `blocks` —
182
+ // must not collide on the block_id primary key. Evict the foreign row (and
183
+ // its FTS entry when live) and drop any pool row: the id is live here now.
184
+ const incoming = [...(resolved.crossDocIds ?? []), ...(resolved.consumedPool ?? [])];
185
+ if (incoming.length > 0)
186
+ evictForeignBlockRows(db, docId, incoming);
187
+ // Refresh current-state blocks (clear + repopulate with carried/minted ids).
188
+ // FTS is external-content: delete old index rows before dropping blocks.
189
+ ftsDeleteDoc(db, docId);
190
+ db.prepare("DELETE FROM blocks WHERE doc_id = ?").run(docId);
191
+ const rows = [];
192
+ flatten(assigned, null, 0, "/", rows);
193
+ const insert = db.prepare(`INSERT INTO blocks
194
+ (block_id, repo_id, doc_id, parent_block, order_key, ordinal, depth,
195
+ ancestor_path, type, attrs, text, raw_hash, norm_hash, trivia_hash, created_commit)
196
+ VALUES
197
+ (@blockId, @repoId, @docId, @parentBlock, @orderKey, @ordinal, @depth,
198
+ @ancestorPath, @type, @attrs, @text, @rawHash, @normHash, @triviaHash, @createdCommit)`);
199
+ for (const r of rows) {
200
+ insert.run({ ...r, repoId, docId, createdCommit: commitId });
201
+ }
202
+ ftsIndexDoc(db, docId);
203
+ rebuildSections(db, docId);
204
+ // Node projection: adapter-provided semantic features from parsed blocks,
205
+ // plus engine-derived `md:section` nodes from the (just-rebuilt) sections
206
+ // table. Written together in one pass (writeDocNodes clears + repopulates).
207
+ const propertyRows = [];
208
+ const sectionNodes = projectSectionNodes(db, docId);
209
+ if (adapter?.projectNodes) {
210
+ // Zip assigned block ids onto the parsed blocks so projectNodes can anchor
211
+ // each node to its own block (the parsed RawBlocks are pre-identity).
212
+ const rawBlocks = zipAssignedIds(rest, assigned);
213
+ const projected = adapter.projectNodes(rawBlocks);
214
+ // Keep only nodes anchored to a real assigned block id.
215
+ const idSet = collectBlockIds(assigned);
216
+ const withIds = projected.map((n) => ({
217
+ ...n,
218
+ blockId: n.blockId && idSet.has(n.blockId) ? n.blockId : "",
219
+ }));
220
+ // Adapters record spans as string indices; the store keeps byte offsets
221
+ // into the block's UTF-8 raw (spec/graph §2.3), so convert here — the one
222
+ // place that has both the nodes and the raws they index.
223
+ const rawById = collectRaws(assigned);
224
+ const byteSpanned = toByteSpans(withIds, (id) => rawById.get(id));
225
+ writeDocNodes(db, repoId, docId, [...byteSpanned, ...sectionNodes]);
226
+ // Inline properties (key:: value) → property rows (source=inline). The
227
+ // authored shape within the inline source drives `card`: a key that occurs
228
+ // exactly once in the document is `scalar` (so `element == "fire"` can
229
+ // match a lone `element:: fire`); a key that repeats is `list`. Repeats
230
+ // accumulate as ord-indexed rows in document order (block ordinal, then
231
+ // in-block position — the order withIds already carries). Effective bare-
232
+ // key cardinality (a frontmatter+inline collision) is decided at query
233
+ // time by the single-value gate on scalar comparisons (see compile.ts),
234
+ // not here — each source stays honest to its own authored shape.
235
+ const inlineFields = withIds.filter((n) => n.kind === "md:inline_field" && n.name);
236
+ const inlineCounts = new Map();
237
+ for (const n of inlineFields)
238
+ inlineCounts.set(n.name, (inlineCounts.get(n.name) ?? 0) + 1);
239
+ const inlineOrd = new Map();
240
+ for (const n of inlineFields) {
241
+ const ord = inlineOrd.get(n.name) ?? 0;
242
+ inlineOrd.set(n.name, ord + 1);
243
+ propertyRows.push({
244
+ source: "inline",
245
+ blockId: n.blockId || null,
246
+ key: n.name,
247
+ card: (inlineCounts.get(n.name) ?? 0) > 1 ? "list" : "scalar",
248
+ ord,
249
+ ...typedInlineValue(n.value),
250
+ });
262
251
  }
263
- // Consumed pool rows (resurrected) are removed.
264
- for (const id of resolved.consumedPool ?? []) {
265
- db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?").run(id);
252
+ }
253
+ else if (sectionNodes.length > 0) {
254
+ // No adapter node projection, but the document still has sections.
255
+ writeDocNodes(db, repoId, docId, sectionNodes);
256
+ }
257
+ // Frontmatter → property rows (source=frontmatter). Flatten nested maps to
258
+ // dotted keys, arrays to ord-indexed list rows, scalars to a scalar row.
259
+ for (const r of flattenFrontmatter(metadata)) {
260
+ propertyRows.push({ source: "frontmatter", blockId: null, ...r });
261
+ }
262
+ // Computed → property rows (source=computed). Engine-derived $-intrinsics
263
+ // ($title, $tags) from the adapter; never claim authored keys.
264
+ if (adapter?.computeProperties) {
265
+ const computed = adapter.computeProperties(rest, metadata);
266
+ for (const r of flattenComputed(computed)) {
267
+ propertyRows.push({ source: "computed", blockId: null, ...r });
266
268
  }
267
- // Update document current pointers + convergence hash.
268
- const fileHash = sha256(content);
269
+ }
270
+ writeDocProperties(db, repoId, docId, commitId, propertyRows);
271
+ // Edge extraction + interval maintenance (05 §2), if the resolver supplies
272
+ // an extractor. Runs in this commit transaction.
273
+ if (resolved.extractEdges) {
274
+ const extracted = resolved.extractEdges(docId, metadata);
275
+ maintainEdges(db, repoId, docId, commitId, extracted);
276
+ }
277
+ // Persist dispositions + block-history projection. With a resolver, use the
278
+ // real reconciliation dispositions; otherwise record every block as inserted.
279
+ const dispositions = resolved.dispositions.length > 0 || opts.resolveIds
280
+ ? resolved.dispositions
281
+ : rows.map((r) => ({ blockId: r.blockId, kind: "inserted", confidence: null, reason: null, matcherV: null, detail: {} }));
282
+ const insDisp = db.prepare(`INSERT OR IGNORE INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail)
283
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
284
+ const bc = db.prepare("INSERT OR IGNORE INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
285
+ for (const d of dispositions) {
286
+ insDisp.run(commitId, d.blockId, d.kind, d.confidence, d.reason, d.matcherV, JSON.stringify(d.detail));
287
+ bc.run(d.blockId, commitId, d.kind);
288
+ }
289
+ // Consumed pool rows (resurrected) are removed.
290
+ for (const id of resolved.consumedPool ?? []) {
291
+ db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?").run(id);
292
+ }
293
+ // Update document current pointers + convergence hash (+ the conflicted
294
+ // flag when the caller is the observe path — one statement, one transaction).
295
+ const fileHash = sha256(content);
296
+ if (opts.conflicted === undefined) {
269
297
  db.prepare("UPDATE docs SET current_rev = ?, file_hash = ? WHERE doc_id = ?").run(rev.revId, fileHash, docId);
270
- // Convergence check (01 §2): file bytes vs rendered revision. Two independent
271
- // round-trips must both reproduce the source exactly:
272
- // 1. the freshly-parsed in-memory tree (validates the parser), and
273
- // 2. a reload from storage (validates that persisted blocks + trivia +
274
- // frontmatter separator tile the source — the fidelity a plain read or
275
- // an `apply` write actually depends on). Rendering only the in-memory
276
- // tree hid store→reload separator loss; reconstructContent closes that.
277
- const renderFn = adapter?.render ?? render;
278
- const converged = fileHash.equals(renderedHash) &&
279
- renderFn(tree) === content &&
280
- reconstructContent(db, docId) === content;
281
- return {
282
- docId,
283
- commitId: commit.commitId,
284
- revId: rev.revId,
285
- blockCount: rows.length,
286
- converged,
287
- };
288
- });
298
+ }
299
+ else {
300
+ db.prepare("UPDATE docs SET current_rev = ?, file_hash = ?, conflicted = ? WHERE doc_id = ?").run(rev.revId, fileHash, opts.conflicted ? 1 : 0, docId);
301
+ }
302
+ // Convergence check (01 §2): file bytes vs rendered revision. Two independent
303
+ // round-trips must both reproduce the source exactly:
304
+ // 1. the freshly-parsed in-memory tree (validates the parser), and
305
+ // 2. a reload from storage (validates that persisted blocks + trivia +
306
+ // frontmatter separator tile the source — the fidelity a plain read or
307
+ // an `apply` write actually depends on). Rendering only the in-memory
308
+ // tree hid store→reload separator loss; reconstructContent closes that.
309
+ const renderFn = adapter?.render ?? render;
310
+ const converged = fileHash.equals(renderedHash) &&
311
+ renderFn(tree) === content &&
312
+ reconstructContent(db, docId) === content;
313
+ return {
314
+ docId,
315
+ commitId: commitId,
316
+ revId: rev.revId,
317
+ blockCount: rows.length,
318
+ converged,
319
+ };
289
320
  }
290
321
  // Remove another document's `blocks` row for each id in `ids` (a cross-doc move
291
322
  // or a resurrection re-homes the id into `docId`). A live foreign LEAF row still