@omgbase/core 0.11.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/core/ids.d.ts +4 -0
- package/dist/src/core/ids.js +17 -0
- package/dist/src/core/ingest.d.ts +39 -3
- package/dist/src/core/ingest.js +233 -202
- package/dist/src/core/store/schema.d.ts +3 -2
- package/dist/src/core/store/schema.js +28 -3
- package/dist/src/core/store/size.d.ts +76 -0
- package/dist/src/core/store/size.js +223 -0
- package/dist/src/core/store/store.d.ts +9 -0
- package/dist/src/core/store/store.js +11 -0
- package/dist/src/graph/history.js +15 -12
- package/dist/src/index.d.ts +12 -5
- package/dist/src/index.js +18 -7
- package/dist/src/mcp/arguments.d.ts +9 -2
- package/dist/src/mcp/arguments.js +15 -2
- package/dist/src/mcp/errors.d.ts +1 -1
- package/dist/src/mcp/errors.js +3 -0
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +27 -6
- package/dist/src/mcp/server.d.ts +56 -1
- package/dist/src/mcp/server.js +365 -203
- package/dist/src/mutate/apply.d.ts +93 -12
- package/dist/src/mutate/apply.js +203 -113
- package/dist/src/mutate/canonical.d.ts +27 -0
- package/dist/src/mutate/canonical.js +138 -0
- package/dist/src/mutate/commit.d.ts +150 -0
- package/dist/src/mutate/commit.js +349 -0
- package/dist/src/mutate/doc-store.d.ts +3 -3
- package/dist/src/mutate/doc-store.js +21 -6
- package/dist/src/mutate/docs.d.ts +42 -87
- package/dist/src/mutate/docs.js +50 -204
- package/dist/src/mutate/effects.d.ts +363 -0
- package/dist/src/mutate/effects.js +1161 -0
- package/dist/src/mutate/materialize.d.ts +49 -0
- package/dist/src/mutate/materialize.js +174 -0
- package/dist/src/mutate/opset.d.ts +27 -2
- package/dist/src/mutate/opset.js +12 -5
- package/dist/src/mutate/plan-update.d.ts +1 -46
- package/dist/src/mutate/plan-update.js +19 -67
- package/dist/src/mutate/plan.d.ts +210 -0
- package/dist/src/mutate/plan.js +512 -0
- package/dist/src/mutate/select.d.ts +89 -0
- package/dist/src/mutate/select.js +879 -0
- package/dist/src/mutate/sugar.d.ts +135 -0
- package/dist/src/mutate/sugar.js +167 -0
- package/dist/src/mutate/tree.d.ts +4 -2
- package/dist/src/oqx-js/planner.js +4 -0
- package/dist/src/oqx-js/run.d.ts +28 -1
- package/dist/src/oqx-js/run.js +85 -22
- package/dist/src/spec-versions.d.ts +6 -6
- package/dist/src/spec-versions.js +5 -5
- package/dist/src/surface-paths.d.ts +17 -26
- package/dist/src/surface-paths.js +27 -22
- package/dist/src/sync/checkpoint.d.ts +5 -2
- package/dist/src/sync/checkpoint.js +10 -6
- package/dist/src/sync/freshness.d.ts +10 -7
- package/dist/src/sync/freshness.js +15 -17
- package/dist/src/sync/observe.d.ts +4 -1
- package/dist/src/sync/observe.js +56 -5
- package/dist/src/sync/recovery.d.ts +4 -0
- package/dist/src/sync/recovery.js +4 -1
- package/dist/src/sync/stat-cache.d.ts +7 -0
- package/dist/src/sync/stat-cache.js +22 -0
- package/dist/src/sync/tombstone.js +23 -19
- package/dist/src/sync/watcher.d.ts +2 -0
- package/dist/src/sync/watcher.js +4 -1
- package/dist/src/sync/writer-lock.d.ts +15 -0
- package/dist/src/sync/writer-lock.js +104 -17
- package/dist/src/version.d.ts +7 -2
- package/dist/src/version.js +5 -2
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -111,7 +111,7 @@ the `Store` first and are synchronous unless noted.
|
|
|
111
111
|
| Search | Ranked "what does the caller mean" resolution; RRF fusion of FTS + vectors; raw vector search | `resolve`, `hybridSearch`, `vectorSearch`, `docVectorSearch` |
|
|
112
112
|
| Embeddings | Out-of-process embedder client, batching worker, post-mutation drainer, task builders | `createExternalProvider`, `EmbeddingWorker`, `EmbedDrainer`, `buildEmbedTasks`, `embeddingSettings`, `SemanticUnavailable` |
|
|
113
113
|
| Format adapters | Register/lookup the parsers that turn a file into blocks + nodes + edges | `registerAdapter`, `adapterForPath`, `detectFormat`, `markdownAdapter`, `yamlAdapter`, `jsonAdapter` |
|
|
114
|
-
| MCP server | Build the full tool surface (
|
|
114
|
+
| MCP server | Build the full tool surface (47 tools: `docs_read`, `query`, `apply`, `observe`, `changes_since`, …) as an `McpServer`; wire it to stdio | `buildServer({ store, repoId, rootPath?, embedQuery?, onMutation? })`, `serveStdio(ctx)` |
|
|
115
115
|
| Admin | Rebuild derived indexes; garbage-collect unreferenced blobs | `rebuildIndex`, `runGc`, `VERSION`, `EngineError` |
|
|
116
116
|
|
|
117
117
|
`omg mcp` is just a host for `serveStdio`: it opens the workspace, primes with
|
package/dist/src/core/ids.d.ts
CHANGED
|
@@ -38,6 +38,10 @@ export declare function sequentialMinter(): IdMinter;
|
|
|
38
38
|
* again, so a case run under it projects exactly as under `sequentialMinter`.
|
|
39
39
|
*/
|
|
40
40
|
export declare function repeatingMinter(): IdMinter;
|
|
41
|
+
/** A mint observer (spec/mutate §11.6, 2.3): told every id `mintId` issues, in order. */
|
|
42
|
+
export type MintObserver = (prefix: IdPrefix, id: string) => void;
|
|
43
|
+
/** Run `body` with `observer` told every id minted meanwhile (also under a nested `withIdMinter`). */
|
|
44
|
+
export declare function observeMints<T>(observer: MintObserver, body: () => T): T;
|
|
41
45
|
/**
|
|
42
46
|
* Mint an id with the given prefix (spec/store §2.1 "Uniqueness at mint"):
|
|
43
47
|
* draw a candidate — from the installed minter, else the CSPRNG — and redraw
|
package/dist/src/core/ids.js
CHANGED
|
@@ -96,6 +96,21 @@ export function repeatingMinter() {
|
|
|
96
96
|
return id;
|
|
97
97
|
};
|
|
98
98
|
}
|
|
99
|
+
// The installed observers. The effect planner's `execute` records the `b`/`d`
|
|
100
|
+
// ids a plan's replay mints so the receipt can zip them, in mint order, onto
|
|
101
|
+
// the plan's symbolic `$new.n` identities — it observes what `mintId` RETURNS
|
|
102
|
+
// (after the in-use check), never a rejected candidate.
|
|
103
|
+
const observers = new Set();
|
|
104
|
+
/** Run `body` with `observer` told every id minted meanwhile (also under a nested `withIdMinter`). */
|
|
105
|
+
export function observeMints(observer, body) {
|
|
106
|
+
observers.add(observer);
|
|
107
|
+
try {
|
|
108
|
+
return body();
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
observers.delete(observer);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
99
114
|
function inUse(prefix, id, issued) {
|
|
100
115
|
if (issued.has(id))
|
|
101
116
|
return true;
|
|
@@ -118,6 +133,8 @@ export function mintId(prefix) {
|
|
|
118
133
|
if (inUse(prefix, candidate, issued))
|
|
119
134
|
continue;
|
|
120
135
|
issued.add(candidate);
|
|
136
|
+
for (const observer of observers)
|
|
137
|
+
observer(prefix, candidate);
|
|
121
138
|
return candidate;
|
|
122
139
|
}
|
|
123
140
|
throw new Error(`mintId(${prefix}): ${MINT_GIVE_UP_AFTER} consecutive candidates were already in use — the ${minter ? "installed" : "production"} minter cannot produce a fresh id`);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Store } from "./store/store.js";
|
|
2
2
|
import { type TreeInputBlock } from "./store/writers.js";
|
|
3
3
|
import type { RawBlock, BlockTree } from "./parse/types.js";
|
|
4
|
+
import type { Database } from "better-sqlite3";
|
|
4
5
|
export interface DispositionRow {
|
|
5
6
|
blockId: string;
|
|
6
7
|
kind: string;
|
|
@@ -48,8 +49,8 @@ export interface IngestResult {
|
|
|
48
49
|
/** convergence: sha256(file bytes) === revision.rendered_hash */
|
|
49
50
|
converged: boolean;
|
|
50
51
|
}
|
|
51
|
-
/**
|
|
52
|
-
export
|
|
52
|
+
/** The options of `ingestFile` — one observed/api/import commit of one file. */
|
|
53
|
+
export interface IngestFileOptions {
|
|
53
54
|
ts?: string;
|
|
54
55
|
/** Commit provenance. `observed` = a file edit picked up by sync; `api` = an intent write (apply/docs_*), which should also carry `actor`/`reason`; `import` = bulk ingest with no retro history. */
|
|
55
56
|
origin?: "observed" | "import" | "api";
|
|
@@ -60,4 +61,39 @@ export declare function ingestFile(store: Store, repoId: string, path: string, c
|
|
|
60
61
|
/** A tree already parsed from `content` (via `parseForIngest`) — skips the
|
|
61
62
|
* re-parse. The caller guarantees it is the parse of these exact bytes. */
|
|
62
63
|
parsed?: BlockTree;
|
|
63
|
-
|
|
64
|
+
/** The document's `conflicted` flag (git conflict markers in the bytes),
|
|
65
|
+
* written in the same statement as the current-revision pointers so the
|
|
66
|
+
* flag and the revision it describes land in one transaction (spec/store
|
|
67
|
+
* §5.4 step 14). Absent ⇒ the column is left untouched — the observe path
|
|
68
|
+
* sets it on every commit; api writes never touch it. */
|
|
69
|
+
conflicted?: boolean;
|
|
70
|
+
}
|
|
71
|
+
/** The options of `ingestInto`: the commit the revision hangs off is the
|
|
72
|
+
* caller's (spec/store §5.4 step 5 ran already), and `ts` is that commit's
|
|
73
|
+
* timestamp (the pool expiry is computed from it). */
|
|
74
|
+
export interface IngestIntoOptions {
|
|
75
|
+
ts: string;
|
|
76
|
+
/** The `commits` row this ingest writes its revision, block rows, dispositions, properties and edges under. */
|
|
77
|
+
commitId: string;
|
|
78
|
+
resolveIds?: IdResolver | undefined;
|
|
79
|
+
format?: string | undefined;
|
|
80
|
+
parsed?: BlockTree | undefined;
|
|
81
|
+
conflicted?: boolean | undefined;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Ingest a single file's content into the store as ONE commit of its own:
|
|
85
|
+
* mint the commit row, then `ingestInto` it, in one transaction. The
|
|
86
|
+
* observe path (`sync/observe.ts`, attach, recovery, the driver) commits one
|
|
87
|
+
* member per call this way; the api path (`mutate/commit.ts`) instead opens
|
|
88
|
+
* one transaction and one commit for a whole request and calls `ingestInto`
|
|
89
|
+
* per document (spec/store §5.4 "The api path", spec/mutate §4 step 5).
|
|
90
|
+
*/
|
|
91
|
+
export declare function ingestFile(store: Store, repoId: string, path: string, content: string, opts?: IngestFileOptions): IngestResult;
|
|
92
|
+
/**
|
|
93
|
+
* spec/store §5.4 steps 1–4 and 6–14 for one document under an EXISTING
|
|
94
|
+
* commit row, inside the caller's transaction (`db` is the connection the
|
|
95
|
+
* caller's `store.write` handed it; nothing here opens one). The revision,
|
|
96
|
+
* the block rows' `created_commit`, the pool snapshot, the dispositions and
|
|
97
|
+
* block changes, the properties and the edge intervals all name `commitId`.
|
|
98
|
+
*/
|
|
99
|
+
export declare function ingestInto(db: Database, repoId: string, path: string, content: string, opts: IngestIntoOptions): IngestResult;
|
package/dist/src/core/ingest.js
CHANGED
|
@@ -74,218 +74,249 @@ function flatten(blocks, parent, depth, ancestorPath, out, quoteDepth = 0) {
|
|
|
74
74
|
}
|
|
75
75
|
});
|
|
76
76
|
}
|
|
77
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Ingest a single file's content into the store as ONE commit of its own:
|
|
79
|
+
* mint the commit row, then `ingestInto` it, in one transaction. The
|
|
80
|
+
* observe path (`sync/observe.ts`, attach, recovery, the driver) commits one
|
|
81
|
+
* member per call this way; the api path (`mutate/commit.ts`) instead opens
|
|
82
|
+
* one transaction and one commit for a whole request and calls `ingestInto`
|
|
83
|
+
* per document (spec/store §5.4 "The api path", spec/mutate §4 step 5).
|
|
84
|
+
*/
|
|
78
85
|
export function ingestFile(store, repoId, path, content, opts = {}) {
|
|
79
86
|
const ts = opts.ts ?? new Date().toISOString();
|
|
80
87
|
const origin = opts.origin ?? "observed";
|
|
81
|
-
const adapter = adapterForPath(path);
|
|
82
|
-
const format = opts.format ?? adapter?.format ?? "markdown";
|
|
83
88
|
return store.write((db) => {
|
|
84
|
-
const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
|
|
85
|
-
const { fmBlock, rest } = extractFrontmatter(tree.children);
|
|
86
|
-
// Frontmatter blob (markdown-specific) preserved for revision history.
|
|
87
|
-
const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
|
|
88
|
-
// The exact separator bytes between the frontmatter block and the first body
|
|
89
|
-
// block (the frontmatter block's trailing trivia). Persisted so the body
|
|
90
|
-
// reconstructs byte-for-byte instead of assuming a canonical blank line.
|
|
91
|
-
// NULL when there is no frontmatter.
|
|
92
|
-
const fmTrivia = fmBlock ? fmBlock.trivia : null;
|
|
93
|
-
// Document property bag: whole-document metadata for the formats whose
|
|
94
|
-
// adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
|
|
95
|
-
// `extractMetadata` — the YAML of the `frontmatter` block and nothing else
|
|
96
|
-
// (spec/properties §3.1: a document whose first block is not `frontmatter`
|
|
97
|
-
// has no frontmatter rows, whatever else its bytes resemble). Flattened into
|
|
98
|
-
// the properties table below (source=frontmatter); also drives frontmatter
|
|
99
|
-
// edge extraction.
|
|
100
|
-
const metadata = adapter?.extractMetadata
|
|
101
|
-
? (adapter.extractMetadata(content) ?? {})
|
|
102
|
-
: parseFrontmatter(fmBlock);
|
|
103
|
-
// Upsert the document row.
|
|
104
|
-
let doc = db.prepare("SELECT doc_id, deleted_commit FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
|
|
105
|
-
const docId = doc?.doc_id ?? mintId("d");
|
|
106
|
-
const isNew = !doc;
|
|
107
|
-
if (!doc) {
|
|
108
|
-
db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
|
|
109
|
-
doc = { doc_id: docId, deleted_commit: null };
|
|
110
|
-
// Adopt phantom edges that pointed at this path so backlinks re-point.
|
|
111
|
-
adoptPhantoms(db, path, docId);
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
// A tombstoned row still owns the path's identity (spec/store §5.6
|
|
115
|
-
// "Re-creation"): bytes observed again at the path revive it — the row is
|
|
116
|
-
// live again, so reads that filter `deleted_commit IS NULL` see it and the
|
|
117
|
-
// next identical observation echo-gates instead of re-ingesting.
|
|
118
|
-
const revived = doc.deleted_commit !== null;
|
|
119
|
-
db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ?, deleted_commit = NULL WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
|
|
120
|
-
// spec/graph §3.5: adoption happens whenever a row BECOMES live, so a
|
|
121
|
-
// revived tombstone also collects the phantom edges minted at its path
|
|
122
|
-
// while it was gone (§8 "Fixed — a revived document adopted no phantoms").
|
|
123
|
-
if (revived)
|
|
124
|
-
adoptPhantoms(db, path, docId);
|
|
125
|
-
}
|
|
126
|
-
// Assign block ids via the resolver when supplied (it reconciles against
|
|
127
|
-
// the prior revision — for a new doc that's an empty old tree, so all blocks
|
|
128
|
-
// mint — and also drives edge extraction). Without a resolver, fresh-mint.
|
|
129
|
-
const resolved = opts.resolveIds
|
|
130
|
-
? opts.resolveIds(rest, isNew ? null : docId)
|
|
131
|
-
: { assigned: assignIds(rest), dispositions: [], deleted: [] };
|
|
132
|
-
const assigned = resolved.assigned;
|
|
133
|
-
const rootTreeHex = writeBlockTree(db, assigned);
|
|
134
89
|
const commit = newCommit(db, { repoId, ts, origin, actor: opts.actor ?? null, reason: opts.reason ?? null });
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
docId,
|
|
138
|
-
rootTreeHex,
|
|
139
|
-
frontmatterBlobHex: fmBlobHex,
|
|
140
|
-
renderedHash,
|
|
141
|
-
path,
|
|
90
|
+
return ingestInto(db, repoId, path, content, {
|
|
91
|
+
ts,
|
|
142
92
|
commitId: commit.commitId,
|
|
93
|
+
resolveIds: opts.resolveIds,
|
|
94
|
+
format: opts.format,
|
|
95
|
+
parsed: opts.parsed,
|
|
96
|
+
conflicted: opts.conflicted,
|
|
143
97
|
});
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* spec/store §5.4 steps 1–4 and 6–14 for one document under an EXISTING
|
|
102
|
+
* commit row, inside the caller's transaction (`db` is the connection the
|
|
103
|
+
* caller's `store.write` handed it; nothing here opens one). The revision,
|
|
104
|
+
* the block rows' `created_commit`, the pool snapshot, the dispositions and
|
|
105
|
+
* block changes, the properties and the edge intervals all name `commitId`.
|
|
106
|
+
*/
|
|
107
|
+
export function ingestInto(db, repoId, path, content, opts) {
|
|
108
|
+
const { ts, commitId } = opts;
|
|
109
|
+
const adapter = adapterForPath(path);
|
|
110
|
+
const format = opts.format ?? adapter?.format ?? "markdown";
|
|
111
|
+
const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
|
|
112
|
+
const { fmBlock, rest } = extractFrontmatter(tree.children);
|
|
113
|
+
// Frontmatter blob (markdown-specific) preserved for revision history.
|
|
114
|
+
const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
|
|
115
|
+
// The exact separator bytes between the frontmatter block and the first body
|
|
116
|
+
// block (the frontmatter block's trailing trivia). Persisted so the body
|
|
117
|
+
// reconstructs byte-for-byte instead of assuming a canonical blank line.
|
|
118
|
+
// NULL when there is no frontmatter.
|
|
119
|
+
const fmTrivia = fmBlock ? fmBlock.trivia : null;
|
|
120
|
+
// Document property bag: whole-document metadata for the formats whose
|
|
121
|
+
// adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
|
|
122
|
+
// `extractMetadata` — the YAML of the `frontmatter` block and nothing else
|
|
123
|
+
// (spec/properties §3.1: a document whose first block is not `frontmatter`
|
|
124
|
+
// has no frontmatter rows, whatever else its bytes resemble). Flattened into
|
|
125
|
+
// the properties table below (source=frontmatter); also drives frontmatter
|
|
126
|
+
// edge extraction.
|
|
127
|
+
const metadata = adapter?.extractMetadata
|
|
128
|
+
? (adapter.extractMetadata(content) ?? {})
|
|
129
|
+
: parseFrontmatter(fmBlock);
|
|
130
|
+
// Upsert the document row.
|
|
131
|
+
let doc = db.prepare("SELECT doc_id, deleted_commit FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
|
|
132
|
+
const docId = doc?.doc_id ?? mintId("d");
|
|
133
|
+
const isNew = !doc;
|
|
134
|
+
if (!doc) {
|
|
135
|
+
db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
|
|
136
|
+
doc = { doc_id: docId, deleted_commit: null };
|
|
137
|
+
// Adopt phantom edges that pointed at this path so backlinks re-point.
|
|
138
|
+
adoptPhantoms(db, path, docId);
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
// A tombstoned row still owns the path's identity (spec/store §5.6
|
|
142
|
+
// "Re-creation"): bytes observed again at the path revive it — the row is
|
|
143
|
+
// live again, so reads that filter `deleted_commit IS NULL` see it and the
|
|
144
|
+
// next identical observation echo-gates instead of re-ingesting.
|
|
145
|
+
const revived = doc.deleted_commit !== null;
|
|
146
|
+
db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ?, deleted_commit = NULL WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
|
|
147
|
+
// spec/graph §3.5: adoption happens whenever a row BECOMES live, so a
|
|
148
|
+
// revived tombstone also collects the phantom edges minted at its path
|
|
149
|
+
// while it was gone (§8 "Fixed — a revived document adopted no phantoms").
|
|
150
|
+
if (revived)
|
|
151
|
+
adoptPhantoms(db, path, docId);
|
|
152
|
+
}
|
|
153
|
+
// Assign block ids via the resolver when supplied (it reconciles against
|
|
154
|
+
// the prior revision — for a new doc that's an empty old tree, so all blocks
|
|
155
|
+
// mint — and also drives edge extraction). Without a resolver, fresh-mint.
|
|
156
|
+
const resolved = opts.resolveIds
|
|
157
|
+
? opts.resolveIds(rest, isNew ? null : docId)
|
|
158
|
+
: { assigned: assignIds(rest), dispositions: [], deleted: [] };
|
|
159
|
+
const assigned = resolved.assigned;
|
|
160
|
+
const rootTreeHex = writeBlockTree(db, assigned);
|
|
161
|
+
const renderedHash = sha256(content);
|
|
162
|
+
const rev = writeRevision(db, {
|
|
163
|
+
docId,
|
|
164
|
+
rootTreeHex,
|
|
165
|
+
frontmatterBlobHex: fmBlobHex,
|
|
166
|
+
renderedHash,
|
|
167
|
+
path,
|
|
168
|
+
commitId,
|
|
169
|
+
});
|
|
170
|
+
// Snapshot deleted blocks into the resurrection pool BEFORE dropping the
|
|
171
|
+
// doc's rows (the ingest rebuilds current-state blocks wholesale).
|
|
172
|
+
if (resolved.deleted.length > 0) {
|
|
173
|
+
const expires = new Date(Date.parse(ts) + 30 * 24 * 3600 * 1000).toISOString();
|
|
174
|
+
const pool = db.prepare(`INSERT OR REPLACE INTO resurrection_pool (block_id, repo_id, doc_id, raw_hash, norm_hash, type, deleted_commit, expires_ts)
|
|
175
|
+
SELECT block_id, repo_id, doc_id, raw_hash, norm_hash, type, ?, ? FROM blocks WHERE block_id = ? AND doc_id = ?`);
|
|
176
|
+
for (const id of resolved.deleted)
|
|
177
|
+
pool.run(commitId, expires, id, docId);
|
|
178
|
+
}
|
|
179
|
+
// Ids arriving from ANOTHER document — a cross-document move whose source
|
|
180
|
+
// commits later in this checkpoint (or was tombstoned in it), or a
|
|
181
|
+
// resurrection out of a tombstoned doc whose rows still sit in `blocks` —
|
|
182
|
+
// must not collide on the block_id primary key. Evict the foreign row (and
|
|
183
|
+
// its FTS entry when live) and drop any pool row: the id is live here now.
|
|
184
|
+
const incoming = [...(resolved.crossDocIds ?? []), ...(resolved.consumedPool ?? [])];
|
|
185
|
+
if (incoming.length > 0)
|
|
186
|
+
evictForeignBlockRows(db, docId, incoming);
|
|
187
|
+
// Refresh current-state blocks (clear + repopulate with carried/minted ids).
|
|
188
|
+
// FTS is external-content: delete old index rows before dropping blocks.
|
|
189
|
+
ftsDeleteDoc(db, docId);
|
|
190
|
+
db.prepare("DELETE FROM blocks WHERE doc_id = ?").run(docId);
|
|
191
|
+
const rows = [];
|
|
192
|
+
flatten(assigned, null, 0, "/", rows);
|
|
193
|
+
const insert = db.prepare(`INSERT INTO blocks
|
|
194
|
+
(block_id, repo_id, doc_id, parent_block, order_key, ordinal, depth,
|
|
195
|
+
ancestor_path, type, attrs, text, raw_hash, norm_hash, trivia_hash, created_commit)
|
|
196
|
+
VALUES
|
|
197
|
+
(@blockId, @repoId, @docId, @parentBlock, @orderKey, @ordinal, @depth,
|
|
198
|
+
@ancestorPath, @type, @attrs, @text, @rawHash, @normHash, @triviaHash, @createdCommit)`);
|
|
199
|
+
for (const r of rows) {
|
|
200
|
+
insert.run({ ...r, repoId, docId, createdCommit: commitId });
|
|
201
|
+
}
|
|
202
|
+
ftsIndexDoc(db, docId);
|
|
203
|
+
rebuildSections(db, docId);
|
|
204
|
+
// Node projection: adapter-provided semantic features from parsed blocks,
|
|
205
|
+
// plus engine-derived `md:section` nodes from the (just-rebuilt) sections
|
|
206
|
+
// table. Written together in one pass (writeDocNodes clears + repopulates).
|
|
207
|
+
const propertyRows = [];
|
|
208
|
+
const sectionNodes = projectSectionNodes(db, docId);
|
|
209
|
+
if (adapter?.projectNodes) {
|
|
210
|
+
// Zip assigned block ids onto the parsed blocks so projectNodes can anchor
|
|
211
|
+
// each node to its own block (the parsed RawBlocks are pre-identity).
|
|
212
|
+
const rawBlocks = zipAssignedIds(rest, assigned);
|
|
213
|
+
const projected = adapter.projectNodes(rawBlocks);
|
|
214
|
+
// Keep only nodes anchored to a real assigned block id.
|
|
215
|
+
const idSet = collectBlockIds(assigned);
|
|
216
|
+
const withIds = projected.map((n) => ({
|
|
217
|
+
...n,
|
|
218
|
+
blockId: n.blockId && idSet.has(n.blockId) ? n.blockId : "",
|
|
219
|
+
}));
|
|
220
|
+
// Adapters record spans as string indices; the store keeps byte offsets
|
|
221
|
+
// into the block's UTF-8 raw (spec/graph §2.3), so convert here — the one
|
|
222
|
+
// place that has both the nodes and the raws they index.
|
|
223
|
+
const rawById = collectRaws(assigned);
|
|
224
|
+
const byteSpanned = toByteSpans(withIds, (id) => rawById.get(id));
|
|
225
|
+
writeDocNodes(db, repoId, docId, [...byteSpanned, ...sectionNodes]);
|
|
226
|
+
// Inline properties (key:: value) → property rows (source=inline). The
|
|
227
|
+
// authored shape within the inline source drives `card`: a key that occurs
|
|
228
|
+
// exactly once in the document is `scalar` (so `element == "fire"` can
|
|
229
|
+
// match a lone `element:: fire`); a key that repeats is `list`. Repeats
|
|
230
|
+
// accumulate as ord-indexed rows in document order (block ordinal, then
|
|
231
|
+
// in-block position — the order withIds already carries). Effective bare-
|
|
232
|
+
// key cardinality (a frontmatter+inline collision) is decided at query
|
|
233
|
+
// time by the single-value gate on scalar comparisons (see compile.ts),
|
|
234
|
+
// not here — each source stays honest to its own authored shape.
|
|
235
|
+
const inlineFields = withIds.filter((n) => n.kind === "md:inline_field" && n.name);
|
|
236
|
+
const inlineCounts = new Map();
|
|
237
|
+
for (const n of inlineFields)
|
|
238
|
+
inlineCounts.set(n.name, (inlineCounts.get(n.name) ?? 0) + 1);
|
|
239
|
+
const inlineOrd = new Map();
|
|
240
|
+
for (const n of inlineFields) {
|
|
241
|
+
const ord = inlineOrd.get(n.name) ?? 0;
|
|
242
|
+
inlineOrd.set(n.name, ord + 1);
|
|
243
|
+
propertyRows.push({
|
|
244
|
+
source: "inline",
|
|
245
|
+
blockId: n.blockId || null,
|
|
246
|
+
key: n.name,
|
|
247
|
+
card: (inlineCounts.get(n.name) ?? 0) > 1 ? "list" : "scalar",
|
|
248
|
+
ord,
|
|
249
|
+
...typedInlineValue(n.value),
|
|
250
|
+
});
|
|
262
251
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
252
|
+
}
|
|
253
|
+
else if (sectionNodes.length > 0) {
|
|
254
|
+
// No adapter node projection, but the document still has sections.
|
|
255
|
+
writeDocNodes(db, repoId, docId, sectionNodes);
|
|
256
|
+
}
|
|
257
|
+
// Frontmatter → property rows (source=frontmatter). Flatten nested maps to
|
|
258
|
+
// dotted keys, arrays to ord-indexed list rows, scalars to a scalar row.
|
|
259
|
+
for (const r of flattenFrontmatter(metadata)) {
|
|
260
|
+
propertyRows.push({ source: "frontmatter", blockId: null, ...r });
|
|
261
|
+
}
|
|
262
|
+
// Computed → property rows (source=computed). Engine-derived $-intrinsics
|
|
263
|
+
// ($title, $tags) from the adapter; never claim authored keys.
|
|
264
|
+
if (adapter?.computeProperties) {
|
|
265
|
+
const computed = adapter.computeProperties(rest, metadata);
|
|
266
|
+
for (const r of flattenComputed(computed)) {
|
|
267
|
+
propertyRows.push({ source: "computed", blockId: null, ...r });
|
|
266
268
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
+
}
|
|
270
|
+
writeDocProperties(db, repoId, docId, commitId, propertyRows);
|
|
271
|
+
// Edge extraction + interval maintenance (05 §2), if the resolver supplies
|
|
272
|
+
// an extractor. Runs in this commit transaction.
|
|
273
|
+
if (resolved.extractEdges) {
|
|
274
|
+
const extracted = resolved.extractEdges(docId, metadata);
|
|
275
|
+
maintainEdges(db, repoId, docId, commitId, extracted);
|
|
276
|
+
}
|
|
277
|
+
// Persist dispositions + block-history projection. With a resolver, use the
|
|
278
|
+
// real reconciliation dispositions; otherwise record every block as inserted.
|
|
279
|
+
const dispositions = resolved.dispositions.length > 0 || opts.resolveIds
|
|
280
|
+
? resolved.dispositions
|
|
281
|
+
: rows.map((r) => ({ blockId: r.blockId, kind: "inserted", confidence: null, reason: null, matcherV: null, detail: {} }));
|
|
282
|
+
const insDisp = db.prepare(`INSERT OR IGNORE INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail)
|
|
283
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
284
|
+
const bc = db.prepare("INSERT OR IGNORE INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
|
|
285
|
+
for (const d of dispositions) {
|
|
286
|
+
insDisp.run(commitId, d.blockId, d.kind, d.confidence, d.reason, d.matcherV, JSON.stringify(d.detail));
|
|
287
|
+
bc.run(d.blockId, commitId, d.kind);
|
|
288
|
+
}
|
|
289
|
+
// Consumed pool rows (resurrected) are removed.
|
|
290
|
+
for (const id of resolved.consumedPool ?? []) {
|
|
291
|
+
db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?").run(id);
|
|
292
|
+
}
|
|
293
|
+
// Update document current pointers + convergence hash (+ the conflicted
|
|
294
|
+
// flag when the caller is the observe path — one statement, one transaction).
|
|
295
|
+
const fileHash = sha256(content);
|
|
296
|
+
if (opts.conflicted === undefined) {
|
|
269
297
|
db.prepare("UPDATE docs SET current_rev = ?, file_hash = ? WHERE doc_id = ?").run(rev.revId, fileHash, docId);
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
298
|
+
}
|
|
299
|
+
else {
|
|
300
|
+
db.prepare("UPDATE docs SET current_rev = ?, file_hash = ?, conflicted = ? WHERE doc_id = ?").run(rev.revId, fileHash, opts.conflicted ? 1 : 0, docId);
|
|
301
|
+
}
|
|
302
|
+
// Convergence check (01 §2): file bytes vs rendered revision. Two independent
|
|
303
|
+
// round-trips must both reproduce the source exactly:
|
|
304
|
+
// 1. the freshly-parsed in-memory tree (validates the parser), and
|
|
305
|
+
// 2. a reload from storage (validates that persisted blocks + trivia +
|
|
306
|
+
// frontmatter separator tile the source — the fidelity a plain read or
|
|
307
|
+
// an `apply` write actually depends on). Rendering only the in-memory
|
|
308
|
+
// tree hid store→reload separator loss; reconstructContent closes that.
|
|
309
|
+
const renderFn = adapter?.render ?? render;
|
|
310
|
+
const converged = fileHash.equals(renderedHash) &&
|
|
311
|
+
renderFn(tree) === content &&
|
|
312
|
+
reconstructContent(db, docId) === content;
|
|
313
|
+
return {
|
|
314
|
+
docId,
|
|
315
|
+
commitId: commitId,
|
|
316
|
+
revId: rev.revId,
|
|
317
|
+
blockCount: rows.length,
|
|
318
|
+
converged,
|
|
319
|
+
};
|
|
289
320
|
}
|
|
290
321
|
// Remove another document's `blocks` row for each id in `ids` (a cross-doc move
|
|
291
322
|
// or a resurrection re-homes the id into `docId`). A live foreign LEAF row still
|