@omgbase/core 0.24.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/src/core/edit-script.d.ts +15 -0
  2. package/dist/src/core/edit-script.js +72 -0
  3. package/dist/src/core/ingest.js +42 -12
  4. package/dist/src/core/read/document.d.ts +7 -0
  5. package/dist/src/core/read/document.js +38 -17
  6. package/dist/src/core/store/archive.d.ts +28 -0
  7. package/dist/src/core/store/archive.js +106 -17
  8. package/dist/src/core/store/delta.d.ts +131 -0
  9. package/dist/src/core/store/delta.js +470 -0
  10. package/dist/src/core/store/gc.d.ts +10 -0
  11. package/dist/src/core/store/gc.js +23 -7
  12. package/dist/src/core/store/retention.d.ts +1 -1
  13. package/dist/src/core/store/retention.js +3 -17
  14. package/dist/src/core/store/schema.d.ts +1 -1
  15. package/dist/src/core/store/schema.js +2 -1
  16. package/dist/src/core/store/store.js +7 -0
  17. package/dist/src/core/store/writers.d.ts +42 -6
  18. package/dist/src/core/store/writers.js +67 -25
  19. package/dist/src/format/json.js +5 -0
  20. package/dist/src/format/yaml.js +15 -3
  21. package/dist/src/graph/history.d.ts +1 -15
  22. package/dist/src/graph/history.js +14 -74
  23. package/dist/src/mcp/arguments.d.ts +11 -0
  24. package/dist/src/mcp/arguments.js +47 -4
  25. package/dist/src/mcp/errors.js +1 -1
  26. package/dist/src/mcp/reference.d.ts +1 -1
  27. package/dist/src/mcp/reference.js +1 -1
  28. package/dist/src/mcp/server.js +4 -2
  29. package/dist/src/mutate/effects.js +87 -13
  30. package/dist/src/mutate/frontmatter-parts.d.ts +71 -0
  31. package/dist/src/mutate/frontmatter-parts.js +237 -0
  32. package/dist/src/mutate/load.js +9 -9
  33. package/dist/src/mutate/select.js +9 -3
  34. package/dist/src/oqx-js/context.js +14 -0
  35. package/dist/src/oqx-js/names.js +1 -1
  36. package/dist/src/oqx-js/run.d.ts +8 -0
  37. package/dist/src/oqx-js/run.js +87 -1
  38. package/dist/src/oqx-js/sql/translate.js +21 -1
  39. package/dist/src/search/cel/parser.d.ts +3 -1
  40. package/dist/src/search/cel/parser.js +4 -1
  41. package/dist/src/spec-versions.d.ts +4 -4
  42. package/dist/src/spec-versions.js +4 -4
  43. package/package.json +1 -1
@@ -0,0 +1,15 @@
1
+ /** One step of the edit script: `keep` consumes a line from both sides,
2
+ * `delete` one from the old text, `insert` one from the new text. */
3
+ export interface EditOp {
4
+ kind: "keep" | "delete" | "insert";
5
+ line: string;
6
+ }
7
+ /**
8
+ * Myers' O(ND) shortest edit script (forward, with a per-`d` trace for the
9
+ * backtrack). `V[k]` is the furthest x on diagonal `k = x - y` reachable with
10
+ * `d` edits. The canonical tie rule, identical in both engines: at each step
11
+ * take the diagonal from `k+1` (moving down — an insertion of `b[y]`) when
12
+ * `k == -d || (k != d && V[k-1] < V[k+1])`, else from `k-1` (moving right —
13
+ * a deletion of `a[x]`). On a tie (`V[k-1] == V[k+1]`) that is the deletion.
14
+ */
15
+ export declare function myersScript(a: string[], b: string[]): EditOp[];
@@ -0,0 +1,72 @@
1
+ // Myers' shortest edit script over two string arrays — the ONE diff algorithm
2
+ // both engines share, so an edit script is deterministic across them. Two
3
+ // consumers: the unified diff of `docs_diff_unified` (spec/surface §3;
4
+ // `graph/history.ts` re-exports it, and `crates/omgbase-surface/src/history.rs`
5
+ // is the twin) and the delta tree nodes of spec/store §4.1 (16.0;
6
+ // `core/store/delta.ts`), whose ops are this script grouped. Kept in `core/`
7
+ // because the store cannot import `graph/`.
8
+ /**
9
+ * Myers' O(ND) shortest edit script (forward, with a per-`d` trace for the
10
+ * backtrack). `V[k]` is the furthest x on diagonal `k = x - y` reachable with
11
+ * `d` edits. The canonical tie rule, identical in both engines: at each step
12
+ * take the diagonal from `k+1` (moving down — an insertion of `b[y]`) when
13
+ * `k == -d || (k != d && V[k-1] < V[k+1])`, else from `k-1` (moving right —
14
+ * a deletion of `a[x]`). On a tie (`V[k-1] == V[k+1]`) that is the deletion.
15
+ */
16
+ export function myersScript(a, b) {
17
+ const n = a.length;
18
+ const m = b.length;
19
+ const max = n + m;
20
+ // V is indexed by k ∈ [-max-1, max+1]; `off` maps it onto a plain array.
21
+ const off = max + 1;
22
+ const v = new Array(2 * max + 3).fill(0);
23
+ const trace = [];
24
+ let found = false;
25
+ for (let d = 0; d <= max && !found; d++) {
26
+ trace.push(v.slice());
27
+ for (let k = -d; k <= d; k += 2) {
28
+ let x;
29
+ if (k === -d || (k !== d && v[off + k - 1] < v[off + k + 1]))
30
+ x = v[off + k + 1];
31
+ else
32
+ x = v[off + k - 1] + 1;
33
+ let y = x - k;
34
+ while (x < n && y < m && a[x] === b[y]) {
35
+ x++;
36
+ y++;
37
+ }
38
+ v[off + k] = x;
39
+ if (x >= n && y >= m) {
40
+ found = true;
41
+ break;
42
+ }
43
+ }
44
+ }
45
+ // Backtrack from (n, m) through the trace, emitting ops newest-first.
46
+ const ops = [];
47
+ let x = n;
48
+ let y = m;
49
+ for (let d = trace.length - 1; d >= 0; d--) {
50
+ const vd = trace[d];
51
+ const k = x - y;
52
+ const prevK = k === -d || (k !== d && vd[off + k - 1] < vd[off + k + 1]) ? k + 1 : k - 1;
53
+ const prevX = vd[off + prevK];
54
+ const prevY = prevX - prevK;
55
+ while (x > prevX && y > prevY) {
56
+ x--;
57
+ y--;
58
+ ops.push({ kind: "keep", line: a[x] });
59
+ }
60
+ if (d > 0) {
61
+ if (x === prevX)
62
+ ops.push({ kind: "insert", line: b[prevY] });
63
+ else
64
+ ops.push({ kind: "delete", line: a[prevX] });
65
+ }
66
+ x = prevX;
67
+ y = prevY;
68
+ }
69
+ ops.reverse();
70
+ return ops;
71
+ }
72
+ //# sourceMappingURL=edit-script.js.map
@@ -2,7 +2,8 @@ import { Store } from "./store/store.js";
2
2
  import { parseTree } from "./parse/tree.js";
3
3
  import { render } from "./parse/render.js";
4
4
  import { reconstructContent } from "./read/document.js";
5
- import { assignIds, writeBlockTree, putBlob, newCommit, writeRevision } from "./store/writers.js";
5
+ import { assignIds, writeBlockTree, writeFrontmatterBlob, isCheckpointRevision, previousRevision, newCommit, writeRevision } from "./store/writers.js";
6
+ import { TreeResolver } from "./store/delta.js";
6
7
  import { sha256, childQuoteDepth, visibleText } from "./hash.js";
7
8
  import { mintId } from "./ids.js";
8
9
  import { keyBetween } from "./order-key.js";
@@ -114,20 +115,22 @@ export function ingestInto(db, repoId, path, content, opts) {
114
115
  const format = opts.format ?? adapter?.format ?? "markdown";
115
116
  const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
116
117
  const { fmBlock, rest } = extractFrontmatter(tree.children);
117
- // Frontmatter blob (markdown-specific) preserved for revision history.
118
- const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
119
118
  // The exact separator bytes between the frontmatter block and the first body
120
119
  // block (the frontmatter block's trailing trivia). Persisted so the body
121
120
  // reconstructs byte-for-byte instead of assuming a canonical blank line.
122
121
  // NULL when there is no frontmatter.
123
122
  const fmTrivia = fmBlock ? fmBlock.trivia : null;
124
- // Document property bag: whole-document metadata for the formats whose
125
- // adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
126
- // `extractMetadata` — the YAML of the `frontmatter` block and nothing else
127
- // (spec/properties §3.1: a document whose first block is not `frontmatter`
128
- // has no frontmatter rows, whatever else its bytes resemble). Flattened into
129
- // the properties table below (source=frontmatter); also drives frontmatter
130
- // edge extraction.
123
+ // Document property bag — the document's AUTHORED MAPPING. For a `yaml` or
124
+ // `json` document it is the whole document's top-level mapping, from the
125
+ // adapter's `extractMetadata` (spec/properties §3.4, 1.2: null for a
126
+ // non-mapping root, a syntax error or an empty file; a YAML stream's first
127
+ // document only); for Markdown — whose adapter has no `extractMetadata` — the
128
+ // YAML of the `frontmatter` block and nothing else (§3.1: a document whose
129
+ // first block is not `frontmatter` has no frontmatter rows, whatever else its
130
+ // bytes resemble). Either way it is flattened into the properties table below
131
+ // under source=frontmatter — the same source, because it is the same thing
132
+ // (§3.4: "frontmatter" names the document's authored mapping) — and drives
133
+ // frontmatter edge extraction.
131
134
  const metadata = adapter?.extractMetadata
132
135
  ? (adapter.extractMetadata(content) ?? {})
133
136
  : parseFrontmatter(fmBlock);
@@ -161,7 +164,29 @@ export function ingestInto(db, repoId, path, content, opts) {
161
164
  ? opts.resolveIds(rest, isNew ? null : docId)
162
165
  : { assigned: assignIds(rest), dispositions: [], deleted: [] };
163
166
  const assigned = resolved.assigned;
164
- const rootTreeHex = writeBlockTree(db, assigned);
167
+ // spec/store §4.4 (16.0): the revision's seq decides whether it is a
168
+ // checkpoint; a non-checkpoint revision writes its tree and frontmatter as
169
+ // deltas against the previous revision's RESOLVED tree and raw where that is
170
+ // smaller. The previous revision is the document's greatest seq — on every
171
+ // path the current revision, never archived (retention rule 1) — but the
172
+ // rule is stated for the general case: no live predecessor ⇒ a checkpoint.
173
+ const previous = previousRevision(db, docId);
174
+ const seq = previous ? previous.seq + 1 : 1;
175
+ const checkpoint = isCheckpointRevision(seq, previous);
176
+ const resolver = new TreeResolver(db);
177
+ const treeCtx = {
178
+ checkpoint,
179
+ base: !checkpoint && previous?.root_tree ? resolver.tree(docId, previous.seq, previous.root_tree.toString("hex")) : null,
180
+ };
181
+ const rootTreeHex = writeBlockTree(db, assigned, treeCtx);
182
+ // Frontmatter blob (markdown-specific) preserved for revision history: the
183
+ // raw at a checkpoint, else the smaller of a line delta over the previous
184
+ // revision's frontmatter and the raw (spec/store §5.4 step 1).
185
+ let fmBlobHex = null;
186
+ if (fmBlock) {
187
+ const baseRaw = !checkpoint && previous?.frontmatter_blob ? resolver.frontmatter(docId, previous.seq, previous.frontmatter_blob.toString("hex")) : null;
188
+ fmBlobHex = writeFrontmatterBlob(db, fmBlock.raw, baseRaw, checkpoint);
189
+ }
165
190
  const renderedHash = sha256(content);
166
191
  const rev = writeRevision(db, {
167
192
  docId,
@@ -170,6 +195,7 @@ export function ingestInto(db, repoId, path, content, opts) {
170
195
  renderedHash,
171
196
  path,
172
197
  commitId,
198
+ seq,
173
199
  });
174
200
  // Snapshot deleted blocks into the resurrection pool BEFORE dropping the
175
201
  // doc's rows (the ingest rebuilds current-state blocks wholesale).
@@ -264,7 +290,11 @@ export function ingestInto(db, repoId, path, content, opts) {
264
290
  propertyRows.push({ source: "frontmatter", blockId: null, ...r });
265
291
  }
266
292
  // Computed → property rows (source=computed). Engine-derived $-intrinsics
267
- // ($title, $tags) from the adapter; never claim authored keys.
293
+ // ($title, $tags) from the adapter; never claim authored keys. Only the
294
+ // Markdown adapter computes (spec/properties §3.3): a `yaml`/`json` document
295
+ // has no body prose to derive a heading or a hashtag from, and its authored
296
+ // `title`/`tags` are reached as any other key of its mapping (§3.4), so it
297
+ // yields no computed rows.
268
298
  if (adapter?.computeProperties) {
269
299
  const computed = adapter.computeProperties(rest, metadata);
270
300
  for (const r of flattenComputed(computed)) {
@@ -32,6 +32,13 @@ export interface DocsReadResult {
32
32
  export interface DocsReadOptions {
33
33
  includeIds?: boolean;
34
34
  }
35
+ /**
36
+ * A live revision's frontmatter raw (spec/store §6.1, 16.0): the blob the row
37
+ * names, resolved when it is a `{"f":…}` delta over the previous revision's
38
+ * (`TreeResolver.frontmatter`, at most 15 steps back). `null` when the
39
+ * revision has no frontmatter.
40
+ */
41
+ export declare function currentFrontmatterRaw(db: Database, revId: string): string | null;
35
42
  /**
36
43
  * Reconstruct a document's full file bytes from its ordered top-level blocks.
37
44
  * Takes a raw Database handle so it can run inside an ingest transaction (the
@@ -3,10 +3,30 @@ import { docPropertiesGrouped } from "../store/properties.js";
3
3
  import { sha256 } from "../hash.js";
4
4
  import { referencePath } from "../paths.js";
5
5
  import { revisionBody } from "../store/archive.js";
6
+ import { TreeResolver } from "../store/delta.js";
7
+ // A revision's root tree node is a JSON array of six-field canonical tuples
8
+ // (see core/hash.ts serializeTreeEntries):
9
+ // [blockId, rawHashHex, childTreeHashHex, type, attrs, triviaHashHex]
10
+ // — or, since 16.0, a delta over the previous revision's (core/store/delta.ts).
11
+ // We read the top-level entries in array order and keep raw + trivia hashes;
12
+ // child subtrees are NOT walked (a container block's `raw` already contains its
13
+ // children verbatim — the same reason reconstructContent walks only roots).
6
14
  function blob(db, hash) {
7
15
  const row = db.prepare("SELECT bytes FROM blobs WHERE hash = ?").get(hash);
8
16
  return row ? row.bytes.toString("utf8") : "";
9
17
  }
18
+ /**
19
+ * A live revision's frontmatter raw (spec/store §6.1, 16.0): the blob the row
20
+ * names, resolved when it is a `{"f":…}` delta over the previous revision's
21
+ * (`TreeResolver.frontmatter`, at most 15 steps back). `null` when the
22
+ * revision has no frontmatter.
23
+ */
24
+ export function currentFrontmatterRaw(db, revId) {
25
+ const rev = db.prepare("SELECT doc_id, seq, frontmatter_blob FROM revisions WHERE rev_id = ?").get(revId);
26
+ if (!rev?.frontmatter_blob)
27
+ return null;
28
+ return new TreeResolver(db).frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex"));
29
+ }
10
30
  /**
11
31
  * Reconstruct a document's full file bytes from its ordered top-level blocks.
12
32
  * Takes a raw Database handle so it can run inside an ingest transaction (the
@@ -20,13 +40,13 @@ export function reconstructContent(db, docId) {
20
40
  return null;
21
41
  const out = [doc.leading_trivia];
22
42
  if (doc.current_rev) {
23
- const rev = db.prepare("SELECT frontmatter_blob FROM revisions WHERE rev_id = ?").get(doc.current_rev);
43
+ const fm = currentFrontmatterRaw(db, doc.current_rev);
24
44
  // Emit the frontmatter block followed by its exact stored separator. Ingest
25
45
  // always captures frontmatter_trivia whenever a frontmatter block exists, so
26
46
  // frontmatter_blob present ⇒ frontmatter_trivia is a real string (possibly
27
47
  // ""); the ?? "" is just a type guard.
28
- if (rev?.frontmatter_blob)
29
- out.push(blob(db, rev.frontmatter_blob) + (doc.frontmatter_trivia ?? ""));
48
+ if (fm !== null)
49
+ out.push(fm + (doc.frontmatter_trivia ?? ""));
30
50
  }
31
51
  // Top-level blocks only: a container's `raw` already contains its nested
32
52
  // children verbatim (03 §2.3), so walking roots reconstructs the whole body.
@@ -190,22 +210,23 @@ export function readDocumentAtRevision(store, docId, revId) {
190
210
  return row ? row.bytes.toString("utf8") : "";
191
211
  };
192
212
  const out = [doc.leading_trivia];
193
- // Frontmatter blob for THIS revision (per-revision), then the doc-level
213
+ // The revision's nodes resolved in its context (spec/store §6.2, 16.0): a
214
+ // `{`-form root or frontmatter walks back through the previous revisions
215
+ // of the document to a checkpoint; a segment holds checkpoint forms only.
216
+ const resolver = new TreeResolver(db, rev.prefix);
217
+ // Frontmatter for THIS revision (per-revision), then the doc-level
194
218
  // separator (best-effort — see fidelity caveat).
195
219
  if (rev.frontmatter_blob)
196
- out.push(blobAt(rev.frontmatter_blob) + (doc.frontmatter_trivia ?? ""));
197
- // Top-level blocks in tree order, from THIS revision's Merkle root tree node.
198
- // A container's raw already holds its children verbatim, so we deliberately do
199
- // NOT recurse into childTreeHashHex (mirrors reconstructContent's
200
- // parent_block IS NULL walk). An absent tree node yields empty body bytes.
201
- const node = db.prepare(`SELECT entries FROM ${at("tree_nodes")} WHERE hash = ?`).get(rev.root_tree);
202
- if (node) {
203
- const entries = JSON.parse(node.entries);
204
- for (const [, rawHashHex, , , , triviaHashHex] of entries) {
205
- out.push(blobAt(Buffer.from(rawHashHex, "hex")));
206
- if (triviaHashHex)
207
- out.push(blobAt(Buffer.from(triviaHashHex, "hex")));
208
- }
220
+ out.push(resolver.frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex")) + (doc.frontmatter_trivia ?? ""));
221
+ // Top-level blocks in tree order, from THIS revision's Merkle root. A
222
+ // container's raw already holds its children verbatim, so we deliberately do
223
+ // NOT recurse into the child trees (mirrors reconstructContent's
224
+ // parent_block IS NULL walk).
225
+ const root = resolver.list(rev.doc_id, rev.seq, rev.root_tree.toString("hex"), null);
226
+ for (const [, rawHashHex, , , , triviaHashHex] of root.entries) {
227
+ out.push(blobAt(Buffer.from(rawHashHex, "hex")));
228
+ if (triviaHashHex)
229
+ out.push(blobAt(Buffer.from(triviaHashHex, "hex")));
209
230
  }
210
231
  const content = out.join("");
211
232
  const renderedHashMatch = sha256(content).equals(rev.rendered_hash);
@@ -1,4 +1,5 @@
1
1
  import type { Store } from "./store.js";
2
+ import { TreeResolver } from "./delta.js";
2
3
  import { type EvaluateOptions, type RetentionPlan, type RetentionPolicy } from "./retention.js";
3
4
  /** spec/store/archive.sql, verbatim — the segment's DDL. */
4
5
  export declare const ARCHIVE_DDL = "-- ---- an archive segment (spec/store \u00A77 \"Retention\", 15.0) --------------------\n-- One file per `history export`, never appended: the bodies of the revisions\n-- the retention policy squashed, self-contained (every blob and tree node\n-- reachable from them), plus copies of their revision, commit, disposition and\n-- block-change rows, and one manifest row. The same DDL subset as schema.sql\n-- minus the foreign keys (a segment holds no repos or docs rows).\nCREATE TABLE IF NOT EXISTS manifest (\n segment_id TEXT PRIMARY KEY,\n repo_id TEXT NOT NULL,\n created_at TEXT NOT NULL,\n policy TEXT NOT NULL,\n seq_lo INTEGER NOT NULL,\n seq_hi INTEGER NOT NULL,\n revision_count INTEGER NOT NULL,\n previous_segment TEXT,\n schema_user_version INTEGER NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS revisions (\n rev_id TEXT PRIMARY KEY,\n doc_id TEXT NOT NULL,\n seq INTEGER NOT NULL,\n root_tree BLOB NOT NULL,\n frontmatter_blob BLOB,\n rendered_hash BLOB NOT NULL,\n path TEXT NOT NULL,\n commit_id TEXT NOT NULL,\n UNIQUE (doc_id, seq)\n);\n\nCREATE TABLE IF NOT EXISTS commits (\n commit_id TEXT PRIMARY KEY,\n repo_id TEXT NOT NULL,\n seq INTEGER NOT NULL,\n ts TEXT NOT NULL,\n origin TEXT NOT NULL CHECK (origin IN ('api','observed','import','projection')),\n actor TEXT,\n reason TEXT,\n checkpoint_id TEXT,\n ops TEXT\n);\n\nCREATE TABLE IF NOT EXISTS blobs (\n hash BLOB PRIMARY KEY,\n bytes BLOB NOT NULL,\n size INTEGER NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS tree_nodes (\n hash BLOB PRIMARY KEY,\n entries TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS dispositions (\n commit_id TEXT NOT NULL,\n block_id TEXT NOT NULL,\n kind TEXT NOT NULL CHECK (kind IN\n ('same','edited','moved','edited_moved','inserted','deleted',\n 'split_from','merged_into','copied_from','resurrected','bulk_rewrite')),\n confidence REAL,\n reason TEXT,\n matcher_v TEXT,\n detail TEXT NOT NULL DEFAULT '{}',\n PRIMARY KEY (commit_id, block_id, kind)\n);\n\nCREATE TABLE IF NOT EXISTS block_changes (\n block_id TEXT NOT NULL,\n commit_id TEXT NOT NULL,\n kind TEXT NOT NULL,\n PRIMARY KEY (block_id, commit_id, kind)\n);\n";
@@ -120,6 +121,8 @@ export declare function archivePrefix(store: Store, rev: {
120
121
  export interface RevisionBody {
121
122
  rev_id: string;
122
123
  doc_id: string;
124
+ /** The per-document seq — the context a `{`-form node is resolved in (spec/store §6, 16.0). */
125
+ seq: number;
123
126
  path: string;
124
127
  rendered_hash: Buffer;
125
128
  root_tree: Buffer;
@@ -135,6 +138,29 @@ export interface RevisionBody {
135
138
  * its segment is unavailable.
136
139
  */
137
140
  export declare function revisionBody(store: Store, revId: string, docId?: string): RevisionBody | undefined;
141
+ /**
142
+ * A live revision's bodies in CHECKPOINT form (spec/store §7.1, 16.0): every
143
+ * list a self-contained `[`-node, the frontmatter its raw — what a segment
144
+ * holds (a segment never walks back) and what the squash re-checkpoints a
145
+ * kept revision to when it archives the revision's base. `nodes` maps each
146
+ * full node's hex hash to its text; `blobs` are the raw/trivia hashes the
147
+ * tree names (every one present in the store while the bases are live);
148
+ * `frontmatter` is the raw and its hash, or null.
149
+ */
150
+ export declare function checkpointBodies(store: Store, rev: {
151
+ doc_id: string;
152
+ seq: number;
153
+ root_tree: Buffer;
154
+ frontmatter_blob: Buffer | null;
155
+ }, resolver?: TreeResolver): {
156
+ rootHex: string;
157
+ nodes: Map<string, string>;
158
+ blobs: Set<string>;
159
+ frontmatter: {
160
+ raw: string;
161
+ hex: string;
162
+ } | null;
163
+ };
138
164
  export interface SegmentInfo {
139
165
  id: string;
140
166
  path: string;
@@ -152,6 +178,8 @@ export interface SquashResult {
152
178
  commits: number;
153
179
  dispositions: number;
154
180
  block_changes: number;
181
+ /** Kept revisions re-written in checkpoint form because the squash archived their base (spec/store §4.4, 16.0). */
182
+ checkpointed: number;
155
183
  }
156
184
  export interface ExportResult {
157
185
  plan: RetentionPlan;
@@ -3,8 +3,11 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, renameSync, statSync, un
3
3
  import { basename, join, resolve } from "node:path";
4
4
  import { SCHEMA_VERSION } from "./schema.js";
5
5
  import { mintId } from "../ids.js";
6
+ import { sha256 } from "../hash.js";
6
7
  import { sweepUnreachable } from "./gc.js";
7
- import { evaluateRetention, parseSettingsJson, ReachMarker, LAST_SEGMENT_SETTINGS_KEY, RETENTION_SETTINGS_KEY, } from "./retention.js";
8
+ import { TreeResolver, entryBlobs } from "./delta.js";
9
+ import { putBlob, putNodeText } from "./writers.js";
10
+ import { evaluateRetention, parseSettingsJson, LAST_SEGMENT_SETTINGS_KEY, RETENTION_SETTINGS_KEY, } from "./retention.js";
8
11
  // Archive segments (spec/store §7 "Retention: plan, export, squash", 15.0).
9
12
  //
10
13
  // A segment is ONE SQLite file per `history export`, never appended: the bodies
@@ -366,18 +369,41 @@ export function archivePrefix(store, rev) {
366
369
  */
367
370
  export function revisionBody(store, revId, docId) {
368
371
  const row = store.db
369
- .prepare(`SELECT rev_id, doc_id, path, rendered_hash, root_tree, frontmatter_blob, archive FROM revisions WHERE rev_id = ?${docId === undefined ? "" : " AND doc_id = ?"}`)
372
+ .prepare(`SELECT rev_id, doc_id, seq, path, rendered_hash, root_tree, frontmatter_blob, archive FROM revisions WHERE rev_id = ?${docId === undefined ? "" : " AND doc_id = ?"}`)
370
373
  .get(...(docId === undefined ? [revId] : [revId, docId]));
371
374
  if (!row)
372
375
  return undefined;
373
376
  if (row.archive === null) {
374
- return { rev_id: row.rev_id, doc_id: row.doc_id, path: row.path, rendered_hash: row.rendered_hash, root_tree: row.root_tree, frontmatter_blob: row.frontmatter_blob, prefix: "", archived: false };
377
+ return { rev_id: row.rev_id, doc_id: row.doc_id, seq: row.seq, path: row.path, rendered_hash: row.rendered_hash, root_tree: row.root_tree, frontmatter_blob: row.frontmatter_blob, prefix: "", archived: false };
375
378
  }
376
379
  const prefix = archivePrefix(store, row);
377
380
  const body = store.db.prepare(`SELECT root_tree, frontmatter_blob FROM ${prefix}revisions WHERE rev_id = ?`).get(revId);
378
381
  if (!body)
379
382
  throw new RevisionArchived(revId, row.archive, archiveRow(store, row.archive)?.path ?? null);
380
- return { rev_id: row.rev_id, doc_id: row.doc_id, path: row.path, rendered_hash: row.rendered_hash, root_tree: body.root_tree, frontmatter_blob: body.frontmatter_blob, prefix, archived: true };
383
+ return { rev_id: row.rev_id, doc_id: row.doc_id, seq: row.seq, path: row.path, rendered_hash: row.rendered_hash, root_tree: body.root_tree, frontmatter_blob: body.frontmatter_blob, prefix, archived: true };
384
+ }
385
+ /**
386
+ * A live revision's bodies in CHECKPOINT form (spec/store §7.1, 16.0): every
387
+ * list a self-contained `[`-node, the frontmatter its raw — what a segment
388
+ * holds (a segment never walks back) and what the squash re-checkpoints a
389
+ * kept revision to when it archives the revision's base. `nodes` maps each
390
+ * full node's hex hash to its text; `blobs` are the raw/trivia hashes the
391
+ * tree names (every one present in the store while the bases are live);
392
+ * `frontmatter` is the raw and its hash, or null.
393
+ */
394
+ export function checkpointBodies(store, rev, resolver = new TreeResolver(store.db)) {
395
+ const tree = resolver.tree(rev.doc_id, rev.seq, rev.root_tree.toString("hex"));
396
+ const { rootHex, nodes } = resolver.checkpointForm(tree);
397
+ const blobs = new Set();
398
+ entryBlobs(tree.root.entries, blobs);
399
+ for (const list of tree.byBlock.values())
400
+ entryBlobs(list.entries, blobs);
401
+ let frontmatter = null;
402
+ if (rev.frontmatter_blob) {
403
+ const raw = resolver.frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex"));
404
+ frontmatter = { raw, hex: sha256(raw).toString("hex") };
405
+ }
406
+ return { rootHex, nodes, blobs, frontmatter };
381
407
  }
382
408
  /** An unrecorded segment under `dest` names both live and non-live revisions (it predates another squash, or belongs elsewhere). */
383
409
  export class StaleSegment extends Error {
@@ -446,14 +472,29 @@ function writeSegment(store, repoId, revIds, plan, dest) {
446
472
  const segmentId = mintId("s");
447
473
  const createdAt = plan.now;
448
474
  const previous = lastSegment(store, repoId);
449
- const marker = new ReachMarker(store);
450
- const trees = new Set();
475
+ // spec/store §7.1 (16.0): a segment holds CHECKPOINT forms — every list a
476
+ // self-contained node, the frontmatter its raw — so a reader of the segment
477
+ // never walks back into revisions that are not in it. The forms are computed
478
+ // from the resolved trees (a delta revision's full nodes and resolved raw
479
+ // were never rows of the store); the raw/trivia blobs they name are.
480
+ const resolver = new TreeResolver(db);
481
+ const nodes = new Map();
451
482
  const blobs = new Set();
452
- marker.mark(revisions, trees, blobs);
483
+ const fmBlobs = new Map();
484
+ const bodies = new Map();
485
+ for (const r of revisions) {
486
+ const cp = checkpointBodies(store, r, resolver);
487
+ for (const [h, text] of cp.nodes)
488
+ nodes.set(h, text);
489
+ for (const b of cp.blobs)
490
+ blobs.add(b);
491
+ if (cp.frontmatter)
492
+ fmBlobs.set(cp.frontmatter.hex, cp.frontmatter.raw);
493
+ bodies.set(r.rev_id, { rootHex: cp.rootHex, fmHex: cp.frontmatter?.hex ?? null });
494
+ }
453
495
  const wholly = whollyArchivedCommits(store, new Set(revIds));
454
496
  const dispRows = db.prepare("SELECT commit_id, block_id, kind, confidence, reason, matcher_v, detail FROM dispositions WHERE commit_id = ? ORDER BY block_id, kind");
455
497
  const bcRows = db.prepare("SELECT block_id, commit_id, kind FROM block_changes WHERE commit_id = ? ORDER BY block_id, kind");
456
- const treeRow = db.prepare("SELECT hash, entries FROM tree_nodes WHERE hash = ?");
457
498
  const blobRow = db.prepare("SELECT hash, bytes, size FROM blobs WHERE hash = ?");
458
499
  mkdirSync(dest, { recursive: true });
459
500
  for (const stale of readdirSync(dest))
@@ -471,23 +512,26 @@ function writeSegment(store, repoId, revIds, plan, dest) {
471
512
  seg.prepare("INSERT INTO manifest (segment_id, repo_id, created_at, policy, seq_lo, seq_hi, revision_count, previous_segment, schema_user_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
472
513
  .run(segmentId, repoId, createdAt, JSON.stringify(plan.policy), seqLo, seqHi, revisions.length, previous, SCHEMA_VERSION);
473
514
  const insRev = seg.prepare("INSERT INTO revisions (rev_id, doc_id, seq, root_tree, frontmatter_blob, rendered_hash, path, commit_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
474
- for (const r of revisions)
475
- insRev.run(r.rev_id, r.doc_id, r.seq, r.root_tree, r.frontmatter_blob, r.rendered_hash, r.path, r.commit_id);
515
+ for (const r of revisions) {
516
+ const body = bodies.get(r.rev_id);
517
+ insRev.run(r.rev_id, r.doc_id, r.seq, Buffer.from(body.rootHex, "hex"), body.fmHex ? Buffer.from(body.fmHex, "hex") : null, r.rendered_hash, r.path, r.commit_id);
518
+ }
476
519
  const insCommit = seg.prepare("INSERT INTO commits (commit_id, repo_id, seq, ts, origin, actor, reason, checkpoint_id, ops) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)");
477
520
  for (const c of commits)
478
521
  insCommit.run(c.commit_id, c.repo_id, c.seq, c.ts, c.origin, c.actor, c.reason, c.checkpoint_id, c.ops);
479
522
  const insTree = seg.prepare("INSERT INTO tree_nodes (hash, entries) VALUES (?, ?)");
480
- for (const t of [...trees].sort()) {
481
- const row = treeRow.get(Buffer.from(t, "hex"));
482
- if (row)
483
- insTree.run(row.hash, row.entries);
484
- }
485
- const insBlob = seg.prepare("INSERT INTO blobs (hash, bytes, size) VALUES (?, ?, ?)");
523
+ for (const h of [...nodes.keys()].sort())
524
+ insTree.run(Buffer.from(h, "hex"), nodes.get(h));
525
+ const insBlob = seg.prepare("INSERT OR IGNORE INTO blobs (hash, bytes, size) VALUES (?, ?, ?)");
486
526
  for (const b of [...blobs].sort()) {
487
527
  const row = blobRow.get(Buffer.from(b, "hex"));
488
528
  if (row)
489
529
  insBlob.run(row.hash, row.bytes, row.size);
490
530
  }
531
+ for (const h of [...fmBlobs.keys()].sort()) {
532
+ const bytes = Buffer.from(fmBlobs.get(h), "utf8");
533
+ insBlob.run(Buffer.from(h, "hex"), bytes, bytes.length);
534
+ }
491
535
  const insDisp = seg.prepare("INSERT INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail) VALUES (?, ?, ?, ?, ?, ?, ?)");
492
536
  const insBc = seg.prepare("INSERT INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
493
537
  for (const c of wholly) {
@@ -520,6 +564,14 @@ function writeSegment(store, repoId, revIds, plan, dest) {
520
564
  function squash(store, repoId, segment, revIds) {
521
565
  return store.write((db) => {
522
566
  const wholly = whollyArchivedCommits(store, new Set(revIds));
567
+ // spec/store §4.4 "base liveness" (16.0): a kept revision whose previous
568
+ // revision this squash archives is re-written in checkpoint form FIRST,
569
+ // while its bases are still readable — its delta nodes and delta
570
+ // frontmatter resolve, the self-contained forms are put, the row repointed.
571
+ // Its own successors are untouched: a delta is over the resolved content,
572
+ // which re-encoding does not change, and only self-contained nodes are
573
+ // ever reused across revisions.
574
+ const checkpointed = checkpointDependents(store, new Set(revIds));
523
575
  const upd = db.prepare("UPDATE revisions SET root_tree = NULL, frontmatter_blob = NULL, archive = ? WHERE rev_id = ?");
524
576
  for (const id of revIds)
525
577
  upd.run(segment.id, id);
@@ -543,11 +595,48 @@ function squash(store, repoId, segment, revIds) {
543
595
  });
544
596
  setLastSegment(store, repoId, segment.id);
545
597
  return {
546
- squashed: { revisions: revIds.length, commits: wholly.length, dispositions, block_changes: blockChanges },
598
+ squashed: { revisions: revIds.length, commits: wholly.length, dispositions, block_changes: blockChanges, checkpointed },
547
599
  gc: { blobs: swept.blobsSwept, tree_nodes: swept.treeNodesSwept },
548
600
  };
549
601
  });
550
602
  }
603
+ /**
604
+ * Re-checkpoint every live revision that stays and whose predecessor
605
+ * (`seq − 1`, same document) is in `archiving`: resolve its tree and
606
+ * frontmatter, put the checkpoint forms, repoint the row. Returns how many
607
+ * rows changed (a revision already in checkpoint form changes nothing).
608
+ * Runs inside the squash transaction, before the bases lose their bodies.
609
+ */
610
+ function checkpointDependents(store, archiving) {
611
+ const db = store.db;
612
+ const revRow = db.prepare("SELECT doc_id, seq FROM revisions WHERE rev_id = ?");
613
+ const nextRow = db.prepare("SELECT rev_id, doc_id, seq, root_tree, frontmatter_blob FROM revisions WHERE doc_id = ? AND seq = ? AND archive IS NULL");
614
+ const upd = db.prepare("UPDATE revisions SET root_tree = ?, frontmatter_blob = ? WHERE rev_id = ?");
615
+ const resolver = new TreeResolver(db);
616
+ const seen = new Set();
617
+ let changed = 0;
618
+ for (const id of [...archiving].sort()) {
619
+ const r = revRow.get(id);
620
+ if (!r)
621
+ continue;
622
+ const next = nextRow.get(r.doc_id, r.seq + 1);
623
+ if (!next || !next.root_tree || archiving.has(next.rev_id) || seen.has(next.rev_id))
624
+ continue;
625
+ seen.add(next.rev_id);
626
+ const cp = checkpointBodies(store, { doc_id: next.doc_id, seq: next.seq, root_tree: next.root_tree, frontmatter_blob: next.frontmatter_blob }, resolver);
627
+ const fmHex = cp.frontmatter ? cp.frontmatter.hex : null;
628
+ const oldFm = next.frontmatter_blob ? next.frontmatter_blob.toString("hex") : null;
629
+ if (cp.rootHex === next.root_tree.toString("hex") && fmHex === oldFm)
630
+ continue;
631
+ for (const text of cp.nodes.values())
632
+ putNodeText(db, text);
633
+ if (cp.frontmatter)
634
+ putBlob(db, cp.frontmatter.raw);
635
+ upd.run(Buffer.from(cp.rootHex, "hex"), fmHex ? Buffer.from(fmHex, "hex") : null, next.rev_id);
636
+ changed++;
637
+ }
638
+ return changed;
639
+ }
551
640
  /**
552
641
  * `omg history export [--prune]`: evaluate the policy, write ONE new segment
553
642
  * for the revisions to archive (or finish an unrecorded one — the crash