@omgbase/core 0.24.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/core/edit-script.d.ts +15 -0
- package/dist/src/core/edit-script.js +72 -0
- package/dist/src/core/ingest.js +42 -12
- package/dist/src/core/read/document.d.ts +7 -0
- package/dist/src/core/read/document.js +38 -17
- package/dist/src/core/store/archive.d.ts +28 -0
- package/dist/src/core/store/archive.js +106 -17
- package/dist/src/core/store/delta.d.ts +131 -0
- package/dist/src/core/store/delta.js +470 -0
- package/dist/src/core/store/gc.d.ts +10 -0
- package/dist/src/core/store/gc.js +23 -7
- package/dist/src/core/store/retention.d.ts +1 -1
- package/dist/src/core/store/retention.js +3 -17
- package/dist/src/core/store/schema.d.ts +1 -1
- package/dist/src/core/store/schema.js +2 -1
- package/dist/src/core/store/store.js +7 -0
- package/dist/src/core/store/writers.d.ts +42 -6
- package/dist/src/core/store/writers.js +67 -25
- package/dist/src/format/json.js +5 -0
- package/dist/src/format/yaml.js +15 -3
- package/dist/src/graph/history.d.ts +1 -15
- package/dist/src/graph/history.js +14 -74
- package/dist/src/mcp/arguments.d.ts +11 -0
- package/dist/src/mcp/arguments.js +47 -4
- package/dist/src/mcp/errors.js +1 -1
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +1 -1
- package/dist/src/mcp/server.js +4 -2
- package/dist/src/mutate/effects.js +87 -13
- package/dist/src/mutate/frontmatter-parts.d.ts +71 -0
- package/dist/src/mutate/frontmatter-parts.js +237 -0
- package/dist/src/mutate/load.js +9 -9
- package/dist/src/mutate/select.js +9 -3
- package/dist/src/oqx-js/context.js +21 -10
- package/dist/src/oqx-js/names.js +1 -1
- package/dist/src/oqx-js/run.d.ts +8 -0
- package/dist/src/oqx-js/run.js +91 -4
- package/dist/src/oqx-js/sql/translate.js +21 -1
- package/dist/src/search/cel/parser.d.ts +3 -1
- package/dist/src/search/cel/parser.js +4 -1
- package/dist/src/spec-versions.d.ts +4 -4
- package/dist/src/spec-versions.js +4 -4
- package/package.json +1 -1
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** One step of the edit script: `keep` consumes a line from both sides,
|
|
2
|
+
* `delete` one from the old text, `insert` one from the new text. */
|
|
3
|
+
export interface EditOp {
|
|
4
|
+
kind: "keep" | "delete" | "insert";
|
|
5
|
+
line: string;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Myers' O(ND) shortest edit script (forward, with a per-`d` trace for the
|
|
9
|
+
* backtrack). `V[k]` is the furthest x on diagonal `k = x - y` reachable with
|
|
10
|
+
* `d` edits. The canonical tie rule, identical in both engines: at each step
|
|
11
|
+
* take the diagonal from `k+1` (moving down — an insertion of `b[y]`) when
|
|
12
|
+
* `k == -d || (k != d && V[k-1] < V[k+1])`, else from `k-1` (moving right —
|
|
13
|
+
* a deletion of `a[x]`). On a tie (`V[k-1] == V[k+1]`) that is the deletion.
|
|
14
|
+
*/
|
|
15
|
+
export declare function myersScript(a: string[], b: string[]): EditOp[];
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Myers' shortest edit script over two string arrays — the ONE diff algorithm
|
|
2
|
+
// both engines share, so an edit script is deterministic across them. Two
|
|
3
|
+
// consumers: the unified diff of `docs_diff_unified` (spec/surface §3;
|
|
4
|
+
// `graph/history.ts` re-exports it, and `crates/omgbase-surface/src/history.rs`
|
|
5
|
+
// is the twin) and the delta tree nodes of spec/store §4.1 (16.0;
|
|
6
|
+
// `core/store/delta.ts`), whose ops are this script grouped. Kept in `core/`
|
|
7
|
+
// because the store cannot import `graph/`.
|
|
8
|
+
/**
|
|
9
|
+
* Myers' O(ND) shortest edit script (forward, with a per-`d` trace for the
|
|
10
|
+
* backtrack). `V[k]` is the furthest x on diagonal `k = x - y` reachable with
|
|
11
|
+
* `d` edits. The canonical tie rule, identical in both engines: at each step
|
|
12
|
+
* take the diagonal from `k+1` (moving down — an insertion of `b[y]`) when
|
|
13
|
+
* `k == -d || (k != d && V[k-1] < V[k+1])`, else from `k-1` (moving right —
|
|
14
|
+
* a deletion of `a[x]`). On a tie (`V[k-1] == V[k+1]`) that is the deletion.
|
|
15
|
+
*/
|
|
16
|
+
export function myersScript(a, b) {
|
|
17
|
+
const n = a.length;
|
|
18
|
+
const m = b.length;
|
|
19
|
+
const max = n + m;
|
|
20
|
+
// V is indexed by k ∈ [-max-1, max+1]; `off` maps it onto a plain array.
|
|
21
|
+
const off = max + 1;
|
|
22
|
+
const v = new Array(2 * max + 3).fill(0);
|
|
23
|
+
const trace = [];
|
|
24
|
+
let found = false;
|
|
25
|
+
for (let d = 0; d <= max && !found; d++) {
|
|
26
|
+
trace.push(v.slice());
|
|
27
|
+
for (let k = -d; k <= d; k += 2) {
|
|
28
|
+
let x;
|
|
29
|
+
if (k === -d || (k !== d && v[off + k - 1] < v[off + k + 1]))
|
|
30
|
+
x = v[off + k + 1];
|
|
31
|
+
else
|
|
32
|
+
x = v[off + k - 1] + 1;
|
|
33
|
+
let y = x - k;
|
|
34
|
+
while (x < n && y < m && a[x] === b[y]) {
|
|
35
|
+
x++;
|
|
36
|
+
y++;
|
|
37
|
+
}
|
|
38
|
+
v[off + k] = x;
|
|
39
|
+
if (x >= n && y >= m) {
|
|
40
|
+
found = true;
|
|
41
|
+
break;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// Backtrack from (n, m) through the trace, emitting ops newest-first.
|
|
46
|
+
const ops = [];
|
|
47
|
+
let x = n;
|
|
48
|
+
let y = m;
|
|
49
|
+
for (let d = trace.length - 1; d >= 0; d--) {
|
|
50
|
+
const vd = trace[d];
|
|
51
|
+
const k = x - y;
|
|
52
|
+
const prevK = k === -d || (k !== d && vd[off + k - 1] < vd[off + k + 1]) ? k + 1 : k - 1;
|
|
53
|
+
const prevX = vd[off + prevK];
|
|
54
|
+
const prevY = prevX - prevK;
|
|
55
|
+
while (x > prevX && y > prevY) {
|
|
56
|
+
x--;
|
|
57
|
+
y--;
|
|
58
|
+
ops.push({ kind: "keep", line: a[x] });
|
|
59
|
+
}
|
|
60
|
+
if (d > 0) {
|
|
61
|
+
if (x === prevX)
|
|
62
|
+
ops.push({ kind: "insert", line: b[prevY] });
|
|
63
|
+
else
|
|
64
|
+
ops.push({ kind: "delete", line: a[prevX] });
|
|
65
|
+
}
|
|
66
|
+
x = prevX;
|
|
67
|
+
y = prevY;
|
|
68
|
+
}
|
|
69
|
+
ops.reverse();
|
|
70
|
+
return ops;
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=edit-script.js.map
|
package/dist/src/core/ingest.js
CHANGED
|
@@ -2,7 +2,8 @@ import { Store } from "./store/store.js";
|
|
|
2
2
|
import { parseTree } from "./parse/tree.js";
|
|
3
3
|
import { render } from "./parse/render.js";
|
|
4
4
|
import { reconstructContent } from "./read/document.js";
|
|
5
|
-
import { assignIds, writeBlockTree,
|
|
5
|
+
import { assignIds, writeBlockTree, writeFrontmatterBlob, isCheckpointRevision, previousRevision, newCommit, writeRevision } from "./store/writers.js";
|
|
6
|
+
import { TreeResolver } from "./store/delta.js";
|
|
6
7
|
import { sha256, childQuoteDepth, visibleText } from "./hash.js";
|
|
7
8
|
import { mintId } from "./ids.js";
|
|
8
9
|
import { keyBetween } from "./order-key.js";
|
|
@@ -114,20 +115,22 @@ export function ingestInto(db, repoId, path, content, opts) {
|
|
|
114
115
|
const format = opts.format ?? adapter?.format ?? "markdown";
|
|
115
116
|
const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
|
|
116
117
|
const { fmBlock, rest } = extractFrontmatter(tree.children);
|
|
117
|
-
// Frontmatter blob (markdown-specific) preserved for revision history.
|
|
118
|
-
const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
|
|
119
118
|
// The exact separator bytes between the frontmatter block and the first body
|
|
120
119
|
// block (the frontmatter block's trailing trivia). Persisted so the body
|
|
121
120
|
// reconstructs byte-for-byte instead of assuming a canonical blank line.
|
|
122
121
|
// NULL when there is no frontmatter.
|
|
123
122
|
const fmTrivia = fmBlock ? fmBlock.trivia : null;
|
|
124
|
-
// Document property bag
|
|
125
|
-
//
|
|
126
|
-
// `extractMetadata`
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
// the
|
|
130
|
-
//
|
|
123
|
+
// Document property bag — the document's AUTHORED MAPPING. For a `yaml` or
|
|
124
|
+
// `json` document it is the whole document's top-level mapping, from the
|
|
125
|
+
// adapter's `extractMetadata` (spec/properties §3.4, 1.2: null for a
|
|
126
|
+
// non-mapping root, a syntax error or an empty file; a YAML stream's first
|
|
127
|
+
// document only); for Markdown — whose adapter has no `extractMetadata` — the
|
|
128
|
+
// YAML of the `frontmatter` block and nothing else (§3.1: a document whose
|
|
129
|
+
// first block is not `frontmatter` has no frontmatter rows, whatever else its
|
|
130
|
+
// bytes resemble). Either way it is flattened into the properties table below
|
|
131
|
+
// under source=frontmatter — the same source, because it is the same thing
|
|
132
|
+
// (§3.4: "frontmatter" names the document's authored mapping) — and drives
|
|
133
|
+
// frontmatter edge extraction.
|
|
131
134
|
const metadata = adapter?.extractMetadata
|
|
132
135
|
? (adapter.extractMetadata(content) ?? {})
|
|
133
136
|
: parseFrontmatter(fmBlock);
|
|
@@ -161,7 +164,29 @@ export function ingestInto(db, repoId, path, content, opts) {
|
|
|
161
164
|
? opts.resolveIds(rest, isNew ? null : docId)
|
|
162
165
|
: { assigned: assignIds(rest), dispositions: [], deleted: [] };
|
|
163
166
|
const assigned = resolved.assigned;
|
|
164
|
-
|
|
167
|
+
// spec/store §4.4 (16.0): the revision's seq decides whether it is a
|
|
168
|
+
// checkpoint; a non-checkpoint revision writes its tree and frontmatter as
|
|
169
|
+
// deltas against the previous revision's RESOLVED tree and raw where that is
|
|
170
|
+
// smaller. The previous revision is the document's greatest seq — on every
|
|
171
|
+
// path the current revision, never archived (retention rule 1) — but the
|
|
172
|
+
// rule is stated for the general case: no live predecessor ⇒ a checkpoint.
|
|
173
|
+
const previous = previousRevision(db, docId);
|
|
174
|
+
const seq = previous ? previous.seq + 1 : 1;
|
|
175
|
+
const checkpoint = isCheckpointRevision(seq, previous);
|
|
176
|
+
const resolver = new TreeResolver(db);
|
|
177
|
+
const treeCtx = {
|
|
178
|
+
checkpoint,
|
|
179
|
+
base: !checkpoint && previous?.root_tree ? resolver.tree(docId, previous.seq, previous.root_tree.toString("hex")) : null,
|
|
180
|
+
};
|
|
181
|
+
const rootTreeHex = writeBlockTree(db, assigned, treeCtx);
|
|
182
|
+
// Frontmatter blob (markdown-specific) preserved for revision history: the
|
|
183
|
+
// raw at a checkpoint, else the smaller of a line delta over the previous
|
|
184
|
+
// revision's frontmatter and the raw (spec/store §5.4 step 1).
|
|
185
|
+
let fmBlobHex = null;
|
|
186
|
+
if (fmBlock) {
|
|
187
|
+
const baseRaw = !checkpoint && previous?.frontmatter_blob ? resolver.frontmatter(docId, previous.seq, previous.frontmatter_blob.toString("hex")) : null;
|
|
188
|
+
fmBlobHex = writeFrontmatterBlob(db, fmBlock.raw, baseRaw, checkpoint);
|
|
189
|
+
}
|
|
165
190
|
const renderedHash = sha256(content);
|
|
166
191
|
const rev = writeRevision(db, {
|
|
167
192
|
docId,
|
|
@@ -170,6 +195,7 @@ export function ingestInto(db, repoId, path, content, opts) {
|
|
|
170
195
|
renderedHash,
|
|
171
196
|
path,
|
|
172
197
|
commitId,
|
|
198
|
+
seq,
|
|
173
199
|
});
|
|
174
200
|
// Snapshot deleted blocks into the resurrection pool BEFORE dropping the
|
|
175
201
|
// doc's rows (the ingest rebuilds current-state blocks wholesale).
|
|
@@ -264,7 +290,11 @@ export function ingestInto(db, repoId, path, content, opts) {
|
|
|
264
290
|
propertyRows.push({ source: "frontmatter", blockId: null, ...r });
|
|
265
291
|
}
|
|
266
292
|
// Computed → property rows (source=computed). Engine-derived $-intrinsics
|
|
267
|
-
// ($title, $tags) from the adapter; never claim authored keys.
|
|
293
|
+
// ($title, $tags) from the adapter; never claim authored keys. Only the
|
|
294
|
+
// Markdown adapter computes (spec/properties §3.3): a `yaml`/`json` document
|
|
295
|
+
// has no body prose to derive a heading or a hashtag from, and its authored
|
|
296
|
+
// `title`/`tags` are reached as any other key of its mapping (§3.4), so it
|
|
297
|
+
// yields no computed rows.
|
|
268
298
|
if (adapter?.computeProperties) {
|
|
269
299
|
const computed = adapter.computeProperties(rest, metadata);
|
|
270
300
|
for (const r of flattenComputed(computed)) {
|
|
@@ -32,6 +32,13 @@ export interface DocsReadResult {
|
|
|
32
32
|
export interface DocsReadOptions {
|
|
33
33
|
includeIds?: boolean;
|
|
34
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* A live revision's frontmatter raw (spec/store §6.1, 16.0): the blob the row
|
|
37
|
+
* names, resolved when it is a `{"f":…}` delta over the previous revision's
|
|
38
|
+
* (`TreeResolver.frontmatter`, at most 15 steps back). `null` when the
|
|
39
|
+
* revision has no frontmatter.
|
|
40
|
+
*/
|
|
41
|
+
export declare function currentFrontmatterRaw(db: Database, revId: string): string | null;
|
|
35
42
|
/**
|
|
36
43
|
* Reconstruct a document's full file bytes from its ordered top-level blocks.
|
|
37
44
|
* Takes a raw Database handle so it can run inside an ingest transaction (the
|
|
@@ -3,10 +3,30 @@ import { docPropertiesGrouped } from "../store/properties.js";
|
|
|
3
3
|
import { sha256 } from "../hash.js";
|
|
4
4
|
import { referencePath } from "../paths.js";
|
|
5
5
|
import { revisionBody } from "../store/archive.js";
|
|
6
|
+
import { TreeResolver } from "../store/delta.js";
|
|
7
|
+
// A revision's root tree node is a JSON array of six-field canonical tuples
|
|
8
|
+
// (see core/hash.ts serializeTreeEntries):
|
|
9
|
+
// [blockId, rawHashHex, childTreeHashHex, type, attrs, triviaHashHex]
|
|
10
|
+
// — or, since 16.0, a delta over the previous revision's (core/store/delta.ts).
|
|
11
|
+
// We read the top-level entries in array order and keep raw + trivia hashes;
|
|
12
|
+
// child subtrees are NOT walked (a container block's `raw` already contains its
|
|
13
|
+
// children verbatim — the same reason reconstructContent walks only roots).
|
|
6
14
|
function blob(db, hash) {
|
|
7
15
|
const row = db.prepare("SELECT bytes FROM blobs WHERE hash = ?").get(hash);
|
|
8
16
|
return row ? row.bytes.toString("utf8") : "";
|
|
9
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* A live revision's frontmatter raw (spec/store §6.1, 16.0): the blob the row
|
|
20
|
+
* names, resolved when it is a `{"f":…}` delta over the previous revision's
|
|
21
|
+
* (`TreeResolver.frontmatter`, at most 15 steps back). `null` when the
|
|
22
|
+
* revision has no frontmatter.
|
|
23
|
+
*/
|
|
24
|
+
export function currentFrontmatterRaw(db, revId) {
|
|
25
|
+
const rev = db.prepare("SELECT doc_id, seq, frontmatter_blob FROM revisions WHERE rev_id = ?").get(revId);
|
|
26
|
+
if (!rev?.frontmatter_blob)
|
|
27
|
+
return null;
|
|
28
|
+
return new TreeResolver(db).frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex"));
|
|
29
|
+
}
|
|
10
30
|
/**
|
|
11
31
|
* Reconstruct a document's full file bytes from its ordered top-level blocks.
|
|
12
32
|
* Takes a raw Database handle so it can run inside an ingest transaction (the
|
|
@@ -20,13 +40,13 @@ export function reconstructContent(db, docId) {
|
|
|
20
40
|
return null;
|
|
21
41
|
const out = [doc.leading_trivia];
|
|
22
42
|
if (doc.current_rev) {
|
|
23
|
-
const
|
|
43
|
+
const fm = currentFrontmatterRaw(db, doc.current_rev);
|
|
24
44
|
// Emit the frontmatter block followed by its exact stored separator. Ingest
|
|
25
45
|
// always captures frontmatter_trivia whenever a frontmatter block exists, so
|
|
26
46
|
// frontmatter_blob present ⇒ frontmatter_trivia is a real string (possibly
|
|
27
47
|
// ""); the ?? "" is just a type guard.
|
|
28
|
-
if (
|
|
29
|
-
out.push(
|
|
48
|
+
if (fm !== null)
|
|
49
|
+
out.push(fm + (doc.frontmatter_trivia ?? ""));
|
|
30
50
|
}
|
|
31
51
|
// Top-level blocks only: a container's `raw` already contains its nested
|
|
32
52
|
// children verbatim (03 §2.3), so walking roots reconstructs the whole body.
|
|
@@ -190,22 +210,23 @@ export function readDocumentAtRevision(store, docId, revId) {
|
|
|
190
210
|
return row ? row.bytes.toString("utf8") : "";
|
|
191
211
|
};
|
|
192
212
|
const out = [doc.leading_trivia];
|
|
193
|
-
//
|
|
213
|
+
// The revision's nodes resolved in its context (spec/store §6.2, 16.0): a
|
|
214
|
+
// `{`-form root or frontmatter walks back through the previous revisions
|
|
215
|
+
// of the document to a checkpoint; a segment holds checkpoint forms only.
|
|
216
|
+
const resolver = new TreeResolver(db, rev.prefix);
|
|
217
|
+
// Frontmatter for THIS revision (per-revision), then the doc-level
|
|
194
218
|
// separator (best-effort — see fidelity caveat).
|
|
195
219
|
if (rev.frontmatter_blob)
|
|
196
|
-
out.push(
|
|
197
|
-
// Top-level blocks in tree order, from THIS revision's Merkle root
|
|
198
|
-
//
|
|
199
|
-
// NOT recurse into
|
|
200
|
-
// parent_block IS NULL walk).
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
out.push(blobAt(Buffer.from(
|
|
206
|
-
if (triviaHashHex)
|
|
207
|
-
out.push(blobAt(Buffer.from(triviaHashHex, "hex")));
|
|
208
|
-
}
|
|
220
|
+
out.push(resolver.frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex")) + (doc.frontmatter_trivia ?? ""));
|
|
221
|
+
// Top-level blocks in tree order, from THIS revision's Merkle root. A
|
|
222
|
+
// container's raw already holds its children verbatim, so we deliberately do
|
|
223
|
+
// NOT recurse into the child trees (mirrors reconstructContent's
|
|
224
|
+
// parent_block IS NULL walk).
|
|
225
|
+
const root = resolver.list(rev.doc_id, rev.seq, rev.root_tree.toString("hex"), null);
|
|
226
|
+
for (const [, rawHashHex, , , , triviaHashHex] of root.entries) {
|
|
227
|
+
out.push(blobAt(Buffer.from(rawHashHex, "hex")));
|
|
228
|
+
if (triviaHashHex)
|
|
229
|
+
out.push(blobAt(Buffer.from(triviaHashHex, "hex")));
|
|
209
230
|
}
|
|
210
231
|
const content = out.join("");
|
|
211
232
|
const renderedHashMatch = sha256(content).equals(rev.rendered_hash);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Store } from "./store.js";
|
|
2
|
+
import { TreeResolver } from "./delta.js";
|
|
2
3
|
import { type EvaluateOptions, type RetentionPlan, type RetentionPolicy } from "./retention.js";
|
|
3
4
|
/** spec/store/archive.sql, verbatim — the segment's DDL. */
|
|
4
5
|
export declare const ARCHIVE_DDL = "-- ---- an archive segment (spec/store \u00A77 \"Retention\", 15.0) --------------------\n-- One file per `history export`, never appended: the bodies of the revisions\n-- the retention policy squashed, self-contained (every blob and tree node\n-- reachable from them), plus copies of their revision, commit, disposition and\n-- block-change rows, and one manifest row. The same DDL subset as schema.sql\n-- minus the foreign keys (a segment holds no repos or docs rows).\nCREATE TABLE IF NOT EXISTS manifest (\n segment_id TEXT PRIMARY KEY,\n repo_id TEXT NOT NULL,\n created_at TEXT NOT NULL,\n policy TEXT NOT NULL,\n seq_lo INTEGER NOT NULL,\n seq_hi INTEGER NOT NULL,\n revision_count INTEGER NOT NULL,\n previous_segment TEXT,\n schema_user_version INTEGER NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS revisions (\n rev_id TEXT PRIMARY KEY,\n doc_id TEXT NOT NULL,\n seq INTEGER NOT NULL,\n root_tree BLOB NOT NULL,\n frontmatter_blob BLOB,\n rendered_hash BLOB NOT NULL,\n path TEXT NOT NULL,\n commit_id TEXT NOT NULL,\n UNIQUE (doc_id, seq)\n);\n\nCREATE TABLE IF NOT EXISTS commits (\n commit_id TEXT PRIMARY KEY,\n repo_id TEXT NOT NULL,\n seq INTEGER NOT NULL,\n ts TEXT NOT NULL,\n origin TEXT NOT NULL CHECK (origin IN ('api','observed','import','projection')),\n actor TEXT,\n reason TEXT,\n checkpoint_id TEXT,\n ops TEXT\n);\n\nCREATE TABLE IF NOT EXISTS blobs (\n hash BLOB PRIMARY KEY,\n bytes BLOB NOT NULL,\n size INTEGER NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS tree_nodes (\n hash BLOB PRIMARY KEY,\n entries TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS dispositions (\n commit_id TEXT NOT NULL,\n block_id TEXT NOT NULL,\n kind TEXT NOT NULL CHECK (kind IN\n ('same','edited','moved','edited_moved','inserted','deleted',\n 'split_from','merged_into','copied_from','resurrected','bulk_rewrite')),\n confidence REAL,\n reason TEXT,\n matcher_v TEXT,\n detail TEXT NOT NULL DEFAULT '{}',\n PRIMARY KEY (commit_id, block_id, kind)\n);\n\nCREATE TABLE IF NOT EXISTS block_changes (\n block_id TEXT NOT NULL,\n commit_id TEXT NOT NULL,\n kind TEXT NOT NULL,\n PRIMARY KEY (block_id, commit_id, kind)\n);\n";
|
|
@@ -120,6 +121,8 @@ export declare function archivePrefix(store: Store, rev: {
|
|
|
120
121
|
export interface RevisionBody {
|
|
121
122
|
rev_id: string;
|
|
122
123
|
doc_id: string;
|
|
124
|
+
/** The per-document seq — the context a `{`-form node is resolved in (spec/store §6, 16.0). */
|
|
125
|
+
seq: number;
|
|
123
126
|
path: string;
|
|
124
127
|
rendered_hash: Buffer;
|
|
125
128
|
root_tree: Buffer;
|
|
@@ -135,6 +138,29 @@ export interface RevisionBody {
|
|
|
135
138
|
* its segment is unavailable.
|
|
136
139
|
*/
|
|
137
140
|
export declare function revisionBody(store: Store, revId: string, docId?: string): RevisionBody | undefined;
|
|
141
|
+
/**
|
|
142
|
+
* A live revision's bodies in CHECKPOINT form (spec/store §7.1, 16.0): every
|
|
143
|
+
* list a self-contained `[`-node, the frontmatter its raw — what a segment
|
|
144
|
+
* holds (a segment never walks back) and what the squash re-checkpoints a
|
|
145
|
+
* kept revision to when it archives the revision's base. `nodes` maps each
|
|
146
|
+
* full node's hex hash to its text; `blobs` are the raw/trivia hashes the
|
|
147
|
+
* tree names (every one present in the store while the bases are live);
|
|
148
|
+
* `frontmatter` is the raw and its hash, or null.
|
|
149
|
+
*/
|
|
150
|
+
export declare function checkpointBodies(store: Store, rev: {
|
|
151
|
+
doc_id: string;
|
|
152
|
+
seq: number;
|
|
153
|
+
root_tree: Buffer;
|
|
154
|
+
frontmatter_blob: Buffer | null;
|
|
155
|
+
}, resolver?: TreeResolver): {
|
|
156
|
+
rootHex: string;
|
|
157
|
+
nodes: Map<string, string>;
|
|
158
|
+
blobs: Set<string>;
|
|
159
|
+
frontmatter: {
|
|
160
|
+
raw: string;
|
|
161
|
+
hex: string;
|
|
162
|
+
} | null;
|
|
163
|
+
};
|
|
138
164
|
export interface SegmentInfo {
|
|
139
165
|
id: string;
|
|
140
166
|
path: string;
|
|
@@ -152,6 +178,8 @@ export interface SquashResult {
|
|
|
152
178
|
commits: number;
|
|
153
179
|
dispositions: number;
|
|
154
180
|
block_changes: number;
|
|
181
|
+
/** Kept revisions re-written in checkpoint form because the squash archived their base (spec/store §4.4, 16.0). */
|
|
182
|
+
checkpointed: number;
|
|
155
183
|
}
|
|
156
184
|
export interface ExportResult {
|
|
157
185
|
plan: RetentionPlan;
|
|
@@ -3,8 +3,11 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, renameSync, statSync, un
|
|
|
3
3
|
import { basename, join, resolve } from "node:path";
|
|
4
4
|
import { SCHEMA_VERSION } from "./schema.js";
|
|
5
5
|
import { mintId } from "../ids.js";
|
|
6
|
+
import { sha256 } from "../hash.js";
|
|
6
7
|
import { sweepUnreachable } from "./gc.js";
|
|
7
|
-
import {
|
|
8
|
+
import { TreeResolver, entryBlobs } from "./delta.js";
|
|
9
|
+
import { putBlob, putNodeText } from "./writers.js";
|
|
10
|
+
import { evaluateRetention, parseSettingsJson, LAST_SEGMENT_SETTINGS_KEY, RETENTION_SETTINGS_KEY, } from "./retention.js";
|
|
8
11
|
// Archive segments (spec/store §7 "Retention: plan, export, squash", 15.0).
|
|
9
12
|
//
|
|
10
13
|
// A segment is ONE SQLite file per `history export`, never appended: the bodies
|
|
@@ -366,18 +369,41 @@ export function archivePrefix(store, rev) {
|
|
|
366
369
|
*/
|
|
367
370
|
export function revisionBody(store, revId, docId) {
|
|
368
371
|
const row = store.db
|
|
369
|
-
.prepare(`SELECT rev_id, doc_id, path, rendered_hash, root_tree, frontmatter_blob, archive FROM revisions WHERE rev_id = ?${docId === undefined ? "" : " AND doc_id = ?"}`)
|
|
372
|
+
.prepare(`SELECT rev_id, doc_id, seq, path, rendered_hash, root_tree, frontmatter_blob, archive FROM revisions WHERE rev_id = ?${docId === undefined ? "" : " AND doc_id = ?"}`)
|
|
370
373
|
.get(...(docId === undefined ? [revId] : [revId, docId]));
|
|
371
374
|
if (!row)
|
|
372
375
|
return undefined;
|
|
373
376
|
if (row.archive === null) {
|
|
374
|
-
return { rev_id: row.rev_id, doc_id: row.doc_id, path: row.path, rendered_hash: row.rendered_hash, root_tree: row.root_tree, frontmatter_blob: row.frontmatter_blob, prefix: "", archived: false };
|
|
377
|
+
return { rev_id: row.rev_id, doc_id: row.doc_id, seq: row.seq, path: row.path, rendered_hash: row.rendered_hash, root_tree: row.root_tree, frontmatter_blob: row.frontmatter_blob, prefix: "", archived: false };
|
|
375
378
|
}
|
|
376
379
|
const prefix = archivePrefix(store, row);
|
|
377
380
|
const body = store.db.prepare(`SELECT root_tree, frontmatter_blob FROM ${prefix}revisions WHERE rev_id = ?`).get(revId);
|
|
378
381
|
if (!body)
|
|
379
382
|
throw new RevisionArchived(revId, row.archive, archiveRow(store, row.archive)?.path ?? null);
|
|
380
|
-
return { rev_id: row.rev_id, doc_id: row.doc_id, path: row.path, rendered_hash: row.rendered_hash, root_tree: body.root_tree, frontmatter_blob: body.frontmatter_blob, prefix, archived: true };
|
|
383
|
+
return { rev_id: row.rev_id, doc_id: row.doc_id, seq: row.seq, path: row.path, rendered_hash: row.rendered_hash, root_tree: body.root_tree, frontmatter_blob: body.frontmatter_blob, prefix, archived: true };
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* A live revision's bodies in CHECKPOINT form (spec/store §7.1, 16.0): every
|
|
387
|
+
* list a self-contained `[`-node, the frontmatter its raw — what a segment
|
|
388
|
+
* holds (a segment never walks back) and what the squash re-checkpoints a
|
|
389
|
+
* kept revision to when it archives the revision's base. `nodes` maps each
|
|
390
|
+
* full node's hex hash to its text; `blobs` are the raw/trivia hashes the
|
|
391
|
+
* tree names (every one present in the store while the bases are live);
|
|
392
|
+
* `frontmatter` is the raw and its hash, or null.
|
|
393
|
+
*/
|
|
394
|
+
export function checkpointBodies(store, rev, resolver = new TreeResolver(store.db)) {
|
|
395
|
+
const tree = resolver.tree(rev.doc_id, rev.seq, rev.root_tree.toString("hex"));
|
|
396
|
+
const { rootHex, nodes } = resolver.checkpointForm(tree);
|
|
397
|
+
const blobs = new Set();
|
|
398
|
+
entryBlobs(tree.root.entries, blobs);
|
|
399
|
+
for (const list of tree.byBlock.values())
|
|
400
|
+
entryBlobs(list.entries, blobs);
|
|
401
|
+
let frontmatter = null;
|
|
402
|
+
if (rev.frontmatter_blob) {
|
|
403
|
+
const raw = resolver.frontmatter(rev.doc_id, rev.seq, rev.frontmatter_blob.toString("hex"));
|
|
404
|
+
frontmatter = { raw, hex: sha256(raw).toString("hex") };
|
|
405
|
+
}
|
|
406
|
+
return { rootHex, nodes, blobs, frontmatter };
|
|
381
407
|
}
|
|
382
408
|
/** An unrecorded segment under `dest` names both live and non-live revisions (it predates another squash, or belongs elsewhere). */
|
|
383
409
|
export class StaleSegment extends Error {
|
|
@@ -446,14 +472,29 @@ function writeSegment(store, repoId, revIds, plan, dest) {
|
|
|
446
472
|
const segmentId = mintId("s");
|
|
447
473
|
const createdAt = plan.now;
|
|
448
474
|
const previous = lastSegment(store, repoId);
|
|
449
|
-
|
|
450
|
-
|
|
475
|
+
// spec/store §7.1 (16.0): a segment holds CHECKPOINT forms — every list a
|
|
476
|
+
// self-contained node, the frontmatter its raw — so a reader of the segment
|
|
477
|
+
// never walks back into revisions that are not in it. The forms are computed
|
|
478
|
+
// from the resolved trees (a delta revision's full nodes and resolved raw
|
|
479
|
+
// were never rows of the store); the raw/trivia blobs they name are.
|
|
480
|
+
const resolver = new TreeResolver(db);
|
|
481
|
+
const nodes = new Map();
|
|
451
482
|
const blobs = new Set();
|
|
452
|
-
|
|
483
|
+
const fmBlobs = new Map();
|
|
484
|
+
const bodies = new Map();
|
|
485
|
+
for (const r of revisions) {
|
|
486
|
+
const cp = checkpointBodies(store, r, resolver);
|
|
487
|
+
for (const [h, text] of cp.nodes)
|
|
488
|
+
nodes.set(h, text);
|
|
489
|
+
for (const b of cp.blobs)
|
|
490
|
+
blobs.add(b);
|
|
491
|
+
if (cp.frontmatter)
|
|
492
|
+
fmBlobs.set(cp.frontmatter.hex, cp.frontmatter.raw);
|
|
493
|
+
bodies.set(r.rev_id, { rootHex: cp.rootHex, fmHex: cp.frontmatter?.hex ?? null });
|
|
494
|
+
}
|
|
453
495
|
const wholly = whollyArchivedCommits(store, new Set(revIds));
|
|
454
496
|
const dispRows = db.prepare("SELECT commit_id, block_id, kind, confidence, reason, matcher_v, detail FROM dispositions WHERE commit_id = ? ORDER BY block_id, kind");
|
|
455
497
|
const bcRows = db.prepare("SELECT block_id, commit_id, kind FROM block_changes WHERE commit_id = ? ORDER BY block_id, kind");
|
|
456
|
-
const treeRow = db.prepare("SELECT hash, entries FROM tree_nodes WHERE hash = ?");
|
|
457
498
|
const blobRow = db.prepare("SELECT hash, bytes, size FROM blobs WHERE hash = ?");
|
|
458
499
|
mkdirSync(dest, { recursive: true });
|
|
459
500
|
for (const stale of readdirSync(dest))
|
|
@@ -471,23 +512,26 @@ function writeSegment(store, repoId, revIds, plan, dest) {
|
|
|
471
512
|
seg.prepare("INSERT INTO manifest (segment_id, repo_id, created_at, policy, seq_lo, seq_hi, revision_count, previous_segment, schema_user_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
|
472
513
|
.run(segmentId, repoId, createdAt, JSON.stringify(plan.policy), seqLo, seqHi, revisions.length, previous, SCHEMA_VERSION);
|
|
473
514
|
const insRev = seg.prepare("INSERT INTO revisions (rev_id, doc_id, seq, root_tree, frontmatter_blob, rendered_hash, path, commit_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
|
|
474
|
-
for (const r of revisions)
|
|
475
|
-
|
|
515
|
+
for (const r of revisions) {
|
|
516
|
+
const body = bodies.get(r.rev_id);
|
|
517
|
+
insRev.run(r.rev_id, r.doc_id, r.seq, Buffer.from(body.rootHex, "hex"), body.fmHex ? Buffer.from(body.fmHex, "hex") : null, r.rendered_hash, r.path, r.commit_id);
|
|
518
|
+
}
|
|
476
519
|
const insCommit = seg.prepare("INSERT INTO commits (commit_id, repo_id, seq, ts, origin, actor, reason, checkpoint_id, ops) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
|
477
520
|
for (const c of commits)
|
|
478
521
|
insCommit.run(c.commit_id, c.repo_id, c.seq, c.ts, c.origin, c.actor, c.reason, c.checkpoint_id, c.ops);
|
|
479
522
|
const insTree = seg.prepare("INSERT INTO tree_nodes (hash, entries) VALUES (?, ?)");
|
|
480
|
-
for (const
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
insTree.run(row.hash, row.entries);
|
|
484
|
-
}
|
|
485
|
-
const insBlob = seg.prepare("INSERT INTO blobs (hash, bytes, size) VALUES (?, ?, ?)");
|
|
523
|
+
for (const h of [...nodes.keys()].sort())
|
|
524
|
+
insTree.run(Buffer.from(h, "hex"), nodes.get(h));
|
|
525
|
+
const insBlob = seg.prepare("INSERT OR IGNORE INTO blobs (hash, bytes, size) VALUES (?, ?, ?)");
|
|
486
526
|
for (const b of [...blobs].sort()) {
|
|
487
527
|
const row = blobRow.get(Buffer.from(b, "hex"));
|
|
488
528
|
if (row)
|
|
489
529
|
insBlob.run(row.hash, row.bytes, row.size);
|
|
490
530
|
}
|
|
531
|
+
for (const h of [...fmBlobs.keys()].sort()) {
|
|
532
|
+
const bytes = Buffer.from(fmBlobs.get(h), "utf8");
|
|
533
|
+
insBlob.run(Buffer.from(h, "hex"), bytes, bytes.length);
|
|
534
|
+
}
|
|
491
535
|
const insDisp = seg.prepare("INSERT INTO dispositions (commit_id, block_id, kind, confidence, reason, matcher_v, detail) VALUES (?, ?, ?, ?, ?, ?, ?)");
|
|
492
536
|
const insBc = seg.prepare("INSERT INTO block_changes (block_id, commit_id, kind) VALUES (?, ?, ?)");
|
|
493
537
|
for (const c of wholly) {
|
|
@@ -520,6 +564,14 @@ function writeSegment(store, repoId, revIds, plan, dest) {
|
|
|
520
564
|
function squash(store, repoId, segment, revIds) {
|
|
521
565
|
return store.write((db) => {
|
|
522
566
|
const wholly = whollyArchivedCommits(store, new Set(revIds));
|
|
567
|
+
// spec/store §4.4 "base liveness" (16.0): a kept revision whose previous
|
|
568
|
+
// revision this squash archives is re-written in checkpoint form FIRST,
|
|
569
|
+
// while its bases are still readable — its delta nodes and delta
|
|
570
|
+
// frontmatter resolve, the self-contained forms are put, the row repointed.
|
|
571
|
+
// Its own successors are untouched: a delta is over the resolved content,
|
|
572
|
+
// which re-encoding does not change, and only self-contained nodes are
|
|
573
|
+
// ever reused across revisions.
|
|
574
|
+
const checkpointed = checkpointDependents(store, new Set(revIds));
|
|
523
575
|
const upd = db.prepare("UPDATE revisions SET root_tree = NULL, frontmatter_blob = NULL, archive = ? WHERE rev_id = ?");
|
|
524
576
|
for (const id of revIds)
|
|
525
577
|
upd.run(segment.id, id);
|
|
@@ -543,11 +595,48 @@ function squash(store, repoId, segment, revIds) {
|
|
|
543
595
|
});
|
|
544
596
|
setLastSegment(store, repoId, segment.id);
|
|
545
597
|
return {
|
|
546
|
-
squashed: { revisions: revIds.length, commits: wholly.length, dispositions, block_changes: blockChanges },
|
|
598
|
+
squashed: { revisions: revIds.length, commits: wholly.length, dispositions, block_changes: blockChanges, checkpointed },
|
|
547
599
|
gc: { blobs: swept.blobsSwept, tree_nodes: swept.treeNodesSwept },
|
|
548
600
|
};
|
|
549
601
|
});
|
|
550
602
|
}
|
|
603
|
+
/**
|
|
604
|
+
* Re-checkpoint every live revision that stays and whose predecessor
|
|
605
|
+
* (`seq − 1`, same document) is in `archiving`: resolve its tree and
|
|
606
|
+
* frontmatter, put the checkpoint forms, repoint the row. Returns how many
|
|
607
|
+
* rows changed (a revision already in checkpoint form changes nothing).
|
|
608
|
+
* Runs inside the squash transaction, before the bases lose their bodies.
|
|
609
|
+
*/
|
|
610
|
+
function checkpointDependents(store, archiving) {
|
|
611
|
+
const db = store.db;
|
|
612
|
+
const revRow = db.prepare("SELECT doc_id, seq FROM revisions WHERE rev_id = ?");
|
|
613
|
+
const nextRow = db.prepare("SELECT rev_id, doc_id, seq, root_tree, frontmatter_blob FROM revisions WHERE doc_id = ? AND seq = ? AND archive IS NULL");
|
|
614
|
+
const upd = db.prepare("UPDATE revisions SET root_tree = ?, frontmatter_blob = ? WHERE rev_id = ?");
|
|
615
|
+
const resolver = new TreeResolver(db);
|
|
616
|
+
const seen = new Set();
|
|
617
|
+
let changed = 0;
|
|
618
|
+
for (const id of [...archiving].sort()) {
|
|
619
|
+
const r = revRow.get(id);
|
|
620
|
+
if (!r)
|
|
621
|
+
continue;
|
|
622
|
+
const next = nextRow.get(r.doc_id, r.seq + 1);
|
|
623
|
+
if (!next || !next.root_tree || archiving.has(next.rev_id) || seen.has(next.rev_id))
|
|
624
|
+
continue;
|
|
625
|
+
seen.add(next.rev_id);
|
|
626
|
+
const cp = checkpointBodies(store, { doc_id: next.doc_id, seq: next.seq, root_tree: next.root_tree, frontmatter_blob: next.frontmatter_blob }, resolver);
|
|
627
|
+
const fmHex = cp.frontmatter ? cp.frontmatter.hex : null;
|
|
628
|
+
const oldFm = next.frontmatter_blob ? next.frontmatter_blob.toString("hex") : null;
|
|
629
|
+
if (cp.rootHex === next.root_tree.toString("hex") && fmHex === oldFm)
|
|
630
|
+
continue;
|
|
631
|
+
for (const text of cp.nodes.values())
|
|
632
|
+
putNodeText(db, text);
|
|
633
|
+
if (cp.frontmatter)
|
|
634
|
+
putBlob(db, cp.frontmatter.raw);
|
|
635
|
+
upd.run(Buffer.from(cp.rootHex, "hex"), fmHex ? Buffer.from(fmHex, "hex") : null, next.rev_id);
|
|
636
|
+
changed++;
|
|
637
|
+
}
|
|
638
|
+
return changed;
|
|
639
|
+
}
|
|
551
640
|
/**
|
|
552
641
|
* `omg history export [--prune]`: evaluate the policy, write ONE new segment
|
|
553
642
|
* for the revisions to archive (or finish an unrecorded one — the crash
|