@omgbase/core 0.24.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/core/edit-script.d.ts +15 -0
- package/dist/src/core/edit-script.js +72 -0
- package/dist/src/core/ingest.js +42 -12
- package/dist/src/core/read/document.d.ts +7 -0
- package/dist/src/core/read/document.js +38 -17
- package/dist/src/core/store/archive.d.ts +28 -0
- package/dist/src/core/store/archive.js +106 -17
- package/dist/src/core/store/delta.d.ts +131 -0
- package/dist/src/core/store/delta.js +470 -0
- package/dist/src/core/store/gc.d.ts +10 -0
- package/dist/src/core/store/gc.js +23 -7
- package/dist/src/core/store/retention.d.ts +1 -1
- package/dist/src/core/store/retention.js +3 -17
- package/dist/src/core/store/schema.d.ts +1 -1
- package/dist/src/core/store/schema.js +2 -1
- package/dist/src/core/store/store.js +7 -0
- package/dist/src/core/store/writers.d.ts +42 -6
- package/dist/src/core/store/writers.js +67 -25
- package/dist/src/format/json.js +5 -0
- package/dist/src/format/yaml.js +15 -3
- package/dist/src/graph/history.d.ts +1 -15
- package/dist/src/graph/history.js +14 -74
- package/dist/src/mcp/arguments.d.ts +11 -0
- package/dist/src/mcp/arguments.js +47 -4
- package/dist/src/mcp/errors.js +1 -1
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +1 -1
- package/dist/src/mcp/server.js +4 -2
- package/dist/src/mutate/effects.js +87 -13
- package/dist/src/mutate/frontmatter-parts.d.ts +71 -0
- package/dist/src/mutate/frontmatter-parts.js +237 -0
- package/dist/src/mutate/load.js +9 -9
- package/dist/src/mutate/select.js +9 -3
- package/dist/src/oqx-js/context.js +21 -10
- package/dist/src/oqx-js/names.js +1 -1
- package/dist/src/oqx-js/run.d.ts +8 -0
- package/dist/src/oqx-js/run.js +91 -4
- package/dist/src/oqx-js/sql/translate.js +21 -1
- package/dist/src/search/cel/parser.d.ts +3 -1
- package/dist/src/search/cel/parser.js +4 -1
- package/dist/src/spec-versions.d.ts +4 -4
- package/dist/src/spec-versions.js +4 -4
- package/package.json +1 -1
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { Database } from "better-sqlite3";
|
|
2
|
+
/** The checkpoint cadence: revision `seq` is a checkpoint when `(seq − 1) % CHECKPOINT_EVERY == 0` (1, 17, 33, …). */
|
|
3
|
+
export declare const CHECKPOINT_EVERY = 16;
|
|
4
|
+
/** The measurement seam (tests only): `1` makes every revision a checkpoint, which is exactly the 15.x encoding. */
|
|
5
|
+
export declare function setCheckpointEvery(n: number | null): void;
|
|
6
|
+
export declare function currentCheckpointEvery(): number;
|
|
7
|
+
/** spec/store §4.4: the cadence rule alone — the writer adds "no base" and "base archived". */
|
|
8
|
+
export declare function isCheckpointSeq(seq: number): boolean;
|
|
9
|
+
/** A malformed or unresolvable delta (ops that do not consume the base, a `{`-node with no base, a delta in a segment). */
|
|
10
|
+
export declare class DeltaError extends Error {
|
|
11
|
+
constructor(message: string);
|
|
12
|
+
}
|
|
13
|
+
/** One op over the base's positions; `x` is the payload's TEXT (an entry's canonical JSON, or a frontmatter line). */
|
|
14
|
+
export type DeltaOp = ["=", number] | ["+", string] | ["-"] | ["~", string];
|
|
15
|
+
/** One §4.1 entry: [block_id, raw_hash_hex, child_tree_hash_hex | null, type, attrs, trivia_hash_hex | null]. */
|
|
16
|
+
export type Entry = [string, string, string | null, string, Record<string, unknown>, string | null];
|
|
17
|
+
/** An entry's canonical text — exactly the six-element array as §4.1 serializes it. */
|
|
18
|
+
export declare function entryText(e: Entry): string;
|
|
19
|
+
/** Parse one entry's canonical text back (the payload of a `+`/`~` op, or a full node's element). */
|
|
20
|
+
export declare function parseEntry(v: unknown): Entry | null;
|
|
21
|
+
/** The full-form text of a sibling list from its entry texts. */
|
|
22
|
+
export declare function fullNodeText(texts: readonly string[]): string;
|
|
23
|
+
/**
|
|
24
|
+
* The delta text of `next` against `base` (both as the payload texts of their
|
|
25
|
+
* items): Myers' shortest edit script over the texts, grouped — a run of keeps
|
|
26
|
+
* is `["=",n]`; inside a run of edits the k-th deletion and the k-th insertion
|
|
27
|
+
* pair into `["~",x]`, the surplus deletions are `["-"]` and the surplus
|
|
28
|
+
* insertions `["+",x]`. `key` is `"d"` for a body node, `"f"` for a
|
|
29
|
+
* frontmatter; `payload` renders an item's text into the op (verbatim for an
|
|
30
|
+
* entry, JSON-quoted for a line).
|
|
31
|
+
*/
|
|
32
|
+
export declare function encodeDelta(base: readonly string[], next: readonly string[], key: "d" | "f", payload: (text: string) => string): string;
|
|
33
|
+
/** The delta of a body sibling list: entry texts spliced verbatim. */
|
|
34
|
+
export declare function encodeBodyDelta(baseTexts: readonly string[], nextTexts: readonly string[]): string;
|
|
35
|
+
/** Lines of a text, each with its own line ending (`\n`, or `\r\n` — the `\r` stays with its line); the last has none. `""` → `[""]`. */
|
|
36
|
+
export declare function splitLines(text: string): string[];
|
|
37
|
+
/** The delta of a frontmatter raw against the previous revision's raw: lines, JSON-quoted (spec/store §4.1 "frontmatter delta"). */
|
|
38
|
+
export declare function encodeFrontmatterDelta(baseRaw: string, nextRaw: string): string;
|
|
39
|
+
/**
|
|
40
|
+
* The measured-and-not-chosen variant (spec/store §10 "Decisions", 16.0): a
|
|
41
|
+
* frontmatter delta whose changed lines are blob references (64 hex) instead
|
|
42
|
+
* of inlined bytes. Returns the text and the lines a writer would have to put
|
|
43
|
+
* as blobs. Never written; the measurement test costs it against the inlined form.
|
|
44
|
+
*/
|
|
45
|
+
export declare function encodeFrontmatterDeltaRefs(baseRaw: string, nextRaw: string): {
|
|
46
|
+
text: string;
|
|
47
|
+
newBlobs: string[];
|
|
48
|
+
};
|
|
49
|
+
/** UTF-8 byte length — the never-larger rule compares bytes, as `blobs.size` and `tree_nodes.entries` are stored. */
|
|
50
|
+
export declare function byteLength(text: string): number;
|
|
51
|
+
export type ParsedNode = {
|
|
52
|
+
form: "full";
|
|
53
|
+
entries: Entry[];
|
|
54
|
+
} | {
|
|
55
|
+
form: "delta";
|
|
56
|
+
ops: DeltaOp[];
|
|
57
|
+
};
|
|
58
|
+
/** Parse a `tree_nodes.entries` text: `[`-form entries or `{"d":…}` ops; null when neither. */
|
|
59
|
+
export declare function parseNodeText(text: string): ParsedNode | null;
|
|
60
|
+
/** Parse a frontmatter blob's bytes: the raw (`-`-form) or `{"f":…}` ops; null when a `{` text is not a valid delta. */
|
|
61
|
+
export declare function parseFrontmatterBlob(text: string): {
|
|
62
|
+
form: "full";
|
|
63
|
+
raw: string;
|
|
64
|
+
} | {
|
|
65
|
+
form: "delta";
|
|
66
|
+
ops: DeltaOp[];
|
|
67
|
+
} | null;
|
|
68
|
+
/** Apply ops to a base (item texts); `DeltaError` unless the ops consume exactly the base. */
|
|
69
|
+
export declare function applyOps(ops: readonly DeltaOp[], base: readonly string[]): string[];
|
|
70
|
+
/** The objects a node's text names on its own: every entry of a `[`-node, the `+`/`~` entries of a `{`-node (spec/store §7, the explicit mark). */
|
|
71
|
+
export declare function explicitEntries(node: ParsedNode): Entry[];
|
|
72
|
+
/** A sibling list resolved in its revision's context. */
|
|
73
|
+
export interface ResolvedList {
|
|
74
|
+
/** The node row's hash (hex) as the revision references it — a `[` or `{` node. */
|
|
75
|
+
hash: string;
|
|
76
|
+
form: "full" | "delta";
|
|
77
|
+
entries: Entry[];
|
|
78
|
+
/** `entries` as canonical texts (what a delta is encoded against). */
|
|
79
|
+
texts: string[];
|
|
80
|
+
/** A `[`-form node whose every child list is self-contained: context-free, reusable by the next revision. */
|
|
81
|
+
selfContained: boolean;
|
|
82
|
+
}
|
|
83
|
+
/** A revision's whole tree resolved: the root list and every container's children by the container's block id. */
|
|
84
|
+
export interface ResolvedTree {
|
|
85
|
+
root: ResolvedList;
|
|
86
|
+
byBlock: Map<string, ResolvedList>;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Resolves nodes in revision context, memoized per connection use. `prefix`
|
|
90
|
+
* is `""` for the store or `"<alias>."` for an archive segment — a segment
|
|
91
|
+
* holds checkpoint forms only (spec/store §7.1), so a `{`-node there is an
|
|
92
|
+
* error rather than a walk-back.
|
|
93
|
+
*/
|
|
94
|
+
export declare class TreeResolver {
|
|
95
|
+
private readonly db;
|
|
96
|
+
private readonly prefix;
|
|
97
|
+
private readonly nodeStmt;
|
|
98
|
+
private readonly revStmt;
|
|
99
|
+
private readonly blobStmt;
|
|
100
|
+
private readonly lists;
|
|
101
|
+
private readonly trees;
|
|
102
|
+
private readonly frontmatters;
|
|
103
|
+
constructor(db: Database, prefix?: string);
|
|
104
|
+
private revision;
|
|
105
|
+
/** The base revision of a delta at `seq`: `seq − 1`, which must exist and be live (spec/store §4.4, §8 I10). */
|
|
106
|
+
private base;
|
|
107
|
+
/**
|
|
108
|
+
* The sibling list stored under `hashHex`, in the context of `(docId, seq)`,
|
|
109
|
+
* at the position `blockId` (null = the root). A `[`-node resolves to its
|
|
110
|
+
* entries; a `{`-node to its ops applied to the same position of `seq − 1`.
|
|
111
|
+
*/
|
|
112
|
+
list(docId: string, seq: number, hashHex: string, blockId: string | null): ResolvedList;
|
|
113
|
+
/** The children list of `blockId` anywhere in the tree rooted at `rootHex` (context `(docId, seq)`), or undefined when the block has no children there. */
|
|
114
|
+
childList(docId: string, seq: number, rootHex: string, blockId: string): ResolvedList | undefined;
|
|
115
|
+
/** The whole tree of a revision, every list resolved in its context. */
|
|
116
|
+
tree(docId: string, seq: number, rootHex: string): ResolvedTree;
|
|
117
|
+
/** A revision's frontmatter raw from its `frontmatter_blob` (a `-`-form blob, or a `{"f":…}` delta over `seq − 1`'s). */
|
|
118
|
+
frontmatter(docId: string, seq: number, blobHex: string): string;
|
|
119
|
+
/**
|
|
120
|
+
* The checkpoint (self-contained) form of a resolved tree: every list as a
|
|
121
|
+
* `[`-node whose child hashes are the full children's — what an archive
|
|
122
|
+
* segment holds (§7.1) and what the squash re-checkpoints a kept revision
|
|
123
|
+
* to. Returns the root's full hash and every full node by hash.
|
|
124
|
+
*/
|
|
125
|
+
checkpointForm(tree: ResolvedTree): {
|
|
126
|
+
rootHex: string;
|
|
127
|
+
nodes: Map<string, string>;
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/** The blob hashes (raw + trivia) a list of entries names — the raws a reconstruct reads, the mark's leaves. */
|
|
131
|
+
export declare function entryBlobs(entries: readonly Entry[], into: Set<string>): void;
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
import { sha256, serializeTreeEntries } from "../hash.js";
|
|
2
|
+
import { myersScript } from "../edit-script.js";
|
|
3
|
+
// Delta tree nodes (spec/store §4.1 "Delta nodes", §4.4 "Checkpoints", §6,
|
|
4
|
+
// 16.0). A sibling list — the body root, a container's children — and a
|
|
5
|
+
// revision's frontmatter may be stored as a DELTA against the same list (the
|
|
6
|
+
// same frontmatter) of the document's PREVIOUS revision (`revisions.seq − 1`
|
|
7
|
+
// of the same `doc_id`; the base is implicit, no base hash is stored):
|
|
8
|
+
//
|
|
9
|
+
// tree_nodes.entries full: [ [block_id, raw_hash_hex, child, type, attrs, trivia], … ] (15.x, unchanged)
|
|
10
|
+
// delta: {"d":[ op, … ]}
|
|
11
|
+
// blobs.bytes (frontmatter) full: the frontmatter block's raw, `---…` (15.x, unchanged)
|
|
12
|
+
// delta: {"f":[ op, … ]}
|
|
13
|
+
//
|
|
14
|
+
// told apart by the FIRST BYTE (`[` / `-` full, `{` delta). An op is one of
|
|
15
|
+
// `["=",n]` (keep the next n entries of the base), `["+",x]` (insert x),
|
|
16
|
+
// `["-"]` (drop the next base entry), `["~",x]` (replace the next base entry
|
|
17
|
+
// with x); x is a six-element entry (body) or a line with its ending
|
|
18
|
+
// (frontmatter), spliced verbatim. The ops consume exactly the base. A node
|
|
19
|
+
// is a delta only when the delta is strictly smaller than the full form, the
|
|
20
|
+
// revision is not a CHECKPOINT (`(seq − 1) % 16 == 0`, the first revision, a
|
|
21
|
+
// revision whose previous one is archived or missing), and a base exists at
|
|
22
|
+
// the same position (the same block id in the previous revision's tree; the
|
|
23
|
+
// root; the previous frontmatter). A delta node hash is `sha256(entries)` like
|
|
24
|
+
// any node — content-addressed, shared by any two revisions producing the same
|
|
25
|
+
// text — and is MEANINGFUL ONLY in the context of the revision that names it:
|
|
26
|
+
// a reader resolves `{`-form at (doc, seq, block) by applying the ops to the
|
|
27
|
+
// resolved list at (doc, seq − 1, block), walking back until a `[`-form node
|
|
28
|
+
// (at most 15 steps, by the checkpoint cadence). A revision therefore never
|
|
29
|
+
// reuses a delta node of its predecessor for an unchanged list: an unchanged
|
|
30
|
+
// list reuses the base's node hash only when that node is SELF-CONTAINED (a
|
|
31
|
+
// `[`-form node whose every child is self-contained — context-free), else it
|
|
32
|
+
// writes the identity delta `{"d":[["=",n]]}` (one shared row, ~17 B).
|
|
33
|
+
//
|
|
34
|
+
// The GC mark (§7) is EXPLICIT: a live revision marks the objects its own
|
|
35
|
+
// nodes name (every entry of a `[`-node, the `+`/`~` entries of a `{`-node, the
|
|
36
|
+
// frontmatter blob whatever its form); what a delta keeps "as the base" is
|
|
37
|
+
// marked by the base revision, which the base-liveness rule keeps live: a
|
|
38
|
+
// delta's previous revision is never archived while the delta is — the squash
|
|
39
|
+
// re-checkpoints a kept revision whose predecessor it archives (archive.ts).
|
|
40
|
+
/** The checkpoint cadence: revision `seq` is a checkpoint when `(seq − 1) % CHECKPOINT_EVERY == 0` (1, 17, 33, …). */
|
|
41
|
+
export const CHECKPOINT_EVERY = 16;
|
|
42
|
+
let checkpointEvery = CHECKPOINT_EVERY;
|
|
43
|
+
/** The measurement seam (tests only): `1` makes every revision a checkpoint, which is exactly the 15.x encoding. */
|
|
44
|
+
export function setCheckpointEvery(n) {
|
|
45
|
+
checkpointEvery = n ?? CHECKPOINT_EVERY;
|
|
46
|
+
}
|
|
47
|
+
export function currentCheckpointEvery() {
|
|
48
|
+
return checkpointEvery;
|
|
49
|
+
}
|
|
50
|
+
/** spec/store §4.4: the cadence rule alone — the writer adds "no base" and "base archived". */
|
|
51
|
+
export function isCheckpointSeq(seq) {
|
|
52
|
+
return (seq - 1) % checkpointEvery === 0;
|
|
53
|
+
}
|
|
54
|
+
/** A malformed or unresolvable delta (ops that do not consume the base, a `{`-node with no base, a delta in a segment). */
|
|
55
|
+
export class DeltaError extends Error {
|
|
56
|
+
constructor(message) {
|
|
57
|
+
super(message);
|
|
58
|
+
this.name = "DeltaError";
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// ---- encodings ----------------------------------------------------------------------
|
|
62
|
+
/** An entry's canonical text — exactly the six-element array as §4.1 serializes it. */
|
|
63
|
+
export function entryText(e) {
|
|
64
|
+
const t = { blockId: e[0], rawHashHex: e[1], childTreeHashHex: e[2], type: e[3], attrs: e[4], triviaHashHex: e[5] };
|
|
65
|
+
return serializeTreeEntries([t]).slice(1, -1);
|
|
66
|
+
}
|
|
67
|
+
/** Parse one entry's canonical text back (the payload of a `+`/`~` op, or a full node's element). */
|
|
68
|
+
export function parseEntry(v) {
|
|
69
|
+
if (!Array.isArray(v) || v.length !== 6)
|
|
70
|
+
return null;
|
|
71
|
+
const [id, raw, child, type, attrs, trivia] = v;
|
|
72
|
+
if (typeof id !== "string" || typeof raw !== "string" || typeof type !== "string")
|
|
73
|
+
return null;
|
|
74
|
+
if (typeof attrs !== "object" || attrs === null || Array.isArray(attrs))
|
|
75
|
+
return null;
|
|
76
|
+
if (child !== null && typeof child !== "string")
|
|
77
|
+
return null;
|
|
78
|
+
if (trivia !== null && typeof trivia !== "string")
|
|
79
|
+
return null;
|
|
80
|
+
return [id, raw, child, type, attrs, trivia];
|
|
81
|
+
}
|
|
82
|
+
/** The full-form text of a sibling list from its entry texts. */
|
|
83
|
+
export function fullNodeText(texts) {
|
|
84
|
+
return `[${texts.join(",")}]`;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The delta text of `next` against `base` (both as the payload texts of their
|
|
88
|
+
* items): Myers' shortest edit script over the texts, grouped — a run of keeps
|
|
89
|
+
* is `["=",n]`; inside a run of edits the k-th deletion and the k-th insertion
|
|
90
|
+
* pair into `["~",x]`, the surplus deletions are `["-"]` and the surplus
|
|
91
|
+
* insertions `["+",x]`. `key` is `"d"` for a body node, `"f"` for a
|
|
92
|
+
* frontmatter; `payload` renders an item's text into the op (verbatim for an
|
|
93
|
+
* entry, JSON-quoted for a line).
|
|
94
|
+
*/
|
|
95
|
+
export function encodeDelta(base, next, key, payload) {
|
|
96
|
+
const script = myersScript([...base], [...next]);
|
|
97
|
+
const ops = [];
|
|
98
|
+
let i = 0;
|
|
99
|
+
while (i < script.length) {
|
|
100
|
+
const op = script[i];
|
|
101
|
+
if (op.kind === "keep") {
|
|
102
|
+
let n = 0;
|
|
103
|
+
while (i < script.length && script[i].kind === "keep") {
|
|
104
|
+
n++;
|
|
105
|
+
i++;
|
|
106
|
+
}
|
|
107
|
+
ops.push(`["=",${n}]`);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
const deletes = [];
|
|
111
|
+
const inserts = [];
|
|
112
|
+
while (i < script.length && script[i].kind !== "keep") {
|
|
113
|
+
if (script[i].kind === "delete")
|
|
114
|
+
deletes.push(script[i].line);
|
|
115
|
+
else
|
|
116
|
+
inserts.push(script[i].line);
|
|
117
|
+
i++;
|
|
118
|
+
}
|
|
119
|
+
const paired = Math.min(deletes.length, inserts.length);
|
|
120
|
+
for (let k = 0; k < paired; k++)
|
|
121
|
+
ops.push(`["~",${payload(inserts[k])}]`);
|
|
122
|
+
for (let k = paired; k < deletes.length; k++)
|
|
123
|
+
ops.push(`["-"]`);
|
|
124
|
+
for (let k = paired; k < inserts.length; k++)
|
|
125
|
+
ops.push(`["+",${payload(inserts[k])}]`);
|
|
126
|
+
}
|
|
127
|
+
return `{"${key}":[${ops.join(",")}]}`;
|
|
128
|
+
}
|
|
129
|
+
/** The delta of a body sibling list: entry texts spliced verbatim. */
|
|
130
|
+
export function encodeBodyDelta(baseTexts, nextTexts) {
|
|
131
|
+
return encodeDelta(baseTexts, nextTexts, "d", (t) => t);
|
|
132
|
+
}
|
|
133
|
+
/** Lines of a text, each with its own line ending (`\n`, or `\r\n` — the `\r` stays with its line); the last has none. `""` → `[""]`. */
|
|
134
|
+
export function splitLines(text) {
|
|
135
|
+
const out = [];
|
|
136
|
+
let start = 0;
|
|
137
|
+
for (let i = 0; i < text.length; i++) {
|
|
138
|
+
if (text.charCodeAt(i) === 10) {
|
|
139
|
+
out.push(text.slice(start, i + 1));
|
|
140
|
+
start = i + 1;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
out.push(text.slice(start));
|
|
144
|
+
return out;
|
|
145
|
+
}
|
|
146
|
+
/** The delta of a frontmatter raw against the previous revision's raw: lines, JSON-quoted (spec/store §4.1 "frontmatter delta"). */
|
|
147
|
+
export function encodeFrontmatterDelta(baseRaw, nextRaw) {
|
|
148
|
+
return encodeDelta(splitLines(baseRaw), splitLines(nextRaw), "f", (t) => JSON.stringify(t));
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The measured-and-not-chosen variant (spec/store §10 "Decisions", 16.0): a
|
|
152
|
+
* frontmatter delta whose changed lines are blob references (64 hex) instead
|
|
153
|
+
* of inlined bytes. Returns the text and the lines a writer would have to put
|
|
154
|
+
* as blobs. Never written; the measurement test costs it against the inlined form.
|
|
155
|
+
*/
|
|
156
|
+
export function encodeFrontmatterDeltaRefs(baseRaw, nextRaw) {
|
|
157
|
+
const newBlobs = [];
|
|
158
|
+
const text = encodeDelta(splitLines(baseRaw), splitLines(nextRaw), "f", (t) => {
|
|
159
|
+
newBlobs.push(t);
|
|
160
|
+
return JSON.stringify(sha256(t).toString("hex"));
|
|
161
|
+
});
|
|
162
|
+
return { text, newBlobs };
|
|
163
|
+
}
|
|
164
|
+
/** UTF-8 byte length — the never-larger rule compares bytes, as `blobs.size` and `tree_nodes.entries` are stored. */
|
|
165
|
+
export function byteLength(text) {
|
|
166
|
+
return Buffer.byteLength(text, "utf8");
|
|
167
|
+
}
|
|
168
|
+
function parseOps(v, payload) {
|
|
169
|
+
if (!Array.isArray(v))
|
|
170
|
+
return null;
|
|
171
|
+
const ops = [];
|
|
172
|
+
for (const raw of v) {
|
|
173
|
+
if (!Array.isArray(raw) || raw.length === 0 || typeof raw[0] !== "string")
|
|
174
|
+
return null;
|
|
175
|
+
const tag = raw[0];
|
|
176
|
+
if (tag === "=") {
|
|
177
|
+
if (raw.length !== 2 || typeof raw[1] !== "number" || !Number.isInteger(raw[1]) || raw[1] <= 0)
|
|
178
|
+
return null;
|
|
179
|
+
ops.push(["=", raw[1]]);
|
|
180
|
+
}
|
|
181
|
+
else if (tag === "-") {
|
|
182
|
+
if (raw.length !== 1)
|
|
183
|
+
return null;
|
|
184
|
+
ops.push(["-"]);
|
|
185
|
+
}
|
|
186
|
+
else if (tag === "+" || tag === "~") {
|
|
187
|
+
if (raw.length !== 2)
|
|
188
|
+
return null;
|
|
189
|
+
const x = payload(raw[1]);
|
|
190
|
+
if (x === null)
|
|
191
|
+
return null;
|
|
192
|
+
ops.push([tag, x]);
|
|
193
|
+
}
|
|
194
|
+
else
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
return ops;
|
|
198
|
+
}
|
|
199
|
+
/** Parse a `tree_nodes.entries` text: `[`-form entries or `{"d":…}` ops; null when neither. */
|
|
200
|
+
export function parseNodeText(text) {
|
|
201
|
+
let parsed;
|
|
202
|
+
try {
|
|
203
|
+
parsed = JSON.parse(text);
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
return null;
|
|
207
|
+
}
|
|
208
|
+
if (Array.isArray(parsed)) {
|
|
209
|
+
const entries = [];
|
|
210
|
+
for (const e of parsed) {
|
|
211
|
+
const entry = parseEntry(e);
|
|
212
|
+
if (!entry)
|
|
213
|
+
return null;
|
|
214
|
+
entries.push(entry);
|
|
215
|
+
}
|
|
216
|
+
return { form: "full", entries };
|
|
217
|
+
}
|
|
218
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
219
|
+
return null;
|
|
220
|
+
const keys = Object.keys(parsed);
|
|
221
|
+
if (keys.length !== 1 || keys[0] !== "d")
|
|
222
|
+
return null;
|
|
223
|
+
const ops = parseOps(parsed.d, (x) => {
|
|
224
|
+
const e = parseEntry(x);
|
|
225
|
+
return e ? entryText(e) : null;
|
|
226
|
+
});
|
|
227
|
+
return ops ? { form: "delta", ops } : null;
|
|
228
|
+
}
|
|
229
|
+
/** Parse a frontmatter blob's bytes: the raw (`-`-form) or `{"f":…}` ops; null when a `{` text is not a valid delta. */
|
|
230
|
+
export function parseFrontmatterBlob(text) {
|
|
231
|
+
if (!text.startsWith("{"))
|
|
232
|
+
return { form: "full", raw: text };
|
|
233
|
+
let parsed;
|
|
234
|
+
try {
|
|
235
|
+
parsed = JSON.parse(text);
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
|
|
241
|
+
return null;
|
|
242
|
+
const keys = Object.keys(parsed);
|
|
243
|
+
if (keys.length !== 1 || keys[0] !== "f")
|
|
244
|
+
return null;
|
|
245
|
+
const ops = parseOps(parsed.f, (x) => (typeof x === "string" ? x : null));
|
|
246
|
+
return ops ? { form: "delta", ops } : null;
|
|
247
|
+
}
|
|
248
|
+
/** Apply ops to a base (item texts); `DeltaError` unless the ops consume exactly the base. */
|
|
249
|
+
export function applyOps(ops, base) {
|
|
250
|
+
const out = [];
|
|
251
|
+
let i = 0;
|
|
252
|
+
for (const op of ops) {
|
|
253
|
+
switch (op[0]) {
|
|
254
|
+
case "=":
|
|
255
|
+
if (i + op[1] > base.length)
|
|
256
|
+
throw new DeltaError(`delta keeps ${op[1]} entries at position ${i} of a base with ${base.length}`);
|
|
257
|
+
for (let k = 0; k < op[1]; k++)
|
|
258
|
+
out.push(base[i++]);
|
|
259
|
+
break;
|
|
260
|
+
case "-":
|
|
261
|
+
if (i >= base.length)
|
|
262
|
+
throw new DeltaError(`delta drops an entry past the end of a base with ${base.length}`);
|
|
263
|
+
i++;
|
|
264
|
+
break;
|
|
265
|
+
case "~":
|
|
266
|
+
if (i >= base.length)
|
|
267
|
+
throw new DeltaError(`delta replaces an entry past the end of a base with ${base.length}`);
|
|
268
|
+
i++;
|
|
269
|
+
out.push(op[1]);
|
|
270
|
+
break;
|
|
271
|
+
case "+":
|
|
272
|
+
out.push(op[1]);
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
if (i !== base.length)
|
|
277
|
+
throw new DeltaError(`delta consumed ${i} of ${base.length} base entries`);
|
|
278
|
+
return out;
|
|
279
|
+
}
|
|
280
|
+
/** The objects a node's text names on its own: every entry of a `[`-node, the `+`/`~` entries of a `{`-node (spec/store §7, the explicit mark). */
|
|
281
|
+
export function explicitEntries(node) {
|
|
282
|
+
if (node.form === "full")
|
|
283
|
+
return node.entries;
|
|
284
|
+
const out = [];
|
|
285
|
+
for (const op of node.ops) {
|
|
286
|
+
if (op[0] === "+" || op[0] === "~") {
|
|
287
|
+
const e = parseEntry(JSON.parse(op[1]));
|
|
288
|
+
if (e)
|
|
289
|
+
out.push(e);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return out;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Resolves nodes in revision context, memoized per connection use. `prefix`
|
|
296
|
+
* is `""` for the store or `"<alias>."` for an archive segment — a segment
|
|
297
|
+
* holds checkpoint forms only (spec/store §7.1), so a `{`-node there is an
|
|
298
|
+
* error rather than a walk-back.
|
|
299
|
+
*/
|
|
300
|
+
export class TreeResolver {
|
|
301
|
+
db;
|
|
302
|
+
prefix;
|
|
303
|
+
nodeStmt;
|
|
304
|
+
revStmt;
|
|
305
|
+
blobStmt;
|
|
306
|
+
lists = new Map();
|
|
307
|
+
trees = new Map();
|
|
308
|
+
frontmatters = new Map();
|
|
309
|
+
constructor(db, prefix = "") {
|
|
310
|
+
this.db = db;
|
|
311
|
+
this.prefix = prefix;
|
|
312
|
+
this.nodeStmt = db.prepare(`SELECT entries FROM ${prefix}tree_nodes WHERE hash = ?`);
|
|
313
|
+
this.revStmt = db.prepare(`SELECT root_tree, frontmatter_blob${prefix === "" ? ", archive" : ", NULL AS archive"} FROM ${prefix}revisions WHERE doc_id = ? AND seq = ?`);
|
|
314
|
+
this.blobStmt = db.prepare(`SELECT bytes FROM ${prefix}blobs WHERE hash = ?`);
|
|
315
|
+
}
|
|
316
|
+
revision(docId, seq) {
|
|
317
|
+
return this.revStmt.get(docId, seq);
|
|
318
|
+
}
|
|
319
|
+
/** The base revision of a delta at `seq`: `seq − 1`, which must exist and be live (spec/store §4.4, §8 I10). */
|
|
320
|
+
base(docId, seq, what) {
|
|
321
|
+
if (this.prefix !== "")
|
|
322
|
+
throw new DeltaError(`${what} of ${docId} at seq ${seq} is a delta inside an archive segment`);
|
|
323
|
+
const prev = this.revision(docId, seq - 1);
|
|
324
|
+
if (!prev)
|
|
325
|
+
throw new DeltaError(`${what} of ${docId} at seq ${seq} is a delta but the document has no revision ${seq - 1}`);
|
|
326
|
+
if (prev.archive !== null)
|
|
327
|
+
throw new DeltaError(`${what} of ${docId} at seq ${seq} is a delta whose base revision ${seq - 1} is archived`);
|
|
328
|
+
return prev;
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* The sibling list stored under `hashHex`, in the context of `(docId, seq)`,
|
|
332
|
+
* at the position `blockId` (null = the root). A `[`-node resolves to its
|
|
333
|
+
* entries; a `{`-node to its ops applied to the same position of `seq − 1`.
|
|
334
|
+
*/
|
|
335
|
+
list(docId, seq, hashHex, blockId) {
|
|
336
|
+
const key = `${docId}${seq}${blockId ?? ""}${hashHex}`;
|
|
337
|
+
const memo = this.lists.get(key);
|
|
338
|
+
if (memo)
|
|
339
|
+
return memo;
|
|
340
|
+
const row = this.nodeStmt.get(Buffer.from(hashHex, "hex"));
|
|
341
|
+
if (!row)
|
|
342
|
+
throw new DeltaError(`tree node ${hashHex} named by ${docId} at seq ${seq} is missing`);
|
|
343
|
+
const node = parseNodeText(row.entries);
|
|
344
|
+
if (!node)
|
|
345
|
+
throw new DeltaError(`tree node ${hashHex} does not parse as a full or delta node`);
|
|
346
|
+
let entries;
|
|
347
|
+
let texts;
|
|
348
|
+
if (node.form === "full") {
|
|
349
|
+
entries = node.entries;
|
|
350
|
+
texts = entries.map(entryText);
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
const prev = this.base(docId, seq, `node ${hashHex}`);
|
|
354
|
+
if (!prev.root_tree)
|
|
355
|
+
throw new DeltaError(`base revision ${seq - 1} of ${docId} has no root tree`);
|
|
356
|
+
const baseList = blockId === null
|
|
357
|
+
? this.list(docId, seq - 1, prev.root_tree.toString("hex"), null)
|
|
358
|
+
: this.childList(docId, seq - 1, prev.root_tree.toString("hex"), blockId);
|
|
359
|
+
if (!baseList)
|
|
360
|
+
throw new DeltaError(`node ${hashHex} of ${docId} at seq ${seq} is a delta for block ${blockId} which has no children at seq ${seq - 1}`);
|
|
361
|
+
texts = applyOps(node.ops, baseList.texts);
|
|
362
|
+
entries = texts.map((t) => {
|
|
363
|
+
const e = parseEntry(JSON.parse(t));
|
|
364
|
+
if (!e)
|
|
365
|
+
throw new DeltaError(`delta node ${hashHex} resolved to a malformed entry`);
|
|
366
|
+
return e;
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
// Self-containment needs the children: a delta is never self-contained; a
|
|
370
|
+
// full node is when every child list is.
|
|
371
|
+
let selfContained = node.form === "full";
|
|
372
|
+
if (selfContained) {
|
|
373
|
+
for (const e of entries) {
|
|
374
|
+
if (e[2] !== null && !this.list(docId, seq, e[2], e[0]).selfContained) {
|
|
375
|
+
selfContained = false;
|
|
376
|
+
break;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const out = { hash: hashHex, form: node.form, entries, texts, selfContained };
|
|
381
|
+
this.lists.set(key, out);
|
|
382
|
+
return out;
|
|
383
|
+
}
|
|
384
|
+
/** The children list of `blockId` anywhere in the tree rooted at `rootHex` (context `(docId, seq)`), or undefined when the block has no children there. */
|
|
385
|
+
childList(docId, seq, rootHex, blockId) {
|
|
386
|
+
return this.tree(docId, seq, rootHex).byBlock.get(blockId);
|
|
387
|
+
}
|
|
388
|
+
/** The whole tree of a revision, every list resolved in its context. */
|
|
389
|
+
tree(docId, seq, rootHex) {
|
|
390
|
+
const key = `${docId}${seq}${rootHex}`;
|
|
391
|
+
const memo = this.trees.get(key);
|
|
392
|
+
if (memo)
|
|
393
|
+
return memo;
|
|
394
|
+
const byBlock = new Map();
|
|
395
|
+
const root = this.list(docId, seq, rootHex, null);
|
|
396
|
+
const walk = (list) => {
|
|
397
|
+
for (const e of list.entries) {
|
|
398
|
+
if (e[2] === null)
|
|
399
|
+
continue;
|
|
400
|
+
const child = this.list(docId, seq, e[2], e[0]);
|
|
401
|
+
byBlock.set(e[0], child);
|
|
402
|
+
walk(child);
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
walk(root);
|
|
406
|
+
const out = { root, byBlock };
|
|
407
|
+
this.trees.set(key, out);
|
|
408
|
+
return out;
|
|
409
|
+
}
|
|
410
|
+
/** A revision's frontmatter raw from its `frontmatter_blob` (a `-`-form blob, or a `{"f":…}` delta over `seq − 1`'s). */
|
|
411
|
+
frontmatter(docId, seq, blobHex) {
|
|
412
|
+
const key = `${docId}${seq}${blobHex}`;
|
|
413
|
+
const memo = this.frontmatters.get(key);
|
|
414
|
+
if (memo !== undefined)
|
|
415
|
+
return memo;
|
|
416
|
+
const row = this.blobStmt.get(Buffer.from(blobHex, "hex"));
|
|
417
|
+
if (!row)
|
|
418
|
+
throw new DeltaError(`frontmatter blob ${blobHex} named by ${docId} at seq ${seq} is missing`);
|
|
419
|
+
const text = row.bytes.toString("utf8");
|
|
420
|
+
const parsed = parseFrontmatterBlob(text);
|
|
421
|
+
if (!parsed)
|
|
422
|
+
throw new DeltaError(`frontmatter blob ${blobHex} does not parse as a raw or a delta`);
|
|
423
|
+
let raw;
|
|
424
|
+
if (parsed.form === "full")
|
|
425
|
+
raw = parsed.raw;
|
|
426
|
+
else {
|
|
427
|
+
const prev = this.base(docId, seq, `frontmatter ${blobHex}`);
|
|
428
|
+
if (!prev.frontmatter_blob)
|
|
429
|
+
throw new DeltaError(`frontmatter ${blobHex} of ${docId} at seq ${seq} is a delta but revision ${seq - 1} has no frontmatter`);
|
|
430
|
+
const baseRaw = this.frontmatter(docId, seq - 1, prev.frontmatter_blob.toString("hex"));
|
|
431
|
+
raw = applyOps(parsed.ops, splitLines(baseRaw)).join("");
|
|
432
|
+
}
|
|
433
|
+
this.frontmatters.set(key, raw);
|
|
434
|
+
return raw;
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* The checkpoint (self-contained) form of a resolved tree: every list as a
|
|
438
|
+
* `[`-node whose child hashes are the full children's — what an archive
|
|
439
|
+
* segment holds (§7.1) and what the squash re-checkpoints a kept revision
|
|
440
|
+
* to. Returns the root's full hash and every full node by hash.
|
|
441
|
+
*/
|
|
442
|
+
checkpointForm(tree) {
|
|
443
|
+
const nodes = new Map();
|
|
444
|
+
const full = (list) => {
|
|
445
|
+
const texts = list.entries.map((e) => {
|
|
446
|
+
if (e[2] === null)
|
|
447
|
+
return entryText(e);
|
|
448
|
+
const child = tree.byBlock.get(e[0]);
|
|
449
|
+
if (!child)
|
|
450
|
+
throw new DeltaError(`block ${e[0]} names a child tree but the resolved tree has none`);
|
|
451
|
+
const childHex = full(child);
|
|
452
|
+
return entryText([e[0], e[1], childHex, e[3], e[4], e[5]]);
|
|
453
|
+
});
|
|
454
|
+
const text = fullNodeText(texts);
|
|
455
|
+
const hex = sha256(text).toString("hex");
|
|
456
|
+
nodes.set(hex, text);
|
|
457
|
+
return hex;
|
|
458
|
+
};
|
|
459
|
+
return { rootHex: full(tree.root), nodes };
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/** The blob hashes (raw + trivia) a list of entries names — the raws a reconstruct reads, the mark's leaves. */
|
|
463
|
+
export function entryBlobs(entries, into) {
|
|
464
|
+
for (const e of entries) {
|
|
465
|
+
into.add(e[1]);
|
|
466
|
+
if (e[5] !== null)
|
|
467
|
+
into.add(e[5]);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
//# sourceMappingURL=delta.js.map
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Database } from "better-sqlite3";
|
|
1
2
|
import type { Store } from "./store.js";
|
|
2
3
|
export interface GcResult {
|
|
3
4
|
blobsSwept: number;
|
|
@@ -16,6 +17,15 @@ export declare function markReachable(store: Store): {
|
|
|
16
17
|
trees: Set<string>;
|
|
17
18
|
blobs: Set<string>;
|
|
18
19
|
};
|
|
20
|
+
/**
|
|
21
|
+
* The EXPLICIT mark (spec/store §7, 16.0): a walk from a node over the objects
|
|
22
|
+
* its text names on its own — every entry of a `[`-node, the `+`/`~` entries of
|
|
23
|
+
* a `{`-node — recursing into the child hashes those entries carry. What a
|
|
24
|
+
* delta keeps "as the base" is the base revision's to mark, and the
|
|
25
|
+
* base-liveness rule (§4.4) keeps that revision live while the delta is.
|
|
26
|
+
* Returns the walker; `trees`/`blobs` collect hex hashes.
|
|
27
|
+
*/
|
|
28
|
+
export declare function explicitWalker(db: Database, trees: Set<string>, blobs: Set<string>, onNode?: (hashHex: string, entries: string) => void): (hashHex: string) => void;
|
|
19
29
|
/**
|
|
20
30
|
* The sweep: delete every `tree_nodes` and `blobs` row the mark did not reach
|
|
21
31
|
* (count only when `dry`). Runs in the caller's transaction — `runGc` and the
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { explicitEntries, parseNodeText } from "./delta.js";
|
|
1
2
|
/**
|
|
2
3
|
* The mark (spec/store §7): every tree node reachable from a live revision's
|
|
3
4
|
* `root_tree` (walking `child_tree_hash_hex`) and every blob those nodes name
|
|
@@ -11,7 +12,22 @@ export function markReachable(store) {
|
|
|
11
12
|
const fmRows = store.db.prepare("SELECT DISTINCT frontmatter_blob FROM revisions WHERE frontmatter_blob IS NOT NULL").all();
|
|
12
13
|
for (const r of fmRows)
|
|
13
14
|
blobs.add(r.frontmatter_blob.toString("hex"));
|
|
14
|
-
const
|
|
15
|
+
const walk = explicitWalker(store.db, trees, blobs);
|
|
16
|
+
const roots = store.db.prepare("SELECT DISTINCT root_tree FROM revisions WHERE root_tree IS NOT NULL").all();
|
|
17
|
+
for (const r of roots)
|
|
18
|
+
walk(r.root_tree.toString("hex"));
|
|
19
|
+
return { trees, blobs };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The EXPLICIT mark (spec/store §7, 16.0): a walk from a node over the objects
|
|
23
|
+
* its text names on its own — every entry of a `[`-node, the `+`/`~` entries of
|
|
24
|
+
* a `{`-node — recursing into the child hashes those entries carry. What a
|
|
25
|
+
* delta keeps "as the base" is the base revision's to mark, and the
|
|
26
|
+
* base-liveness rule (§4.4) keeps that revision live while the delta is.
|
|
27
|
+
* Returns the walker; `trees`/`blobs` collect hex hashes.
|
|
28
|
+
*/
|
|
29
|
+
export function explicitWalker(db, trees, blobs, onNode) {
|
|
30
|
+
const treeNode = db.prepare("SELECT entries FROM tree_nodes WHERE hash = ?");
|
|
15
31
|
const walk = (hashHex) => {
|
|
16
32
|
if (trees.has(hashHex))
|
|
17
33
|
return;
|
|
@@ -19,8 +35,11 @@ export function markReachable(store) {
|
|
|
19
35
|
const node = treeNode.get(Buffer.from(hashHex, "hex"));
|
|
20
36
|
if (!node)
|
|
21
37
|
return;
|
|
22
|
-
|
|
23
|
-
|
|
38
|
+
onNode?.(hashHex, node.entries);
|
|
39
|
+
const parsed = parseNodeText(node.entries);
|
|
40
|
+
if (!parsed)
|
|
41
|
+
return;
|
|
42
|
+
for (const [, rawHashHex, childHashHex, , , triviaHashHex] of explicitEntries(parsed)) {
|
|
24
43
|
blobs.add(rawHashHex);
|
|
25
44
|
if (triviaHashHex)
|
|
26
45
|
blobs.add(triviaHashHex);
|
|
@@ -28,10 +47,7 @@ export function markReachable(store) {
|
|
|
28
47
|
walk(childHashHex);
|
|
29
48
|
}
|
|
30
49
|
};
|
|
31
|
-
|
|
32
|
-
for (const r of roots)
|
|
33
|
-
walk(r.root_tree.toString("hex"));
|
|
34
|
-
return { trees, blobs };
|
|
50
|
+
return walk;
|
|
35
51
|
}
|
|
36
52
|
/**
|
|
37
53
|
* The sweep: delete every `tree_nodes` and `blobs` row the mark did not reach
|