@omgbase/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -0
- package/dist/src/cli/eval-matcher.d.ts +6 -1
- package/dist/src/cli/eval-matcher.js +14 -2
- package/dist/src/core/hash.d.ts +25 -1
- package/dist/src/core/hash.js +110 -10
- package/dist/src/core/ids.d.ts +12 -0
- package/dist/src/core/ids.js +43 -1
- package/dist/src/core/ingest.d.ts +22 -2
- package/dist/src/core/ingest.js +92 -18
- package/dist/src/core/parse/parse.d.ts +1 -1
- package/dist/src/core/parse/parse.js +31 -9
- package/dist/src/core/store/nodes.d.ts +13 -0
- package/dist/src/core/store/nodes.js +24 -0
- package/dist/src/core/utf8.d.ts +29 -0
- package/dist/src/core/utf8.js +96 -0
- package/dist/src/format/adapter.d.ts +7 -1
- package/dist/src/format/markdown.js +9 -18
- package/dist/src/graph/history.d.ts +6 -0
- package/dist/src/graph/history.js +13 -2
- package/dist/src/index.d.ts +8 -5
- package/dist/src/index.js +8 -5
- package/dist/src/mcp/graph.js +8 -4
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +17 -6
- package/dist/src/mcp/server.js +12 -7
- package/dist/src/mutate/apply.d.ts +7 -0
- package/dist/src/mutate/apply.js +17 -7
- package/dist/src/mutate/docs.d.ts +2 -0
- package/dist/src/mutate/docs.js +10 -6
- package/dist/src/mutate/known-ids.js +9 -1
- package/dist/src/mutate/macros.js +9 -2
- package/dist/src/mutate/ops.d.ts +2 -1
- package/dist/src/mutate/ops.js +24 -2
- package/dist/src/mutate/plan-update.d.ts +7 -1
- package/dist/src/mutate/plan-update.js +3 -1
- package/dist/src/oqx-js/context.js +43 -6
- package/dist/src/reconcile/eval/generator.d.ts +24 -1
- package/dist/src/reconcile/eval/generator.js +248 -0
- package/dist/src/reconcile/eval/metrics.d.ts +33 -9
- package/dist/src/reconcile/eval/metrics.js +55 -32
- package/dist/src/reconcile/flatten.js +7 -5
- package/dist/src/reconcile/phase5.d.ts +2 -1
- package/dist/src/reconcile/phase5.js +27 -10
- package/dist/src/reconcile/phase6a.js +17 -5
- package/dist/src/reconcile/phases.d.ts +44 -6
- package/dist/src/reconcile/phases.js +0 -0
- package/dist/src/reconcile/reconcile.js +16 -9
- package/dist/src/reconcile/types.d.ts +4 -1
- package/dist/src/reconcile/types.js +11 -2
- package/dist/src/search/external.d.ts +10 -0
- package/dist/src/search/external.js +18 -1
- package/dist/src/search/provider.d.ts +7 -3
- package/dist/src/search/provider.js +8 -7
- package/dist/src/search/rrf.js +3 -0
- package/dist/src/search/vector.js +21 -6
- package/dist/src/sync/attach.d.ts +7 -2
- package/dist/src/sync/attach.js +10 -8
- package/dist/src/sync/checkpoint.d.ts +18 -1
- package/dist/src/sync/checkpoint.js +31 -29
- package/dist/src/sync/driver.d.ts +15 -4
- package/dist/src/sync/driver.js +18 -43
- package/dist/src/sync/external-source.js +7 -1
- package/dist/src/sync/freshness.d.ts +65 -14
- package/dist/src/sync/freshness.js +107 -96
- package/dist/src/sync/fs-util.d.ts +19 -0
- package/dist/src/sync/fs-util.js +24 -1
- package/dist/src/sync/observe.d.ts +53 -5
- package/dist/src/sync/observe.js +162 -21
- package/dist/src/sync/reconciling-ingest.d.ts +42 -2
- package/dist/src/sync/reconciling-ingest.js +82 -10
- package/dist/src/sync/recovery.d.ts +5 -1
- package/dist/src/sync/recovery.js +8 -8
- package/dist/src/sync/workspace.d.ts +11 -3
- package/dist/src/sync/workspace.js +39 -30
- package/dist/src/sync/writer-lock.js +16 -26
- package/package.json +3 -3
package/dist/src/sync/observe.js
CHANGED
|
@@ -1,31 +1,151 @@
|
|
|
1
1
|
import { sha256 } from "../core/hash.js";
|
|
2
2
|
import { ingestFile } from "../core/ingest.js";
|
|
3
|
-
import { makeReconcilingResolver } from "./reconciling-ingest.js";
|
|
3
|
+
import { makeReconcilingResolver, prepareReconcile, loadOldMatchBlocks, loadPool } from "./reconciling-ingest.js";
|
|
4
4
|
import { hasConflictMarkers } from "./git-heuristics.js";
|
|
5
5
|
import { tombstoneObservedDeletion } from "./tombstone.js";
|
|
6
6
|
import { sweepResurrectionPool } from "../core/store/gc.js";
|
|
7
|
+
import { crossDocMatch, applyCrossDocMatches } from "../reconcile/crossdoc.js";
|
|
8
|
+
import { DEFAULT_CONFIG } from "../reconcile/types.js";
|
|
7
9
|
/**
|
|
8
|
-
* The reconcile primitive
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* The batch reconcile primitive — one checkpoint's worth of members, in order.
|
|
11
|
+
* Two passes with the cross-document phase between them (reconciliation-spec §8):
|
|
12
|
+
*
|
|
13
|
+
* 1. every member is echo-gated and, when it differs from the stored revision,
|
|
14
|
+
* parsed + reconciled against its own prior tree WITHOUT committing
|
|
15
|
+
* (`prepareReconcile`; members gone from the source contribute their whole
|
|
16
|
+
* live tree as deleted). All members see one resurrection-pool snapshot with
|
|
17
|
+
* a shared consumed-set, so no pooled id resurrects twice in a batch.
|
|
18
|
+
* 2. the per-doc leftovers are pooled — deleted old blocks × inserted new
|
|
19
|
+
* blocks, across documents only — and matched at theta_xdoc
|
|
20
|
+
* (`crossDocMatch`). Each accepted pair is applied to both results: the
|
|
21
|
+
* destination's minted id becomes the carried id (`moved`/`edited_moved`,
|
|
22
|
+
* `detail.fromDoc`), the source's `deleted` disposition and pool entry for
|
|
23
|
+
* it vanish (`applyCrossDocMatches`).
|
|
24
|
+
* 3. every member commits, in batch order, each in its own transaction:
|
|
25
|
+
* `ingestFile` with the finished result (a carried-in id evicts the row the
|
|
26
|
+
* source doc may still hold — see `IdResolver.crossDocIds` — so the order
|
|
27
|
+
* of A and B in the batch does not matter), or `tombstoneObservedDeletion`.
|
|
28
|
+
*
|
|
29
|
+
* A moved block is therefore never `deleted` + `inserted`, never pooled, and
|
|
30
|
+
* never `resurrected` inside one batch. Across batches the pool still does that
|
|
31
|
+
* job (spec/reconcile §5 phase 6b). Does NOT sweep the pool — the caller does,
|
|
32
|
+
* once per batch. Does NOT write a file.
|
|
12
33
|
*/
|
|
13
|
-
export function
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
34
|
+
export function observeBatch(store, repoId, items, ts, opts = {}) {
|
|
35
|
+
const db = store.db;
|
|
36
|
+
const config = opts.config ?? DEFAULT_CONFIG;
|
|
37
|
+
const liveDoc = db.prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_commit IS NULL");
|
|
38
|
+
// ---- pass 1: echo gate + reconcile, no commits -------------------------------
|
|
39
|
+
const pool = loadPool(db, repoId, ts);
|
|
40
|
+
const consumed = new Set();
|
|
41
|
+
const pending = [];
|
|
42
|
+
for (const it of items) {
|
|
43
|
+
const existing = liveDoc.get(repoId, it.path);
|
|
44
|
+
const oldHashHex = existing?.file_hash?.toString("hex") ?? null;
|
|
45
|
+
if (it.content === null) {
|
|
46
|
+
const oldBlocks = existing ? loadOldMatchBlocks(db, existing.doc_id) : [];
|
|
47
|
+
const result = existing
|
|
48
|
+
? { assignment: new Map(), dispositions: [], deleted: oldBlocks.map((b) => b.blockId), consumedPool: [] }
|
|
49
|
+
: null;
|
|
50
|
+
pending.push({ kind: "gone", path: it.path, docId: existing?.doc_id ?? null, oldHashHex, oldBlocks, result });
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const diskHash = sha256(it.content);
|
|
54
|
+
const newHashHex = diskHash.toString("hex");
|
|
55
|
+
// Echo: the stored revision already equals these bytes → no commit. The
|
|
56
|
+
// loop-breaker the coordinator relies on (§6): re-observing what the engine
|
|
57
|
+
// already holds is a no-op.
|
|
58
|
+
if (existing && existing.file_hash && existing.file_hash.equals(diskHash)) {
|
|
59
|
+
pending.push({
|
|
60
|
+
kind: "echo",
|
|
61
|
+
outcome: { kind: "observed", docId: existing.doc_id, path: it.path, rev: null, commitId: null, converged: true, echo: true, conflicted: false, dispositions: [], oldHashHex, newHashHex },
|
|
62
|
+
});
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
const prepared = prepareReconcile(store, repoId, it.path, it.content, { ts, config, pool, consumed });
|
|
66
|
+
pending.push({ kind: "ingest", prepared, oldHashHex, newHashHex });
|
|
67
|
+
}
|
|
68
|
+
// ---- cross-document phase (spec/reconcile §7) --------------------------------
|
|
69
|
+
if (pending.length > 1)
|
|
70
|
+
crossDocPhase(pending, config);
|
|
71
|
+
// ---- pass 2: commit, in batch order ------------------------------------------
|
|
72
|
+
const out = [];
|
|
73
|
+
for (const p of pending) {
|
|
74
|
+
if (p.kind === "echo") {
|
|
75
|
+
out.push(p.outcome);
|
|
76
|
+
}
|
|
77
|
+
else if (p.kind === "gone") {
|
|
78
|
+
if (p.docId)
|
|
79
|
+
tombstoneObservedDeletion(store, repoId, p.docId, ts);
|
|
80
|
+
out.push({ kind: "deleted", path: p.path, docId: p.docId, oldHashHex: p.oldHashHex });
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
out.push({ kind: "observed", ...commitPrepared(store, repoId, p.prepared, ts, p.oldHashHex, p.newHashHex) });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
// Pool each reconciled member's leftovers (README §7: deleted = old blocks whose
|
|
89
|
+
// ids are in `deleted`, in that order; inserted = new blocks whose disposition
|
|
90
|
+
// is `inserted`, with the minted id, in disposition order), match across
|
|
91
|
+
// documents, and apply. Members are keyed by doc id; a brand-new path has none
|
|
92
|
+
// yet, so it is keyed by path — it can only ever be a destination, and
|
|
93
|
+
// `detail.fromDoc` is always a real doc id.
|
|
94
|
+
function crossDocPhase(pending, config) {
|
|
95
|
+
const docs = [];
|
|
96
|
+
const results = new Map();
|
|
97
|
+
const prepared = new Map();
|
|
98
|
+
for (const p of pending) {
|
|
99
|
+
if (p.kind === "echo")
|
|
100
|
+
continue;
|
|
101
|
+
if (p.kind === "gone") {
|
|
102
|
+
if (!p.docId || !p.result)
|
|
103
|
+
continue;
|
|
104
|
+
docs.push({ docId: p.docId, deleted: p.oldBlocks.map((block) => ({ block })), inserted: [] });
|
|
105
|
+
results.set(p.docId, p.result);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const key = p.prepared.docId ?? `new:${p.prepared.path}`;
|
|
109
|
+
const oldById = new Map(p.prepared.oldBlocks.map((b) => [b.blockId, b]));
|
|
110
|
+
const newByKey = new Map(p.prepared.newBlocks.map((b) => [b.key, b]));
|
|
111
|
+
const keyOfId = new Map();
|
|
112
|
+
for (const [k, id] of p.prepared.result.assignment)
|
|
113
|
+
keyOfId.set(id, k);
|
|
114
|
+
const deleted = [];
|
|
115
|
+
for (const id of p.prepared.result.deleted) {
|
|
116
|
+
const block = oldById.get(id);
|
|
117
|
+
if (block)
|
|
118
|
+
deleted.push({ block });
|
|
119
|
+
}
|
|
120
|
+
const inserted = [];
|
|
121
|
+
for (const d of p.prepared.result.dispositions) {
|
|
122
|
+
if (d.kind !== "inserted")
|
|
123
|
+
continue;
|
|
124
|
+
const k = keyOfId.get(d.blockId);
|
|
125
|
+
const block = k ? newByKey.get(k) : undefined;
|
|
126
|
+
if (block)
|
|
127
|
+
inserted.push({ block, mintedId: d.blockId });
|
|
128
|
+
}
|
|
129
|
+
docs.push({ docId: key, deleted, inserted });
|
|
130
|
+
results.set(key, p.prepared.result);
|
|
131
|
+
prepared.set(key, p.prepared);
|
|
25
132
|
}
|
|
133
|
+
if (docs.length < 2)
|
|
134
|
+
return;
|
|
135
|
+
const matches = crossDocMatch(docs, config);
|
|
136
|
+
if (matches.length === 0)
|
|
137
|
+
return;
|
|
138
|
+
applyCrossDocMatches(results, matches, config.matcherV);
|
|
139
|
+
for (const m of matches)
|
|
140
|
+
prepared.get(m.toDoc)?.crossDocIds.push(m.carriedId);
|
|
141
|
+
}
|
|
142
|
+
// Pass 2 for one reconciled member: commit the finished result as an observed
|
|
143
|
+
// revision (one transaction), flag conflict markers, summarize dispositions.
|
|
144
|
+
function commitPrepared(store, repoId, prepared, ts, oldHashHex, newHashHex) {
|
|
145
|
+
const { path, content } = prepared;
|
|
26
146
|
const conflicted = hasConflictMarkers(content);
|
|
27
|
-
const resolveIds = makeReconcilingResolver(store, repoId, { ts, path });
|
|
28
|
-
const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds });
|
|
147
|
+
const resolveIds = makeReconcilingResolver(store, repoId, { ts, path, prepared });
|
|
148
|
+
const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds, parsed: prepared.tree });
|
|
29
149
|
store.db.prepare("UPDATE docs SET conflicted = ? WHERE repo_id = ? AND path = ?").run(conflicted ? 1 : 0, repoId, path);
|
|
30
150
|
const rows = store.db
|
|
31
151
|
.prepare("SELECT kind, count(*) n FROM dispositions WHERE commit_id = ? GROUP BY kind ORDER BY kind")
|
|
@@ -43,6 +163,20 @@ export function observeOne(store, repoId, path, content, ts) {
|
|
|
43
163
|
newHashHex,
|
|
44
164
|
};
|
|
45
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* The single-member reconcile primitive: echo-gate `content` for `path` and,
|
|
168
|
+
* when it differs from the stored revision, ingest it as an observed commit
|
|
169
|
+
* (reconciling block identity against the prior revision). A batch of one — so
|
|
170
|
+
* no cross-document matching; use `observeBatch` for that. Does NOT sweep the
|
|
171
|
+
* resurrection pool — the caller does that once per batch. Does NOT write a file.
|
|
172
|
+
*/
|
|
173
|
+
export function observeOne(store, repoId, path, content, ts) {
|
|
174
|
+
const [outcome] = observeBatch(store, repoId, [{ path, content }], ts);
|
|
175
|
+
if (!outcome || outcome.kind !== "observed")
|
|
176
|
+
throw new Error(`observeOne: unexpected outcome for ${path}`);
|
|
177
|
+
const { kind: _k, ...rest } = outcome;
|
|
178
|
+
return rest;
|
|
179
|
+
}
|
|
46
180
|
function toPublic(r) {
|
|
47
181
|
const { oldHashHex: _o, newHashHex: _n, ...pub } = r;
|
|
48
182
|
return pub;
|
|
@@ -55,10 +189,17 @@ export function observeFile(store, repoId, path, content, opts = {}) {
|
|
|
55
189
|
return toPublic(r);
|
|
56
190
|
}
|
|
57
191
|
/** Observe a batch of files under one timestamp + one resurrection sweep. The
|
|
58
|
-
* batch form of `observeFile`; each item is echo-gated independently
|
|
192
|
+
* batch form of `observeFile`; each item is echo-gated independently, and a
|
|
193
|
+
* block cut from one file and pasted into another in the same batch keeps its
|
|
194
|
+
* id (`moved`/`edited_moved`) via the cross-document phase of `observeBatch`. */
|
|
59
195
|
export function observeMany(store, repoId, items, opts = {}) {
|
|
60
196
|
const ts = opts.ts ?? new Date().toISOString();
|
|
61
|
-
const out =
|
|
197
|
+
const out = observeBatch(store, repoId, items, ts).map((o) => {
|
|
198
|
+
if (o.kind !== "observed")
|
|
199
|
+
throw new Error(`observeMany: unexpected outcome for ${o.path}`);
|
|
200
|
+
const { kind: _k, ...rest } = o;
|
|
201
|
+
return toPublic(rest);
|
|
202
|
+
});
|
|
62
203
|
sweepResurrectionPool(store, ts);
|
|
63
204
|
return out;
|
|
64
205
|
}
|
|
@@ -1,12 +1,52 @@
|
|
|
1
1
|
import type { Database } from "better-sqlite3";
|
|
2
2
|
import type { Store } from "../core/store/store.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { BlockTree } from "../core/parse/types.js";
|
|
4
|
+
import { type IdResolver } from "../core/ingest.js";
|
|
5
|
+
import { type ResurrectionCandidate, type DocReconcileResult } from "../reconcile/reconcile.js";
|
|
4
6
|
import { type MatchBlock, type ReconcileConfig } from "../reconcile/types.js";
|
|
5
7
|
export declare function loadOldMatchBlocks(db: Database, docId: string): MatchBlock[];
|
|
8
|
+
/** The live resurrection pool for a repo (rows not yet expired at `ts`). A batch
|
|
9
|
+
* caller loads it ONCE per checkpoint and threads a shared consumed-set through
|
|
10
|
+
* `prepareReconcile`, so two documents reconciled before either commits cannot
|
|
11
|
+
* both resurrect the same pooled id. */
|
|
12
|
+
export declare function loadPool(db: Database, repoId: string, ts: string): ResurrectionCandidate[];
|
|
13
|
+
/** One document parsed + reconciled but NOT committed: the first pass of a
|
|
14
|
+
* checkpoint (reconciliation-spec §8). The cross-document phase reads
|
|
15
|
+
* `oldBlocks`/`newBlocks`/`result` across every prepared doc, rewrites `result`
|
|
16
|
+
* in place (`applyCrossDocMatches`) and records the ids it carried in; the
|
|
17
|
+
* second pass commits by handing this to `makeReconcilingResolver({ prepared })`
|
|
18
|
+
* + `ingestFile({ parsed: tree })`. */
|
|
19
|
+
export interface PreparedReconcile {
|
|
20
|
+
path: string;
|
|
21
|
+
content: string;
|
|
22
|
+
/** the doc row at `path` (live or tombstoned), or null when the path is new —
|
|
23
|
+
* exactly the docId ingestFile will pass the resolver. */
|
|
24
|
+
docId: string | null;
|
|
25
|
+
tree: BlockTree;
|
|
26
|
+
oldBlocks: MatchBlock[];
|
|
27
|
+
newBlocks: MatchBlock[];
|
|
28
|
+
result: DocReconcileResult;
|
|
29
|
+
/** ids carried into this document by the cross-doc phase (filled by it). */
|
|
30
|
+
crossDocIds: string[];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Pass 1 of an observed ingest: parse `content`, load the document's current
|
|
34
|
+
* tree, and reconcile — without writing. `pool`/`consumed` let a batch share one
|
|
35
|
+
* pool snapshot: candidates already in `consumed` are hidden from this document,
|
|
36
|
+
* and the ids this document resurrects are added to it.
|
|
37
|
+
*/
|
|
38
|
+
export declare function prepareReconcile(store: Store, repoId: string, path: string, content: string, opts?: {
|
|
39
|
+
ts?: string;
|
|
40
|
+
config?: ReconcileConfig;
|
|
41
|
+
pool?: ResurrectionCandidate[];
|
|
42
|
+
consumed?: Set<string>;
|
|
43
|
+
}): PreparedReconcile;
|
|
6
44
|
/** Build an IdResolver bound to this store + repo, reconciling against the
|
|
7
|
-
* document's current revision.
|
|
45
|
+
* document's current revision. With `prepared` (from `prepareReconcile`, after
|
|
46
|
+
* the cross-doc phase) the resolver skips the reconcile and applies that result. */
|
|
8
47
|
export declare function makeReconcilingResolver(store: Store, repoId: string, opts?: {
|
|
9
48
|
ts?: string;
|
|
10
49
|
config?: ReconcileConfig;
|
|
11
50
|
path?: string;
|
|
51
|
+
prepared?: PreparedReconcile;
|
|
12
52
|
}): IdResolver;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { parseForIngest } from "../core/ingest.js";
|
|
1
2
|
import { mintId } from "../core/ids.js";
|
|
2
3
|
import { extractFromBlock, extractFromFrontmatter, resolveRelativePath } from "../graph/extract.js";
|
|
3
4
|
import { resolveExternal, resolveDocPath } from "../core/store/edges.js";
|
|
4
|
-
import { sha256,
|
|
5
|
+
import { sha256, childQuoteDepth, visibleText } from "../core/hash.js";
|
|
5
6
|
import { reconcileDocument } from "../reconcile/reconcile.js";
|
|
6
7
|
import { flatten } from "../reconcile/flatten.js";
|
|
7
8
|
import { DEFAULT_CONFIG } from "../reconcile/types.js";
|
|
@@ -35,11 +36,41 @@ export function loadOldMatchBlocks(db, docId) {
|
|
|
35
36
|
parentKeyOf.set(r.block_id, parentKey);
|
|
36
37
|
return key;
|
|
37
38
|
};
|
|
39
|
+
// `text` follows spec/format §4.1, which needs the tree: containers compose
|
|
40
|
+
// from their children and nested raws lose up to <blockquote-depth> `> `
|
|
41
|
+
// prefixes. Rebuild the shape (children by parent, in ordinal order) and
|
|
42
|
+
// compute text top-down with the same helper ingest used to store it.
|
|
38
43
|
const blobBytes = db.prepare("SELECT bytes FROM blobs WHERE hash = ?");
|
|
44
|
+
const rawOf = new Map();
|
|
45
|
+
for (const r of rows) {
|
|
46
|
+
rawOf.set(r.block_id, blobBytes.get(r.raw_hash)?.bytes.toString("utf8") ?? "");
|
|
47
|
+
}
|
|
48
|
+
const childrenOf = new Map();
|
|
49
|
+
for (const r of rows) {
|
|
50
|
+
const parent = r.parent_block && byId.has(r.parent_block) ? r.parent_block : null;
|
|
51
|
+
let list = childrenOf.get(parent);
|
|
52
|
+
if (!list)
|
|
53
|
+
childrenOf.set(parent, (list = []));
|
|
54
|
+
list.push(r);
|
|
55
|
+
}
|
|
56
|
+
for (const list of childrenOf.values())
|
|
57
|
+
list.sort((a, b) => a.ordinal - b.ordinal);
|
|
58
|
+
const toTree = (r) => ({
|
|
59
|
+
type: r.type,
|
|
60
|
+
raw: rawOf.get(r.block_id) ?? "",
|
|
61
|
+
children: (childrenOf.get(r.block_id) ?? []).map(toTree),
|
|
62
|
+
});
|
|
63
|
+
const textOf = new Map();
|
|
64
|
+
const walk = (list, quoteDepth) => {
|
|
65
|
+
for (const r of list) {
|
|
66
|
+
textOf.set(r.block_id, visibleText(toTree(r), quoteDepth));
|
|
67
|
+
walk(childrenOf.get(r.block_id) ?? [], childQuoteDepth(r.type, quoteDepth));
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
walk(childrenOf.get(null) ?? [], 0);
|
|
39
71
|
return rows.map((r) => {
|
|
40
72
|
const key = resolveKey(r);
|
|
41
|
-
const
|
|
42
|
-
const text = normalizeVisibleText(raw, r.type);
|
|
73
|
+
const text = textOf.get(r.block_id) ?? "";
|
|
43
74
|
return {
|
|
44
75
|
blockId: r.block_id,
|
|
45
76
|
type: r.type,
|
|
@@ -53,7 +84,11 @@ export function loadOldMatchBlocks(db, docId) {
|
|
|
53
84
|
};
|
|
54
85
|
});
|
|
55
86
|
}
|
|
56
|
-
|
|
87
|
+
/** The live resurrection pool for a repo (rows not yet expired at `ts`). A batch
|
|
88
|
+
* caller loads it ONCE per checkpoint and threads a shared consumed-set through
|
|
89
|
+
* `prepareReconcile`, so two documents reconciled before either commits cannot
|
|
90
|
+
* both resurrect the same pooled id. */
|
|
91
|
+
export function loadPool(db, repoId, ts) {
|
|
57
92
|
const rows = db
|
|
58
93
|
.prepare("SELECT block_id, raw_hash, norm_hash, type FROM resurrection_pool WHERE repo_id = ? AND expires_ts > ?")
|
|
59
94
|
.all(repoId, ts);
|
|
@@ -74,8 +109,37 @@ function assignFromMap(blocks, assignment) {
|
|
|
74
109
|
});
|
|
75
110
|
return walk(blocks, null);
|
|
76
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* Pass 1 of an observed ingest: parse `content`, load the document's current
|
|
114
|
+
* tree, and reconcile — without writing. `pool`/`consumed` let a batch share one
|
|
115
|
+
* pool snapshot: candidates already in `consumed` are hidden from this document,
|
|
116
|
+
* and the ids this document resurrects are added to it.
|
|
117
|
+
*/
|
|
118
|
+
export function prepareReconcile(store, repoId, path, content, opts = {}) {
|
|
119
|
+
const db = store.db;
|
|
120
|
+
const ts = opts.ts ?? new Date().toISOString();
|
|
121
|
+
const config = opts.config ?? DEFAULT_CONFIG;
|
|
122
|
+
// Same lookup ingestFile does (a tombstoned row still owns the path's identity).
|
|
123
|
+
const doc = db.prepare("SELECT doc_id FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
|
|
124
|
+
const docId = doc?.doc_id ?? null;
|
|
125
|
+
const { tree, rest } = parseForIngest(path, content);
|
|
126
|
+
const oldBlocks = docId ? loadOldMatchBlocks(db, docId) : [];
|
|
127
|
+
const newBlocks = flatten(rest.map(toFlatSource));
|
|
128
|
+
let pool = [];
|
|
129
|
+
if (docId) {
|
|
130
|
+
pool = opts.pool ?? loadPool(db, repoId, ts);
|
|
131
|
+
if (opts.consumed)
|
|
132
|
+
pool = pool.filter((c) => !opts.consumed.has(c.blockId));
|
|
133
|
+
}
|
|
134
|
+
const result = reconcileDocument(oldBlocks, newBlocks, { config, pool });
|
|
135
|
+
if (opts.consumed)
|
|
136
|
+
for (const id of result.consumedPool)
|
|
137
|
+
opts.consumed.add(id);
|
|
138
|
+
return { path, content, docId, tree, oldBlocks, newBlocks, result, crossDocIds: [] };
|
|
139
|
+
}
|
|
77
140
|
/** Build an IdResolver bound to this store + repo, reconciling against the
|
|
78
|
-
* document's current revision.
|
|
141
|
+
* document's current revision. With `prepared` (from `prepareReconcile`, after
|
|
142
|
+
* the cross-doc phase) the resolver skips the reconcile and applies that result. */
|
|
79
143
|
export function makeReconcilingResolver(store, repoId, opts = {}) {
|
|
80
144
|
const ts = opts.ts ?? new Date().toISOString();
|
|
81
145
|
const config = opts.config ?? DEFAULT_CONFIG;
|
|
@@ -83,10 +147,18 @@ export function makeReconcilingResolver(store, repoId, opts = {}) {
|
|
|
83
147
|
const docDir = opts.path ? opts.path.replace(/[^/]*$/, "") : "";
|
|
84
148
|
return (rest, docId) => {
|
|
85
149
|
const db = store.db;
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
150
|
+
let result;
|
|
151
|
+
let crossDocIds = [];
|
|
152
|
+
if (opts.prepared) {
|
|
153
|
+
result = opts.prepared.result;
|
|
154
|
+
crossDocIds = opts.prepared.crossDocIds;
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
const oldBlocks = docId ? loadOldMatchBlocks(db, docId) : [];
|
|
158
|
+
const newBlocks = flatten(rest.map(toFlatSource));
|
|
159
|
+
const pool = docId ? loadPool(db, repoId, ts) : [];
|
|
160
|
+
result = reconcileDocument(oldBlocks, newBlocks, { config, pool });
|
|
161
|
+
}
|
|
90
162
|
const assigned = assignFromMap(rest, result.assignment);
|
|
91
163
|
const dispositions = result.dispositions.map((d) => ({
|
|
92
164
|
blockId: d.blockId,
|
|
@@ -124,7 +196,7 @@ export function makeReconcilingResolver(store, repoId, opts = {}) {
|
|
|
124
196
|
}
|
|
125
197
|
return out;
|
|
126
198
|
};
|
|
127
|
-
return { assigned, dispositions, deleted: result.deleted, consumedPool: result.consumedPool, extractEdges };
|
|
199
|
+
return { assigned, dispositions, deleted: result.deleted, consumedPool: result.consumedPool, crossDocIds, extractEdges };
|
|
128
200
|
};
|
|
129
201
|
}
|
|
130
202
|
// Resolve an extracted edge's target to a node id (external mint, doc path or
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import type { Store } from "../core/store/store.js";
|
|
2
|
+
import { type SyncFs } from "./fs-util.js";
|
|
2
3
|
export interface RecoveryResult {
|
|
3
4
|
healed: string[];
|
|
4
5
|
missing: string[];
|
|
5
6
|
}
|
|
6
|
-
export declare function recoverRepo(store: Store, repoId: string, rootPath: string
|
|
7
|
+
export declare function recoverRepo(store: Store, repoId: string, rootPath: string, opts?: {
|
|
8
|
+
ts?: string;
|
|
9
|
+
fs?: SyncFs;
|
|
10
|
+
}): RecoveryResult;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { readFileSync, existsSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
3
1
|
import { sha256 } from "../core/hash.js";
|
|
4
2
|
import { ingestFile } from "../core/ingest.js";
|
|
5
3
|
import { makeReconcilingResolver } from "./reconciling-ingest.js";
|
|
6
|
-
|
|
4
|
+
import { nodeFs } from "./fs-util.js";
|
|
5
|
+
export function recoverRepo(store, repoId, rootPath, opts = {}) {
|
|
6
|
+
const fs = opts.fs ?? nodeFs;
|
|
7
7
|
const docs = store.db
|
|
8
8
|
.prepare(`SELECT d.doc_id, d.path, d.file_hash, r.rendered_hash
|
|
9
9
|
FROM docs d LEFT JOIN revisions r ON r.rev_id = d.current_rev
|
|
@@ -11,18 +11,18 @@ export function recoverRepo(store, repoId, rootPath) {
|
|
|
11
11
|
.all(repoId);
|
|
12
12
|
const healed = [];
|
|
13
13
|
const missing = [];
|
|
14
|
-
const ts = new Date().toISOString();
|
|
14
|
+
const ts = opts.ts ?? new Date().toISOString();
|
|
15
15
|
for (const doc of docs) {
|
|
16
|
-
const
|
|
17
|
-
if (
|
|
16
|
+
const bytes = fs.read(rootPath, doc.path);
|
|
17
|
+
if (bytes === null) {
|
|
18
18
|
missing.push(doc.path);
|
|
19
19
|
continue;
|
|
20
20
|
}
|
|
21
|
-
const onDisk = sha256(
|
|
21
|
+
const onDisk = sha256(bytes);
|
|
22
22
|
const recorded = doc.rendered_hash ?? doc.file_hash;
|
|
23
23
|
if (!recorded || !onDisk.equals(recorded)) {
|
|
24
24
|
// Divergence: the file moved ahead of (or behind) the store. Re-ingest.
|
|
25
|
-
ingestFile(store, repoId, doc.path,
|
|
25
|
+
ingestFile(store, repoId, doc.path, bytes, {
|
|
26
26
|
ts,
|
|
27
27
|
resolveIds: makeReconcilingResolver(store, repoId, { ts, path: doc.path }),
|
|
28
28
|
});
|
|
@@ -20,13 +20,21 @@ export declare class Workspace {
|
|
|
20
20
|
repos(): RepoRow[];
|
|
21
21
|
repoBySlug(slug: string): RepoRow | null;
|
|
22
22
|
/**
|
|
23
|
-
* Select the active repo
|
|
24
|
-
*
|
|
25
|
-
* or several match, throws AmbiguousRepo with the candidate slugs.
|
|
23
|
+
* Select the active repo (spec/sync §1 "Repo selection"): {@link selectRepo}
|
|
24
|
+
* over this workspace's repos.
|
|
26
25
|
*/
|
|
27
26
|
selectRepo(cwd: string, slug?: string): RepoRow;
|
|
28
27
|
close(): void;
|
|
29
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Repo selection (spec/sync §1) over a plain `repos` list. With an explicit
|
|
31
|
+
* slug, that repo (or `repo_not_found` with the candidate slugs). Else with
|
|
32
|
+
* exactly one repo, that repo. Else the repos whose root path contains `cwd`
|
|
33
|
+
* (path-prefix containment after resolving both; a sourceless repo never
|
|
34
|
+
* matches): exactly one → it; none → `repo_not_found`; several (nested roots)
|
|
35
|
+
* → the one with the longest resolved root path.
|
|
36
|
+
*/
|
|
37
|
+
export declare function selectRepo(repos: RepoRow[], cwd: string, slug?: string): RepoRow;
|
|
30
38
|
export declare class RepoSelectionError extends Error {
|
|
31
39
|
readonly code: "repo_not_found";
|
|
32
40
|
readonly candidates: string[];
|
|
@@ -70,43 +70,52 @@ export class Workspace {
|
|
|
70
70
|
return this.repos().find((r) => r.slug === slug) ?? null;
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
73
|
-
* Select the active repo
|
|
74
|
-
*
|
|
75
|
-
* or several match, throws AmbiguousRepo with the candidate slugs.
|
|
73
|
+
* Select the active repo (spec/sync §1 "Repo selection"): {@link selectRepo}
|
|
74
|
+
* over this workspace's repos.
|
|
76
75
|
*/
|
|
77
76
|
selectRepo(cwd, slug) {
|
|
78
|
-
|
|
79
|
-
if (slug) {
|
|
80
|
-
const found = repos.find((r) => r.slug === slug);
|
|
81
|
-
if (!found) {
|
|
82
|
-
throw new RepoSelectionError("repo_not_found", `no repo with slug '${slug}'`, repos.map((r) => r.slug));
|
|
83
|
-
}
|
|
84
|
-
return found;
|
|
85
|
-
}
|
|
86
|
-
if (repos.length === 1)
|
|
87
|
-
return repos[0];
|
|
88
|
-
const here = resolve(cwd);
|
|
89
|
-
// Only filesystem-backed repos (a derived rootPath) can contain the cwd; a
|
|
90
|
-
// sourceless/headless repo has no location, so it never matches by cwd.
|
|
91
|
-
const containing = repos.filter((r) => {
|
|
92
|
-
if (r.rootPath == null)
|
|
93
|
-
return false;
|
|
94
|
-
const rel = relative(resolve(r.rootPath), here);
|
|
95
|
-
return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
|
|
96
|
-
});
|
|
97
|
-
if (containing.length === 1)
|
|
98
|
-
return containing[0];
|
|
99
|
-
if (containing.length === 0) {
|
|
100
|
-
throw new RepoSelectionError("repo_not_found", `no repo contains ${here}; select one with --repo`, repos.map((r) => r.slug));
|
|
101
|
-
}
|
|
102
|
-
// Several contain cwd (nested roots): pick the deepest (longest root path).
|
|
103
|
-
containing.sort((a, b) => resolve(b.rootPath).length - resolve(a.rootPath).length);
|
|
104
|
-
return containing[0];
|
|
77
|
+
return selectRepo(this.repos(), cwd, slug);
|
|
105
78
|
}
|
|
106
79
|
close() {
|
|
107
80
|
this.store.close();
|
|
108
81
|
}
|
|
109
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* Repo selection (spec/sync §1) over a plain `repos` list. With an explicit
|
|
85
|
+
* slug, that repo (or `repo_not_found` with the candidate slugs). Else with
|
|
86
|
+
* exactly one repo, that repo. Else the repos whose root path contains `cwd`
|
|
87
|
+
* (path-prefix containment after resolving both; a sourceless repo never
|
|
88
|
+
* matches): exactly one → it; none → `repo_not_found`; several (nested roots)
|
|
89
|
+
* → the one with the longest resolved root path.
|
|
90
|
+
*/
|
|
91
|
+
export function selectRepo(repos, cwd, slug) {
|
|
92
|
+
if (slug) {
|
|
93
|
+
const found = repos.find((r) => r.slug === slug);
|
|
94
|
+
if (!found) {
|
|
95
|
+
throw new RepoSelectionError("repo_not_found", `no repo with slug '${slug}'`, repos.map((r) => r.slug));
|
|
96
|
+
}
|
|
97
|
+
return found;
|
|
98
|
+
}
|
|
99
|
+
if (repos.length === 1)
|
|
100
|
+
return repos[0];
|
|
101
|
+
const here = resolve(cwd);
|
|
102
|
+
// Only filesystem-backed repos (a derived rootPath) can contain the cwd; a
|
|
103
|
+
// sourceless/headless repo has no location, so it never matches by cwd.
|
|
104
|
+
const containing = repos.filter((r) => {
|
|
105
|
+
if (r.rootPath == null)
|
|
106
|
+
return false;
|
|
107
|
+
const rel = relative(resolve(r.rootPath), here);
|
|
108
|
+
return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
|
|
109
|
+
});
|
|
110
|
+
if (containing.length === 1)
|
|
111
|
+
return containing[0];
|
|
112
|
+
if (containing.length === 0) {
|
|
113
|
+
throw new RepoSelectionError("repo_not_found", `no repo contains ${here}; select one with --repo`, repos.map((r) => r.slug));
|
|
114
|
+
}
|
|
115
|
+
// Several contain cwd (nested roots): pick the deepest (longest root path).
|
|
116
|
+
containing.sort((a, b) => resolve(b.rootPath).length - resolve(a.rootPath).length);
|
|
117
|
+
return containing[0];
|
|
118
|
+
}
|
|
110
119
|
export class RepoSelectionError extends Error {
|
|
111
120
|
code;
|
|
112
121
|
candidates;
|
|
@@ -8,12 +8,12 @@ import { join } from "node:path";
|
|
|
8
8
|
// Node core exposes no flock(2), and the design forbids new runtime deps (11
|
|
9
9
|
// §8). We implement the same advisory contract with an O_EXCL lockfile: exclusive
|
|
10
10
|
// creation is atomic on local filesystems, the holder writes its pid for
|
|
11
|
-
// liveness, and a lock whose pid is dead
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
11
|
+
// liveness, and a lock whose pid is dead (or whose body does not parse) is
|
|
12
|
+
// stolen — never one whose holder is alive, however old the record (spec/sync
|
|
13
|
+
// §7; no daemon, no lease sweeper — the same "probe by trying" spirit as the
|
|
14
|
+
// watch lease in §3.4). This is an as-built substitution for flock; the
|
|
15
|
+
// file-CAS + ingest-and-replay behavior in the write protocol is unchanged.
|
|
15
16
|
const LOCK_NAME = "writer.lock";
|
|
16
|
-
const STALE_MS = 30_000;
|
|
17
17
|
function pidAlive(pid) {
|
|
18
18
|
try {
|
|
19
19
|
process.kill(pid, 0);
|
|
@@ -40,20 +40,9 @@ function tryAcquire(lockPath) {
|
|
|
40
40
|
catch (err) {
|
|
41
41
|
if (err.code !== "EEXIST")
|
|
42
42
|
throw err;
|
|
43
|
-
// Lock exists: steal it if the holder is dead or the record is
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
const rec = JSON.parse(readFileSync(lockPath, "utf8"));
|
|
48
|
-
holderPid = rec.pid ?? null;
|
|
49
|
-
ts = rec.ts ?? 0;
|
|
50
|
-
}
|
|
51
|
-
catch {
|
|
52
|
-
// Unreadable/partial lock record — treat as stale.
|
|
53
|
-
}
|
|
54
|
-
const dead = holderPid != null && !pidAlive(holderPid);
|
|
55
|
-
const stale = Date.now() - ts > STALE_MS;
|
|
56
|
-
if (dead || stale) {
|
|
43
|
+
// Lock exists: steal it only if the holder is dead or the record is unreadable.
|
|
44
|
+
const holderPid = readHolderPid(lockPath);
|
|
45
|
+
if (holderPid == null || !pidAlive(holderPid)) {
|
|
57
46
|
try {
|
|
58
47
|
unlinkSync(lockPath);
|
|
59
48
|
}
|
|
@@ -73,14 +62,15 @@ function readHolderPid(lockPath) {
|
|
|
73
62
|
return null;
|
|
74
63
|
}
|
|
75
64
|
}
|
|
76
|
-
//
|
|
77
|
-
//
|
|
78
|
-
//
|
|
65
|
+
// Blocking sleep between polls. Writers hold the lock only for the brief
|
|
66
|
+
// file-CAS + ingest window, so contention waits are short, and the lock API
|
|
67
|
+
// stays synchronous like the rest of the write path. `Atomics.wait` parks the
|
|
68
|
+
// thread in the kernel for `ms` — unlike a `Date.now()` spin it burns no CPU,
|
|
69
|
+
// which matters precisely when several writers contend on a loaded machine
|
|
70
|
+
// (each spinning waiter would otherwise steal cycles from the lock holder).
|
|
71
|
+
const SLEEP_CELL = new Int32Array(new SharedArrayBuffer(4));
|
|
79
72
|
function sleepSync(ms) {
|
|
80
|
-
|
|
81
|
-
while (Date.now() < until) {
|
|
82
|
-
/* spin */
|
|
83
|
-
}
|
|
73
|
+
Atomics.wait(SLEEP_CELL, 0, 0, ms);
|
|
84
74
|
}
|
|
85
75
|
/**
|
|
86
76
|
* Run fn while holding the workspace writer lock. Blocks (polling) until the
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@omgbase/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Open Markdown Graph Base — embedded engine",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@modelcontextprotocol/sdk": "^1.30.0",
|
|
58
|
-
"@omgbase/oqx": "^0.
|
|
58
|
+
"@omgbase/oqx": "^0.12.0",
|
|
59
59
|
"better-sqlite3": "^13.0.3",
|
|
60
60
|
"mdast-util-from-markdown": "^2.0.2",
|
|
61
61
|
"mdast-util-frontmatter": "^2.0.1",
|
|
@@ -73,7 +73,7 @@
|
|
|
73
73
|
"scripts": {
|
|
74
74
|
"build": "tsc -p tsconfig.json",
|
|
75
75
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
76
|
-
"test": "vitest run",
|
|
76
|
+
"test": "vitest run --exclude src/core/perf.test.ts && vitest run src/core/perf.test.ts",
|
|
77
77
|
"lint": "eslint .",
|
|
78
78
|
"lint:fix": "eslint . --fix"
|
|
79
79
|
}
|