@omgbase/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -0
- package/dist/src/cli/eval-matcher.d.ts +6 -1
- package/dist/src/cli/eval-matcher.js +14 -2
- package/dist/src/core/clock.d.ts +9 -0
- package/dist/src/core/clock.js +46 -0
- package/dist/src/core/hash.d.ts +25 -1
- package/dist/src/core/hash.js +110 -10
- package/dist/src/core/ids.d.ts +12 -0
- package/dist/src/core/ids.js +43 -1
- package/dist/src/core/ingest.d.ts +22 -2
- package/dist/src/core/ingest.js +92 -18
- package/dist/src/core/parse/parse.d.ts +1 -1
- package/dist/src/core/parse/parse.js +31 -9
- package/dist/src/core/store/nodes.d.ts +13 -0
- package/dist/src/core/store/nodes.js +24 -0
- package/dist/src/core/store/writers.d.ts +6 -0
- package/dist/src/core/store/writers.js +1 -0
- package/dist/src/core/utf8.d.ts +29 -0
- package/dist/src/core/utf8.js +96 -0
- package/dist/src/format/adapter.d.ts +7 -1
- package/dist/src/format/markdown.js +36 -41
- package/dist/src/graph/extract.d.ts +41 -1
- package/dist/src/graph/extract.js +59 -1
- package/dist/src/graph/history.d.ts +39 -1
- package/dist/src/graph/history.js +154 -20
- package/dist/src/index.d.ts +10 -6
- package/dist/src/index.js +11 -6
- package/dist/src/mcp/graph.js +8 -4
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +17 -6
- package/dist/src/mcp/server.js +15 -10
- package/dist/src/mutate/apply.d.ts +9 -0
- package/dist/src/mutate/apply.js +24 -11
- package/dist/src/mutate/docs.d.ts +2 -0
- package/dist/src/mutate/docs.js +10 -6
- package/dist/src/mutate/known-ids.js +11 -3
- package/dist/src/mutate/macros.js +9 -2
- package/dist/src/mutate/ops.d.ts +5 -4
- package/dist/src/mutate/ops.js +62 -15
- package/dist/src/mutate/plan-update.d.ts +7 -1
- package/dist/src/mutate/plan-update.js +3 -1
- package/dist/src/oqx-js/context.js +57 -7
- package/dist/src/reconcile/eval/generator.d.ts +24 -1
- package/dist/src/reconcile/eval/generator.js +248 -0
- package/dist/src/reconcile/eval/metrics.d.ts +33 -9
- package/dist/src/reconcile/eval/metrics.js +55 -32
- package/dist/src/reconcile/flatten.js +7 -5
- package/dist/src/reconcile/phase5.d.ts +2 -1
- package/dist/src/reconcile/phase5.js +27 -10
- package/dist/src/reconcile/phase6a.js +17 -5
- package/dist/src/reconcile/phases.d.ts +44 -6
- package/dist/src/reconcile/phases.js +0 -0
- package/dist/src/reconcile/reconcile.js +16 -9
- package/dist/src/reconcile/types.d.ts +4 -1
- package/dist/src/reconcile/types.js +11 -2
- package/dist/src/search/external.d.ts +10 -0
- package/dist/src/search/external.js +18 -1
- package/dist/src/search/provider.d.ts +7 -3
- package/dist/src/search/provider.js +8 -7
- package/dist/src/search/rrf.js +4 -0
- package/dist/src/search/tasks.d.ts +28 -0
- package/dist/src/search/tasks.js +36 -23
- package/dist/src/search/vector.js +30 -14
- package/dist/src/sync/attach.d.ts +7 -2
- package/dist/src/sync/attach.js +10 -8
- package/dist/src/sync/checkpoint.d.ts +18 -1
- package/dist/src/sync/checkpoint.js +31 -29
- package/dist/src/sync/driver.d.ts +15 -4
- package/dist/src/sync/driver.js +18 -43
- package/dist/src/sync/external-source.js +7 -1
- package/dist/src/sync/freshness.d.ts +73 -15
- package/dist/src/sync/freshness.js +128 -97
- package/dist/src/sync/fs-util.d.ts +19 -0
- package/dist/src/sync/fs-util.js +24 -1
- package/dist/src/sync/observe.d.ts +53 -5
- package/dist/src/sync/observe.js +162 -21
- package/dist/src/sync/reconciling-ingest.d.ts +42 -2
- package/dist/src/sync/reconciling-ingest.js +87 -13
- package/dist/src/sync/recovery.d.ts +5 -1
- package/dist/src/sync/recovery.js +8 -8
- package/dist/src/sync/workspace.d.ts +11 -3
- package/dist/src/sync/workspace.js +39 -30
- package/dist/src/sync/writer-lock.js +16 -26
- package/package.json +3 -3
|
@@ -1,101 +1,121 @@
|
|
|
1
|
-
import { statSync, readFileSync, existsSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
3
1
|
import { sha256 } from "../core/hash.js";
|
|
4
2
|
import { processCheckpoint } from "./checkpoint.js";
|
|
5
|
-
import {
|
|
6
|
-
/**
|
|
7
|
-
export function
|
|
8
|
-
|
|
3
|
+
import { nodeFs } from "./fs-util.js";
|
|
4
|
+
/** Step 1 of §4.3 alone: the stat-mismatched or new paths, without hashing anything. */
|
|
5
|
+
export function sweepCandidates(cache, snapshot) {
|
|
6
|
+
const out = [];
|
|
7
|
+
for (const e of snapshot) {
|
|
8
|
+
const cached = cache.get(e.path);
|
|
9
|
+
if (!cached || cached.mtime_ns !== e.mtime_ns || cached.size !== e.size)
|
|
10
|
+
out.push(e.path);
|
|
11
|
+
}
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
/** spec/sync §4.3 steps 1–2 as a pure function (README §8 `sweep_plan`). */
|
|
15
|
+
export function sweepPlan(cache, snapshot) {
|
|
16
|
+
const byPath = new Map(snapshot.map((e) => [e.path, e]));
|
|
17
|
+
const candidates = sweepCandidates(cache, snapshot);
|
|
18
|
+
const deletions = [...cache.keys()].filter((p) => !byPath.has(p));
|
|
19
|
+
const changed = [];
|
|
20
|
+
const refreshed = [];
|
|
21
|
+
const hashes = new Map();
|
|
22
|
+
for (const path of candidates) {
|
|
23
|
+
const hash = byPath.get(path).hash();
|
|
24
|
+
hashes.set(path, hash);
|
|
25
|
+
const cached = cache.get(path);
|
|
26
|
+
if (!cached || !cached.hash.equals(hash))
|
|
27
|
+
changed.push(path);
|
|
28
|
+
else
|
|
29
|
+
refreshed.push(path);
|
|
30
|
+
}
|
|
31
|
+
return { candidates, changed, deletions, refreshed, hashes };
|
|
32
|
+
}
|
|
33
|
+
/** The repo's `file_stats` rows keyed by path. Read with safe integers: an `mtime_ns` is a
|
|
34
|
+
* 64-bit count a JS number cannot hold exactly, and a rounded value would never equal a stat. */
|
|
35
|
+
export function loadStatCache(store, repoId) {
|
|
36
|
+
const cache = new Map();
|
|
37
|
+
const rows = store.db.prepare("SELECT path, mtime_ns, size, hash FROM file_stats WHERE repo_id = ?").safeIntegers(true).all(repoId);
|
|
38
|
+
for (const row of rows)
|
|
39
|
+
cache.set(row.path, { path: row.path, mtime_ns: row.mtime_ns, size: Number(row.size), hash: row.hash });
|
|
40
|
+
return cache;
|
|
41
|
+
}
|
|
42
|
+
/** The current filesystem snapshot: the §4.2 walk with each file's stat and a lazy hash. */
|
|
43
|
+
export function snapshotOf(fs, rootPath) {
|
|
44
|
+
const out = [];
|
|
45
|
+
for (const path of fs.walk(rootPath)) {
|
|
46
|
+
const st = fs.stat(rootPath, path);
|
|
47
|
+
if (!st)
|
|
48
|
+
continue; // vanished between the walk and the stat
|
|
49
|
+
let memo = null;
|
|
50
|
+
out.push({
|
|
51
|
+
path,
|
|
52
|
+
mtime_ns: st.mtimeNs,
|
|
53
|
+
size: st.size,
|
|
54
|
+
hash: () => (memo ??= sha256(fs.read(rootPath, path) ?? "")),
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
/** Write a file's stat cache row from a stat (`INSERT … ON CONFLICT DO UPDATE`); a null stat (the
|
|
60
|
+
* file vanished) deletes the row. The DB half of `record_file_stat` (spec/sync §4.3 step 4). */
|
|
61
|
+
export function recordStat(store, repoId, path, st, hash) {
|
|
62
|
+
if (!st) {
|
|
9
63
|
store.db.prepare("DELETE FROM file_stats WHERE repo_id = ? AND path = ?").run(repoId, path);
|
|
10
64
|
return;
|
|
11
65
|
}
|
|
12
|
-
const st = statSync(abs, { bigint: true });
|
|
13
66
|
store.db
|
|
14
67
|
.prepare(`INSERT INTO file_stats (repo_id, path, mtime_ns, size, hash) VALUES (?, ?, ?, ?, ?)
|
|
15
68
|
ON CONFLICT(repo_id, path) DO UPDATE SET mtime_ns = excluded.mtime_ns, size = excluded.size, hash = excluded.hash`)
|
|
16
|
-
.run(repoId, path, st.mtimeNs
|
|
69
|
+
.run(repoId, path, st.mtimeNs, st.size, hash);
|
|
70
|
+
}
|
|
71
|
+
/** Record/refresh a file's stat cache row from its absolute path. Call after any ingest of the file. */
|
|
72
|
+
export function recordFileStat(store, repoId, path, abs, hash) {
|
|
73
|
+
recordStat(store, repoId, path, nodeFs.stat(abs, ""), hash);
|
|
17
74
|
}
|
|
18
75
|
/**
|
|
19
|
-
* Run the freshness sweep for a repo. Returns the checkpoint
|
|
20
|
-
* counters. Idempotent; safe alongside a live watcher
|
|
21
|
-
* suppression makes a double ingest a no-op).
|
|
76
|
+
* Run the freshness sweep for a repo (spec/sync §4.3). Returns the checkpoint
|
|
77
|
+
* result plus scan counters. Idempotent; safe alongside a live watcher
|
|
78
|
+
* (hash-based echo suppression makes a double ingest a no-op). `opts.ts` pins
|
|
79
|
+
* the batch timestamp (default now); `opts.fs` the filesystem seam.
|
|
22
80
|
*/
|
|
23
|
-
export function freshnessSweep(store, repoId, rootPath) {
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
81
|
+
export function freshnessSweep(store, repoId, rootPath, opts = {}) {
|
|
82
|
+
const fs = opts.fs ?? nodeFs;
|
|
83
|
+
const cache = loadStatCache(store, repoId);
|
|
84
|
+
const snapshot = snapshotOf(fs, rootPath);
|
|
85
|
+
const plan = sweepPlan(cache, snapshot);
|
|
86
|
+
const statOf = new Map(snapshot.map((e) => [e.path, { mtimeNs: e.mtime_ns, size: e.size }]));
|
|
87
|
+
// Content unchanged; refresh the stat cache so we don't re-hash next time.
|
|
88
|
+
for (const path of plan.refreshed) {
|
|
89
|
+
const st = statOf.get(path);
|
|
90
|
+
store.db.prepare("UPDATE file_stats SET mtime_ns = ?, size = ? WHERE repo_id = ? AND path = ?").run(st.mtimeNs, st.size, repoId, path);
|
|
29
91
|
}
|
|
30
|
-
const
|
|
31
|
-
const seen = new Set(paths);
|
|
32
|
-
// Candidates: files whose (mtime_ns, size) differs from cache, or are new.
|
|
33
|
-
const candidates = [];
|
|
34
|
-
const freshStats = new Map();
|
|
35
|
-
for (const path of paths) {
|
|
36
|
-
const st = statSync(join(rootPath, path), { bigint: true });
|
|
37
|
-
freshStats.set(path, { mtimeNs: st.mtimeNs, size: Number(st.size) });
|
|
38
|
-
const cached = cache.get(path);
|
|
39
|
-
if (!cached || cached.mtime_ns !== st.mtimeNs || cached.size !== Number(st.size)) {
|
|
40
|
-
candidates.push(path);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
// Deletions: cached paths no longer on disk.
|
|
44
|
-
const deletedPaths = [...cache.keys()].filter((p) => !seen.has(p));
|
|
45
|
-
// Hash candidates; only those whose content hash actually differs from the
|
|
46
|
-
// cached hash need ingesting (mtime can change without content changing).
|
|
47
|
-
const changedPaths = [];
|
|
48
|
-
const candHashes = new Map();
|
|
49
|
-
for (const path of candidates) {
|
|
50
|
-
const hash = sha256(readFileSync(join(rootPath, path), "utf8"));
|
|
51
|
-
candHashes.set(path, hash);
|
|
52
|
-
const cached = cache.get(path);
|
|
53
|
-
if (!cached || !cached.hash.equals(hash))
|
|
54
|
-
changedPaths.push(path);
|
|
55
|
-
else {
|
|
56
|
-
// Content unchanged; refresh the stat cache so we don't re-hash next time.
|
|
57
|
-
const fs = freshStats.get(path);
|
|
58
|
-
store.db
|
|
59
|
-
.prepare("UPDATE file_stats SET mtime_ns = ?, size = ? WHERE repo_id = ? AND path = ?")
|
|
60
|
-
.run(fs.mtimeNs.toString(), fs.size, repoId, path);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
const toIngest = [...changedPaths, ...deletedPaths];
|
|
64
|
-
const result = processCheckpoint(store, repoId, rootPath, toIngest.map((path) => ({ path })));
|
|
92
|
+
const result = processCheckpoint(store, repoId, rootPath, [...plan.changed, ...plan.deletions].map((path) => ({ path })), { fs, ...(opts.ts !== undefined ? { ts: opts.ts } : {}), ...(opts.gitHead !== undefined ? { gitHead: opts.gitHead } : {}) });
|
|
65
93
|
// Refresh stat cache for everything we touched (ingested or echo-suppressed).
|
|
66
|
-
for (const path of
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
for (const path of deletedPaths) {
|
|
94
|
+
for (const path of plan.changed)
|
|
95
|
+
recordStat(store, repoId, path, fs.stat(rootPath, path), plan.hashes.get(path));
|
|
96
|
+
for (const path of plan.deletions)
|
|
70
97
|
store.db.prepare("DELETE FROM file_stats WHERE repo_id = ? AND path = ?").run(repoId, path);
|
|
71
|
-
}
|
|
72
98
|
return {
|
|
73
99
|
...result,
|
|
74
|
-
scanned:
|
|
75
|
-
candidates: candidates.length,
|
|
100
|
+
scanned: snapshot.length,
|
|
101
|
+
candidates: plan.candidates.length,
|
|
76
102
|
changed: result.ingested.length > 0 || result.deleted.length > 0 || result.conflicted.length > 0,
|
|
77
103
|
};
|
|
78
104
|
}
|
|
79
105
|
/**
|
|
80
|
-
* READ-ONLY disk-drift detector
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
106
|
+
* READ-ONLY disk-drift detector (spec/sync §4.3 "Disk drift"). Mirrors
|
|
107
|
+
* freshnessSweep's staged cheap approach — the same `sweepCandidates` over the
|
|
108
|
+
* same cache and snapshot, hashing only the stat-mismatched candidates — but
|
|
109
|
+
* never ingests, commits, or touches file_stats. It answers a single question:
|
|
110
|
+
* does the DB still agree with what's on disk right now?
|
|
84
111
|
*
|
|
85
112
|
* `changed` is decided by comparing a candidate's content hash to the doc's
|
|
86
113
|
* stored `file_hash` (the same sha256(bytes) checkpoint writes), so a stat
|
|
87
|
-
* change without a content change is NOT counted as drift.
|
|
88
|
-
* (≤10⁴ docs) the clean case is a directory walk plus stats (tens of ms);
|
|
89
|
-
* hashing is bounded to files whose (mtime_ns, size) moved.
|
|
114
|
+
* change without a content change is NOT counted as drift.
|
|
90
115
|
*/
|
|
91
|
-
export function detectDiskDrift(store, repoId, rootPath) {
|
|
92
|
-
|
|
93
|
-
const cache =
|
|
94
|
-
for (const row of store.db
|
|
95
|
-
.prepare("SELECT path, mtime_ns, size, hash FROM file_stats WHERE repo_id = ?")
|
|
96
|
-
.all(repoId)) {
|
|
97
|
-
cache.set(row.path, { mtime_ns: BigInt(row.mtime_ns), size: row.size, hash: row.hash });
|
|
98
|
-
}
|
|
116
|
+
export function detectDiskDrift(store, repoId, rootPath, opts = {}) {
|
|
117
|
+
const fs = opts.fs ?? nodeFs;
|
|
118
|
+
const cache = loadStatCache(store, repoId);
|
|
99
119
|
// Live docs keyed by path, with their durable file_hash (the convergence
|
|
100
120
|
// signal). This is the authority for changed/deleted/untracked, independent
|
|
101
121
|
// of the derived file_stats cache.
|
|
@@ -105,47 +125,58 @@ export function detectDiskDrift(store, repoId, rootPath) {
|
|
|
105
125
|
.all(repoId)) {
|
|
106
126
|
docs.set(row.path, row.file_hash);
|
|
107
127
|
}
|
|
108
|
-
const
|
|
109
|
-
const
|
|
110
|
-
// Candidates whose (mtime_ns, size) differs from the stat cache (or are new).
|
|
111
|
-
const candidates = [];
|
|
112
|
-
for (const path of paths) {
|
|
113
|
-
const st = statSync(join(rootPath, path), { bigint: true });
|
|
114
|
-
const cached = cache.get(path);
|
|
115
|
-
if (!cached || cached.mtime_ns !== st.mtimeNs || cached.size !== Number(st.size)) {
|
|
116
|
-
candidates.push(path);
|
|
117
|
-
}
|
|
118
|
-
}
|
|
128
|
+
const snapshot = snapshotOf(fs, rootPath);
|
|
129
|
+
const byPath = new Map(snapshot.map((e) => [e.path, e]));
|
|
119
130
|
let changed = 0;
|
|
120
131
|
let untracked = 0;
|
|
121
|
-
for (const path of
|
|
132
|
+
for (const path of sweepCandidates(cache, snapshot)) {
|
|
122
133
|
const doc = docs.get(path);
|
|
123
134
|
if (doc === undefined) {
|
|
124
135
|
// No non-deleted doc for an on-disk *.md ⇒ untracked (new file).
|
|
125
136
|
untracked += 1;
|
|
126
137
|
continue;
|
|
127
138
|
}
|
|
128
|
-
const hash = sha256(readFileSync(join(rootPath, path), "utf8"));
|
|
129
139
|
// Genuinely drifted only if content hash differs from the doc's file_hash.
|
|
130
|
-
if (!doc || !doc.equals(hash))
|
|
140
|
+
if (!doc || !doc.equals(byPath.get(path).hash()))
|
|
131
141
|
changed += 1;
|
|
132
142
|
}
|
|
133
143
|
// A doc present in the DB whose path isn't on disk anymore is a pending delete.
|
|
134
144
|
let deleted = 0;
|
|
135
|
-
for (const path of docs.keys())
|
|
136
|
-
if (!
|
|
145
|
+
for (const path of docs.keys())
|
|
146
|
+
if (!byPath.has(path))
|
|
137
147
|
deleted += 1;
|
|
138
|
-
}
|
|
139
148
|
return { changed, deleted, untracked };
|
|
140
149
|
}
|
|
141
|
-
/**
|
|
142
|
-
|
|
150
|
+
/**
|
|
151
|
+
* Rebuild `file_stats` from scratch (spec/sync §4.3, since 1.1): delete the repo's rows, then
|
|
152
|
+
* walk the tree and record a row only for a file whose bytes' hash equals its live doc's
|
|
153
|
+
* `file_hash` — the cache is allowed to say "known" only about bytes the store already holds.
|
|
154
|
+
* A file with no live doc, or whose bytes differ from what was ingested, gets no row, so the
|
|
155
|
+
* next sweep still treats it as a candidate and drift still reports it (a cache rebuild must
|
|
156
|
+
* never change what the next sweep does). Returns the number of files walked, recorded or not.
|
|
157
|
+
*/
|
|
158
|
+
export function rebuildFileStats(store, repoId, rootPath, opts = {}) {
|
|
159
|
+
const fs = opts.fs ?? nodeFs;
|
|
143
160
|
store.db.prepare("DELETE FROM file_stats WHERE repo_id = ?").run(repoId);
|
|
144
|
-
|
|
161
|
+
// The live docs' durable convergence signal, keyed by path (one query).
|
|
162
|
+
const live = new Map();
|
|
163
|
+
for (const row of store.db
|
|
164
|
+
.prepare("SELECT path, file_hash FROM docs WHERE repo_id = ? AND deleted_commit IS NULL AND file_hash IS NOT NULL")
|
|
165
|
+
.all(repoId)) {
|
|
166
|
+
live.set(row.path, row.file_hash);
|
|
167
|
+
}
|
|
168
|
+
const paths = fs.walk(rootPath);
|
|
145
169
|
for (const path of paths) {
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
170
|
+
const want = live.get(path);
|
|
171
|
+
if (!want)
|
|
172
|
+
continue; // no live doc: nothing the cache may vouch for
|
|
173
|
+
const bytes = fs.read(rootPath, path);
|
|
174
|
+
if (bytes === null)
|
|
175
|
+
continue; // vanished between the walk and the read
|
|
176
|
+
const hash = sha256(bytes);
|
|
177
|
+
if (!hash.equals(want))
|
|
178
|
+
continue; // a pending edit: leave it visible
|
|
179
|
+
recordStat(store, repoId, path, fs.stat(rootPath, path), hash);
|
|
149
180
|
}
|
|
150
181
|
return paths.length;
|
|
151
182
|
}
|
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
/** Walk `root` for *.md files, returning repo-relative canonical paths. */
|
|
2
2
|
export declare function walkMarkdown(root: string): string[];
|
|
3
|
+
/** The two stat fields the freshness cache keys on (spec/sync §4.3). */
|
|
4
|
+
export interface FileStat {
|
|
5
|
+
mtimeNs: bigint;
|
|
6
|
+
size: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* The filesystem as the sync paths see it — a walk, a stat and a read, all
|
|
10
|
+
* keyed by (root, repo-relative path). `nodeFs` is the production default.
|
|
11
|
+
*/
|
|
12
|
+
export interface SyncFs {
|
|
13
|
+
/** spec/sync §4.2: every `*.md` under `root`, repo-relative, `/` separators, readdir order, depth-first. */
|
|
14
|
+
walk(root: string): string[];
|
|
15
|
+
/** `(mtime_ns, size)` of `root/path`, or null when the file is absent. */
|
|
16
|
+
stat(root: string, path: string): FileStat | null;
|
|
17
|
+
/** The UTF-8 bytes of `root/path`, or null when the file is absent. */
|
|
18
|
+
read(root: string, path: string): string | null;
|
|
19
|
+
}
|
|
20
|
+
/** The `node:fs` filesystem. */
|
|
21
|
+
export declare const nodeFs: SyncFs;
|
|
3
22
|
/**
|
|
4
23
|
* Async variant of {@link walkMarkdown}: identical result, but yields to the
|
|
5
24
|
* event loop between directories and invokes `onFound(count)` as matches
|
package/dist/src/sync/fs-util.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
-
import { readdirSync, statSync } from "node:fs";
|
|
1
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
2
|
import { readdir, stat } from "node:fs/promises";
|
|
3
3
|
import { join, relative, sep } from "node:path";
|
|
4
4
|
// Shared filesystem walk for the in-process one-shot/reconcile paths (attach,
|
|
5
5
|
// checkpoint, freshness). This is NOT the live watcher — chokidar lives only in
|
|
6
6
|
// the external @omgbase/fs-adapter (sync-plugins). node:fs (a builtin) reads
|
|
7
7
|
// are retained here for the one-shot ingest/freshness fast-path.
|
|
8
|
+
//
|
|
9
|
+
// `SyncFs` is the seam those paths read the filesystem through (spec/sync §4,
|
|
10
|
+
// §8): the sweep, the checkpoint and recovery take an optional `fs` and default
|
|
11
|
+
// to `nodeFs`, so a conformance runner can supply an in-memory filesystem with
|
|
12
|
+
// explicit `mtime_ns` and the fixtures stay free of real I/O.
|
|
8
13
|
const IGNORED_DIRS = new Set([".omgbase", ".git", "node_modules"]);
|
|
9
14
|
/** Walk `root` for *.md files, returning repo-relative canonical paths. */
|
|
10
15
|
export function walkMarkdown(root) {
|
|
@@ -23,6 +28,24 @@ export function walkMarkdown(root) {
|
|
|
23
28
|
recur(root);
|
|
24
29
|
return out;
|
|
25
30
|
}
|
|
31
|
+
/** The `node:fs` filesystem. */
|
|
32
|
+
export const nodeFs = {
|
|
33
|
+
walk: walkMarkdown,
|
|
34
|
+
stat(root, path) {
|
|
35
|
+
const st = statSync(join(root, path), { bigint: true, throwIfNoEntry: false });
|
|
36
|
+
return st ? { mtimeNs: st.mtimeNs, size: Number(st.size) } : null;
|
|
37
|
+
},
|
|
38
|
+
read(root, path) {
|
|
39
|
+
try {
|
|
40
|
+
return readFileSync(join(root, path), "utf8");
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
if (err.code === "ENOENT")
|
|
44
|
+
return null;
|
|
45
|
+
throw err;
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
};
|
|
26
49
|
/**
|
|
27
50
|
* Async variant of {@link walkMarkdown}: identical result, but yields to the
|
|
28
51
|
* event loop between directories and invokes `onFound(count)` as matches
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Store } from "../core/store/store.js";
|
|
2
|
+
import { type ReconcileConfig } from "../reconcile/types.js";
|
|
2
3
|
export interface ObserveResult {
|
|
3
4
|
docId: string;
|
|
4
5
|
path: string;
|
|
@@ -25,11 +26,56 @@ export interface ObserveOneResult extends ObserveResult {
|
|
|
25
26
|
/** sha256(content) (hex) — the new file hash. */
|
|
26
27
|
newHashHex: string;
|
|
27
28
|
}
|
|
29
|
+
/** One member of a batch: its current bytes, or `null` when it left the source
|
|
30
|
+
* scope (deleted on disk / adapter fetch returned null). */
|
|
31
|
+
export interface BatchItem {
|
|
32
|
+
path: string;
|
|
33
|
+
content: string | null;
|
|
34
|
+
}
|
|
35
|
+
/** Outcome of one batch member: an observation (echo or commit) or a deletion. */
|
|
36
|
+
export type BatchOutcome = ({
|
|
37
|
+
kind: "observed";
|
|
38
|
+
} & ObserveOneResult) | {
|
|
39
|
+
kind: "deleted";
|
|
40
|
+
path: string;
|
|
41
|
+
/** the tombstoned doc, or null when no live doc existed at `path` (no-op). */
|
|
42
|
+
docId: string | null;
|
|
43
|
+
oldHashHex: string | null;
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* The batch reconcile primitive — one checkpoint's worth of members, in order.
|
|
47
|
+
* Two passes with the cross-document phase between them (reconciliation-spec §8):
|
|
48
|
+
*
|
|
49
|
+
* 1. every member is echo-gated and, when it differs from the stored revision,
|
|
50
|
+
* parsed + reconciled against its own prior tree WITHOUT committing
|
|
51
|
+
* (`prepareReconcile`; members gone from the source contribute their whole
|
|
52
|
+
* live tree as deleted). All members see one resurrection-pool snapshot with
|
|
53
|
+
* a shared consumed-set, so no pooled id resurrects twice in a batch.
|
|
54
|
+
* 2. the per-doc leftovers are pooled — deleted old blocks × inserted new
|
|
55
|
+
* blocks, across documents only — and matched at theta_xdoc
|
|
56
|
+
* (`crossDocMatch`). Each accepted pair is applied to both results: the
|
|
57
|
+
* destination's minted id becomes the carried id (`moved`/`edited_moved`,
|
|
58
|
+
* `detail.fromDoc`), the source's `deleted` disposition and pool entry for
|
|
59
|
+
* it vanish (`applyCrossDocMatches`).
|
|
60
|
+
* 3. every member commits, in batch order, each in its own transaction:
|
|
61
|
+
* `ingestFile` with the finished result (a carried-in id evicts the row the
|
|
62
|
+
* source doc may still hold — see `IdResolver.crossDocIds` — so the order
|
|
63
|
+
* of A and B in the batch does not matter), or `tombstoneObservedDeletion`.
|
|
64
|
+
*
|
|
65
|
+
* A moved block is therefore never `deleted` + `inserted`, never pooled, and
|
|
66
|
+
* never `resurrected` inside one batch. Across batches the pool still does that
|
|
67
|
+
* job (spec/reconcile §5 phase 6b). Does NOT sweep the pool — the caller does,
|
|
68
|
+
* once per batch. Does NOT write a file.
|
|
69
|
+
*/
|
|
70
|
+
export declare function observeBatch(store: Store, repoId: string, items: BatchItem[], ts: string, opts?: {
|
|
71
|
+
config?: ReconcileConfig;
|
|
72
|
+
}): BatchOutcome[];
|
|
28
73
|
/**
|
|
29
|
-
* The reconcile primitive: echo-gate `content` for `path` and,
|
|
30
|
-
* from the stored revision, ingest it as an observed commit
|
|
31
|
-
* identity against the prior revision).
|
|
32
|
-
*
|
|
74
|
+
* The single-member reconcile primitive: echo-gate `content` for `path` and,
|
|
75
|
+
* when it differs from the stored revision, ingest it as an observed commit
|
|
76
|
+
* (reconciling block identity against the prior revision). A batch of one — so
|
|
77
|
+
* no cross-document matching; use `observeBatch` for that. Does NOT sweep the
|
|
78
|
+
* resurrection pool — the caller does that once per batch. Does NOT write a file.
|
|
33
79
|
*/
|
|
34
80
|
export declare function observeOne(store: Store, repoId: string, path: string, content: string, ts: string): ObserveOneResult;
|
|
35
81
|
/** Observe one file's authoritative bytes as an observed commit (echo-suppressed). */
|
|
@@ -37,7 +83,9 @@ export declare function observeFile(store: Store, repoId: string, path: string,
|
|
|
37
83
|
ts?: string;
|
|
38
84
|
}): ObserveResult;
|
|
39
85
|
/** Observe a batch of files under one timestamp + one resurrection sweep. The
|
|
40
|
-
* batch form of `observeFile`; each item is echo-gated independently
|
|
86
|
+
* batch form of `observeFile`; each item is echo-gated independently, and a
|
|
87
|
+
* block cut from one file and pasted into another in the same batch keeps its
|
|
88
|
+
* id (`moved`/`edited_moved`) via the cross-document phase of `observeBatch`. */
|
|
41
89
|
export declare function observeMany(store: Store, repoId: string, items: {
|
|
42
90
|
path: string;
|
|
43
91
|
content: string;
|
package/dist/src/sync/observe.js
CHANGED
|
@@ -1,31 +1,151 @@
|
|
|
1
1
|
import { sha256 } from "../core/hash.js";
|
|
2
2
|
import { ingestFile } from "../core/ingest.js";
|
|
3
|
-
import { makeReconcilingResolver } from "./reconciling-ingest.js";
|
|
3
|
+
import { makeReconcilingResolver, prepareReconcile, loadOldMatchBlocks, loadPool } from "./reconciling-ingest.js";
|
|
4
4
|
import { hasConflictMarkers } from "./git-heuristics.js";
|
|
5
5
|
import { tombstoneObservedDeletion } from "./tombstone.js";
|
|
6
6
|
import { sweepResurrectionPool } from "../core/store/gc.js";
|
|
7
|
+
import { crossDocMatch, applyCrossDocMatches } from "../reconcile/crossdoc.js";
|
|
8
|
+
import { DEFAULT_CONFIG } from "../reconcile/types.js";
|
|
7
9
|
/**
|
|
8
|
-
* The reconcile primitive
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* The batch reconcile primitive — one checkpoint's worth of members, in order.
|
|
11
|
+
* Two passes with the cross-document phase between them (reconciliation-spec §8):
|
|
12
|
+
*
|
|
13
|
+
* 1. every member is echo-gated and, when it differs from the stored revision,
|
|
14
|
+
* parsed + reconciled against its own prior tree WITHOUT committing
|
|
15
|
+
* (`prepareReconcile`; members gone from the source contribute their whole
|
|
16
|
+
* live tree as deleted). All members see one resurrection-pool snapshot with
|
|
17
|
+
* a shared consumed-set, so no pooled id resurrects twice in a batch.
|
|
18
|
+
* 2. the per-doc leftovers are pooled — deleted old blocks × inserted new
|
|
19
|
+
* blocks, across documents only — and matched at theta_xdoc
|
|
20
|
+
* (`crossDocMatch`). Each accepted pair is applied to both results: the
|
|
21
|
+
* destination's minted id becomes the carried id (`moved`/`edited_moved`,
|
|
22
|
+
* `detail.fromDoc`), the source's `deleted` disposition and pool entry for
|
|
23
|
+
* it vanish (`applyCrossDocMatches`).
|
|
24
|
+
* 3. every member commits, in batch order, each in its own transaction:
|
|
25
|
+
* `ingestFile` with the finished result (a carried-in id evicts the row the
|
|
26
|
+
* source doc may still hold — see `IdResolver.crossDocIds` — so the order
|
|
27
|
+
* of A and B in the batch does not matter), or `tombstoneObservedDeletion`.
|
|
28
|
+
*
|
|
29
|
+
* A moved block is therefore never `deleted` + `inserted`, never pooled, and
|
|
30
|
+
* never `resurrected` inside one batch. Across batches the pool still does that
|
|
31
|
+
* job (spec/reconcile §5 phase 6b). Does NOT sweep the pool — the caller does,
|
|
32
|
+
* once per batch. Does NOT write a file.
|
|
12
33
|
*/
|
|
13
|
-
export function
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
34
|
+
export function observeBatch(store, repoId, items, ts, opts = {}) {
|
|
35
|
+
const db = store.db;
|
|
36
|
+
const config = opts.config ?? DEFAULT_CONFIG;
|
|
37
|
+
const liveDoc = db.prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_commit IS NULL");
|
|
38
|
+
// ---- pass 1: echo gate + reconcile, no commits -------------------------------
|
|
39
|
+
const pool = loadPool(db, repoId, ts);
|
|
40
|
+
const consumed = new Set();
|
|
41
|
+
const pending = [];
|
|
42
|
+
for (const it of items) {
|
|
43
|
+
const existing = liveDoc.get(repoId, it.path);
|
|
44
|
+
const oldHashHex = existing?.file_hash?.toString("hex") ?? null;
|
|
45
|
+
if (it.content === null) {
|
|
46
|
+
const oldBlocks = existing ? loadOldMatchBlocks(db, existing.doc_id) : [];
|
|
47
|
+
const result = existing
|
|
48
|
+
? { assignment: new Map(), dispositions: [], deleted: oldBlocks.map((b) => b.blockId), consumedPool: [] }
|
|
49
|
+
: null;
|
|
50
|
+
pending.push({ kind: "gone", path: it.path, docId: existing?.doc_id ?? null, oldHashHex, oldBlocks, result });
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const diskHash = sha256(it.content);
|
|
54
|
+
const newHashHex = diskHash.toString("hex");
|
|
55
|
+
// Echo: the stored revision already equals these bytes → no commit. The
|
|
56
|
+
// loop-breaker the coordinator relies on (§6): re-observing what the engine
|
|
57
|
+
// already holds is a no-op.
|
|
58
|
+
if (existing && existing.file_hash && existing.file_hash.equals(diskHash)) {
|
|
59
|
+
pending.push({
|
|
60
|
+
kind: "echo",
|
|
61
|
+
outcome: { kind: "observed", docId: existing.doc_id, path: it.path, rev: null, commitId: null, converged: true, echo: true, conflicted: false, dispositions: [], oldHashHex, newHashHex },
|
|
62
|
+
});
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
const prepared = prepareReconcile(store, repoId, it.path, it.content, { ts, config, pool, consumed });
|
|
66
|
+
pending.push({ kind: "ingest", prepared, oldHashHex, newHashHex });
|
|
67
|
+
}
|
|
68
|
+
// ---- cross-document phase (spec/reconcile §7) --------------------------------
|
|
69
|
+
if (pending.length > 1)
|
|
70
|
+
crossDocPhase(pending, config);
|
|
71
|
+
// ---- pass 2: commit, in batch order ------------------------------------------
|
|
72
|
+
const out = [];
|
|
73
|
+
for (const p of pending) {
|
|
74
|
+
if (p.kind === "echo") {
|
|
75
|
+
out.push(p.outcome);
|
|
76
|
+
}
|
|
77
|
+
else if (p.kind === "gone") {
|
|
78
|
+
if (p.docId)
|
|
79
|
+
tombstoneObservedDeletion(store, repoId, p.docId, ts);
|
|
80
|
+
out.push({ kind: "deleted", path: p.path, docId: p.docId, oldHashHex: p.oldHashHex });
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
out.push({ kind: "observed", ...commitPrepared(store, repoId, p.prepared, ts, p.oldHashHex, p.newHashHex) });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
// Pool each reconciled member's leftovers (README §7: deleted = old blocks whose
|
|
89
|
+
// ids are in `deleted`, in that order; inserted = new blocks whose disposition
|
|
90
|
+
// is `inserted`, with the minted id, in disposition order), match across
|
|
91
|
+
// documents, and apply. Members are keyed by doc id; a brand-new path has none
|
|
92
|
+
// yet, so it is keyed by path — it can only ever be a destination, and
|
|
93
|
+
// `detail.fromDoc` is always a real doc id.
|
|
94
|
+
function crossDocPhase(pending, config) {
|
|
95
|
+
const docs = [];
|
|
96
|
+
const results = new Map();
|
|
97
|
+
const prepared = new Map();
|
|
98
|
+
for (const p of pending) {
|
|
99
|
+
if (p.kind === "echo")
|
|
100
|
+
continue;
|
|
101
|
+
if (p.kind === "gone") {
|
|
102
|
+
if (!p.docId || !p.result)
|
|
103
|
+
continue;
|
|
104
|
+
docs.push({ docId: p.docId, deleted: p.oldBlocks.map((block) => ({ block })), inserted: [] });
|
|
105
|
+
results.set(p.docId, p.result);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const key = p.prepared.docId ?? `new:${p.prepared.path}`;
|
|
109
|
+
const oldById = new Map(p.prepared.oldBlocks.map((b) => [b.blockId, b]));
|
|
110
|
+
const newByKey = new Map(p.prepared.newBlocks.map((b) => [b.key, b]));
|
|
111
|
+
const keyOfId = new Map();
|
|
112
|
+
for (const [k, id] of p.prepared.result.assignment)
|
|
113
|
+
keyOfId.set(id, k);
|
|
114
|
+
const deleted = [];
|
|
115
|
+
for (const id of p.prepared.result.deleted) {
|
|
116
|
+
const block = oldById.get(id);
|
|
117
|
+
if (block)
|
|
118
|
+
deleted.push({ block });
|
|
119
|
+
}
|
|
120
|
+
const inserted = [];
|
|
121
|
+
for (const d of p.prepared.result.dispositions) {
|
|
122
|
+
if (d.kind !== "inserted")
|
|
123
|
+
continue;
|
|
124
|
+
const k = keyOfId.get(d.blockId);
|
|
125
|
+
const block = k ? newByKey.get(k) : undefined;
|
|
126
|
+
if (block)
|
|
127
|
+
inserted.push({ block, mintedId: d.blockId });
|
|
128
|
+
}
|
|
129
|
+
docs.push({ docId: key, deleted, inserted });
|
|
130
|
+
results.set(key, p.prepared.result);
|
|
131
|
+
prepared.set(key, p.prepared);
|
|
25
132
|
}
|
|
133
|
+
if (docs.length < 2)
|
|
134
|
+
return;
|
|
135
|
+
const matches = crossDocMatch(docs, config);
|
|
136
|
+
if (matches.length === 0)
|
|
137
|
+
return;
|
|
138
|
+
applyCrossDocMatches(results, matches, config.matcherV);
|
|
139
|
+
for (const m of matches)
|
|
140
|
+
prepared.get(m.toDoc)?.crossDocIds.push(m.carriedId);
|
|
141
|
+
}
|
|
142
|
+
// Pass 2 for one reconciled member: commit the finished result as an observed
|
|
143
|
+
// revision (one transaction), flag conflict markers, summarize dispositions.
|
|
144
|
+
function commitPrepared(store, repoId, prepared, ts, oldHashHex, newHashHex) {
|
|
145
|
+
const { path, content } = prepared;
|
|
26
146
|
const conflicted = hasConflictMarkers(content);
|
|
27
|
-
const resolveIds = makeReconcilingResolver(store, repoId, { ts, path });
|
|
28
|
-
const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds });
|
|
147
|
+
const resolveIds = makeReconcilingResolver(store, repoId, { ts, path, prepared });
|
|
148
|
+
const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds, parsed: prepared.tree });
|
|
29
149
|
store.db.prepare("UPDATE docs SET conflicted = ? WHERE repo_id = ? AND path = ?").run(conflicted ? 1 : 0, repoId, path);
|
|
30
150
|
const rows = store.db
|
|
31
151
|
.prepare("SELECT kind, count(*) n FROM dispositions WHERE commit_id = ? GROUP BY kind ORDER BY kind")
|
|
@@ -43,6 +163,20 @@ export function observeOne(store, repoId, path, content, ts) {
|
|
|
43
163
|
newHashHex,
|
|
44
164
|
};
|
|
45
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* The single-member reconcile primitive: echo-gate `content` for `path` and,
|
|
168
|
+
* when it differs from the stored revision, ingest it as an observed commit
|
|
169
|
+
* (reconciling block identity against the prior revision). A batch of one — so
|
|
170
|
+
* no cross-document matching; use `observeBatch` for that. Does NOT sweep the
|
|
171
|
+
* resurrection pool — the caller does that once per batch. Does NOT write a file.
|
|
172
|
+
*/
|
|
173
|
+
export function observeOne(store, repoId, path, content, ts) {
|
|
174
|
+
const [outcome] = observeBatch(store, repoId, [{ path, content }], ts);
|
|
175
|
+
if (!outcome || outcome.kind !== "observed")
|
|
176
|
+
throw new Error(`observeOne: unexpected outcome for ${path}`);
|
|
177
|
+
const { kind: _k, ...rest } = outcome;
|
|
178
|
+
return rest;
|
|
179
|
+
}
|
|
46
180
|
function toPublic(r) {
|
|
47
181
|
const { oldHashHex: _o, newHashHex: _n, ...pub } = r;
|
|
48
182
|
return pub;
|
|
@@ -55,10 +189,17 @@ export function observeFile(store, repoId, path, content, opts = {}) {
|
|
|
55
189
|
return toPublic(r);
|
|
56
190
|
}
|
|
57
191
|
/** Observe a batch of files under one timestamp + one resurrection sweep. The
|
|
58
|
-
* batch form of `observeFile`; each item is echo-gated independently
|
|
192
|
+
* batch form of `observeFile`; each item is echo-gated independently, and a
|
|
193
|
+
* block cut from one file and pasted into another in the same batch keeps its
|
|
194
|
+
* id (`moved`/`edited_moved`) via the cross-document phase of `observeBatch`. */
|
|
59
195
|
export function observeMany(store, repoId, items, opts = {}) {
|
|
60
196
|
const ts = opts.ts ?? new Date().toISOString();
|
|
61
|
-
const out =
|
|
197
|
+
const out = observeBatch(store, repoId, items, ts).map((o) => {
|
|
198
|
+
if (o.kind !== "observed")
|
|
199
|
+
throw new Error(`observeMany: unexpected outcome for ${o.path}`);
|
|
200
|
+
const { kind: _k, ...rest } = o;
|
|
201
|
+
return toPublic(rest);
|
|
202
|
+
});
|
|
62
203
|
sweepResurrectionPool(store, ts);
|
|
63
204
|
return out;
|
|
64
205
|
}
|