@omgbase/core 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +172 -0
  2. package/dist/src/cli/eval-matcher.d.ts +6 -1
  3. package/dist/src/cli/eval-matcher.js +14 -2
  4. package/dist/src/core/clock.d.ts +9 -0
  5. package/dist/src/core/clock.js +46 -0
  6. package/dist/src/core/hash.d.ts +25 -1
  7. package/dist/src/core/hash.js +110 -10
  8. package/dist/src/core/ids.d.ts +12 -0
  9. package/dist/src/core/ids.js +43 -1
  10. package/dist/src/core/ingest.d.ts +22 -2
  11. package/dist/src/core/ingest.js +92 -18
  12. package/dist/src/core/parse/parse.d.ts +1 -1
  13. package/dist/src/core/parse/parse.js +31 -9
  14. package/dist/src/core/store/nodes.d.ts +13 -0
  15. package/dist/src/core/store/nodes.js +24 -0
  16. package/dist/src/core/store/writers.d.ts +6 -0
  17. package/dist/src/core/store/writers.js +1 -0
  18. package/dist/src/core/utf8.d.ts +29 -0
  19. package/dist/src/core/utf8.js +96 -0
  20. package/dist/src/format/adapter.d.ts +7 -1
  21. package/dist/src/format/markdown.js +36 -41
  22. package/dist/src/graph/extract.d.ts +41 -1
  23. package/dist/src/graph/extract.js +59 -1
  24. package/dist/src/graph/history.d.ts +39 -1
  25. package/dist/src/graph/history.js +154 -20
  26. package/dist/src/index.d.ts +10 -6
  27. package/dist/src/index.js +11 -6
  28. package/dist/src/mcp/graph.js +8 -4
  29. package/dist/src/mcp/reference.d.ts +1 -1
  30. package/dist/src/mcp/reference.js +17 -6
  31. package/dist/src/mcp/server.js +15 -10
  32. package/dist/src/mutate/apply.d.ts +9 -0
  33. package/dist/src/mutate/apply.js +24 -11
  34. package/dist/src/mutate/docs.d.ts +2 -0
  35. package/dist/src/mutate/docs.js +10 -6
  36. package/dist/src/mutate/known-ids.js +11 -3
  37. package/dist/src/mutate/macros.js +9 -2
  38. package/dist/src/mutate/ops.d.ts +5 -4
  39. package/dist/src/mutate/ops.js +62 -15
  40. package/dist/src/mutate/plan-update.d.ts +7 -1
  41. package/dist/src/mutate/plan-update.js +3 -1
  42. package/dist/src/oqx-js/context.js +57 -7
  43. package/dist/src/reconcile/eval/generator.d.ts +24 -1
  44. package/dist/src/reconcile/eval/generator.js +248 -0
  45. package/dist/src/reconcile/eval/metrics.d.ts +33 -9
  46. package/dist/src/reconcile/eval/metrics.js +55 -32
  47. package/dist/src/reconcile/flatten.js +7 -5
  48. package/dist/src/reconcile/phase5.d.ts +2 -1
  49. package/dist/src/reconcile/phase5.js +27 -10
  50. package/dist/src/reconcile/phase6a.js +17 -5
  51. package/dist/src/reconcile/phases.d.ts +44 -6
  52. package/dist/src/reconcile/phases.js +0 -0
  53. package/dist/src/reconcile/reconcile.js +16 -9
  54. package/dist/src/reconcile/types.d.ts +4 -1
  55. package/dist/src/reconcile/types.js +11 -2
  56. package/dist/src/search/external.d.ts +10 -0
  57. package/dist/src/search/external.js +18 -1
  58. package/dist/src/search/provider.d.ts +7 -3
  59. package/dist/src/search/provider.js +8 -7
  60. package/dist/src/search/rrf.js +4 -0
  61. package/dist/src/search/tasks.d.ts +28 -0
  62. package/dist/src/search/tasks.js +36 -23
  63. package/dist/src/search/vector.js +30 -14
  64. package/dist/src/sync/attach.d.ts +7 -2
  65. package/dist/src/sync/attach.js +10 -8
  66. package/dist/src/sync/checkpoint.d.ts +18 -1
  67. package/dist/src/sync/checkpoint.js +31 -29
  68. package/dist/src/sync/driver.d.ts +15 -4
  69. package/dist/src/sync/driver.js +18 -43
  70. package/dist/src/sync/external-source.js +7 -1
  71. package/dist/src/sync/freshness.d.ts +73 -15
  72. package/dist/src/sync/freshness.js +128 -97
  73. package/dist/src/sync/fs-util.d.ts +19 -0
  74. package/dist/src/sync/fs-util.js +24 -1
  75. package/dist/src/sync/observe.d.ts +53 -5
  76. package/dist/src/sync/observe.js +162 -21
  77. package/dist/src/sync/reconciling-ingest.d.ts +42 -2
  78. package/dist/src/sync/reconciling-ingest.js +87 -13
  79. package/dist/src/sync/recovery.d.ts +5 -1
  80. package/dist/src/sync/recovery.js +8 -8
  81. package/dist/src/sync/workspace.d.ts +11 -3
  82. package/dist/src/sync/workspace.js +39 -30
  83. package/dist/src/sync/writer-lock.js +16 -26
  84. package/package.json +3 -3
@@ -1,101 +1,121 @@
1
- import { statSync, readFileSync, existsSync } from "node:fs";
2
- import { join } from "node:path";
3
1
  import { sha256 } from "../core/hash.js";
4
2
  import { processCheckpoint } from "./checkpoint.js";
5
- import { walkMarkdown } from "./fs-util.js";
6
- /** Record/refresh a file's stat cache row. Call after any ingest of the file. */
7
- export function recordFileStat(store, repoId, path, abs, hash) {
8
- if (!existsSync(abs)) {
3
+ import { nodeFs } from "./fs-util.js";
4
+ /** Step 1 of §4.3 alone: the stat-mismatched or new paths, without hashing anything. */
5
+ export function sweepCandidates(cache, snapshot) {
6
+ const out = [];
7
+ for (const e of snapshot) {
8
+ const cached = cache.get(e.path);
9
+ if (!cached || cached.mtime_ns !== e.mtime_ns || cached.size !== e.size)
10
+ out.push(e.path);
11
+ }
12
+ return out;
13
+ }
14
+ /** spec/sync §4.3 steps 1–2 as a pure function (README §8 `sweep_plan`). */
15
+ export function sweepPlan(cache, snapshot) {
16
+ const byPath = new Map(snapshot.map((e) => [e.path, e]));
17
+ const candidates = sweepCandidates(cache, snapshot);
18
+ const deletions = [...cache.keys()].filter((p) => !byPath.has(p));
19
+ const changed = [];
20
+ const refreshed = [];
21
+ const hashes = new Map();
22
+ for (const path of candidates) {
23
+ const hash = byPath.get(path).hash();
24
+ hashes.set(path, hash);
25
+ const cached = cache.get(path);
26
+ if (!cached || !cached.hash.equals(hash))
27
+ changed.push(path);
28
+ else
29
+ refreshed.push(path);
30
+ }
31
+ return { candidates, changed, deletions, refreshed, hashes };
32
+ }
33
+ /** The repo's `file_stats` rows keyed by path. Read with safe integers: an `mtime_ns` is a
34
+ * 64-bit count a JS number cannot hold exactly, and a rounded value would never equal a stat. */
35
+ export function loadStatCache(store, repoId) {
36
+ const cache = new Map();
37
+ const rows = store.db.prepare("SELECT path, mtime_ns, size, hash FROM file_stats WHERE repo_id = ?").safeIntegers(true).all(repoId);
38
+ for (const row of rows)
39
+ cache.set(row.path, { path: row.path, mtime_ns: row.mtime_ns, size: Number(row.size), hash: row.hash });
40
+ return cache;
41
+ }
42
+ /** The current filesystem snapshot: the §4.2 walk with each file's stat and a lazy hash. */
43
+ export function snapshotOf(fs, rootPath) {
44
+ const out = [];
45
+ for (const path of fs.walk(rootPath)) {
46
+ const st = fs.stat(rootPath, path);
47
+ if (!st)
48
+ continue; // vanished between the walk and the stat
49
+ let memo = null;
50
+ out.push({
51
+ path,
52
+ mtime_ns: st.mtimeNs,
53
+ size: st.size,
54
+ hash: () => (memo ??= sha256(fs.read(rootPath, path) ?? "")),
55
+ });
56
+ }
57
+ return out;
58
+ }
59
+ /** Write a file's stat cache row from a stat (`INSERT … ON CONFLICT DO UPDATE`); a null stat (the
60
+ * file vanished) deletes the row. The DB half of `record_file_stat` (spec/sync §4.3 step 4). */
61
+ export function recordStat(store, repoId, path, st, hash) {
62
+ if (!st) {
9
63
  store.db.prepare("DELETE FROM file_stats WHERE repo_id = ? AND path = ?").run(repoId, path);
10
64
  return;
11
65
  }
12
- const st = statSync(abs, { bigint: true });
13
66
  store.db
14
67
  .prepare(`INSERT INTO file_stats (repo_id, path, mtime_ns, size, hash) VALUES (?, ?, ?, ?, ?)
15
68
  ON CONFLICT(repo_id, path) DO UPDATE SET mtime_ns = excluded.mtime_ns, size = excluded.size, hash = excluded.hash`)
16
- .run(repoId, path, st.mtimeNs.toString(), Number(st.size), hash);
69
+ .run(repoId, path, st.mtimeNs, st.size, hash);
70
+ }
71
+ /** Record/refresh a file's stat cache row from its absolute path. Call after any ingest of the file. */
72
+ export function recordFileStat(store, repoId, path, abs, hash) {
73
+ recordStat(store, repoId, path, nodeFs.stat(abs, ""), hash);
17
74
  }
18
75
  /**
19
- * Run the freshness sweep for a repo. Returns the checkpoint result plus scan
20
- * counters. Idempotent; safe alongside a live watcher (hash-based echo
21
- * suppression makes a double ingest a no-op).
76
+ * Run the freshness sweep for a repo (spec/sync §4.3). Returns the checkpoint
77
+ * result plus scan counters. Idempotent; safe alongside a live watcher
78
+ * (hash-based echo suppression makes a double ingest a no-op). `opts.ts` pins
79
+ * the batch timestamp (default now); `opts.fs` the filesystem seam.
22
80
  */
23
- export function freshnessSweep(store, repoId, rootPath) {
24
- const cache = new Map();
25
- for (const row of store.db
26
- .prepare("SELECT path, mtime_ns, size, hash FROM file_stats WHERE repo_id = ?")
27
- .all(repoId)) {
28
- cache.set(row.path, { mtime_ns: BigInt(row.mtime_ns), size: row.size, hash: row.hash });
81
+ export function freshnessSweep(store, repoId, rootPath, opts = {}) {
82
+ const fs = opts.fs ?? nodeFs;
83
+ const cache = loadStatCache(store, repoId);
84
+ const snapshot = snapshotOf(fs, rootPath);
85
+ const plan = sweepPlan(cache, snapshot);
86
+ const statOf = new Map(snapshot.map((e) => [e.path, { mtimeNs: e.mtime_ns, size: e.size }]));
87
+ // Content unchanged; refresh the stat cache so we don't re-hash next time.
88
+ for (const path of plan.refreshed) {
89
+ const st = statOf.get(path);
90
+ store.db.prepare("UPDATE file_stats SET mtime_ns = ?, size = ? WHERE repo_id = ? AND path = ?").run(st.mtimeNs, st.size, repoId, path);
29
91
  }
30
- const paths = walkMarkdown(rootPath);
31
- const seen = new Set(paths);
32
- // Candidates: files whose (mtime_ns, size) differs from cache, or are new.
33
- const candidates = [];
34
- const freshStats = new Map();
35
- for (const path of paths) {
36
- const st = statSync(join(rootPath, path), { bigint: true });
37
- freshStats.set(path, { mtimeNs: st.mtimeNs, size: Number(st.size) });
38
- const cached = cache.get(path);
39
- if (!cached || cached.mtime_ns !== st.mtimeNs || cached.size !== Number(st.size)) {
40
- candidates.push(path);
41
- }
42
- }
43
- // Deletions: cached paths no longer on disk.
44
- const deletedPaths = [...cache.keys()].filter((p) => !seen.has(p));
45
- // Hash candidates; only those whose content hash actually differs from the
46
- // cached hash need ingesting (mtime can change without content changing).
47
- const changedPaths = [];
48
- const candHashes = new Map();
49
- for (const path of candidates) {
50
- const hash = sha256(readFileSync(join(rootPath, path), "utf8"));
51
- candHashes.set(path, hash);
52
- const cached = cache.get(path);
53
- if (!cached || !cached.hash.equals(hash))
54
- changedPaths.push(path);
55
- else {
56
- // Content unchanged; refresh the stat cache so we don't re-hash next time.
57
- const fs = freshStats.get(path);
58
- store.db
59
- .prepare("UPDATE file_stats SET mtime_ns = ?, size = ? WHERE repo_id = ? AND path = ?")
60
- .run(fs.mtimeNs.toString(), fs.size, repoId, path);
61
- }
62
- }
63
- const toIngest = [...changedPaths, ...deletedPaths];
64
- const result = processCheckpoint(store, repoId, rootPath, toIngest.map((path) => ({ path })));
92
+ const result = processCheckpoint(store, repoId, rootPath, [...plan.changed, ...plan.deletions].map((path) => ({ path })), { fs, ...(opts.ts !== undefined ? { ts: opts.ts } : {}), ...(opts.gitHead !== undefined ? { gitHead: opts.gitHead } : {}) });
65
93
  // Refresh stat cache for everything we touched (ingested or echo-suppressed).
66
- for (const path of changedPaths) {
67
- recordFileStat(store, repoId, path, join(rootPath, path), candHashes.get(path));
68
- }
69
- for (const path of deletedPaths) {
94
+ for (const path of plan.changed)
95
+ recordStat(store, repoId, path, fs.stat(rootPath, path), plan.hashes.get(path));
96
+ for (const path of plan.deletions)
70
97
  store.db.prepare("DELETE FROM file_stats WHERE repo_id = ? AND path = ?").run(repoId, path);
71
- }
72
98
  return {
73
99
  ...result,
74
- scanned: paths.length,
75
- candidates: candidates.length,
100
+ scanned: snapshot.length,
101
+ candidates: plan.candidates.length,
76
102
  changed: result.ingested.length > 0 || result.deleted.length > 0 || result.conflicted.length > 0,
77
103
  };
78
104
  }
79
105
  /**
80
- * READ-ONLY disk-drift detector. Mirrors freshnessSweep's staged cheap approach
81
- * — stat-compare against file_stats first, hash only the stat-mismatched
82
- * candidates — but never ingests, commits, or touches file_stats. It answers a
83
- * single question: does the DB still agree with what's on disk right now?
106
+ * READ-ONLY disk-drift detector (spec/sync §4.3 "Disk drift"). Mirrors
107
+ * freshnessSweep's staged cheap approach — the same `sweepCandidates` over the
108
+ * same cache and snapshot, hashing only the stat-mismatched candidates — but
109
+ * never ingests, commits, or touches file_stats. It answers a single question:
110
+ * does the DB still agree with what's on disk right now?
84
111
  *
85
112
  * `changed` is decided by comparing a candidate's content hash to the doc's
86
113
  * stored `file_hash` (the same sha256(bytes) checkpoint writes), so a stat
87
- * change without a content change is NOT counted as drift. At the envelope
88
- * (≤10⁴ docs) the clean case is a directory walk plus stats (tens of ms);
89
- * hashing is bounded to files whose (mtime_ns, size) moved.
114
+ * change without a content change is NOT counted as drift.
90
115
  */
91
- export function detectDiskDrift(store, repoId, rootPath) {
92
- // Stat cache keyed by path, for cheap change detection.
93
- const cache = new Map();
94
- for (const row of store.db
95
- .prepare("SELECT path, mtime_ns, size, hash FROM file_stats WHERE repo_id = ?")
96
- .all(repoId)) {
97
- cache.set(row.path, { mtime_ns: BigInt(row.mtime_ns), size: row.size, hash: row.hash });
98
- }
116
+ export function detectDiskDrift(store, repoId, rootPath, opts = {}) {
117
+ const fs = opts.fs ?? nodeFs;
118
+ const cache = loadStatCache(store, repoId);
99
119
  // Live docs keyed by path, with their durable file_hash (the convergence
100
120
  // signal). This is the authority for changed/deleted/untracked, independent
101
121
  // of the derived file_stats cache.
@@ -105,47 +125,58 @@ export function detectDiskDrift(store, repoId, rootPath) {
105
125
  .all(repoId)) {
106
126
  docs.set(row.path, row.file_hash);
107
127
  }
108
- const paths = walkMarkdown(rootPath);
109
- const seen = new Set(paths);
110
- // Candidates whose (mtime_ns, size) differs from the stat cache (or are new).
111
- const candidates = [];
112
- for (const path of paths) {
113
- const st = statSync(join(rootPath, path), { bigint: true });
114
- const cached = cache.get(path);
115
- if (!cached || cached.mtime_ns !== st.mtimeNs || cached.size !== Number(st.size)) {
116
- candidates.push(path);
117
- }
118
- }
128
+ const snapshot = snapshotOf(fs, rootPath);
129
+ const byPath = new Map(snapshot.map((e) => [e.path, e]));
119
130
  let changed = 0;
120
131
  let untracked = 0;
121
- for (const path of candidates) {
132
+ for (const path of sweepCandidates(cache, snapshot)) {
122
133
  const doc = docs.get(path);
123
134
  if (doc === undefined) {
124
135
  // No non-deleted doc for an on-disk *.md ⇒ untracked (new file).
125
136
  untracked += 1;
126
137
  continue;
127
138
  }
128
- const hash = sha256(readFileSync(join(rootPath, path), "utf8"));
129
139
  // Genuinely drifted only if content hash differs from the doc's file_hash.
130
- if (!doc || !doc.equals(hash))
140
+ if (!doc || !doc.equals(byPath.get(path).hash()))
131
141
  changed += 1;
132
142
  }
133
143
  // A doc present in the DB whose path isn't on disk anymore is a pending delete.
134
144
  let deleted = 0;
135
- for (const path of docs.keys()) {
136
- if (!seen.has(path))
145
+ for (const path of docs.keys())
146
+ if (!byPath.has(path))
137
147
  deleted += 1;
138
- }
139
148
  return { changed, deleted, untracked };
140
149
  }
141
- /** Rebuild file_stats from scratch by re-statting + re-hashing every file. */
142
- export function rebuildFileStats(store, repoId, rootPath) {
150
+ /**
151
+ * Rebuild `file_stats` from scratch (spec/sync §4.3, since 1.1): delete the repo's rows, then
152
+ * walk the tree and record a row only for a file whose bytes' hash equals its live doc's
153
+ * `file_hash` — the cache is allowed to say "known" only about bytes the store already holds.
154
+ * A file with no live doc, or whose bytes differ from what was ingested, gets no row, so the
155
+ * next sweep still treats it as a candidate and drift still reports it (a cache rebuild must
156
+ * never change what the next sweep does). Returns the number of files walked, recorded or not.
157
+ */
158
+ export function rebuildFileStats(store, repoId, rootPath, opts = {}) {
159
+ const fs = opts.fs ?? nodeFs;
143
160
  store.db.prepare("DELETE FROM file_stats WHERE repo_id = ?").run(repoId);
144
- const paths = walkMarkdown(rootPath);
161
+ // The live docs' durable convergence signal, keyed by path (one query).
162
+ const live = new Map();
163
+ for (const row of store.db
164
+ .prepare("SELECT path, file_hash FROM docs WHERE repo_id = ? AND deleted_commit IS NULL AND file_hash IS NOT NULL")
165
+ .all(repoId)) {
166
+ live.set(row.path, row.file_hash);
167
+ }
168
+ const paths = fs.walk(rootPath);
145
169
  for (const path of paths) {
146
- const abs = join(rootPath, path);
147
- const hash = sha256(readFileSync(abs, "utf8"));
148
- recordFileStat(store, repoId, path, abs, hash);
170
+ const want = live.get(path);
171
+ if (!want)
172
+ continue; // no live doc: nothing the cache may vouch for
173
+ const bytes = fs.read(rootPath, path);
174
+ if (bytes === null)
175
+ continue; // vanished between the walk and the read
176
+ const hash = sha256(bytes);
177
+ if (!hash.equals(want))
178
+ continue; // a pending edit: leave it visible
179
+ recordStat(store, repoId, path, fs.stat(rootPath, path), hash);
149
180
  }
150
181
  return paths.length;
151
182
  }
@@ -1,5 +1,24 @@
1
1
  /** Walk `root` for *.md files, returning repo-relative canonical paths. */
2
2
  export declare function walkMarkdown(root: string): string[];
3
+ /** The two stat fields the freshness cache keys on (spec/sync §4.3). */
4
+ export interface FileStat {
5
+ mtimeNs: bigint;
6
+ size: number;
7
+ }
8
+ /**
9
+ * The filesystem as the sync paths see it — a walk, a stat and a read, all
10
+ * keyed by (root, repo-relative path). `nodeFs` is the production default.
11
+ */
12
+ export interface SyncFs {
13
+ /** spec/sync §4.2: every `*.md` under `root`, repo-relative, `/` separators, readdir order, depth-first. */
14
+ walk(root: string): string[];
15
+ /** `(mtime_ns, size)` of `root/path`, or null when the file is absent. */
16
+ stat(root: string, path: string): FileStat | null;
17
+ /** The UTF-8 bytes of `root/path`, or null when the file is absent. */
18
+ read(root: string, path: string): string | null;
19
+ }
20
+ /** The `node:fs` filesystem. */
21
+ export declare const nodeFs: SyncFs;
3
22
  /**
4
23
  * Async variant of {@link walkMarkdown}: identical result, but yields to the
5
24
  * event loop between directories and invokes `onFound(count)` as matches
@@ -1,10 +1,15 @@
1
- import { readdirSync, statSync } from "node:fs";
1
+ import { readdirSync, readFileSync, statSync } from "node:fs";
2
2
  import { readdir, stat } from "node:fs/promises";
3
3
  import { join, relative, sep } from "node:path";
4
4
  // Shared filesystem walk for the in-process one-shot/reconcile paths (attach,
5
5
  // checkpoint, freshness). This is NOT the live watcher — chokidar lives only in
6
6
  // the external @omgbase/fs-adapter (sync-plugins). node:fs (a builtin) reads
7
7
  // are retained here for the one-shot ingest/freshness fast-path.
8
+ //
9
+ // `SyncFs` is the seam those paths read the filesystem through (spec/sync §4,
10
+ // §8): the sweep, the checkpoint and recovery take an optional `fs` and default
11
+ // to `nodeFs`, so a conformance runner can supply an in-memory filesystem with
12
+ // explicit `mtime_ns` and the fixtures stay free of real I/O.
8
13
  const IGNORED_DIRS = new Set([".omgbase", ".git", "node_modules"]);
9
14
  /** Walk `root` for *.md files, returning repo-relative canonical paths. */
10
15
  export function walkMarkdown(root) {
@@ -23,6 +28,24 @@ export function walkMarkdown(root) {
23
28
  recur(root);
24
29
  return out;
25
30
  }
31
+ /** The `node:fs` filesystem. */
32
+ export const nodeFs = {
33
+ walk: walkMarkdown,
34
+ stat(root, path) {
35
+ const st = statSync(join(root, path), { bigint: true, throwIfNoEntry: false });
36
+ return st ? { mtimeNs: st.mtimeNs, size: Number(st.size) } : null;
37
+ },
38
+ read(root, path) {
39
+ try {
40
+ return readFileSync(join(root, path), "utf8");
41
+ }
42
+ catch (err) {
43
+ if (err.code === "ENOENT")
44
+ return null;
45
+ throw err;
46
+ }
47
+ },
48
+ };
26
49
  /**
27
50
  * Async variant of {@link walkMarkdown}: identical result, but yields to the
28
51
  * event loop between directories and invokes `onFound(count)` as matches
@@ -1,4 +1,5 @@
1
1
  import type { Store } from "../core/store/store.js";
2
+ import { type ReconcileConfig } from "../reconcile/types.js";
2
3
  export interface ObserveResult {
3
4
  docId: string;
4
5
  path: string;
@@ -25,11 +26,56 @@ export interface ObserveOneResult extends ObserveResult {
25
26
  /** sha256(content) (hex) — the new file hash. */
26
27
  newHashHex: string;
27
28
  }
29
+ /** One member of a batch: its current bytes, or `null` when it left the source
30
+ * scope (deleted on disk / adapter fetch returned null). */
31
+ export interface BatchItem {
32
+ path: string;
33
+ content: string | null;
34
+ }
35
+ /** Outcome of one batch member: an observation (echo or commit) or a deletion. */
36
+ export type BatchOutcome = ({
37
+ kind: "observed";
38
+ } & ObserveOneResult) | {
39
+ kind: "deleted";
40
+ path: string;
41
+ /** the tombstoned doc, or null when no live doc existed at `path` (no-op). */
42
+ docId: string | null;
43
+ oldHashHex: string | null;
44
+ };
45
+ /**
46
+ * The batch reconcile primitive — one checkpoint's worth of members, in order.
47
+ * Two passes with the cross-document phase between them (reconciliation-spec §8):
48
+ *
49
+ * 1. every member is echo-gated and, when it differs from the stored revision,
50
+ * parsed + reconciled against its own prior tree WITHOUT committing
51
+ * (`prepareReconcile`; members gone from the source contribute their whole
52
+ * live tree as deleted). All members see one resurrection-pool snapshot with
53
+ * a shared consumed-set, so no pooled id resurrects twice in a batch.
54
+ * 2. the per-doc leftovers are pooled — deleted old blocks × inserted new
55
+ * blocks, across documents only — and matched at theta_xdoc
56
+ * (`crossDocMatch`). Each accepted pair is applied to both results: the
57
+ * destination's minted id becomes the carried id (`moved`/`edited_moved`,
58
+ * `detail.fromDoc`), the source's `deleted` disposition and pool entry for
59
+ * it vanish (`applyCrossDocMatches`).
60
+ * 3. every member commits, in batch order, each in its own transaction:
61
+ * `ingestFile` with the finished result (a carried-in id evicts the row the
62
+ * source doc may still hold — see `IdResolver.crossDocIds` — so the order
63
+ * of A and B in the batch does not matter), or `tombstoneObservedDeletion`.
64
+ *
65
+ * A moved block is therefore never `deleted` + `inserted`, never pooled, and
66
+ * never `resurrected` inside one batch. Across batches the pool still does that
67
+ * job (spec/reconcile §5 phase 6b). Does NOT sweep the pool — the caller does,
68
+ * once per batch. Does NOT write a file.
69
+ */
70
+ export declare function observeBatch(store: Store, repoId: string, items: BatchItem[], ts: string, opts?: {
71
+ config?: ReconcileConfig;
72
+ }): BatchOutcome[];
28
73
  /**
29
- * The reconcile primitive: echo-gate `content` for `path` and, when it differs
30
- * from the stored revision, ingest it as an observed commit (reconciling block
31
- * identity against the prior revision). Does NOT sweep the resurrection pool —
32
- * the caller does that once per batch. Does NOT write a file.
74
+ * The single-member reconcile primitive: echo-gate `content` for `path` and,
75
+ * when it differs from the stored revision, ingest it as an observed commit
76
+ * (reconciling block identity against the prior revision). A batch of one — so
77
+ * no cross-document matching; use `observeBatch` for that. Does NOT sweep the
78
+ * resurrection pool — the caller does that once per batch. Does NOT write a file.
33
79
  */
34
80
  export declare function observeOne(store: Store, repoId: string, path: string, content: string, ts: string): ObserveOneResult;
35
81
  /** Observe one file's authoritative bytes as an observed commit (echo-suppressed). */
@@ -37,7 +83,9 @@ export declare function observeFile(store: Store, repoId: string, path: string,
37
83
  ts?: string;
38
84
  }): ObserveResult;
39
85
  /** Observe a batch of files under one timestamp + one resurrection sweep. The
40
- * batch form of `observeFile`; each item is echo-gated independently. */
86
+ * batch form of `observeFile`; each item is echo-gated independently, and a
87
+ * block cut from one file and pasted into another in the same batch keeps its
88
+ * id (`moved`/`edited_moved`) via the cross-document phase of `observeBatch`. */
41
89
  export declare function observeMany(store: Store, repoId: string, items: {
42
90
  path: string;
43
91
  content: string;
@@ -1,31 +1,151 @@
1
1
  import { sha256 } from "../core/hash.js";
2
2
  import { ingestFile } from "../core/ingest.js";
3
- import { makeReconcilingResolver } from "./reconciling-ingest.js";
3
+ import { makeReconcilingResolver, prepareReconcile, loadOldMatchBlocks, loadPool } from "./reconciling-ingest.js";
4
4
  import { hasConflictMarkers } from "./git-heuristics.js";
5
5
  import { tombstoneObservedDeletion } from "./tombstone.js";
6
6
  import { sweepResurrectionPool } from "../core/store/gc.js";
7
+ import { crossDocMatch, applyCrossDocMatches } from "../reconcile/crossdoc.js";
8
+ import { DEFAULT_CONFIG } from "../reconcile/types.js";
7
9
  /**
8
- * The reconcile primitive: echo-gate `content` for `path` and, when it differs
9
- * from the stored revision, ingest it as an observed commit (reconciling block
10
- * identity against the prior revision). Does NOT sweep the resurrection pool —
11
- * the caller does that once per batch. Does NOT write a file.
10
+ * The batch reconcile primitive — one checkpoint's worth of members, in order.
11
+ * Two passes with the cross-document phase between them (reconciliation-spec §8):
12
+ *
13
+ * 1. every member is echo-gated and, when it differs from the stored revision,
14
+ * parsed + reconciled against its own prior tree WITHOUT committing
15
+ * (`prepareReconcile`; members gone from the source contribute their whole
16
+ * live tree as deleted). All members see one resurrection-pool snapshot with
17
+ * a shared consumed-set, so no pooled id resurrects twice in a batch.
18
+ * 2. the per-doc leftovers are pooled — deleted old blocks × inserted new
19
+ * blocks, across documents only — and matched at theta_xdoc
20
+ * (`crossDocMatch`). Each accepted pair is applied to both results: the
21
+ * destination's minted id becomes the carried id (`moved`/`edited_moved`,
22
+ * `detail.fromDoc`), the source's `deleted` disposition and pool entry for
23
+ * it vanish (`applyCrossDocMatches`).
24
+ * 3. every member commits, in batch order, each in its own transaction:
25
+ * `ingestFile` with the finished result (a carried-in id evicts the row the
26
+ * source doc may still hold — see `IdResolver.crossDocIds` — so the order
27
+ * of A and B in the batch does not matter), or `tombstoneObservedDeletion`.
28
+ *
29
+ * A moved block is therefore never `deleted` + `inserted`, never pooled, and
30
+ * never `resurrected` inside one batch. Across batches the pool still does that
31
+ * job (spec/reconcile §5 phase 6b). Does NOT sweep the pool — the caller does,
32
+ * once per batch. Does NOT write a file.
12
33
  */
13
- export function observeOne(store, repoId, path, content, ts) {
14
- const existing = store.db
15
- .prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_commit IS NULL")
16
- .get(repoId, path);
17
- const oldHashHex = existing?.file_hash?.toString("hex") ?? null;
18
- const diskHash = sha256(content);
19
- const newHashHex = diskHash.toString("hex");
20
- // Echo: the stored revision already equals these bytes → no commit. The
21
- // loop-breaker the coordinator relies on (§6): re-observing what the engine
22
- // already holds is a no-op.
23
- if (existing && existing.file_hash && existing.file_hash.equals(diskHash)) {
24
- return { docId: existing.doc_id, path, rev: null, commitId: null, converged: true, echo: true, conflicted: false, dispositions: [], oldHashHex, newHashHex };
34
+ export function observeBatch(store, repoId, items, ts, opts = {}) {
35
+ const db = store.db;
36
+ const config = opts.config ?? DEFAULT_CONFIG;
37
+ const liveDoc = db.prepare("SELECT doc_id, file_hash FROM docs WHERE repo_id = ? AND path = ? AND deleted_commit IS NULL");
38
+ // ---- pass 1: echo gate + reconcile, no commits -------------------------------
39
+ const pool = loadPool(db, repoId, ts);
40
+ const consumed = new Set();
41
+ const pending = [];
42
+ for (const it of items) {
43
+ const existing = liveDoc.get(repoId, it.path);
44
+ const oldHashHex = existing?.file_hash?.toString("hex") ?? null;
45
+ if (it.content === null) {
46
+ const oldBlocks = existing ? loadOldMatchBlocks(db, existing.doc_id) : [];
47
+ const result = existing
48
+ ? { assignment: new Map(), dispositions: [], deleted: oldBlocks.map((b) => b.blockId), consumedPool: [] }
49
+ : null;
50
+ pending.push({ kind: "gone", path: it.path, docId: existing?.doc_id ?? null, oldHashHex, oldBlocks, result });
51
+ continue;
52
+ }
53
+ const diskHash = sha256(it.content);
54
+ const newHashHex = diskHash.toString("hex");
55
+ // Echo: the stored revision already equals these bytes → no commit. The
56
+ // loop-breaker the coordinator relies on (§6): re-observing what the engine
57
+ // already holds is a no-op.
58
+ if (existing && existing.file_hash && existing.file_hash.equals(diskHash)) {
59
+ pending.push({
60
+ kind: "echo",
61
+ outcome: { kind: "observed", docId: existing.doc_id, path: it.path, rev: null, commitId: null, converged: true, echo: true, conflicted: false, dispositions: [], oldHashHex, newHashHex },
62
+ });
63
+ continue;
64
+ }
65
+ const prepared = prepareReconcile(store, repoId, it.path, it.content, { ts, config, pool, consumed });
66
+ pending.push({ kind: "ingest", prepared, oldHashHex, newHashHex });
67
+ }
68
+ // ---- cross-document phase (spec/reconcile §7) --------------------------------
69
+ if (pending.length > 1)
70
+ crossDocPhase(pending, config);
71
+ // ---- pass 2: commit, in batch order ------------------------------------------
72
+ const out = [];
73
+ for (const p of pending) {
74
+ if (p.kind === "echo") {
75
+ out.push(p.outcome);
76
+ }
77
+ else if (p.kind === "gone") {
78
+ if (p.docId)
79
+ tombstoneObservedDeletion(store, repoId, p.docId, ts);
80
+ out.push({ kind: "deleted", path: p.path, docId: p.docId, oldHashHex: p.oldHashHex });
81
+ }
82
+ else {
83
+ out.push({ kind: "observed", ...commitPrepared(store, repoId, p.prepared, ts, p.oldHashHex, p.newHashHex) });
84
+ }
85
+ }
86
+ return out;
87
+ }
88
+ // Pool each reconciled member's leftovers (README §7: deleted = old blocks whose
89
+ // ids are in `deleted`, in that order; inserted = new blocks whose disposition
90
+ // is `inserted`, with the minted id, in disposition order), match across
91
+ // documents, and apply. Members are keyed by doc id; a brand-new path has none
92
+ // yet, so it is keyed by path — it can only ever be a destination, and
93
+ // `detail.fromDoc` is always a real doc id.
94
+ function crossDocPhase(pending, config) {
95
+ const docs = [];
96
+ const results = new Map();
97
+ const prepared = new Map();
98
+ for (const p of pending) {
99
+ if (p.kind === "echo")
100
+ continue;
101
+ if (p.kind === "gone") {
102
+ if (!p.docId || !p.result)
103
+ continue;
104
+ docs.push({ docId: p.docId, deleted: p.oldBlocks.map((block) => ({ block })), inserted: [] });
105
+ results.set(p.docId, p.result);
106
+ continue;
107
+ }
108
+ const key = p.prepared.docId ?? `new:${p.prepared.path}`;
109
+ const oldById = new Map(p.prepared.oldBlocks.map((b) => [b.blockId, b]));
110
+ const newByKey = new Map(p.prepared.newBlocks.map((b) => [b.key, b]));
111
+ const keyOfId = new Map();
112
+ for (const [k, id] of p.prepared.result.assignment)
113
+ keyOfId.set(id, k);
114
+ const deleted = [];
115
+ for (const id of p.prepared.result.deleted) {
116
+ const block = oldById.get(id);
117
+ if (block)
118
+ deleted.push({ block });
119
+ }
120
+ const inserted = [];
121
+ for (const d of p.prepared.result.dispositions) {
122
+ if (d.kind !== "inserted")
123
+ continue;
124
+ const k = keyOfId.get(d.blockId);
125
+ const block = k ? newByKey.get(k) : undefined;
126
+ if (block)
127
+ inserted.push({ block, mintedId: d.blockId });
128
+ }
129
+ docs.push({ docId: key, deleted, inserted });
130
+ results.set(key, p.prepared.result);
131
+ prepared.set(key, p.prepared);
25
132
  }
133
+ if (docs.length < 2)
134
+ return;
135
+ const matches = crossDocMatch(docs, config);
136
+ if (matches.length === 0)
137
+ return;
138
+ applyCrossDocMatches(results, matches, config.matcherV);
139
+ for (const m of matches)
140
+ prepared.get(m.toDoc)?.crossDocIds.push(m.carriedId);
141
+ }
142
+ // Pass 2 for one reconciled member: commit the finished result as an observed
143
+ // revision (one transaction), flag conflict markers, summarize dispositions.
144
+ function commitPrepared(store, repoId, prepared, ts, oldHashHex, newHashHex) {
145
+ const { path, content } = prepared;
26
146
  const conflicted = hasConflictMarkers(content);
27
- const resolveIds = makeReconcilingResolver(store, repoId, { ts, path });
28
- const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds });
147
+ const resolveIds = makeReconcilingResolver(store, repoId, { ts, path, prepared });
148
+ const res = ingestFile(store, repoId, path, content, { ts, origin: "observed", resolveIds, parsed: prepared.tree });
29
149
  store.db.prepare("UPDATE docs SET conflicted = ? WHERE repo_id = ? AND path = ?").run(conflicted ? 1 : 0, repoId, path);
30
150
  const rows = store.db
31
151
  .prepare("SELECT kind, count(*) n FROM dispositions WHERE commit_id = ? GROUP BY kind ORDER BY kind")
@@ -43,6 +163,20 @@ export function observeOne(store, repoId, path, content, ts) {
43
163
  newHashHex,
44
164
  };
45
165
  }
166
+ /**
167
+ * The single-member reconcile primitive: echo-gate `content` for `path` and,
168
+ * when it differs from the stored revision, ingest it as an observed commit
169
+ * (reconciling block identity against the prior revision). A batch of one — so
170
+ * no cross-document matching; use `observeBatch` for that. Does NOT sweep the
171
+ * resurrection pool — the caller does that once per batch. Does NOT write a file.
172
+ */
173
+ export function observeOne(store, repoId, path, content, ts) {
174
+ const [outcome] = observeBatch(store, repoId, [{ path, content }], ts);
175
+ if (!outcome || outcome.kind !== "observed")
176
+ throw new Error(`observeOne: unexpected outcome for ${path}`);
177
+ const { kind: _k, ...rest } = outcome;
178
+ return rest;
179
+ }
46
180
  function toPublic(r) {
47
181
  const { oldHashHex: _o, newHashHex: _n, ...pub } = r;
48
182
  return pub;
@@ -55,10 +189,17 @@ export function observeFile(store, repoId, path, content, opts = {}) {
55
189
  return toPublic(r);
56
190
  }
57
191
  /** Observe a batch of files under one timestamp + one resurrection sweep. The
58
- * batch form of `observeFile`; each item is echo-gated independently. */
192
+ * batch form of `observeFile`; each item is echo-gated independently, and a
193
+ * block cut from one file and pasted into another in the same batch keeps its
194
+ * id (`moved`/`edited_moved`) via the cross-document phase of `observeBatch`. */
59
195
  export function observeMany(store, repoId, items, opts = {}) {
60
196
  const ts = opts.ts ?? new Date().toISOString();
61
- const out = items.map((it) => toPublic(observeOne(store, repoId, it.path, it.content, ts)));
197
+ const out = observeBatch(store, repoId, items, ts).map((o) => {
198
+ if (o.kind !== "observed")
199
+ throw new Error(`observeMany: unexpected outcome for ${o.path}`);
200
+ const { kind: _k, ...rest } = o;
201
+ return toPublic(rest);
202
+ });
62
203
  sweepResurrectionPool(store, ts);
63
204
  return out;
64
205
  }