@omgbase/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -0
- package/dist/src/cli/eval-matcher.d.ts +6 -1
- package/dist/src/cli/eval-matcher.js +14 -2
- package/dist/src/core/clock.d.ts +9 -0
- package/dist/src/core/clock.js +46 -0
- package/dist/src/core/hash.d.ts +25 -1
- package/dist/src/core/hash.js +110 -10
- package/dist/src/core/ids.d.ts +12 -0
- package/dist/src/core/ids.js +43 -1
- package/dist/src/core/ingest.d.ts +22 -2
- package/dist/src/core/ingest.js +92 -18
- package/dist/src/core/parse/parse.d.ts +1 -1
- package/dist/src/core/parse/parse.js +31 -9
- package/dist/src/core/store/nodes.d.ts +13 -0
- package/dist/src/core/store/nodes.js +24 -0
- package/dist/src/core/store/writers.d.ts +6 -0
- package/dist/src/core/store/writers.js +1 -0
- package/dist/src/core/utf8.d.ts +29 -0
- package/dist/src/core/utf8.js +96 -0
- package/dist/src/format/adapter.d.ts +7 -1
- package/dist/src/format/markdown.js +36 -41
- package/dist/src/graph/extract.d.ts +41 -1
- package/dist/src/graph/extract.js +59 -1
- package/dist/src/graph/history.d.ts +39 -1
- package/dist/src/graph/history.js +154 -20
- package/dist/src/index.d.ts +10 -6
- package/dist/src/index.js +11 -6
- package/dist/src/mcp/graph.js +8 -4
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +17 -6
- package/dist/src/mcp/server.js +15 -10
- package/dist/src/mutate/apply.d.ts +9 -0
- package/dist/src/mutate/apply.js +24 -11
- package/dist/src/mutate/docs.d.ts +2 -0
- package/dist/src/mutate/docs.js +10 -6
- package/dist/src/mutate/known-ids.js +11 -3
- package/dist/src/mutate/macros.js +9 -2
- package/dist/src/mutate/ops.d.ts +5 -4
- package/dist/src/mutate/ops.js +62 -15
- package/dist/src/mutate/plan-update.d.ts +7 -1
- package/dist/src/mutate/plan-update.js +3 -1
- package/dist/src/oqx-js/context.js +57 -7
- package/dist/src/reconcile/eval/generator.d.ts +24 -1
- package/dist/src/reconcile/eval/generator.js +248 -0
- package/dist/src/reconcile/eval/metrics.d.ts +33 -9
- package/dist/src/reconcile/eval/metrics.js +55 -32
- package/dist/src/reconcile/flatten.js +7 -5
- package/dist/src/reconcile/phase5.d.ts +2 -1
- package/dist/src/reconcile/phase5.js +27 -10
- package/dist/src/reconcile/phase6a.js +17 -5
- package/dist/src/reconcile/phases.d.ts +44 -6
- package/dist/src/reconcile/phases.js +0 -0
- package/dist/src/reconcile/reconcile.js +16 -9
- package/dist/src/reconcile/types.d.ts +4 -1
- package/dist/src/reconcile/types.js +11 -2
- package/dist/src/search/external.d.ts +10 -0
- package/dist/src/search/external.js +18 -1
- package/dist/src/search/provider.d.ts +7 -3
- package/dist/src/search/provider.js +8 -7
- package/dist/src/search/rrf.js +4 -0
- package/dist/src/search/tasks.d.ts +28 -0
- package/dist/src/search/tasks.js +36 -23
- package/dist/src/search/vector.js +30 -14
- package/dist/src/sync/attach.d.ts +7 -2
- package/dist/src/sync/attach.js +10 -8
- package/dist/src/sync/checkpoint.d.ts +18 -1
- package/dist/src/sync/checkpoint.js +31 -29
- package/dist/src/sync/driver.d.ts +15 -4
- package/dist/src/sync/driver.js +18 -43
- package/dist/src/sync/external-source.js +7 -1
- package/dist/src/sync/freshness.d.ts +73 -15
- package/dist/src/sync/freshness.js +128 -97
- package/dist/src/sync/fs-util.d.ts +19 -0
- package/dist/src/sync/fs-util.js +24 -1
- package/dist/src/sync/observe.d.ts +53 -5
- package/dist/src/sync/observe.js +162 -21
- package/dist/src/sync/reconciling-ingest.d.ts +42 -2
- package/dist/src/sync/reconciling-ingest.js +87 -13
- package/dist/src/sync/recovery.d.ts +5 -1
- package/dist/src/sync/recovery.js +8 -8
- package/dist/src/sync/workspace.d.ts +11 -3
- package/dist/src/sync/workspace.js +39 -30
- package/dist/src/sync/writer-lock.js +16 -26
- package/package.json +3 -3
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import { type EditClass, type GeneratedCase } from "./generator.js";
|
|
1
|
+
import { type ReconcileConfig } from "../types.js";
|
|
2
|
+
import { type EditClass, type GeneratedCase, type StructuredEditClass } from "./generator.js";
|
|
3
3
|
export interface ClassMetrics {
|
|
4
|
+
/** new blocks of this class (carried or minted in truth) */
|
|
5
|
+
newBlocks: number;
|
|
4
6
|
truePairs: number;
|
|
5
7
|
carriedPairs: number;
|
|
6
8
|
correctCarries: number;
|
|
@@ -9,24 +11,46 @@ export interface ClassMetrics {
|
|
|
9
11
|
}
|
|
10
12
|
export interface EvalReport {
|
|
11
13
|
cases: number;
|
|
14
|
+
mode: EvalMode;
|
|
12
15
|
overall: ClassMetrics;
|
|
13
16
|
perClass: Record<string, ClassMetrics>;
|
|
17
|
+
/** old ids the truth deletes, and how many of them the matcher carried anyway */
|
|
18
|
+
deleted: {
|
|
19
|
+
expected: number;
|
|
20
|
+
wronglyCarried: number;
|
|
21
|
+
};
|
|
22
|
+
/** the effective config the suite ran under */
|
|
14
23
|
config: ReconcileConfig;
|
|
15
24
|
}
|
|
16
|
-
|
|
25
|
+
export type EvalMode = "flat" | "structured";
|
|
26
|
+
type Agg = {
|
|
27
|
+
newBlocks: number;
|
|
17
28
|
truePairs: number;
|
|
18
29
|
carriedPairs: number;
|
|
19
30
|
correctCarries: number;
|
|
20
31
|
};
|
|
21
32
|
export declare function evaluateCase(c: GeneratedCase, config?: ReconcileConfig): {
|
|
22
|
-
overall:
|
|
23
|
-
perClass: Map<string,
|
|
33
|
+
overall: Agg;
|
|
34
|
+
perClass: Map<string, Agg>;
|
|
35
|
+
deleted: {
|
|
36
|
+
expected: number;
|
|
37
|
+
wronglyCarried: number;
|
|
38
|
+
};
|
|
24
39
|
};
|
|
25
|
-
|
|
26
|
-
export declare function runEval(opts?: {
|
|
40
|
+
export interface RunEvalOptions {
|
|
27
41
|
n?: number;
|
|
42
|
+
/** flat: top-level paragraphs per doc (12); structured: top-level blocks per doc (10) */
|
|
28
43
|
size?: number;
|
|
29
44
|
intensity?: number;
|
|
45
|
+
/** structured mode only: words per list item (absent ⇒ 2–6 per item) */
|
|
46
|
+
itemWords?: number;
|
|
47
|
+
/** "flat" (default; the release-gate suite) or "structured" (lists of short items) */
|
|
48
|
+
mode?: EvalMode;
|
|
49
|
+
/** full config; defaults to DEFAULT_CONFIG */
|
|
30
50
|
config?: ReconcileConfig;
|
|
31
|
-
})
|
|
32
|
-
|
|
51
|
+
/** fields to override on top of `config` (e.g. `{ contextSimFloor: 0.3 }`) */
|
|
52
|
+
configOverrides?: Partial<ReconcileConfig>;
|
|
53
|
+
}
|
|
54
|
+
/** Run the synthetic suite over `n` seeded cases and produce the report. */
|
|
55
|
+
export declare function runEval(opts?: RunEvalOptions): EvalReport;
|
|
56
|
+
export type { EditClass, StructuredEditClass };
|
|
@@ -1,38 +1,51 @@
|
|
|
1
1
|
import { reconcileDocument } from "../reconcile.js";
|
|
2
|
-
import {
|
|
2
|
+
import { DEFAULT_CONFIG } from "../types.js";
|
|
3
|
+
import { generateCase, generateStructuredCase, } from "./generator.js";
|
|
3
4
|
function blank() {
|
|
4
|
-
return { truePairs: 0, carriedPairs: 0, correctCarries: 0 };
|
|
5
|
+
return { newBlocks: 0, truePairs: 0, carriedPairs: 0, correctCarries: 0 };
|
|
6
|
+
}
|
|
7
|
+
function add(into, from) {
|
|
8
|
+
into.newBlocks += from.newBlocks;
|
|
9
|
+
into.truePairs += from.truePairs;
|
|
10
|
+
into.carriedPairs += from.carriedPairs;
|
|
11
|
+
into.correctCarries += from.correctCarries;
|
|
5
12
|
}
|
|
6
13
|
export function evaluateCase(c, config) {
|
|
7
14
|
const res = reconcileDocument(c.old, c.neu, config ? { config } : {});
|
|
8
15
|
const overall = blank();
|
|
9
16
|
const perClass = new Map();
|
|
10
17
|
const bucket = (cls) => perClass.get(cls) ?? perClass.set(cls, blank()).get(cls);
|
|
18
|
+
// Flat mode leaves untouched blocks unclassified and they count as "edit"
|
|
19
|
+
// (kept: the release gates were tuned against that bucketing).
|
|
20
|
+
const classOf = (key) => c.truth.classOf.get(key) ?? "edit";
|
|
21
|
+
for (const b of c.neu) {
|
|
22
|
+
overall.newBlocks++;
|
|
23
|
+
bucket(classOf(b.key)).newBlocks++;
|
|
24
|
+
}
|
|
11
25
|
// True pairs: ground-truth carries.
|
|
12
26
|
for (const [key, oldId] of c.truth.carries) {
|
|
13
27
|
overall.truePairs++;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
const got = res.assignment.get(key);
|
|
17
|
-
if (got === oldId) {
|
|
28
|
+
bucket(classOf(key)).truePairs++;
|
|
29
|
+
if (res.assignment.get(key) === oldId) {
|
|
18
30
|
overall.correctCarries++;
|
|
19
|
-
bucket(
|
|
31
|
+
bucket(classOf(key)).correctCarries++;
|
|
20
32
|
}
|
|
21
33
|
}
|
|
22
34
|
// Carried pairs: assignments where the matcher reused an OLD id (not minted).
|
|
23
35
|
const oldIds = new Set(c.old.map((b) => b.blockId));
|
|
24
|
-
const
|
|
36
|
+
const carriedOld = new Set();
|
|
25
37
|
for (const [key, id] of res.assignment) {
|
|
26
38
|
if (oldIds.has(id)) {
|
|
27
39
|
overall.carriedPairs++;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
// correctness already tallied above when it matches truth; recount for
|
|
31
|
-
// precision denominators handled via correctCarries computed there.
|
|
32
|
-
void truthByKey;
|
|
40
|
+
bucket(classOf(key)).carriedPairs++;
|
|
41
|
+
carriedOld.add(id);
|
|
33
42
|
}
|
|
34
43
|
}
|
|
35
|
-
|
|
44
|
+
let wronglyCarried = 0;
|
|
45
|
+
for (const id of c.truth.deleted)
|
|
46
|
+
if (carriedOld.has(id))
|
|
47
|
+
wronglyCarried++;
|
|
48
|
+
return { overall, perClass, deleted: { expected: c.truth.deleted.size, wronglyCarried } };
|
|
36
49
|
}
|
|
37
50
|
function finalize(agg) {
|
|
38
51
|
return {
|
|
@@ -44,31 +57,41 @@ function finalize(agg) {
|
|
|
44
57
|
/** Run the synthetic suite over `n` seeded cases and produce the report. */
|
|
45
58
|
export function runEval(opts = {}) {
|
|
46
59
|
const n = opts.n ?? 200;
|
|
60
|
+
const mode = opts.mode ?? "flat";
|
|
61
|
+
const config = { ...(opts.config ?? DEFAULT_CONFIG), ...(opts.configOverrides ?? {}) };
|
|
47
62
|
const overall = blank();
|
|
48
63
|
const perClass = new Map();
|
|
64
|
+
const deleted = { expected: 0, wronglyCarried: 0 };
|
|
49
65
|
for (let seed = 1; seed <= n; seed++) {
|
|
50
|
-
|
|
51
|
-
if (
|
|
52
|
-
genOpts
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
66
|
+
let c;
|
|
67
|
+
if (mode === "structured") {
|
|
68
|
+
const genOpts = {};
|
|
69
|
+
if (opts.size !== undefined)
|
|
70
|
+
genOpts.size = opts.size;
|
|
71
|
+
if (opts.intensity !== undefined)
|
|
72
|
+
genOpts.intensity = opts.intensity;
|
|
73
|
+
if (opts.itemWords !== undefined)
|
|
74
|
+
genOpts.itemWords = opts.itemWords;
|
|
75
|
+
c = generateStructuredCase(seed, genOpts);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
const genOpts = {};
|
|
79
|
+
if (opts.size !== undefined)
|
|
80
|
+
genOpts.size = opts.size;
|
|
81
|
+
if (opts.intensity !== undefined)
|
|
82
|
+
genOpts.intensity = opts.intensity;
|
|
83
|
+
c = generateCase(seed, genOpts);
|
|
65
84
|
}
|
|
85
|
+
const r = evaluateCase(c, config);
|
|
86
|
+
add(overall, r.overall);
|
|
87
|
+
for (const [cls, agg] of r.perClass)
|
|
88
|
+
add(perClass.get(cls) ?? perClass.set(cls, blank()).get(cls), agg);
|
|
89
|
+
deleted.expected += r.deleted.expected;
|
|
90
|
+
deleted.wronglyCarried += r.deleted.wronglyCarried;
|
|
66
91
|
}
|
|
67
92
|
const perClassOut = {};
|
|
68
93
|
for (const [cls, agg] of perClass)
|
|
69
94
|
perClassOut[cls] = finalize(agg);
|
|
70
|
-
|
|
71
|
-
const cfg = opts.config ?? undefined;
|
|
72
|
-
return { cases: n, overall: finalize(overall), perClass: perClassOut, config: cfg };
|
|
95
|
+
return { cases: n, mode, overall: finalize(overall), perClass: perClassOut, deleted, config };
|
|
73
96
|
}
|
|
74
97
|
//# sourceMappingURL=metrics.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { sha256,
|
|
1
|
+
import { sha256, childQuoteDepth, visibleText } from "../core/hash.js";
|
|
2
2
|
/** Adapt a parsed RawBlock-ish tree (no ids) to FlatSource. */
|
|
3
3
|
export function fromInput(blocks) {
|
|
4
4
|
return blocks.map((b) => ({
|
|
@@ -10,10 +10,12 @@ export function fromInput(blocks) {
|
|
|
10
10
|
}
|
|
11
11
|
export function flatten(blocks) {
|
|
12
12
|
const out = [];
|
|
13
|
-
|
|
13
|
+
// quoteDepth = blockquote ancestors; text follows spec/format §4.1 (containers
|
|
14
|
+
// compose from children, nested raws lose their `> ` prefixes).
|
|
15
|
+
const walk = (list, parentKey, quoteDepth) => {
|
|
14
16
|
list.forEach((b, index) => {
|
|
15
17
|
const key = `${parentKey ?? ""}/${index}`;
|
|
16
|
-
const text =
|
|
18
|
+
const text = visibleText(b, quoteDepth);
|
|
17
19
|
const mb = {
|
|
18
20
|
type: b.type,
|
|
19
21
|
rawHashHex: sha256(b.raw).toString("hex"),
|
|
@@ -28,10 +30,10 @@ export function flatten(blocks) {
|
|
|
28
30
|
mb.blockId = b.blockId;
|
|
29
31
|
out.push(mb);
|
|
30
32
|
if (b.children.length > 0)
|
|
31
|
-
walk(b.children, key);
|
|
33
|
+
walk(b.children, key, childQuoteDepth(b.type, quoteDepth));
|
|
32
34
|
});
|
|
33
35
|
};
|
|
34
|
-
walk(blocks, null);
|
|
36
|
+
walk(blocks, null, 0);
|
|
35
37
|
return out;
|
|
36
38
|
}
|
|
37
39
|
//# sourceMappingURL=flatten.js.map
|
|
@@ -3,6 +3,7 @@ import type { PhaseState } from "./phases.js";
|
|
|
3
3
|
* Phase 5 — scored assignment. Greedy-by-score with R3 order constraints:
|
|
4
4
|
* accept the highest-scoring candidate, discard conflicting candidates
|
|
5
5
|
* (same old or new, or order-crossing with an already-accepted same-parent
|
|
6
|
-
* pair), repeat. Accept
|
|
6
|
+
* pair), repeat. Accept a candidate when score ≥ θ (θ_small for tiny blocks);
|
|
7
|
+
* a candidate below its θ is skipped, not a stopping point.
|
|
7
8
|
*/
|
|
8
9
|
export declare function phase5Scored(state: PhaseState): void;
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { shingles, dice, tokenCount } from "./similarity.js";
|
|
2
2
|
// score(o,n) = 0.55 text_sim + 0.15 neighbor_ctx + 0.10 parent_match
|
|
3
3
|
// + 0.10 position_prior + 0.10 anchor_evidence
|
|
4
|
-
function scorePair(o, n, state,
|
|
4
|
+
function scorePair(o, n, state, oldSiblings, newSiblings) {
|
|
5
5
|
const textSimVal = dice(shingles(o.text), shingles(n.text));
|
|
6
6
|
// neighbor_ctx: fraction of {prev,next} siblings that are matched pairs.
|
|
7
7
|
const neighborCtx = neighborContext(o, n, state);
|
|
8
8
|
// parent_match: 1 if parents are a matched pair (or both roots).
|
|
9
9
|
const parentMatch = parentsMatched(o, n, state) ? 1 : 0;
|
|
10
|
-
// position_prior: 1 - |
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const positionPrior = 1 - Math.abs(
|
|
10
|
+
// position_prior: 1 - |rel(o) - rel(n)| with rel(b) = b.index / (siblings - 1)
|
|
11
|
+
// over b's own sibling count (spec §5; m2.1 — m2.0 divided by the size of
|
|
12
|
+
// the whole flattened list, compressing the prior for nested blocks).
|
|
13
|
+
const positionPrior = 1 - Math.abs(relPos(o, oldSiblings) - relPos(n, newSiblings));
|
|
14
14
|
// anchor_evidence: shared anchors / code-fence info / heading prefix.
|
|
15
15
|
const anchorEvidence = sharedAnchorEvidence(o, n);
|
|
16
16
|
return (0.55 * textSimVal +
|
|
@@ -19,6 +19,18 @@ function scorePair(o, n, state, oldCount, newCount) {
|
|
|
19
19
|
0.1 * positionPrior +
|
|
20
20
|
0.1 * anchorEvidence);
|
|
21
21
|
}
|
|
22
|
+
function siblingCounts(blocks) {
|
|
23
|
+
const counts = new Map();
|
|
24
|
+
for (const b of blocks) {
|
|
25
|
+
const k = b.parentKey ?? "";
|
|
26
|
+
counts.set(k, (counts.get(k) ?? 0) + 1);
|
|
27
|
+
}
|
|
28
|
+
return counts;
|
|
29
|
+
}
|
|
30
|
+
function relPos(b, counts) {
|
|
31
|
+
const siblings = counts.get(b.parentKey ?? "") ?? 1;
|
|
32
|
+
return siblings > 1 ? b.index / (siblings - 1) : 0;
|
|
33
|
+
}
|
|
22
34
|
function siblingAt(blocks, parentKey, index) {
|
|
23
35
|
return blocks.find((b) => b.parentKey === parentKey && b.index === index);
|
|
24
36
|
}
|
|
@@ -98,7 +110,8 @@ function unmatchedNew(state) {
|
|
|
98
110
|
* Phase 5 — scored assignment. Greedy-by-score with R3 order constraints:
|
|
99
111
|
* accept the highest-scoring candidate, discard conflicting candidates
|
|
100
112
|
* (same old or new, or order-crossing with an already-accepted same-parent
|
|
101
|
-
* pair), repeat. Accept
|
|
113
|
+
* pair), repeat. Accept a candidate when score ≥ θ (θ_small for tiny blocks);
|
|
114
|
+
* a candidate below its θ is skipped, not a stopping point.
|
|
102
115
|
*/
|
|
103
116
|
export function phase5Scored(state) {
|
|
104
117
|
const olds = unmatchedOld(state);
|
|
@@ -108,11 +121,12 @@ export function phase5Scored(state) {
|
|
|
108
121
|
// Bulk path guard: too many unmatched ⇒ skip (handled by give-up logic 2.5).
|
|
109
122
|
if (olds.length + news.length > state.config.maxScoredBlocks * 2)
|
|
110
123
|
return;
|
|
111
|
-
|
|
112
|
-
const
|
|
124
|
+
// Sibling counts once per tree (position_prior).
|
|
125
|
+
const oldSiblings = siblingCounts(state.old);
|
|
126
|
+
const newSiblings = siblingCounts(state.neu);
|
|
113
127
|
const candidates = pruneCandidates(olds, news);
|
|
114
128
|
for (const c of candidates)
|
|
115
|
-
c.score = scorePair(c.old, c.neu, state,
|
|
129
|
+
c.score = scorePair(c.old, c.neu, state, oldSiblings, newSiblings);
|
|
116
130
|
// Deterministic order: score desc, then old key, then new key.
|
|
117
131
|
candidates.sort((a, b) => b.score - a.score || cmp(a.old.key, b.old.key) || cmp(a.neu.key, b.neu.key));
|
|
118
132
|
// All scored candidates per new key, for near-miss recording (R5).
|
|
@@ -125,8 +139,11 @@ export function phase5Scored(state) {
|
|
|
125
139
|
if (state.usedOld.has(c.old.blockId) || state.usedNew.has(c.neu.key))
|
|
126
140
|
continue;
|
|
127
141
|
const threshold = tokenCount(c.neu.text) < state.config.smallBlockTokens ? state.config.thetaSmall : state.config.thetaAccept;
|
|
142
|
+
// Sorted desc, but θ differs per candidate (θ_small for tiny new blocks), so a
|
|
143
|
+
// sub-threshold tiny-block candidate must not end the walk for the regular
|
|
144
|
+
// candidates sorted below it (m2.1; spec §5 phase 5 step 2).
|
|
128
145
|
if (c.score < threshold)
|
|
129
|
-
|
|
146
|
+
continue;
|
|
130
147
|
// R3: within the same new parent, accepted pairs must not cross in order.
|
|
131
148
|
if (c.old.parentKey === c.neu.parentKey) {
|
|
132
149
|
const list = acceptedByParent.get(c.neu.parentKey) ?? [];
|
|
@@ -2,6 +2,9 @@ import { shingles, dice, tokenCount, tokenize } from "./similarity.js";
|
|
|
2
2
|
// Phase 6a — compound classification (03 §4): split, merge, copy over the
|
|
3
3
|
// still-unmatched. Split/merge use dominant-fragment inheritance (tunable via
|
|
4
4
|
// split.dominant_share; set to 1.01 to disable). Copies never steal identity.
|
|
5
|
+
// Each pass proceeds in document order over the blocks that were unmatched when
|
|
6
|
+
// it began and are still unmatched when reached, resolving every split (merge)
|
|
7
|
+
// it finds (m2.1; m2.0 returned after the first).
|
|
5
8
|
function unmatchedOld(state) {
|
|
6
9
|
return state.old.filter((b) => !state.usedOld.has(b.blockId));
|
|
7
10
|
}
|
|
@@ -44,11 +47,15 @@ function noteLineage(state, newB, kind, counterpartOldId) {
|
|
|
44
47
|
function detectSplits(state) {
|
|
45
48
|
const { splitCoverage, splitDominantShare } = state.config;
|
|
46
49
|
for (const o of unmatchedOld(state)) {
|
|
50
|
+
if (state.usedOld.has(o.blockId))
|
|
51
|
+
continue; // consumed earlier in this pass
|
|
47
52
|
const news = unmatchedNew(state)
|
|
48
53
|
.filter((n) => n.parentKey === o.parentKey && n.type === o.type)
|
|
49
54
|
.sort((a, b) => a.index - b.index);
|
|
50
|
-
// find a contiguous run (by index) of length ≥2 covering O
|
|
51
|
-
|
|
55
|
+
// find a contiguous run (by index) of length ≥2 covering O; the first hit
|
|
56
|
+
// (start ascending, longest window first) is the split.
|
|
57
|
+
let found = false;
|
|
58
|
+
for (let start = 0; start < news.length && !found; start++) {
|
|
52
59
|
for (let end = news.length; end > start + 1; end--) {
|
|
53
60
|
const run = news.slice(start, end);
|
|
54
61
|
if (!isContiguous(run))
|
|
@@ -76,7 +83,8 @@ function detectSplits(state) {
|
|
|
76
83
|
for (const n of run)
|
|
77
84
|
noteLineage(state, n, "split_from", o.blockId);
|
|
78
85
|
}
|
|
79
|
-
|
|
86
|
+
found = true;
|
|
87
|
+
break; // on to the next unmatched old block
|
|
80
88
|
}
|
|
81
89
|
}
|
|
82
90
|
}
|
|
@@ -85,10 +93,13 @@ function detectSplits(state) {
|
|
|
85
93
|
function detectMerges(state) {
|
|
86
94
|
const { splitCoverage, splitDominantShare } = state.config;
|
|
87
95
|
for (const n of unmatchedNew(state)) {
|
|
96
|
+
if (state.usedNew.has(n.key))
|
|
97
|
+
continue; // consumed earlier in this pass
|
|
88
98
|
const olds = unmatchedOld(state)
|
|
89
99
|
.filter((o) => o.parentKey === n.parentKey && o.type === n.type)
|
|
90
100
|
.sort((a, b) => a.index - b.index);
|
|
91
|
-
|
|
101
|
+
let found = false;
|
|
102
|
+
for (let start = 0; start < olds.length && !found; start++) {
|
|
92
103
|
for (let end = olds.length; end > start + 1; end--) {
|
|
93
104
|
const run = olds.slice(start, end);
|
|
94
105
|
if (!isContiguous(run))
|
|
@@ -116,7 +127,8 @@ function detectMerges(state) {
|
|
|
116
127
|
}
|
|
117
128
|
noteLineage(state, n, "merged_into", first.blockId);
|
|
118
129
|
}
|
|
119
|
-
|
|
130
|
+
found = true;
|
|
131
|
+
break; // on to the next unmatched new block
|
|
120
132
|
}
|
|
121
133
|
}
|
|
122
134
|
}
|
|
@@ -18,11 +18,49 @@ export declare function phase2Normalized(state: PhaseState): void;
|
|
|
18
18
|
/** Phase 3 — anchor lock: same authored ^ref, unique on both sides. */
|
|
19
19
|
export declare function phase3Anchor(state: PhaseState): void;
|
|
20
20
|
/**
|
|
21
|
-
* Phase
|
|
22
|
-
* stranger between them: within a gap bounded by matched blocks, if
|
|
23
|
-
* unmatched OLD child remains, pair it with its best-matching
|
|
24
|
-
* child in that same gap (text_sim ≥ floor, unique best).
|
|
25
|
-
* do not block the carry — they simply lose to the
|
|
26
|
-
* which is why an inserted block never steals an
|
|
21
|
+
* Phase 4a — context propagation (spec/reconcile §5). Matched siblings vouch
|
|
22
|
+
* for the stranger between them: within a gap bounded by matched blocks, if
|
|
23
|
+
* exactly one unmatched OLD child remains, pair it with its best-matching
|
|
24
|
+
* unmatched NEW child in that same gap (text_sim ≥ floor, unique best).
|
|
25
|
+
* Insertions in the gap do not block the carry — they simply lose to the
|
|
26
|
+
* better-matching candidate, which is why an inserted block never steals an
|
|
27
|
+
* edited block's identity.
|
|
28
|
+
*
|
|
29
|
+
* Step 5, the singleton rule (m2.3): when the floor test fails (or there is
|
|
30
|
+
* no best), a carried container (not the root pair) with exactly one unmatched
|
|
31
|
+
* old child and exactly one unmatched new child of its type carries them
|
|
32
|
+
* without a text floor, provided the new child sits in the SAME SLOT — same
|
|
33
|
+
* index, or the siblings just before both (or just after both) are a carried
|
|
34
|
+
* pair — and anchors do not veto (if either block has anchors they must share
|
|
35
|
+
* one). Reason `context_unique`, confidence 0.75 + 0.2 × text_sim, whatever
|
|
36
|
+
* text_sim is (0 for a two-word item whose one word changed). Root-level
|
|
37
|
+
* singletons are excluded: an unrelated replacement paragraph is common there
|
|
38
|
+
* and paragraphs carry the block references R4 protects.
|
|
27
39
|
*/
|
|
28
40
|
export declare function phase4Context(state: PhaseState): void;
|
|
41
|
+
/**
|
|
42
|
+
* Phase 4b — children vouch for their parent (03 §4). Phase 4 lets a matched
|
|
43
|
+
* parent vouch for the stranger among its children; this is the converse. For
|
|
44
|
+
* each unmatched OLD container, look at where its already-carried children
|
|
45
|
+
* landed. If a clear majority of them (≥ childrenVouchFrac of ALL its children)
|
|
46
|
+
* sit under one unmatched NEW block of the same type, and that block is the
|
|
47
|
+
* unique destination for them, and the old container is in turn the unique
|
|
48
|
+
* source of that block's carried children (mutual best by children), and the
|
|
49
|
+
* two containers' own parents do not contradict the pairing (a matched pair,
|
|
50
|
+
* both roots, or both still undecided), the containers pair. Reason
|
|
51
|
+
* `context_children`, confidence 0.75 + 0.2 × fraction.
|
|
52
|
+
*
|
|
53
|
+
* Text is deliberately not consulted: a container's visible text is its
|
|
54
|
+
* children's text, so a one-word-per-item list has no shingle evidence to give
|
|
55
|
+
* — its children's carried ids are the evidence. Returns true if any pair was
|
|
56
|
+
* made (phase4Propagate iterates to a fixed point).
|
|
57
|
+
*/
|
|
58
|
+
export declare function phase4bChildren(state: PhaseState): boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Phase 4 fixed point: context propagation (parents/siblings vouch for a
|
|
61
|
+
* child) and children vouching (children vouch for their parent) feed each
|
|
62
|
+
* other — a newly paired list unlocks its lone edited item; a newly paired
|
|
63
|
+
* inner list unlocks the item that contains it, then the outer list. Iterate
|
|
64
|
+
* until neither adds a pair. Terminates: every round consumes ≥ 1 old block.
|
|
65
|
+
*/
|
|
66
|
+
export declare function phase4Propagate(state: PhaseState): void;
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { mintId } from "../core/ids.js";
|
|
2
2
|
import { DEFAULT_CONFIG } from "./types.js";
|
|
3
|
-
import { phase1Exact, phase2Normalized, phase3Anchor,
|
|
3
|
+
import { phase1Exact, phase2Normalized, phase3Anchor, phase4Propagate } from "./phases.js";
|
|
4
4
|
import { phase5Scored } from "./phase5.js";
|
|
5
5
|
import { phase6aCompound } from "./phase6a.js";
|
|
6
6
|
function newState(old, neu, config) {
|
|
@@ -50,7 +50,7 @@ export function reconcileDocument(oldBlocks, newBlocks, opts = {}) {
|
|
|
50
50
|
return bulkRewrite(state);
|
|
51
51
|
}
|
|
52
52
|
phase3Anchor(state);
|
|
53
|
-
|
|
53
|
+
phase4Propagate(state); // phases 4 + 4b to a fixed point
|
|
54
54
|
phase5Scored(state);
|
|
55
55
|
phase6aCompound(state);
|
|
56
56
|
if (opts.pool && opts.pool.length > 0)
|
|
@@ -84,15 +84,24 @@ function finalize(state, consumedPool) {
|
|
|
84
84
|
dispositions.push({ blockId: minted, kind: "inserted", confidence: null, reason: null, matcherV: state.config.matcherV, detail: {} });
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
-
// Remaining old blocks (not carried) → deleted.
|
|
88
|
-
const deleted = [];
|
|
87
|
+
// Remaining old blocks (not carried, not otherwise disposed) → deleted.
|
|
89
88
|
for (const o of state.old) {
|
|
90
89
|
if (!state.usedOld.has(o.blockId)) {
|
|
91
|
-
deleted.push(o.blockId);
|
|
92
90
|
dispositions.push({ blockId: o.blockId, kind: "deleted", confidence: null, reason: "tombstone", matcherV: state.config.matcherV, detail: {} });
|
|
93
91
|
}
|
|
94
92
|
}
|
|
95
|
-
return { assignment, dispositions, deleted, consumedPool };
|
|
93
|
+
return { assignment, dispositions, deleted: deletedIds(state.old, dispositions), consumedPool };
|
|
94
|
+
}
|
|
95
|
+
/** `deleted` = every old id whose disposition kind is `deleted` — the phase 7
|
|
96
|
+
* tombstones and the phase 6a non-dominant split tombstones — in old document
|
|
97
|
+
* order (spec §2; m2.1 — m2.0 listed only the phase 7 ones, so a split
|
|
98
|
+
* tombstone never reached the resurrection pool). */
|
|
99
|
+
function deletedIds(old, dispositions) {
|
|
100
|
+
const deletedSet = new Set();
|
|
101
|
+
for (const d of dispositions)
|
|
102
|
+
if (d.kind === "deleted")
|
|
103
|
+
deletedSet.add(d.blockId);
|
|
104
|
+
return old.filter((o) => deletedSet.has(o.blockId)).map((o) => o.blockId);
|
|
96
105
|
}
|
|
97
106
|
function bulkRewrite(state) {
|
|
98
107
|
// Mint everything; emit one doc-scoped bulk_rewrite + deleted for all old.
|
|
@@ -105,11 +114,9 @@ function bulkRewrite(state) {
|
|
|
105
114
|
assignment.set(n.key, minted);
|
|
106
115
|
dispositions.push({ blockId: minted, kind: "inserted", confidence: null, reason: null, matcherV: state.config.matcherV, detail: {} });
|
|
107
116
|
}
|
|
108
|
-
const deleted = [];
|
|
109
117
|
for (const o of state.old) {
|
|
110
|
-
deleted.push(o.blockId);
|
|
111
118
|
dispositions.push({ blockId: o.blockId, kind: "deleted", confidence: null, reason: "tombstone", matcherV: state.config.matcherV, detail: {} });
|
|
112
119
|
}
|
|
113
|
-
return { assignment, dispositions, deleted, consumedPool: [] };
|
|
120
|
+
return { assignment, dispositions, deleted: deletedIds(state.old, dispositions), consumedPool: [] };
|
|
114
121
|
}
|
|
115
122
|
//# sourceMappingURL=reconcile.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type DispositionKind = "same" | "edited" | "moved" | "edited_moved" | "inserted" | "deleted" | "split_from" | "merged_into" | "copied_from" | "resurrected" | "bulk_rewrite";
|
|
2
|
-
export type Reason = "exact_hash" | "normalized_hash" | "anchor" | "context_unique" | "scored" | "tombstone" | "api";
|
|
2
|
+
export type Reason = "exact_hash" | "normalized_hash" | "anchor" | "context_unique" | "context_children" | "scored" | "tombstone" | "api";
|
|
3
3
|
export interface MatchBlock {
|
|
4
4
|
/** present on old blocks; undefined on new blocks until assigned */
|
|
5
5
|
blockId?: string;
|
|
@@ -34,6 +34,9 @@ export interface ReconcileConfig {
|
|
|
34
34
|
maxScoredBlocks: number;
|
|
35
35
|
smallBlockTokens: number;
|
|
36
36
|
contextSimFloor: number;
|
|
37
|
+
/** phase-4b: minimum fraction of an old container's children that must have
|
|
38
|
+
* carried into one new container for the children to vouch for it (0.5). */
|
|
39
|
+
childrenVouchFrac: number;
|
|
37
40
|
}
|
|
38
41
|
export declare const DEFAULT_CONFIG: ReconcileConfig;
|
|
39
42
|
/** The result of matching one document's old tree against new blocks. */
|
|
@@ -3,9 +3,17 @@
|
|
|
3
3
|
// without ids) and produces an assignment: each new block either CARRIES an
|
|
4
4
|
// existing id or is MINTED, plus a disposition per carry/mint.
|
|
5
5
|
export const DEFAULT_CONFIG = {
|
|
6
|
-
|
|
6
|
+
// "m" + spec/reconcile/VERSION. m2.0: phase 4b (children vouch for their
|
|
7
|
+
// parent) and the phase-4 fixed point. m2.1: the four spec §10 fixes — phase 5
|
|
8
|
+
// skips (not stops at) a sub-threshold candidate, position_prior over the
|
|
9
|
+
// sibling count, every split and merge per run, split tombstones listed in
|
|
10
|
+
// `deleted`. m2.2: thetaSmall 0.80 → 0.62 (eval harness, 2026-09-26). m2.3:
|
|
11
|
+
// phase 4a's singleton rule — the lone unmatched child of a carried container
|
|
12
|
+
// carries into the lone unmatched child in the same slot without a text floor
|
|
13
|
+
// (2026-09-26). Rule refinements bump the minor; phase changes bump the major.
|
|
14
|
+
matcherV: "m2.3",
|
|
7
15
|
thetaAccept: 0.62,
|
|
8
|
-
thetaSmall: 0.
|
|
16
|
+
thetaSmall: 0.62,
|
|
9
17
|
thetaXdoc: 0.8,
|
|
10
18
|
splitCoverage: 0.8,
|
|
11
19
|
splitDominantShare: 0.7,
|
|
@@ -15,5 +23,6 @@ export const DEFAULT_CONFIG = {
|
|
|
15
23
|
maxScoredBlocks: 2000,
|
|
16
24
|
smallBlockTokens: 8,
|
|
17
25
|
contextSimFloor: 0.35,
|
|
26
|
+
childrenVouchFrac: 0.5,
|
|
18
27
|
};
|
|
19
28
|
//# sourceMappingURL=types.js.map
|
|
@@ -11,3 +11,13 @@ export interface ExternalProvider {
|
|
|
11
11
|
* failure. Callers must call close() when done.
|
|
12
12
|
*/
|
|
13
13
|
export declare function createExternalProvider(settings: EmbeddingSettings): Promise<ExternalProvider | null>;
|
|
14
|
+
/**
|
|
15
|
+
* The environment a stdio embedder is spawned with: our own env, with the
|
|
16
|
+
* repo's explicit `embedding.model` / `dim` / `maxInputTokens` settings laid over
|
|
17
|
+
* it as the `OMGBASE_EMBEDDER_*` variables `omgbase-embedder` reads. Explicit
|
|
18
|
+
* settings win over an inherited variable (config is the deliberate per-repo
|
|
19
|
+
* choice; the env is ambient); an unset setting leaves the inherited value (or
|
|
20
|
+
* the embedder's own default) in place. The handshake still has the last word on
|
|
21
|
+
* what the provider reports back (see connectStdio).
|
|
22
|
+
*/
|
|
23
|
+
export declare function embedderEnv(settings: EmbeddingSettings, base?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
@@ -67,7 +67,7 @@ async function connectStdio(settings) {
|
|
|
67
67
|
// even after we exit. A private pipe we own severs that coupling.
|
|
68
68
|
let child;
|
|
69
69
|
try {
|
|
70
|
-
child = spawn(cmd, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
70
|
+
child = spawn(cmd, args, { stdio: ["pipe", "pipe", "pipe"], env: embedderEnv(settings) });
|
|
71
71
|
}
|
|
72
72
|
catch (err) {
|
|
73
73
|
throw new Error(`embedding command '${settings.provider}' failed to spawn: ${err.message}`);
|
|
@@ -153,6 +153,23 @@ async function connectStdio(settings) {
|
|
|
153
153
|
};
|
|
154
154
|
return { provider, close };
|
|
155
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* The environment a stdio embedder is spawned with: our own env, with the
|
|
158
|
+
* repo's explicit `embedding.model` / `dim` / `maxInputTokens` settings laid over
|
|
159
|
+
* it as the `OMGBASE_EMBEDDER_*` variables `omgbase-embedder` reads. Explicit
|
|
160
|
+
* settings win over an inherited variable (config is the deliberate per-repo
|
|
161
|
+
* choice; the env is ambient); an unset setting leaves the inherited value (or
|
|
162
|
+
* the embedder's own default) in place. The handshake still has the last word on
|
|
163
|
+
* what the provider reports back (see connectStdio).
|
|
164
|
+
*/
|
|
165
|
+
export function embedderEnv(settings, base = process.env) {
|
|
166
|
+
return {
|
|
167
|
+
...base,
|
|
168
|
+
...(settings.model !== undefined ? { OMGBASE_EMBEDDER_MODEL: settings.model } : {}),
|
|
169
|
+
...(settings.dim !== undefined ? { OMGBASE_EMBEDDER_DIM: String(settings.dim) } : {}),
|
|
170
|
+
...(settings.maxInputTokens !== undefined ? { OMGBASE_EMBEDDER_MAX_TOKENS: String(settings.maxInputTokens) } : {}),
|
|
171
|
+
};
|
|
172
|
+
}
|
|
156
173
|
// A pull-based async line reader over a readline Interface: next() resolves with
|
|
157
174
|
// the next line, buffering lines that arrive before they're requested.
|
|
158
175
|
function lineQueue(rl) {
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
export interface EmbeddingSettings {
|
|
2
|
-
/**
|
|
2
|
+
/** The embedder: a command to spawn (e.g. "omgbase-embedder") or an http(s)
|
|
3
|
+
* URL. Absent ⇒ no provider (`semantic_unavailable`). */
|
|
3
4
|
provider?: string;
|
|
4
|
-
/** Optional model id
|
|
5
|
+
/** Optional model id. For a spawned command it is exported as
|
|
6
|
+
* OMGBASE_EMBEDDER_MODEL (explicit setting wins over an inherited variable);
|
|
7
|
+
* the provider's handshake/metadata reply overrides it for reporting. */
|
|
5
8
|
model?: string;
|
|
6
|
-
/** Optional dimension
|
|
9
|
+
/** Optional dimension. For a spawned command it is exported as
|
|
10
|
+
* OMGBASE_EMBEDDER_DIM; the handshake/metadata reply overrides it. */
|
|
7
11
|
dim?: number;
|
|
8
12
|
/**
|
|
9
13
|
* Optional max input length (tokens) the provider's model accepts before it
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
// Provider configuration (05 §6). The embedding provider is
|
|
2
|
-
// repo settings
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
1
|
+
// Provider configuration (05 §6). The embedding provider is an external
|
|
2
|
+
// *process or endpoint* named in repo settings — `embedding.provider` is either
|
|
3
|
+
// a shell command (spawned and spoken to over the stdio JSON protocol) or an
|
|
4
|
+
// http(s) URL (GET metadata, POST embed) — see `search/external.ts`. Nothing is
|
|
5
|
+
// imported in-process, so core carries no ML dependency and the embedder can be
|
|
6
|
+
// any language. `@omgbase/embedder`'s `omgbase-embedder` binary is the default
|
|
7
|
+
// local implementation; an HTTP endpoint is the same contract over the wire —
|
|
8
|
+
// nothing privileges local vs. remote.
|
|
8
9
|
/** Read the embedding.* block out of a repo's parsed settings JSON. */
|
|
9
10
|
export function embeddingSettings(settings) {
|
|
10
11
|
const e = settings.embedding;
|
package/dist/src/search/rrf.js
CHANGED
|
@@ -50,7 +50,11 @@ export function hybridSearch(store, input) {
|
|
|
50
50
|
const vecRanks = new Map();
|
|
51
51
|
const cosineByBlock = new Map();
|
|
52
52
|
if (input.vector) {
|
|
53
|
+
// First occurrence wins, exactly as for FTS (spec/search §4). Since search
|
|
54
|
+
// 1.1 vectorSearch yields one candidate per block, so this is a no-op guard.
|
|
53
55
|
vectorSearch(store, input.repoId, input.vector.model, input.vector.vec, { limit: 200 }).forEach((h, i) => {
|
|
56
|
+
if (vecRanks.has(h.blockId))
|
|
57
|
+
return;
|
|
54
58
|
vecRanks.set(h.blockId, i + 1);
|
|
55
59
|
cosineByBlock.set(h.blockId, h.cosine);
|
|
56
60
|
});
|