@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
|
@@ -7,13 +7,19 @@
|
|
|
7
7
|
// not a hand-rolled tally. The result is one or more independent points of
|
|
8
8
|
// attention for the rest of the pipeline to follow.
|
|
9
9
|
import { isChunk } from "../sema.js";
|
|
10
|
-
import { lightestDerivation
|
|
10
|
+
import { lightestDerivation } from "../derive/src/index.js";
|
|
11
11
|
import { consensusFloor, dominates, estimatorNoise } from "../geometry.js";
|
|
12
12
|
import { foldTree, gistOf, perceive, read } from "./primitives.js";
|
|
13
13
|
import { recognise } from "./recognition.js";
|
|
14
14
|
import { leafIdRun } from "./canonical.js";
|
|
15
15
|
import { corpusN, edgeAncestors } from "./traverse.js";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
cachedRead,
|
|
18
|
+
junctionContainersFrom,
|
|
19
|
+
junctionSeeds,
|
|
20
|
+
junctionSynonyms,
|
|
21
|
+
walkCache,
|
|
22
|
+
} from "./junction.js";
|
|
17
23
|
import { indexOf } from "../bytes.js";
|
|
18
24
|
import { rNode, traceDerivation } from "./trace.js";
|
|
19
25
|
// ── Public entry points ───────────────────────────────────────────────────
|
|
@@ -21,297 +27,330 @@ import { rNode, traceDerivation } from "./trace.js";
|
|
|
21
27
|
* resonance, pool the evidence, and return only the ROOT points of
|
|
22
28
|
* attention — those that cleared commitVotes' significance floor. */
|
|
23
29
|
export async function climbAttention(ctx, query, k, mode = "inverse") {
|
|
24
|
-
|
|
30
|
+
return (await climbAttentionAll(ctx, query, k, mode)).roots;
|
|
25
31
|
}
|
|
26
32
|
/** Full read-out of one consensus climb: both the roots (dominant points of
|
|
27
33
|
* attention) and the entire ranked list. Cached via ctx.climbMemo when
|
|
28
34
|
* ctx.trace is null. */
|
|
29
35
|
export async function climbAttentionAll(ctx, query, k, mode = "inverse") {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const hit = byRead.get(key);
|
|
36
|
-
if (hit !== undefined)
|
|
37
|
-
return hit;
|
|
38
|
-
const read = await computeAttention(ctx, query, k, mode);
|
|
39
|
-
byRead.set(key, read);
|
|
40
|
-
return read;
|
|
36
|
+
if (ctx.climbMemo && !ctx.trace) {
|
|
37
|
+
const key = `${k}:${mode}`;
|
|
38
|
+
let byRead = ctx.climbMemo.get(query);
|
|
39
|
+
if (byRead === undefined) {
|
|
40
|
+
ctx.climbMemo.set(query, byRead = new Map());
|
|
41
41
|
}
|
|
42
|
-
|
|
42
|
+
const hit = byRead.get(key);
|
|
43
|
+
if (hit !== undefined) {
|
|
44
|
+
return hit;
|
|
45
|
+
}
|
|
46
|
+
const read = await computeAttention(ctx, query, k, mode);
|
|
47
|
+
byRead.set(key, read);
|
|
48
|
+
return read;
|
|
49
|
+
}
|
|
50
|
+
return computeAttention(ctx, query, k, mode);
|
|
43
51
|
}
|
|
44
52
|
// ── Pipeline ──────────────────────────────────────────────────────────────
|
|
45
53
|
export async function computeAttention(ctx, query, k, mode) {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
54
|
+
const regions = collectRegions(ctx, query);
|
|
55
|
+
// Recognised sites carry structural evidence that perceived sub-regions
|
|
56
|
+
// miss: a word crossing a W-boundary is split into chunks whose partial
|
|
57
|
+
// gists may not resonate distinctively, but the SITE (content-addressed,
|
|
58
|
+
// exact) names the whole form. Adding sites as climb regions lets the
|
|
59
|
+
// consensus vote with the full word, at zero cost — recognition is already
|
|
60
|
+
// memoised per response (ctx.recogniseMemo), and gistOf for short sites is
|
|
61
|
+
// O(|span|·D). Sites that overlap perceived regions add corroborating
|
|
62
|
+
// evidence; sites in gaps (like cross-boundary words) fill them.
|
|
63
|
+
const rec = recognise(ctx, query);
|
|
64
|
+
for (const s of rec.sites) {
|
|
65
|
+
regions.push({
|
|
66
|
+
v: gistOf(ctx, query.subarray(s.start, s.end)),
|
|
67
|
+
start: s.start,
|
|
68
|
+
end: s.end,
|
|
69
|
+
chunk: false,
|
|
70
|
+
known: true, // a recognised site IS a stored form
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (regions.length === 0) {
|
|
74
|
+
return { roots: [], ranked: [] };
|
|
75
|
+
}
|
|
76
|
+
const N = corpusN(ctx);
|
|
77
|
+
// One climb per distinct anchor for the WHOLE query: regions sharing a
|
|
78
|
+
// chunk, and canonicalChunkId's prefix probes, all hit this memo instead of
|
|
79
|
+
// re-reading the anchor's full edge fan-out from the store.
|
|
80
|
+
const reachMemo = new Map();
|
|
81
|
+
const rvs = await voteRegions(ctx, query, regions, k, mode, N, reachMemo);
|
|
82
|
+
// ── Cross-region: DIRECT region-to-region interaction ─────────────────
|
|
83
|
+
// Two regions whose individual climbs land on DIFFERENT contexts leave
|
|
84
|
+
// their JOINT context — the learnt whole that contains BOTH — with no
|
|
85
|
+
// vote. crossRegionVotes recovers it by the bridge's content-addressed
|
|
86
|
+
// junction ascent (see the note above the function).
|
|
87
|
+
const cross = await crossRegionVotes(
|
|
88
|
+
ctx,
|
|
89
|
+
query,
|
|
90
|
+
regions,
|
|
91
|
+
rvs,
|
|
92
|
+
k,
|
|
93
|
+
N,
|
|
94
|
+
reachMemo,
|
|
95
|
+
);
|
|
96
|
+
// A vote SUPERSEDED by exact joint evidence (its bytes literally live
|
|
97
|
+
// inside the joint container, yet it climbed elsewhere — grid aliasing)
|
|
98
|
+
// is dropped, not down-weighted: the joint container explains it away.
|
|
99
|
+
const allVotes = cross.votes.length > 0
|
|
100
|
+
? [
|
|
101
|
+
...rvs.votes.filter((v) => !cross.superseded.has(v)),
|
|
102
|
+
...cross.votes,
|
|
103
|
+
]
|
|
104
|
+
: rvs.votes;
|
|
105
|
+
// ──────────────────────────────────────────────────────────────────────
|
|
106
|
+
if (allVotes.length === 0) {
|
|
107
|
+
traceAttention(ctx, regions, rvs.voters, []);
|
|
108
|
+
return { roots: [], ranked: [] };
|
|
109
|
+
}
|
|
110
|
+
const sat = detectSaturated(ctx, regions, rvs.saturated);
|
|
111
|
+
const pooled = poolVotes(ctx, allVotes, sat, N);
|
|
112
|
+
return commitVotes(ctx, pooled, sat, regions, rvs.voters, N);
|
|
96
113
|
}
|
|
97
114
|
export function collectRegions(ctx, query) {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}
|
|
125
|
-
|
|
115
|
+
const regions = [];
|
|
116
|
+
// A region that DOMINATES the query (covers more than half — the shared
|
|
117
|
+
// {@link dominates} test liftAnswer uses for a span that swallows its
|
|
118
|
+
// surroundings) can never itself discriminate between several topics the
|
|
119
|
+
// query weaves; voting with it only when it is the sole structure (no
|
|
120
|
+
// narrower region exists) keeps a flat/short query's single point of
|
|
121
|
+
// attention intact without letting a broad, non-discriminative wrapper
|
|
122
|
+
// dilute a multi-topic query's vote or masquerade as a genuine second
|
|
123
|
+
// point of attention.
|
|
124
|
+
// foldTree (not walkTree): the same post-order walk, but each node also
|
|
125
|
+
// resolves content-addressed against the store — `known` is what lets the
|
|
126
|
+
// climb keep exact evidence at full weight while margin-damping the
|
|
127
|
+
// approximate kind (see voteRegions). One findLeaf/findBranch per tree
|
|
128
|
+
// node, the same lookups a deposit pays.
|
|
129
|
+
foldTree(ctx, perceive(ctx, query), 0, (n, start, end, node) => {
|
|
130
|
+
if (n.kids === null) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (!dominates(end - start, query.length) || regions.length === 0) {
|
|
134
|
+
regions.push({
|
|
135
|
+
v: n.v,
|
|
136
|
+
start,
|
|
137
|
+
end,
|
|
138
|
+
chunk: isChunk(n),
|
|
139
|
+
known: node !== null,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
return regions;
|
|
126
144
|
}
|
|
127
145
|
export async function voteRegions(ctx, query, regions, k, mode, N, reachMemo) {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
146
|
+
const regionSaturated = new Array(regions.length).fill(false);
|
|
147
|
+
const regionVotes = [];
|
|
148
|
+
const regionVoter = ctx.trace ? regions.map(() => null) : [];
|
|
149
|
+
for (let ri = 0; ri < regions.length; ri++) {
|
|
150
|
+
const { v, start, end, chunk, known } = regions[ri];
|
|
151
|
+
// EXACT-FIRST: a chunk whose canonical anchor is content-addressed needs
|
|
152
|
+
// no estimator — identity is exact, so its score is 1 BY DEFINITION (the
|
|
153
|
+
// estimated cosine of a form with itself, minus quantisation noise, and
|
|
154
|
+
// the caveat atop geometry.ts forbids trusting the estimate over the
|
|
155
|
+
// exact resolution anyway). The ANN query is deferred behind
|
|
156
|
+
// `ensureHits` and paid only when actually consulted: the orphan
|
|
157
|
+
// fallback, the contrastive margin (approximate regions only), or a
|
|
158
|
+
// region with no usable canonical. On chunk-heavy queries this removes
|
|
159
|
+
// the resonate() call for most exact regions — the single largest
|
|
160
|
+
// remaining inference sink — with the anchor choice unchanged (the
|
|
161
|
+
// canonical branch already ignored hits[0]).
|
|
162
|
+
const canonicalId = chunk
|
|
163
|
+
? canonicalChunkId(ctx, query.subarray(start, end), N, reachMemo)
|
|
164
|
+
: null;
|
|
165
|
+
const canonicalUsable = canonicalId !== null &&
|
|
166
|
+
(ctx.store.hasParents(canonicalId) ||
|
|
167
|
+
ctx.store.hasContainers(canonicalId));
|
|
168
|
+
let hits = null;
|
|
169
|
+
const ensureHits = async () => hits ??= await ctx.store.resonate(v, k);
|
|
170
|
+
const canonicalFailed = chunk && canonicalId === null;
|
|
171
|
+
let voterId;
|
|
172
|
+
let score;
|
|
173
|
+
let scoreId; // the node the score was measured against
|
|
174
|
+
if (canonicalUsable) {
|
|
175
|
+
voterId = canonicalId;
|
|
176
|
+
score = 1;
|
|
177
|
+
scoreId = canonicalId;
|
|
178
|
+
} else {
|
|
179
|
+
const h = await ensureHits();
|
|
180
|
+
if (h.length === 0) {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
voterId = h[0].id;
|
|
184
|
+
score = h[0].score;
|
|
185
|
+
scoreId = h[0].id;
|
|
186
|
+
}
|
|
187
|
+
let reach = edgeAncestors(ctx, voterId, N, reachMemo);
|
|
188
|
+
// A region's vote must not die with the TOP hit: `hits[1..k]` were
|
|
189
|
+
// already fetched, and the top-ranked anchor being a structural orphan
|
|
190
|
+
// (no edge-bearing ancestors) is an accident of the approximate ranking,
|
|
191
|
+
// not evidence the region relates to nothing. Walk the remaining hits —
|
|
192
|
+
// nearest first, climbs memoised — until one climbs. A SATURATED reach
|
|
193
|
+
// is not an orphan: it is a deliberate abstention, kept as-is.
|
|
194
|
+
if (reach.roots.length === 0 && !reach.saturated) {
|
|
195
|
+
for (const h of await ensureHits()) {
|
|
196
|
+
if (h.id === voterId) {
|
|
197
|
+
continue;
|
|
160
198
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
voterId
|
|
166
|
-
|
|
167
|
-
|
|
199
|
+
const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
|
|
200
|
+
if (r2.saturated || r2.roots.length > 0) {
|
|
201
|
+
ctx.trace?.step(
|
|
202
|
+
"anchorFallback",
|
|
203
|
+
[rNode(ctx, voterId, "orphan-anchor", score)],
|
|
204
|
+
[rNode(ctx, h.id, "anchor", h.score)],
|
|
205
|
+
"the top-ranked anchor climbs to no context — a lower-ranked hit votes instead",
|
|
206
|
+
);
|
|
207
|
+
reach = r2;
|
|
208
|
+
voterId = h.id;
|
|
209
|
+
score = h.score;
|
|
210
|
+
scoreId = h.id;
|
|
211
|
+
break;
|
|
168
212
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
213
|
+
}
|
|
214
|
+
} else if (!canonicalUsable && reach.saturated) {
|
|
215
|
+
// TIE-BAND saturation fallback. A saturated top hit abstains the whole
|
|
216
|
+
// region (a hub's reach concludes nothing) — but the hub may only CLAIM
|
|
217
|
+
// that abstention when it is DISTINGUISHABLY the nearest anchor. The
|
|
218
|
+
// resonance ranking is an estimate: the difference between two scores
|
|
219
|
+
// against the same query carries √2× the estimator's per-score error,
|
|
220
|
+
// ≈ 1/√D ({@link estimatorNoise}) — so any hit within that band of the
|
|
221
|
+
// top is the SAME rank at measurement resolution, and letting the hub
|
|
222
|
+
// win the tie decides the region by quantisation accident (observed:
|
|
223
|
+
// a 0.1σ rank inversion flipped a pinned behaviour when the query
|
|
224
|
+
// estimator sharpened from 4 to 8 bits). Walk the tied hits, nearest
|
|
225
|
+
// first; the first that climbs somewhere non-saturated votes for the
|
|
226
|
+
// region. Beyond the band the hub is genuinely nearest and its
|
|
227
|
+
// abstention stands. A KNOWN (content-addressed) region never enters:
|
|
228
|
+
// its anchor is exact, not an estimate.
|
|
229
|
+
const band = estimatorNoise(ctx.store.D);
|
|
230
|
+
for (const h of await ensureHits()) {
|
|
231
|
+
if (h.id === voterId) {
|
|
232
|
+
continue;
|
|
190
233
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
// region (a hub's reach concludes nothing) — but the hub may only CLAIM
|
|
194
|
-
// that abstention when it is DISTINGUISHABLY the nearest anchor. The
|
|
195
|
-
// resonance ranking is an estimate: the difference between two scores
|
|
196
|
-
// against the same query carries √2× the estimator's per-score error,
|
|
197
|
-
// ≈ 1/√D ({@link estimatorNoise}) — so any hit within that band of the
|
|
198
|
-
// top is the SAME rank at measurement resolution, and letting the hub
|
|
199
|
-
// win the tie decides the region by quantisation accident (observed:
|
|
200
|
-
// a 0.1σ rank inversion flipped a pinned behaviour when the query
|
|
201
|
-
// estimator sharpened from 4 to 8 bits). Walk the tied hits, nearest
|
|
202
|
-
// first; the first that climbs somewhere non-saturated votes for the
|
|
203
|
-
// region. Beyond the band the hub is genuinely nearest and its
|
|
204
|
-
// abstention stands. A KNOWN (content-addressed) region never enters:
|
|
205
|
-
// its anchor is exact, not an estimate.
|
|
206
|
-
const band = estimatorNoise(ctx.store.D);
|
|
207
|
-
for (const h of await ensureHits()) {
|
|
208
|
-
if (h.id === voterId)
|
|
209
|
-
continue;
|
|
210
|
-
if (h.score < score - band)
|
|
211
|
-
break; // hits are nearest-first
|
|
212
|
-
const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
|
|
213
|
-
if (!r2.saturated && r2.roots.length > 0) {
|
|
214
|
-
ctx.trace?.step("anchorFallback", [rNode(ctx, voterId, "saturated-anchor", score)], [rNode(ctx, h.id, "anchor", h.score)], "the top-ranked anchor is a saturated hub tied within estimator noise — the tied hit votes instead");
|
|
215
|
-
reach = r2;
|
|
216
|
-
voterId = h.id;
|
|
217
|
-
score = h.score;
|
|
218
|
-
scoreId = h.id;
|
|
219
|
-
break;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
234
|
+
if (h.score < score - band) {
|
|
235
|
+
break; // hits are nearest-first
|
|
222
236
|
}
|
|
223
|
-
|
|
224
|
-
if (
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
// fold demands, applied to APPROXIMATE evidence only. Under the linear
|
|
237
|
-
// fold a resonance score reads "fraction of aligned shared bytes", so a
|
|
238
|
-
// NOVEL span sharing a frame with several stored exemplars scores high
|
|
239
|
-
// against each of them without being evidence of ANY of them: the shared
|
|
240
|
-
// scaffolding, not the span's own content, carries the similarity. Such a
|
|
241
|
-
// frame region resonates ~equally to every framed exemplar, so its top hit
|
|
242
|
-
// barely beats the best DIFFERENT-conclusion rival (a different climb
|
|
243
|
-
// root-set) — its discriminative margin, score MINUS that rival, collapses
|
|
244
|
-
// toward zero. A region votes only when that margin clears the estimator's
|
|
245
|
-
// own noise floor (1/√D — see {@link estimatorNoise}); below it the margin
|
|
246
|
-
// is quantisation noise, not evidence. A KNOWN region (content-addressed,
|
|
247
|
-
// exact) skips the contrast: it IS learnt content, not an approximation.
|
|
248
|
-
//
|
|
249
|
-
// The margin GATES; it does NOT scale the weight. A surviving region votes
|
|
250
|
-
// at its genuine strength (score²·wf) — the SAME scale {@link
|
|
251
|
-
// consensusFloor} is derived for. Using the margin as a MULTIPLIER
|
|
252
|
-
// (score·margin) conflated "discriminative" with "strong": a genuinely
|
|
253
|
-
// discriminative span whose frame-rival happened to score close got a tiny
|
|
254
|
-
// vote, systematically compressing correct scaffolding-dominated groundings
|
|
255
|
-
// (reordered / paraphrased queries) below the floor so they grounded
|
|
256
|
-
// nothing. Gating at the noise floor keeps frame-echo suppression (a frame
|
|
257
|
-
// region's margin ≈ 0 is gated out) without penalising honest evidence.
|
|
258
|
-
if (!known) {
|
|
259
|
-
let margin = score;
|
|
260
|
-
for (const h of await ensureHits()) {
|
|
261
|
-
if (h.id === voterId)
|
|
262
|
-
continue;
|
|
263
|
-
const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
|
|
264
|
-
if (r2.saturated || r2.roots.length === 0)
|
|
265
|
-
continue; // concludes nothing
|
|
266
|
-
if (sameRoots(r2.roots, reach.roots))
|
|
267
|
-
continue; // same conclusion
|
|
268
|
-
margin = score - h.score; // hits are nearest-first: the best rival
|
|
269
|
-
break;
|
|
270
|
-
}
|
|
271
|
-
if (margin <= estimatorNoise(ctx.store.D))
|
|
272
|
-
continue;
|
|
237
|
+
const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
|
|
238
|
+
if (!r2.saturated && r2.roots.length > 0) {
|
|
239
|
+
ctx.trace?.step(
|
|
240
|
+
"anchorFallback",
|
|
241
|
+
[rNode(ctx, voterId, "saturated-anchor", score)],
|
|
242
|
+
[rNode(ctx, h.id, "anchor", h.score)],
|
|
243
|
+
"the top-ranked anchor is a saturated hub tied within estimator noise — the tied hit votes instead",
|
|
244
|
+
);
|
|
245
|
+
reach = r2;
|
|
246
|
+
voterId = h.id;
|
|
247
|
+
score = h.score;
|
|
248
|
+
scoreId = h.id;
|
|
249
|
+
break;
|
|
273
250
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
regionSaturated[ri] = reach.saturated;
|
|
254
|
+
if (reach.roots.length === 0) {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (reach.saturated) {
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
// One IDF per region — dfWeight() and the focus weight used to compute
|
|
261
|
+
// the same logarithm independently.
|
|
262
|
+
const idf = Math.log(N / Math.max(1, reach.contextsReached));
|
|
263
|
+
const df = Math.log(1 + reach.contextsReached);
|
|
264
|
+
const wf = mode === "direct" ? df : mode === "combined" ? idf + df : idf;
|
|
265
|
+
if (wf <= 0) {
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
// CONTRASTIVE-MARGIN GATE — the compensation the linear (byte-proportional)
|
|
269
|
+
// fold demands, applied to APPROXIMATE evidence only. Under the linear
|
|
270
|
+
// fold a resonance score reads "fraction of aligned shared bytes", so a
|
|
271
|
+
// NOVEL span sharing a frame with several stored exemplars scores high
|
|
272
|
+
// against each of them without being evidence of ANY of them: the shared
|
|
273
|
+
// scaffolding, not the span's own content, carries the similarity. Such a
|
|
274
|
+
// frame region resonates ~equally to every framed exemplar, so its top hit
|
|
275
|
+
// barely beats the best DIFFERENT-conclusion rival (a different climb
|
|
276
|
+
// root-set) — its discriminative margin, score MINUS that rival, collapses
|
|
277
|
+
// toward zero. A region votes only when that margin clears the estimator's
|
|
278
|
+
// own noise floor (1/√D — see {@link estimatorNoise}); below it the margin
|
|
279
|
+
// is quantisation noise, not evidence. A KNOWN region (content-addressed,
|
|
280
|
+
// exact) skips the contrast: it IS learnt content, not an approximation.
|
|
281
|
+
//
|
|
282
|
+
// The margin GATES; it does NOT scale the weight. A surviving region votes
|
|
283
|
+
// at its genuine strength (score²·wf) — the SAME scale {@link
|
|
284
|
+
// consensusFloor} is derived for. Using the margin as a MULTIPLIER
|
|
285
|
+
// (score·margin) conflated "discriminative" with "strong": a genuinely
|
|
286
|
+
// discriminative span whose frame-rival happened to score close got a tiny
|
|
287
|
+
// vote, systematically compressing correct scaffolding-dominated groundings
|
|
288
|
+
// (reordered / paraphrased queries) below the floor so they grounded
|
|
289
|
+
// nothing. Gating at the noise floor keeps frame-echo suppression (a frame
|
|
290
|
+
// region's margin ≈ 0 is gated out) without penalising honest evidence.
|
|
291
|
+
if (!known) {
|
|
292
|
+
let margin = score;
|
|
293
|
+
for (const h of await ensureHits()) {
|
|
294
|
+
if (h.id === voterId) {
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
const r2 = edgeAncestors(ctx, h.id, N, reachMemo);
|
|
298
|
+
if (r2.saturated || r2.roots.length === 0) {
|
|
299
|
+
continue; // concludes nothing
|
|
300
|
+
}
|
|
301
|
+
if (sameRoots(r2.roots, reach.roots)) {
|
|
302
|
+
continue; // same conclusion
|
|
308
303
|
}
|
|
304
|
+
margin = score - h.score; // hits are nearest-first: the best rival
|
|
305
|
+
break;
|
|
306
|
+
}
|
|
307
|
+
if (margin <= estimatorNoise(ctx.store.D)) {
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
309
310
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
311
|
+
// MUTUAL-EXPLANATION WEIGHT (angle + magnitude). Under the linear fold
|
|
312
|
+
// cos = shared/(‖r‖·‖h‖) with ‖·‖² = content bytes, so the old score²
|
|
313
|
+
// was already — implicitly — (shared/len_r)·(shared/len_h): the fraction
|
|
314
|
+
// of the REGION the hit explains times the fraction of the HIT the
|
|
315
|
+
// region pins down. Made explicit, each factor is computed from the two
|
|
316
|
+
// magnitudes (the region's own span; the hit's, read from the store —
|
|
317
|
+
// contentLen, √bytes being the linear fold's gist norm) and CAPPED at 1:
|
|
318
|
+
// the estimated cosine can imply more shared content than the smaller
|
|
319
|
+
// side even holds, and the uncapped square silently credited that
|
|
320
|
+
// impossible surplus — a small region echoing inside a large context, or
|
|
321
|
+
// the reverse, voted above its physical evidence. In the uncapped
|
|
322
|
+
// regime this is exactly score², the scale {@link consensusFloor} is
|
|
323
|
+
// derived for. (The margin gate above deliberately stays in raw cosine
|
|
324
|
+
// units: it tests the ESTIMATOR's noise floor, which lives in cosine
|
|
325
|
+
// space; converting each side by its own hit's magnitude would compare
|
|
326
|
+
// noise floors of different scales.)
|
|
327
|
+
const lenR = Math.max(1, end - start);
|
|
328
|
+
// Cap the magnitude read at lenR·D: past it s/ratio ≤ s/√D — below the
|
|
329
|
+
// estimator's own noise floor — so the mutual weight is ~0 regardless
|
|
330
|
+
// and the clamped value yields exactly that; no full walk of a huge hit.
|
|
331
|
+
const ratio = Math.sqrt(
|
|
332
|
+
Math.max(1, ctx.store.contentLen(scoreId, lenR * ctx.store.D)) / lenR,
|
|
333
|
+
);
|
|
334
|
+
const mutual = Math.min(1, score * ratio) * Math.min(1, score / ratio);
|
|
335
|
+
const w = (mutual * wf) / reach.roots.length;
|
|
336
|
+
const wFocus = (mutual * idf) / reach.roots.length;
|
|
337
|
+
regionVotes.push({
|
|
338
|
+
start,
|
|
339
|
+
end,
|
|
340
|
+
canonicalFailed,
|
|
341
|
+
roots: reach.roots,
|
|
342
|
+
w,
|
|
343
|
+
wFocus,
|
|
344
|
+
});
|
|
345
|
+
if (ctx.trace) {
|
|
346
|
+
regionVoter[ri] = { id: voterId, score, w: wf };
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return {
|
|
350
|
+
votes: regionVotes,
|
|
351
|
+
saturated: regionSaturated,
|
|
352
|
+
voters: regionVoter,
|
|
353
|
+
};
|
|
315
354
|
}
|
|
316
355
|
/** The consensus vote as EVIDENCE POOLING, not shortest path: each surviving
|
|
317
356
|
* region is an axiom; it contributes to every root it climbed to (or, for a
|
|
@@ -326,267 +365,286 @@ export async function voteRegions(ctx, query, regions, k, mode, N, reachMemo) {
|
|
|
326
365
|
* read-outs {@link commitVotes} always gated on; only how they accumulate
|
|
327
366
|
* changed. */
|
|
328
367
|
export function poolVotes(ctx, regionVotes, sat, N) {
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
368
|
+
const eligible = [];
|
|
369
|
+
for (let ri = 0; ri < regionVotes.length; ri++) {
|
|
370
|
+
const rv = regionVotes[ri];
|
|
371
|
+
if (
|
|
372
|
+
rv.canonicalFailed &&
|
|
373
|
+
sat.intervals.some((iv) => rv.start >= iv.start && rv.end <= iv.end)
|
|
374
|
+
) {
|
|
375
|
+
continue;
|
|
337
376
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
steps.push({
|
|
415
|
-
order: order++,
|
|
416
|
-
move: "pool-vote",
|
|
417
|
-
premises,
|
|
418
|
-
conclusion: { kind: "form", span: [-1, -1], node: pc.item.id },
|
|
419
|
-
cost: pc.cost,
|
|
420
|
-
producers: [],
|
|
421
|
-
});
|
|
377
|
+
eligible.push(ri);
|
|
378
|
+
}
|
|
379
|
+
const key = (it) =>
|
|
380
|
+
it.kind === "region"
|
|
381
|
+
? `r${it.ri}`
|
|
382
|
+
: it.kind === "anchor"
|
|
383
|
+
? `a${it.id}`
|
|
384
|
+
: `x${it.id}`;
|
|
385
|
+
const pool = new Map();
|
|
386
|
+
const system = {
|
|
387
|
+
key,
|
|
388
|
+
*axioms() {
|
|
389
|
+
for (const ri of eligible) {
|
|
390
|
+
yield { item: { kind: "region", ri }, cost: 0 };
|
|
391
|
+
}
|
|
392
|
+
},
|
|
393
|
+
isGoal: () => false, // exhaust every axiom; there is no single goal to stop at
|
|
394
|
+
// Every region axiom ties at cost 0, so the agenda's pop order among them
|
|
395
|
+
// is otherwise unspecified; ordering by `ri` here only steers the HEAP
|
|
396
|
+
// (never added to a stored cost — see relax's use of h) so pooling fires
|
|
397
|
+
// in exactly the regionVotes array order the original loop used, byte-for-
|
|
398
|
+
// byte reproducing its accumulation and tie-break order.
|
|
399
|
+
heuristic: (it) => it.kind === "region" ? it.ri : 0,
|
|
400
|
+
*rules(it) {
|
|
401
|
+
if (it.kind !== "region") {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
const rv = regionVotes[it.ri];
|
|
405
|
+
// The same hub bound the rest of the system uses (edgeAncestors' parent
|
|
406
|
+
// cutoff, chooseNext's candidate cap): a terminal answer followed by
|
|
407
|
+
// more than √N contexts is a non-discriminative hub — spreading a
|
|
408
|
+
// region's vote across its FULL corpus-sized fan-in yields O(corpus)
|
|
409
|
+
// rule applications per region and near-zero per-target weight anyway.
|
|
410
|
+
// Cap the redistribution at the first √N contexts (insertion order,
|
|
411
|
+
// the same convention chooseNext caps by).
|
|
412
|
+
const hubBound = Math.ceil(Math.sqrt(N));
|
|
413
|
+
for (const r of rv.roots) {
|
|
414
|
+
// CAPPED read: only the first hubBound targets are ever credited, so
|
|
415
|
+
// only they are read — a common continuation's full reverse fan-in
|
|
416
|
+
// is corpus-sized and is never materialised.
|
|
417
|
+
const pv = ctx.store.prevFirst(r, hubBound);
|
|
418
|
+
const isAnswer = pv.length > 0 && !ctx.store.hasNext(r);
|
|
419
|
+
const targets = isAnswer ? pv : [r];
|
|
420
|
+
for (const t of targets) {
|
|
421
|
+
yield {
|
|
422
|
+
premises: [it],
|
|
423
|
+
conclusion: { kind: "anchor", id: t },
|
|
424
|
+
cost: rv.w / targets.length,
|
|
425
|
+
combine: "sum",
|
|
426
|
+
};
|
|
427
|
+
yield {
|
|
428
|
+
premises: [it],
|
|
429
|
+
conclusion: { kind: "anchorFocus", id: t },
|
|
430
|
+
cost: rv.wFocus / targets.length,
|
|
431
|
+
combine: "sum",
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
},
|
|
436
|
+
pool,
|
|
437
|
+
};
|
|
438
|
+
lightestDerivation(system);
|
|
439
|
+
const votes = new Map();
|
|
440
|
+
const votesIdf = new Map();
|
|
441
|
+
const support = new Map();
|
|
442
|
+
const steps = [];
|
|
443
|
+
let order = 0;
|
|
444
|
+
for (const pc of pool.values()) {
|
|
445
|
+
if (pc.item.kind === "anchor") {
|
|
446
|
+
votes.set(pc.item.id, pc.cost);
|
|
447
|
+
const premises = [];
|
|
448
|
+
const seenRi = new Set();
|
|
449
|
+
for (const c of pc.contributions) {
|
|
450
|
+
const p0 = c.premises[0].item;
|
|
451
|
+
if (p0.kind !== "region" || seenRi.has(p0.ri)) {
|
|
452
|
+
continue;
|
|
422
453
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
454
|
+
seenRi.add(p0.ri);
|
|
455
|
+
const rv = regionVotes[p0.ri];
|
|
456
|
+
premises.push({ kind: "form", span: [rv.start, rv.end] });
|
|
457
|
+
}
|
|
458
|
+
steps.push({
|
|
459
|
+
order: order++,
|
|
460
|
+
move: "pool-vote",
|
|
461
|
+
premises,
|
|
462
|
+
conclusion: { kind: "form", span: [-1, -1], node: pc.item.id },
|
|
463
|
+
cost: pc.cost,
|
|
464
|
+
producers: [],
|
|
465
|
+
});
|
|
466
|
+
} else if (pc.item.kind === "anchorFocus") {
|
|
467
|
+
votesIdf.set(pc.item.id, pc.cost);
|
|
468
|
+
let bestRv = null;
|
|
469
|
+
for (const c of pc.contributions) {
|
|
470
|
+
const p0 = c.premises[0].item;
|
|
471
|
+
if (p0.kind !== "region") {
|
|
472
|
+
continue;
|
|
441
473
|
}
|
|
474
|
+
const rv = regionVotes[p0.ri];
|
|
475
|
+
if (!bestRv || rv.wFocus > bestRv.wFocus) {
|
|
476
|
+
bestRv = rv;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
if (bestRv) {
|
|
480
|
+
support.set(pc.item.id, {
|
|
481
|
+
start: bestRv.start,
|
|
482
|
+
end: bestRv.end,
|
|
483
|
+
w: bestRv.wFocus,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
442
486
|
}
|
|
443
|
-
|
|
487
|
+
}
|
|
488
|
+
return { votes, votesIdf, support, steps };
|
|
444
489
|
}
|
|
445
490
|
export function commitVotes(ctx, pooled, sat, regions, regionVoter, N) {
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
491
|
+
const { votes, votesIdf, support, steps } = pooled;
|
|
492
|
+
if (votes.size === 0) {
|
|
493
|
+
traceAttention(ctx, regions, regionVoter, [], steps);
|
|
494
|
+
return { roots: [], ranked: [] };
|
|
495
|
+
}
|
|
496
|
+
const ranked = [...votes.entries()]
|
|
497
|
+
.map(([anchor, vote]) => {
|
|
498
|
+
const s = support.get(anchor);
|
|
499
|
+
return { anchor, vote, start: s.start, end: s.end };
|
|
455
500
|
})
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
placed.push(point);
|
|
501
|
+
.sort((a, b) => b.vote - a.vote);
|
|
502
|
+
const overlaps = (a, b) => a.start < b.end && b.start < a.end;
|
|
503
|
+
const idfDesc = [...votesIdf.values()].sort((a, b) => b - a);
|
|
504
|
+
const rootCut = naturalBreak(idfDesc);
|
|
505
|
+
// A FURTHER point of attention (beyond the dominant one, which always
|
|
506
|
+
// grounds) must clear the same absolute significance floor
|
|
507
|
+
// recallByResonance trusts a climb anchor with — log(N) + 1/2, three-ish
|
|
508
|
+
// halvings of confidence above pure chance at this corpus scale — not
|
|
509
|
+
// merely beat whatever its immediate neighbour in the ratio happens to be.
|
|
510
|
+
// Without it, naturalBreak's ratio is scale-free but not FLOOR-free: on a
|
|
511
|
+
// large, topic-diverse corpus the steepest ratio in a long noise tail can
|
|
512
|
+
// sit far below any real signal, admitting scaffolding echoes as if they
|
|
513
|
+
// were genuine further topics.
|
|
514
|
+
const floor = consensusFloor(N);
|
|
515
|
+
const placed = [];
|
|
516
|
+
const roots = [];
|
|
517
|
+
for (const point of ranked) {
|
|
518
|
+
const absorbed = placed.some((p) => overlaps(point, p));
|
|
519
|
+
if (!absorbed) {
|
|
520
|
+
const pastLeading = !sat.hasLeading ||
|
|
521
|
+
roots.length === 0 || point.start >= sat.leadingEnd;
|
|
522
|
+
const vote = votesIdf.get(point.anchor) ?? 0;
|
|
523
|
+
if (
|
|
524
|
+
(roots.length === 0 || (vote >= rootCut && vote >= floor)) &&
|
|
525
|
+
pastLeading
|
|
526
|
+
) {
|
|
527
|
+
roots.push(point);
|
|
528
|
+
} else {
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
487
531
|
}
|
|
488
|
-
|
|
489
|
-
|
|
532
|
+
placed.push(point);
|
|
533
|
+
}
|
|
534
|
+
traceAttention(ctx, regions, regionVoter, roots, steps);
|
|
535
|
+
return { roots, ranked };
|
|
490
536
|
}
|
|
491
537
|
export function detectSaturated(ctx, regions, saturated) {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
if (saturated[ri]) {
|
|
512
|
-
if (intStart === -1)
|
|
513
|
-
intStart = r.start;
|
|
514
|
-
intEnd = r.end;
|
|
515
|
-
}
|
|
516
|
-
else {
|
|
517
|
-
if (intStart !== -1) {
|
|
518
|
-
intervals.push({ start: intStart, end: intEnd });
|
|
519
|
-
intStart = -1;
|
|
520
|
-
}
|
|
521
|
-
}
|
|
538
|
+
// Intervals are built from CHUNK regions only. collectRegions emits the
|
|
539
|
+
// tree in POST-ORDER — a parent region arrives AFTER its children and
|
|
540
|
+
// shares its first child's `start` — so the raw array is not monotone in
|
|
541
|
+
// byte position, and a saturated parent would fuse with a later saturated
|
|
542
|
+
// chunk into an interval swallowing a NON-saturated child. Chunk regions
|
|
543
|
+
// (leaf-parents) are disjoint and already in byte order, and saturation
|
|
544
|
+
// masking exists to drop canonicalFailed CHUNK votes (see poolVotes), so
|
|
545
|
+
// chunks are both the sufficient and the safe basis. A region without a
|
|
546
|
+
// `chunk` flag (a bare {start,end} from a direct caller) is treated as a
|
|
547
|
+
// chunk.
|
|
548
|
+
const intervals = [];
|
|
549
|
+
let intStart = -1;
|
|
550
|
+
let intEnd = -1;
|
|
551
|
+
let totalLen = 0;
|
|
552
|
+
for (let ri = 0; ri < regions.length; ri++) {
|
|
553
|
+
const r = regions[ri];
|
|
554
|
+
totalLen = Math.max(totalLen, r.end);
|
|
555
|
+
if (r.chunk === false) {
|
|
556
|
+
continue;
|
|
522
557
|
}
|
|
523
|
-
if (
|
|
558
|
+
if (saturated[ri]) {
|
|
559
|
+
if (intStart === -1) {
|
|
560
|
+
intStart = r.start;
|
|
561
|
+
}
|
|
562
|
+
intEnd = r.end;
|
|
563
|
+
} else {
|
|
564
|
+
if (intStart !== -1) {
|
|
524
565
|
intervals.push({ start: intStart, end: intEnd });
|
|
566
|
+
intStart = -1;
|
|
567
|
+
}
|
|
525
568
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
569
|
+
}
|
|
570
|
+
if (intStart !== -1) {
|
|
571
|
+
intervals.push({ start: intStart, end: intEnd });
|
|
572
|
+
}
|
|
573
|
+
const leading = intervals.length > 0 && intervals[0].start === 0
|
|
574
|
+
? intervals[0]
|
|
575
|
+
: null;
|
|
576
|
+
const hasLeading = leading !== null &&
|
|
577
|
+
leading.end >= ctx.space.maxGroup &&
|
|
578
|
+
leading.end < totalLen;
|
|
579
|
+
const leadingEnd = leading !== null ? leading.end : 0;
|
|
580
|
+
return { leadingEnd, hasLeading, intervals };
|
|
534
581
|
}
|
|
535
582
|
/** Set equality of two climb root lists (the "same conclusion" test the
|
|
536
583
|
* contrastive margin skips rivals by). */
|
|
537
584
|
function sameRoots(a, b) {
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
585
|
+
if (a.length !== b.length) {
|
|
586
|
+
return false;
|
|
587
|
+
}
|
|
588
|
+
const s = new Set(a);
|
|
589
|
+
for (const x of b) {
|
|
590
|
+
if (!s.has(x)) {
|
|
591
|
+
return false;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
return true;
|
|
545
595
|
}
|
|
546
596
|
export function canonicalChunkId(ctx, regionBytes, N, reachMemo) {
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
return flatId;
|
|
557
|
-
let bestId = flatId;
|
|
558
|
-
let bestReach = edgeAncestors(ctx, flatId, N, reachMemo);
|
|
559
|
-
for (let k2 = 1; k2 < len; k2++) {
|
|
560
|
-
const shortIds = ids.slice(0, len - k2);
|
|
561
|
-
const shortId = ctx.store.findBranch(shortIds);
|
|
562
|
-
if (shortId === null)
|
|
563
|
-
continue;
|
|
564
|
-
const shortReach = edgeAncestors(ctx, shortId, N, reachMemo);
|
|
565
|
-
if (shortReach.saturated ||
|
|
566
|
-
shortReach.contextsReached > bestReach.contextsReached) {
|
|
567
|
-
bestId = shortId;
|
|
568
|
-
bestReach = shortReach;
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
return bestId;
|
|
597
|
+
const len = Math.min(regionBytes.length, ctx.space.maxGroup);
|
|
598
|
+
for (let off = 0; off + len <= regionBytes.length; off++) {
|
|
599
|
+
const ids = leafIdRun(ctx, regionBytes, off, off + len);
|
|
600
|
+
if (ids === null) {
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
const flatId = ctx.store.findBranch(ids);
|
|
604
|
+
if (flatId === null) {
|
|
605
|
+
continue;
|
|
572
606
|
}
|
|
573
|
-
|
|
607
|
+
if (len < 2) {
|
|
608
|
+
return flatId;
|
|
609
|
+
}
|
|
610
|
+
let bestId = flatId;
|
|
611
|
+
let bestReach = edgeAncestors(ctx, flatId, N, reachMemo);
|
|
612
|
+
for (let k2 = 1; k2 < len; k2++) {
|
|
613
|
+
const shortIds = ids.slice(0, len - k2);
|
|
614
|
+
const shortId = ctx.store.findBranch(shortIds);
|
|
615
|
+
if (shortId === null) {
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
const shortReach = edgeAncestors(ctx, shortId, N, reachMemo);
|
|
619
|
+
if (
|
|
620
|
+
shortReach.saturated ||
|
|
621
|
+
shortReach.contextsReached > bestReach.contextsReached
|
|
622
|
+
) {
|
|
623
|
+
bestId = shortId;
|
|
624
|
+
bestReach = shortReach;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
return bestId;
|
|
628
|
+
}
|
|
629
|
+
return null;
|
|
574
630
|
}
|
|
575
631
|
export function naturalBreak(votes) {
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
if (ratio < steepest) {
|
|
585
|
-
steepest = ratio;
|
|
586
|
-
breakAt = i;
|
|
587
|
-
}
|
|
632
|
+
if (votes.length <= 1) {
|
|
633
|
+
return votes[0] ?? 0;
|
|
634
|
+
}
|
|
635
|
+
let breakAt = 1;
|
|
636
|
+
let steepest = Infinity;
|
|
637
|
+
for (let i = 1; i < votes.length; i++) {
|
|
638
|
+
if (votes[i - 1] <= 0) {
|
|
639
|
+
break;
|
|
588
640
|
}
|
|
589
|
-
|
|
641
|
+
const ratio = votes[i] / votes[i - 1];
|
|
642
|
+
if (ratio < steepest) {
|
|
643
|
+
steepest = ratio;
|
|
644
|
+
breakAt = i;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
return votes[breakAt - 1];
|
|
590
648
|
}
|
|
591
649
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
592
650
|
// Cross-region attention — DIRECT region-to-region interaction.
|
|
@@ -645,250 +703,314 @@ export function naturalBreak(votes) {
|
|
|
645
703
|
// container does not hold (a genuine second topic) are untouched.
|
|
646
704
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
647
705
|
async function crossRegionVotes(ctx, query, regions, rvs, k, N, reachMemo) {
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
706
|
+
// Candidate regions: every region that ALREADY CAST ITS OWN VOTE in
|
|
707
|
+
// voteRegions — individually idf > 0, genuinely discriminative on its own,
|
|
708
|
+
// just not necessarily for the SAME context as its partner. This is the
|
|
709
|
+
// exact shape of the binding problem: "red" alone votes for `red square`,
|
|
710
|
+
// "circle" alone for `circle` — each independently informative, disagreeing
|
|
711
|
+
// on the conclusion — and only their CONJUNCTION resolves to the one
|
|
712
|
+
// context, `red circle`, that actually holds both.
|
|
713
|
+
//
|
|
714
|
+
// A region that never voted (idf == 0 — e.g. a repeated system-prompt
|
|
715
|
+
// prefix shared by every deposit) carries NO individual signal, and must be
|
|
716
|
+
// excluded here too: ascending from a non-discriminative fragment's seeds
|
|
717
|
+
// can still land on some deeper, incidentally-unique DESCENDANT container —
|
|
718
|
+
// its rarity would come entirely from context OUTSIDE the fragments
|
|
719
|
+
// actually composed, manufacturing confidence the query gave no reason to
|
|
720
|
+
// have. Requiring a prior individual vote is the same discipline the noise
|
|
721
|
+
// drop already applies to single regions, extended to compositions — with
|
|
722
|
+
// one graded relaxation: a KNOWN region that did NOT vote (saturated, or
|
|
723
|
+
// idf ≤ 0) may still serve as the WEAK side of a pair whose other side DID
|
|
724
|
+
// vote. Saturation is an abstention about where the region CLIMBS; its
|
|
725
|
+
// content-addressed identity is still exact, and the junction asks a
|
|
726
|
+
// different question — "which whole holds both?" — whose conclusion the
|
|
727
|
+
// container's own idf gate below still guards. Two non-voting regions
|
|
728
|
+
// never pair (that is exactly the shared-prefix trap above), so at least
|
|
729
|
+
// one side is always individually discriminative.
|
|
730
|
+
//
|
|
731
|
+
// Only MAXIMAL spans compose: a span contained in another candidate is a
|
|
732
|
+
// fragment of that candidate's evidence, never independent of it.
|
|
733
|
+
const votedSpans = new Set();
|
|
734
|
+
for (const rv of rvs.votes) {
|
|
735
|
+
votedSpans.add(`${rv.start},${rv.end}`);
|
|
736
|
+
}
|
|
737
|
+
const seen = new Set();
|
|
738
|
+
const eligible = [];
|
|
739
|
+
const strong = new Set();
|
|
740
|
+
for (let ri = 0; ri < regions.length; ri++) {
|
|
741
|
+
const r = regions[ri];
|
|
742
|
+
const key = `${r.start},${r.end}`;
|
|
743
|
+
const isStrong = votedSpans.has(key);
|
|
744
|
+
if ((!isStrong && !r.known) || seen.has(key)) {
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
seen.add(key);
|
|
748
|
+
eligible.push(ri);
|
|
749
|
+
if (isStrong) {
|
|
750
|
+
strong.add(ri);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
const cand = eligible.filter((x) =>
|
|
754
|
+
!eligible.some((y) =>
|
|
755
|
+
y !== x &&
|
|
756
|
+
regions[y].start <= regions[x].start &&
|
|
757
|
+
regions[x].end <= regions[y].end &&
|
|
758
|
+
regions[y].end - regions[y].start > regions[x].end - regions[x].start
|
|
759
|
+
)
|
|
760
|
+
);
|
|
761
|
+
const none = { votes: [], superseded: new Set() };
|
|
762
|
+
if (cand.length < 2) {
|
|
763
|
+
return none;
|
|
764
|
+
}
|
|
765
|
+
cand.sort((x, y) =>
|
|
766
|
+
regions[x].start - regions[y].start || regions[x].end - regions[y].end
|
|
767
|
+
);
|
|
768
|
+
const dec = (b) => new TextDecoder().decode(b).replace(/\s+/g, " ").trim();
|
|
769
|
+
const cache = walkCache(ctx);
|
|
770
|
+
// One junctionSeeds per candidate for the WHOLE pairing loop — a candidate
|
|
771
|
+
// recurs in up to |cand|−1 pairs, and its seeds are a pure function of its
|
|
772
|
+
// bytes.
|
|
773
|
+
const seedsMemo = new Map();
|
|
774
|
+
const seedsOf = (ri) => {
|
|
775
|
+
let s = seedsMemo.get(ri);
|
|
776
|
+
if (s === undefined) {
|
|
777
|
+
const r = regions[ri];
|
|
778
|
+
s = junctionSeeds(ctx, query.subarray(r.start, r.end));
|
|
779
|
+
seedsMemo.set(ri, s);
|
|
780
|
+
}
|
|
781
|
+
return s;
|
|
782
|
+
};
|
|
783
|
+
const overlapsSpan = (e, s) => e.start < s.end && s.start < e.end;
|
|
784
|
+
const out = [];
|
|
785
|
+
const superseded = new Set();
|
|
786
|
+
// A candidate consumed by one junction does not seed another: its evidence
|
|
787
|
+
// is already composed at full joint strength, and re-pairing it would vote
|
|
788
|
+
// the same container (or a sub-container of it) twice.
|
|
789
|
+
const consumed = new Set();
|
|
790
|
+
let probes = 0;
|
|
791
|
+
for (let a = 0; a < cand.length && probes < k; a++) {
|
|
792
|
+
if (consumed.has(cand[a])) {
|
|
793
|
+
continue;
|
|
691
794
|
}
|
|
692
|
-
const
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
795
|
+
const ra = regions[cand[a]];
|
|
796
|
+
for (let b = a + 1; b < cand.length && probes < k; b++) {
|
|
797
|
+
if (consumed.has(cand[b])) {
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
const rb = regions[cand[b]];
|
|
801
|
+
if (!strong.has(cand[a]) && !strong.has(cand[b])) {
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
if (ra.end >= rb.start) {
|
|
805
|
+
continue; // overlap or adjacent — nothing between
|
|
806
|
+
}
|
|
807
|
+
// A single KNOWN region covering both: the whole form is already a
|
|
808
|
+
// stored identity that votes directly; its pieces add nothing.
|
|
809
|
+
if (
|
|
810
|
+
regions.some((r) => r.known && r.start <= ra.start && rb.end <= r.end)
|
|
811
|
+
) {
|
|
812
|
+
continue;
|
|
813
|
+
}
|
|
814
|
+
probes++;
|
|
815
|
+
const left = query.subarray(ra.start, ra.end);
|
|
816
|
+
const right = query.subarray(rb.start, rb.end);
|
|
817
|
+
// Phrase-scale contract, exactly as the bridge: the glue between the two
|
|
818
|
+
// forms may be up to W× the content it joins.
|
|
819
|
+
const maxInterior = (left.length + right.length) * ctx.space.maxGroup;
|
|
820
|
+
const cap = left.length + right.length + maxInterior;
|
|
821
|
+
// Tier 1 — exact containers (both forms as substrings, either order, by
|
|
822
|
+
// DAG ascent). Exact evidence first; only falls through to synonyms
|
|
823
|
+
// when the exact ascent finds nothing — the SAME graded ladder the
|
|
824
|
+
// bridge uses.
|
|
825
|
+
let containers = junctionContainersFrom(
|
|
826
|
+
ctx,
|
|
827
|
+
left,
|
|
828
|
+
right,
|
|
829
|
+
cap,
|
|
830
|
+
seedsOf(cand[a]),
|
|
831
|
+
seedsOf(cand[b]),
|
|
832
|
+
undefined,
|
|
833
|
+
true,
|
|
834
|
+
);
|
|
835
|
+
if (containers.length === 0) {
|
|
836
|
+
// Tier 2.5 — synonym containers (halo sibling of one side + the other).
|
|
837
|
+
containers = await junctionSynonyms(
|
|
838
|
+
ctx,
|
|
839
|
+
left,
|
|
840
|
+
right,
|
|
841
|
+
maxInterior,
|
|
842
|
+
true,
|
|
843
|
+
);
|
|
844
|
+
}
|
|
845
|
+
if (containers.length === 0) {
|
|
846
|
+
continue;
|
|
847
|
+
}
|
|
848
|
+
// N-ARY selection: the container covering the MOST remaining candidate
|
|
849
|
+
// forms wins (then tightest interior, then lowest id). Reads are
|
|
850
|
+
// cache hits — every container's bytes were already read by the walk.
|
|
851
|
+
//
|
|
852
|
+
// SELF-EVIDENCE GUARD: a junction is BINDING evidence only when the
|
|
853
|
+
// container joins forms the query mentions APART. When the container's
|
|
854
|
+
// own joined occurrence (left..right including its interior) is a
|
|
855
|
+
// literal substring of the query, the query already spells that phrase
|
|
856
|
+
// out contiguously — perception already voted with it, and grid shards
|
|
857
|
+
// of one phrase pairing "around" a gap chunk would merely rediscover
|
|
858
|
+
// the phrase they are shards of, then explain away its rivals.
|
|
859
|
+
let best = null;
|
|
860
|
+
let bestExtras = [];
|
|
861
|
+
let bestCov = -1;
|
|
862
|
+
for (const c of containers) {
|
|
863
|
+
const bytes = cachedRead(ctx, cache, c.id, cap);
|
|
864
|
+
const li = indexOf(bytes, left, 0);
|
|
865
|
+
const ri = indexOf(bytes, right, 0);
|
|
866
|
+
if (li >= 0 && ri >= 0) {
|
|
867
|
+
const joined = bytes.subarray(
|
|
868
|
+
Math.min(li, ri),
|
|
869
|
+
Math.max(li + left.length, ri + right.length),
|
|
870
|
+
);
|
|
871
|
+
if (indexOf(query, joined, 0) >= 0) {
|
|
872
|
+
continue; // query says it itself
|
|
873
|
+
}
|
|
712
874
|
}
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
const superseded = new Set();
|
|
718
|
-
// A candidate consumed by one junction does not seed another: its evidence
|
|
719
|
-
// is already composed at full joint strength, and re-pairing it would vote
|
|
720
|
-
// the same container (or a sub-container of it) twice.
|
|
721
|
-
const consumed = new Set();
|
|
722
|
-
let probes = 0;
|
|
723
|
-
for (let a = 0; a < cand.length && probes < k; a++) {
|
|
724
|
-
if (consumed.has(cand[a]))
|
|
875
|
+
let cov = left.length + right.length;
|
|
876
|
+
const extras = [];
|
|
877
|
+
for (const ei of cand) {
|
|
878
|
+
if (ei === cand[a] || ei === cand[b] || consumed.has(ei)) {
|
|
725
879
|
continue;
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
best = c;
|
|
799
|
-
bestExtras = extras;
|
|
800
|
-
bestCov = cov;
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
if (best === null)
|
|
804
|
-
continue; // every container was self-evidence
|
|
805
|
-
const reach = edgeAncestors(ctx, best.id, N, reachMemo);
|
|
806
|
-
if (reach.saturated || reach.roots.length === 0)
|
|
807
|
-
continue;
|
|
808
|
-
const idf = Math.log(N / Math.max(1, reach.contextsReached));
|
|
809
|
-
if (idf <= 0)
|
|
810
|
-
continue;
|
|
811
|
-
// EXACT joint evidence (score = 1): the container literally contains
|
|
812
|
-
// every composed form. Mutual-explanation weight over their COMBINED
|
|
813
|
-
// byte length — the same magnitude reading voteRegions uses, here with
|
|
814
|
-
// the estimator collapsed to certainty, so mutual = min(ratio, 1/ratio).
|
|
815
|
-
const lenR = Math.max(1, bestCov);
|
|
816
|
-
const ratio = Math.sqrt(Math.max(1, ctx.store.contentLen(best.id, lenR * ctx.store.D)) / lenR);
|
|
817
|
-
const mutual = Math.min(1, ratio) * Math.min(1, 1 / ratio);
|
|
818
|
-
const w = (mutual * idf) / reach.roots.length;
|
|
819
|
-
let spanStart = ra.start;
|
|
820
|
-
let spanEnd = rb.end;
|
|
821
|
-
for (const ei of bestExtras) {
|
|
822
|
-
spanStart = Math.min(spanStart, regions[ei].start);
|
|
823
|
-
spanEnd = Math.max(spanEnd, regions[ei].end);
|
|
824
|
-
}
|
|
825
|
-
out.push({
|
|
826
|
-
start: spanStart,
|
|
827
|
-
end: spanEnd,
|
|
828
|
-
canonicalFailed: false, // content-addressed: never saturation-masked
|
|
829
|
-
roots: reach.roots,
|
|
830
|
-
w,
|
|
831
|
-
wFocus: w,
|
|
832
|
-
});
|
|
833
|
-
consumed.add(cand[a]);
|
|
834
|
-
consumed.add(cand[b]);
|
|
835
|
-
for (const ei of bestExtras)
|
|
836
|
-
consumed.add(ei);
|
|
837
|
-
// EXPLAINING AWAY — see the block comment above the function. Byte
|
|
838
|
-
// containment in the joint container is the relatedness test (the
|
|
839
|
-
// vote's bytes are literally part of the learnt whole), and FULL root
|
|
840
|
-
// disjointness is the disagreement test: a vote sharing even one root
|
|
841
|
-
// with the junction corroborates it and keeps its say elsewhere.
|
|
842
|
-
const containerBytes = cachedRead(ctx, cache, best.id, cap);
|
|
843
|
-
const jointRoots = new Set(reach.roots);
|
|
844
|
-
for (const rv of rvs.votes) {
|
|
845
|
-
if (rv.roots.some((r) => jointRoots.has(r)))
|
|
846
|
-
continue;
|
|
847
|
-
const bytes = query.subarray(rv.start, rv.end);
|
|
848
|
-
if (indexOf(containerBytes, bytes, 0) >= 0)
|
|
849
|
-
superseded.add(rv);
|
|
850
|
-
}
|
|
851
|
-
const label = [cand[a], cand[b], ...bestExtras]
|
|
852
|
-
.sort((x, y) => regions[x].start - regions[y].start)
|
|
853
|
-
.map((ri) => dec(query.subarray(regions[ri].start, regions[ri].end)))
|
|
854
|
-
.join(" ▸ ");
|
|
855
|
-
ctx.trace?.step("crossRegion", [{ text: label, role: "pair" }], reach.roots.map((r) => ({
|
|
856
|
-
text: dec(read(ctx, r)).slice(0, 60),
|
|
857
|
-
node: r,
|
|
858
|
-
role: "joint-context",
|
|
859
|
-
})), `${label} → junction node ${best.id}` +
|
|
860
|
-
(best.interior.length === 0
|
|
861
|
-
? " (adjacent)"
|
|
862
|
-
: ` (interior "${dec(best.interior)}")`) +
|
|
863
|
-
` → ${reach.roots.length} context(s), by content-addressed ascent` +
|
|
864
|
-
(superseded.size > 0
|
|
865
|
-
? `; ${superseded.size} aliasing vote(s) explained away`
|
|
866
|
-
: ""));
|
|
867
|
-
break; // ra is consumed — move to the next unconsumed candidate
|
|
880
|
+
}
|
|
881
|
+
const e = regions[ei];
|
|
882
|
+
if (overlapsSpan(e, ra) || overlapsSpan(e, rb)) {
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
const eb = query.subarray(e.start, e.end);
|
|
886
|
+
if (indexOf(bytes, eb, 0) >= 0) {
|
|
887
|
+
extras.push(ei);
|
|
888
|
+
cov += eb.length;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
if (
|
|
892
|
+
cov > bestCov ||
|
|
893
|
+
(cov === bestCov && best !== null &&
|
|
894
|
+
(c.interior.length < best.interior.length ||
|
|
895
|
+
(c.interior.length === best.interior.length && c.id < best.id)))
|
|
896
|
+
) {
|
|
897
|
+
best = c;
|
|
898
|
+
bestExtras = extras;
|
|
899
|
+
bestCov = cov;
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
if (best === null) {
|
|
903
|
+
continue; // every container was self-evidence
|
|
904
|
+
}
|
|
905
|
+
const reach = edgeAncestors(ctx, best.id, N, reachMemo);
|
|
906
|
+
if (reach.saturated || reach.roots.length === 0) {
|
|
907
|
+
continue;
|
|
908
|
+
}
|
|
909
|
+
const idf = Math.log(N / Math.max(1, reach.contextsReached));
|
|
910
|
+
if (idf <= 0) {
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
// EXACT joint evidence (score = 1): the container literally contains
|
|
914
|
+
// every composed form. Mutual-explanation weight over their COMBINED
|
|
915
|
+
// byte length — the same magnitude reading voteRegions uses, here with
|
|
916
|
+
// the estimator collapsed to certainty, so mutual = min(ratio, 1/ratio).
|
|
917
|
+
const lenR = Math.max(1, bestCov);
|
|
918
|
+
const ratio = Math.sqrt(
|
|
919
|
+
Math.max(1, ctx.store.contentLen(best.id, lenR * ctx.store.D)) / lenR,
|
|
920
|
+
);
|
|
921
|
+
const mutual = Math.min(1, ratio) * Math.min(1, 1 / ratio);
|
|
922
|
+
const w = (mutual * idf) / reach.roots.length;
|
|
923
|
+
let spanStart = ra.start;
|
|
924
|
+
let spanEnd = rb.end;
|
|
925
|
+
for (const ei of bestExtras) {
|
|
926
|
+
spanStart = Math.min(spanStart, regions[ei].start);
|
|
927
|
+
spanEnd = Math.max(spanEnd, regions[ei].end);
|
|
928
|
+
}
|
|
929
|
+
out.push({
|
|
930
|
+
start: spanStart,
|
|
931
|
+
end: spanEnd,
|
|
932
|
+
canonicalFailed: false, // content-addressed: never saturation-masked
|
|
933
|
+
roots: reach.roots,
|
|
934
|
+
w,
|
|
935
|
+
wFocus: w,
|
|
936
|
+
});
|
|
937
|
+
consumed.add(cand[a]);
|
|
938
|
+
consumed.add(cand[b]);
|
|
939
|
+
for (const ei of bestExtras) {
|
|
940
|
+
consumed.add(ei);
|
|
941
|
+
}
|
|
942
|
+
// EXPLAINING AWAY — see the block comment above the function. Byte
|
|
943
|
+
// containment in the joint container is the relatedness test (the
|
|
944
|
+
// vote's bytes are literally part of the learnt whole), and FULL root
|
|
945
|
+
// disjointness is the disagreement test: a vote sharing even one root
|
|
946
|
+
// with the junction corroborates it and keeps its say elsewhere.
|
|
947
|
+
const containerBytes = cachedRead(ctx, cache, best.id, cap);
|
|
948
|
+
const jointRoots = new Set(reach.roots);
|
|
949
|
+
for (const rv of rvs.votes) {
|
|
950
|
+
if (rv.roots.some((r) => jointRoots.has(r))) {
|
|
951
|
+
continue;
|
|
868
952
|
}
|
|
953
|
+
const bytes = query.subarray(rv.start, rv.end);
|
|
954
|
+
if (indexOf(containerBytes, bytes, 0) >= 0) {
|
|
955
|
+
superseded.add(rv);
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
const label = [cand[a], cand[b], ...bestExtras]
|
|
959
|
+
.sort((x, y) => regions[x].start - regions[y].start)
|
|
960
|
+
.map((ri) => dec(query.subarray(regions[ri].start, regions[ri].end)))
|
|
961
|
+
.join(" ▸ ");
|
|
962
|
+
ctx.trace?.step(
|
|
963
|
+
"crossRegion",
|
|
964
|
+
[{ text: label, role: "pair" }],
|
|
965
|
+
reach.roots.map((r) => ({
|
|
966
|
+
text: dec(read(ctx, r)).slice(0, 60),
|
|
967
|
+
node: r,
|
|
968
|
+
role: "joint-context",
|
|
969
|
+
})),
|
|
970
|
+
`${label} → junction node ${best.id}` +
|
|
971
|
+
(best.interior.length === 0
|
|
972
|
+
? " (adjacent)"
|
|
973
|
+
: ` (interior "${dec(best.interior)}")`) +
|
|
974
|
+
` → ${reach.roots.length} context(s), by content-addressed ascent` +
|
|
975
|
+
(superseded.size > 0
|
|
976
|
+
? `; ${superseded.size} aliasing vote(s) explained away`
|
|
977
|
+
: ""),
|
|
978
|
+
);
|
|
979
|
+
break; // ra is consumed — move to the next unconsumed candidate
|
|
869
980
|
}
|
|
870
|
-
|
|
981
|
+
}
|
|
982
|
+
return { votes: out, superseded };
|
|
871
983
|
}
|
|
872
984
|
export function traceAttention(ctx, regions, regionVoter, roots, steps = []) {
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
item.text = `${item.text} (df-w ${rv.w.toFixed(2)})`;
|
|
882
|
-
voters.push(item);
|
|
985
|
+
if (!ctx.trace) {
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
const voters = [];
|
|
989
|
+
for (let i = 0; i < regions.length; i++) {
|
|
990
|
+
const rv = regionVoter[i];
|
|
991
|
+
if (rv == null) {
|
|
992
|
+
continue;
|
|
883
993
|
}
|
|
884
|
-
const
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
994
|
+
const item = rNode(ctx, rv.id, "sub-region", rv.score);
|
|
995
|
+
item.text = `${item.text} (df-w ${rv.w.toFixed(2)})`;
|
|
996
|
+
voters.push(item);
|
|
997
|
+
}
|
|
998
|
+
const t = ctx.trace.enter("climbConsensus", voters);
|
|
999
|
+
// The pooled-evidence decision, one DerivationStep per anchor — the same
|
|
1000
|
+
// shape {@link GraphSearch}'s own cover steps take (see traceDerivation).
|
|
1001
|
+
if (steps.length > 0) {
|
|
1002
|
+
traceDerivation(ctx, steps);
|
|
1003
|
+
}
|
|
1004
|
+
t.done(
|
|
1005
|
+
roots.map((r) => rNode(ctx, r.anchor, "anchor", r.vote)),
|
|
1006
|
+
roots.length === 0
|
|
1007
|
+
? `${regions.length} sub-regions climbed the DAG, but none agreed on a context`
|
|
1008
|
+
: roots.length === 1
|
|
1009
|
+
? `${voters.length} of ${regions.length} sub-regions voted; IDF-weighted consensus picked one context (vote ${
|
|
1010
|
+
roots[0].vote.toFixed(2)
|
|
1011
|
+
})`
|
|
1012
|
+
: `${voters.length} of ${regions.length} sub-regions voted; consensus ordered ${roots.length} INDEPENDENT points of attention (votes ${
|
|
1013
|
+
roots.map((r) => r.vote.toFixed(2)).join(", ")
|
|
1014
|
+
})`,
|
|
1015
|
+
);
|
|
894
1016
|
}
|