@hviana/sema 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +290 -77
- package/HOW_IT_WORKS.md +2170 -735
- package/dist/example/train_base.d.ts +9 -3
- package/dist/example/train_base.js +21 -4
- package/dist/src/canon.d.ts +19 -0
- package/dist/src/canon.js +28 -0
- package/dist/src/geometry.d.ts +52 -0
- package/dist/src/geometry.js +87 -1
- package/dist/src/mind/bridge.js +27 -1
- package/dist/src/mind/frame-filler.d.ts +15 -0
- package/dist/src/mind/frame-filler.js +535 -0
- package/dist/src/mind/learning.js +6 -11
- package/dist/src/mind/mechanisms/cast.js +72 -2
- package/dist/src/mind/mechanisms/cover.js +6 -1
- package/dist/src/mind/mechanisms/extraction.js +27 -0
- package/dist/src/mind/mechanisms/recall.js +214 -34
- package/dist/src/mind/mind.d.ts +49 -1
- package/dist/src/mind/mind.js +137 -10
- package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
- package/dist/src/mind/pipeline.js +29 -1
- package/dist/src/mind/prefix-completion.d.ts +59 -0
- package/dist/src/mind/prefix-completion.js +270 -0
- package/dist/src/mind/primitives.d.ts +29 -10
- package/dist/src/mind/primitives.js +52 -61
- package/dist/src/mind/recognition.js +119 -9
- package/dist/src/mind/traverse.d.ts +32 -0
- package/dist/src/mind/traverse.js +52 -0
- package/dist/src/mind/types.d.ts +55 -16
- package/dist/src/mind/types.js +68 -19
- package/dist/src/store.d.ts +21 -0
- package/dist/src/store.js +21 -0
- package/example/train_base.ts +21 -4
- package/package.json +1 -1
- package/src/canon.ts +28 -0
- package/src/geometry.ts +100 -1
- package/src/mind/bridge.ts +34 -0
- package/src/mind/frame-filler.ts +604 -0
- package/src/mind/learning.ts +5 -9
- package/src/mind/mechanisms/cast.ts +70 -2
- package/src/mind/mechanisms/cover.ts +6 -1
- package/src/mind/mechanisms/extraction.ts +27 -0
- package/src/mind/mechanisms/recall.ts +236 -37
- package/src/mind/mind.ts +154 -14
- package/src/mind/pipeline-mechanism.ts +7 -0
- package/src/mind/pipeline.ts +33 -1
- package/src/mind/prefix-completion.ts +314 -0
- package/src/mind/primitives.ts +59 -70
- package/src/mind/recognition.ts +117 -6
- package/src/mind/traverse.ts +52 -0
- package/src/mind/types.ts +98 -42
- package/src/store.ts +25 -0
- package/test/13-conversation.test.mjs +13 -0
- package/test/57-fusion-order.test.mjs +65 -0
- package/test/66-query-edge-whitespace.test.mjs +99 -0
- package/test/67-climb-anchor-breadth.test.mjs +113 -0
- package/test/68-extraction-unanchored.test.mjs +79 -0
- package/test/69-frame-filler.test.mjs +115 -0
- package/test/70-prefix-completion.test.mjs +170 -0
- package/test/71-embedded-canon-equivalence.test.mjs +121 -0
- package/test/72-prefix-candidate-supply.test.mjs +114 -0
- package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
- package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
- package/test/75-multiturn-context-optimisation.test.mjs +1082 -0
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
// frame-filler.ts — compositional substitution grounding (recall's last tier
|
|
2
|
+
// before refusal, after the substitution bridge).
|
|
3
|
+
//
|
|
4
|
+
// THE GAP (analyze_training.ts section G): `What is the capital of the country
|
|
5
|
+
// where the Eiffel Tower is?` returns silence, while the answer sits ONE edge
|
|
6
|
+
// away — `resolve("What is the capital of France?")` is a trained form with a
|
|
7
|
+
// continuation. The query differs from it by a single contiguous span: a
|
|
8
|
+
// DEFINITE DESCRIPTION (`the country where the Eiffel Tower is`, 37 B) stands
|
|
9
|
+
// where a proper noun (`France`, 6 B) stands in the trained question.
|
|
10
|
+
//
|
|
11
|
+
// Every existing tier correctly declines. Recall tier 0b needs the constituent
|
|
12
|
+
// to be an edge SOURCE (`Eiffel Tower` is out=0, in=1). Every gist tier is
|
|
13
|
+
// blind here: cos(query, the trained form) = 0.0076, and `capital of Spain`
|
|
14
|
+
// scores 0.0174 — HIGHER than the correct one — so the target is not even in the
|
|
15
|
+
// ANN top-24. The substitution bridge finds the alignment and then refuses it on
|
|
16
|
+
// RAW BALANCE, `dominates(6, 37)`, which it must: a short span standing for a
|
|
17
|
+
// long one is exactly how `France` → `Spain si(nce)` once voiced a wrong fact.
|
|
18
|
+
//
|
|
19
|
+
// THE REFRAMING. The bridge asks whether the two spans are SIMILAR. A definite
|
|
20
|
+
// description and the noun it denotes are not similar, they are CO-REFERENTIAL,
|
|
21
|
+
// so no similarity threshold can separate this case from that fabrication. So
|
|
22
|
+
// this tier does not try. Instead:
|
|
23
|
+
//
|
|
24
|
+
// We invent a lookup KEY, never an answer.
|
|
25
|
+
//
|
|
26
|
+
// Build the query with the candidate filler in the description's place and
|
|
27
|
+
// require the STORE ITSELF to already hold that key, byte-exactly, by content
|
|
28
|
+
// address. The answer is then the trained continuation of a form the store
|
|
29
|
+
// verifiably has — the same grounding tier 0 performs — so nothing is
|
|
30
|
+
// synthesised. What is constructed is only a lookup key, and a key the store
|
|
31
|
+
// does not hold is discarded.
|
|
32
|
+
//
|
|
33
|
+
// FOUR GUARDS, each falsified into existence on the 15.7M-node store. Dropping
|
|
34
|
+
// any one of them reintroduces a wrong answer or outright fabrication:
|
|
35
|
+
//
|
|
36
|
+
// 1. The evidence hit must literally contain the description's RAREST unit.
|
|
37
|
+
// Pooling fillers from every ranked hit gives the twohop query 9 resolving
|
|
38
|
+
// keys, dominated by `What is the capital of India?` → "New Delhi.". And
|
|
39
|
+
// qualifying on any SHARED unit rather than the rarest gives
|
|
40
|
+
// `Can you write a short poem?` exactly one key,
|
|
41
|
+
// `Can you write hello world in C?` — a confident wrong answer earned on
|
|
42
|
+
// the scaffolding unit "write".
|
|
43
|
+
// 2. The frame must be NON-EMPTY: the description is a proper sub-span.
|
|
44
|
+
// Otherwise two of the twohop keys replace the WHOLE query (`Immanuel
|
|
45
|
+
// Kant`, `Africa`), which is not substitution at all.
|
|
46
|
+
// 3. The key must RESOLVE byte-exactly and lead somewhere.
|
|
47
|
+
// 4. Exactly ONE stored form may survive. `What is the capital of Zamunda?`
|
|
48
|
+
// produces 24 resolving keys in the weaker variants (Chile, India, Japan,
|
|
49
|
+
// Italy …) — fabrication, refused here by ambiguity. This is the same
|
|
50
|
+
// discipline tier 0b applies ("two distinct maximal arguments mean the
|
|
51
|
+
// query asks about neither alone").
|
|
52
|
+
//
|
|
53
|
+
// A NOTE ON WHY RESOLUTION ALONE IS NOT THE SAFETY ARGUMENT. Holding the frame
|
|
54
|
+
// fixed and varying only the filler makes byte-exact resolution look like a
|
|
55
|
+
// perfect filter — every wrong filler tried returned null. That is misleading:
|
|
56
|
+
// when the DESCRIPTION is searched too, 95,836 candidate keys were tried for the
|
|
57
|
+
// twohop query and 9 resolved. Resolution is necessary, never sufficient; the
|
|
58
|
+
// guards above are what make it sound.
|
|
59
|
+
//
|
|
60
|
+
// COST — nothing on any answering path (this runs only where the alternative was
|
|
61
|
+
// silence), and no new retrieval: the ranked hits are the ones recall already
|
|
62
|
+
// computed. On the refusal path, measured over 19 queries (every currently-empty
|
|
63
|
+
// battery probe, all three silence probes, three dialogue turns, and a 125-byte
|
|
64
|
+
// worst case): mean 22.7 ms, worst 452 probes / 102 ms, and 10 of the 19 need
|
|
65
|
+
// ZERO probes because guard 1 exits first. Two things keep it there and both are
|
|
66
|
+
// load-bearing, not optimisations:
|
|
67
|
+
//
|
|
68
|
+
// • fillers are MAXIMAL absent runs, not every sub-span of a hit (8,020 probes
|
|
69
|
+
// → 452). The twohop win survives because a sub-quantum unit (< W)
|
|
70
|
+
// terminates a run: in `The most well-known landmark in France is the Eiffel
|
|
71
|
+
// Tower.`, `in` breaks the run, so `France` IS itself a maximal run.
|
|
72
|
+
// • rarity is memoised per response. On the 125-byte query 3,882 rarity reads
|
|
73
|
+
// collapse to 10 lookups, and that query drops from 309 ms to 22 ms.
|
|
74
|
+
//
|
|
75
|
+
// The probe budget is `hubBound` (√N), the same breadth every other bounded
|
|
76
|
+
// search here self-limits to. Exhausting it REFUSES rather than answers:
|
|
77
|
+
// truncating the search would leave uniqueness (guard 4) unestablished, and an
|
|
78
|
+
// unestablished uniqueness claim is exactly the ambiguity the guard exists to
|
|
79
|
+
// catch.
|
|
80
|
+
import { indexOf } from "../bytes.js";
|
|
81
|
+
import { leafIdRun } from "./canonical.js";
|
|
82
|
+
import { hubBound } from "./traverse.js";
|
|
83
|
+
import { alignRuns } from "./match.js";
|
|
84
|
+
import { dominates } from "../geometry.js";
|
|
85
|
+
import { foldTree, perceive } from "./primitives.js";
|
|
86
|
+
import { rItem } from "./trace.js";
|
|
87
|
+
/** A stable memo key for a byte span — latin1, so no UTF-8 validation and no
|
|
88
|
+
* allocation beyond the string itself. */
|
|
89
|
+
function spanKey(bytes, from, to) {
|
|
90
|
+
let s = "";
|
|
91
|
+
for (let i = from; i < to; i++)
|
|
92
|
+
s += String.fromCharCode(bytes[i]);
|
|
93
|
+
return s;
|
|
94
|
+
}
|
|
95
|
+
/** Find the query's own most discriminative unit and the trained contexts that
|
|
96
|
+
* hold it, then try the store for the query with a candidate filler in the
|
|
97
|
+
* described span's place. Returns the sole surviving stored form, or null. */
|
|
98
|
+
export function frameFillerSubstitution(ctx, query, ranked) {
|
|
99
|
+
const W = ctx.space.maxGroup;
|
|
100
|
+
const _t0 = Date.now();
|
|
101
|
+
const t = ctx.trace?.enter("frameFiller", [rItem(query, "query")]);
|
|
102
|
+
const done = (hit, note, data) => {
|
|
103
|
+
t?.done(hit === null ? [] : [rItem(hit.filler, "filler", hit.id)], note, data);
|
|
104
|
+
return hit;
|
|
105
|
+
};
|
|
106
|
+
// ── CONSTITUENCY BY AGREEMENT ─────────────────────────────────────────
|
|
107
|
+
// Read the candidate contexts once, bounded to phrase scale (a stored span
|
|
108
|
+
// can run to hundreds of kilobytes, and a form an order of magnitude longer
|
|
109
|
+
// than the question is not a candidate for BEING it with one span replaced).
|
|
110
|
+
const capBytes = query.length * W;
|
|
111
|
+
const hitMemo = new Map();
|
|
112
|
+
const hitBytes = (sid) => {
|
|
113
|
+
const seen = hitMemo.get(sid);
|
|
114
|
+
if (seen !== undefined)
|
|
115
|
+
return seen;
|
|
116
|
+
const b = ctx.store.bytesPrefix(sid, capBytes + 1);
|
|
117
|
+
const v = b.length === 0 || b.length > capBytes ? null : b;
|
|
118
|
+
hitMemo.set(sid, v);
|
|
119
|
+
return v;
|
|
120
|
+
};
|
|
121
|
+
const contexts = [];
|
|
122
|
+
for (const sid of ranked) {
|
|
123
|
+
const h = hitBytes(sid);
|
|
124
|
+
if (h !== null)
|
|
125
|
+
contexts.push(h);
|
|
126
|
+
}
|
|
127
|
+
/** How many of `others` an alignment covers each byte of `subject` with —
|
|
128
|
+
* the same per-byte depth the weave accumulates, over the same literal
|
|
129
|
+
* alignment. */
|
|
130
|
+
const depthOver = (subject, others) => {
|
|
131
|
+
const depth = new Uint16Array(subject.length);
|
|
132
|
+
for (const other of others) {
|
|
133
|
+
if (other === subject)
|
|
134
|
+
continue;
|
|
135
|
+
for (const r of alignRuns(ctx, subject, other)) {
|
|
136
|
+
for (let i = r.qs; i < r.qe && i < depth.length; i++)
|
|
137
|
+
depth[i]++;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return depth;
|
|
141
|
+
};
|
|
142
|
+
/** The maximal runs of `subject` that the majority does NOT share — its
|
|
143
|
+
* content, as opposed to the frame. This is the constituent notion: a span
|
|
144
|
+
* no character class produced, only agreement. */
|
|
145
|
+
const contentRuns = (subject, others, from = 0, to = subject.length) => {
|
|
146
|
+
// A single exemplar agrees with nothing, so nothing can be called frame
|
|
147
|
+
// and no constituent is established — the honest reading is "none".
|
|
148
|
+
if (others.length < 2)
|
|
149
|
+
return [];
|
|
150
|
+
const depth = depthOver(subject, others);
|
|
151
|
+
const out = [];
|
|
152
|
+
let start = -1;
|
|
153
|
+
for (let i = from; i < to; i++) {
|
|
154
|
+
const frame = dominates(depth[i], others.length);
|
|
155
|
+
if (!frame) {
|
|
156
|
+
if (start < 0)
|
|
157
|
+
start = i;
|
|
158
|
+
}
|
|
159
|
+
else if (start >= 0) {
|
|
160
|
+
out.push([start, i]);
|
|
161
|
+
start = -1;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (start >= 0)
|
|
165
|
+
out.push([start, to]);
|
|
166
|
+
return out;
|
|
167
|
+
};
|
|
168
|
+
// THE FRAME COHORT. A frame is only established among exemplars that are
|
|
169
|
+
// instances of the SAME frame — "more than half the aligned structures share
|
|
170
|
+
// it" says nothing when the structures share nothing to begin with. Two
|
|
171
|
+
// readings were measured and BOTH fail:
|
|
172
|
+
//
|
|
173
|
+
// • ALL resonance candidates. They are merely near in gist, so a majority
|
|
174
|
+
// never forms, every byte reads as content, any span becomes a filler —
|
|
175
|
+
// and it does not merely fail to answer, it FABRICATES ("Tell me the name
|
|
176
|
+
// of the biggest planet orbiting our sun." grounded a list of animals).
|
|
177
|
+
// • candidates holding the query's discriminative content. These are
|
|
178
|
+
// exemplars about the same THING, not instances of the same FRAME: the
|
|
179
|
+
// seven holding "Eiffel" share `" the Eiffel Tower"` and nothing else, so
|
|
180
|
+
// a whole clause reads as content and no constituent is isolated.
|
|
181
|
+
//
|
|
182
|
+
// The cohort of a subject is its STRUCTURAL NEIGHBOURS — the candidates whose
|
|
183
|
+
// alignment covers the most of it — because instances of one frame are
|
|
184
|
+
// exactly the forms that share that frame's bytes. Two weaker readings were
|
|
185
|
+
// measured and both fail: ALL candidates share nothing, so a majority never
|
|
186
|
+
// forms, every byte reads as content and the tier FABRICATES; candidates
|
|
187
|
+
// holding the query's rarest content are exemplars about the same THING, not
|
|
188
|
+
// the same FRAME (the seven holding "Eiffel" share `" the Eiffel Tower"` and
|
|
189
|
+
// nothing else), so no constituent is isolated.
|
|
190
|
+
//
|
|
191
|
+
// THE CUT is half-dominance against the BEST neighbour, not against the
|
|
192
|
+
// subject's length. Coverage is bounded by how much frame two forms can
|
|
193
|
+
// share at all — measured, the best neighbour covered 23 of 59 bytes — so
|
|
194
|
+
// `dominates(n, subject.length)` can never fire and the cohort is always
|
|
195
|
+
// empty. Read against the best coverage the profile actually offers, the
|
|
196
|
+
// same convention becomes scale-free and needs no constant: a form sharing
|
|
197
|
+
// more than half of what the closest instance shares is another instance.
|
|
198
|
+
const coverageOf = (subject, other, keepEdges = null) => {
|
|
199
|
+
const seen = new Uint8Array(subject.length);
|
|
200
|
+
for (const r of alignRuns(ctx, subject, other)) {
|
|
201
|
+
if (keepEdges !== null) {
|
|
202
|
+
keepEdges.add(r.qs);
|
|
203
|
+
keepEdges.add(r.qe);
|
|
204
|
+
}
|
|
205
|
+
for (let i = r.qs; i < r.qe && i < seen.length; i++)
|
|
206
|
+
seen[i] = 1;
|
|
207
|
+
}
|
|
208
|
+
let n = 0;
|
|
209
|
+
for (const x of seen)
|
|
210
|
+
n += x;
|
|
211
|
+
return n;
|
|
212
|
+
};
|
|
213
|
+
// WHERE ANY EXEMPLAR'S SHARED MATERIAL STARTS OR STOPS. Collected over EVERY
|
|
214
|
+
// candidate, not just the cohort: the cut decides whose AGREEMENT establishes
|
|
215
|
+
// the frame, which is a different question from where a boundary EXISTS. One
|
|
216
|
+
// exemplar ending a run at an offset is already evidence of an edge there,
|
|
217
|
+
// however unrelated it is otherwise — measured, drawing edges from the cohort
|
|
218
|
+
// alone loses the boundary between `"is"` and `"?"` (2 candidates of 563
|
|
219
|
+
// attest it, 0 of 131 in the cohort), and without it the two-hop description
|
|
220
|
+
// cannot be expressed at all.
|
|
221
|
+
const edgesMemo = new Map();
|
|
222
|
+
const cohortMemo = new Map();
|
|
223
|
+
const cohortOf = (subject) => {
|
|
224
|
+
const seen = cohortMemo.get(subject);
|
|
225
|
+
if (seen !== undefined)
|
|
226
|
+
return seen;
|
|
227
|
+
const eset = new Set([0, subject.length]);
|
|
228
|
+
edgesMemo.set(subject, eset);
|
|
229
|
+
let best = 0;
|
|
230
|
+
const scored = [];
|
|
231
|
+
for (const c of contexts) {
|
|
232
|
+
if (c === subject)
|
|
233
|
+
continue;
|
|
234
|
+
const n = coverageOf(subject, c, eset);
|
|
235
|
+
if (n < W)
|
|
236
|
+
continue;
|
|
237
|
+
if (n > best)
|
|
238
|
+
best = n;
|
|
239
|
+
scored.push({ c, n });
|
|
240
|
+
}
|
|
241
|
+
const out = scored.filter((x) => dominates(x.n, best)).map((x) => x.c);
|
|
242
|
+
cohortMemo.set(subject, out);
|
|
243
|
+
return out;
|
|
244
|
+
};
|
|
245
|
+
/** Split each run at the boundaries other exemplars attest for `subject` — a
|
|
246
|
+
* run is a stretch the cohort does not share, not itself a constituent, so
|
|
247
|
+
* taking it whole asks the evidence for a span no exemplar holds. */
|
|
248
|
+
const cutAtEdges = (subject, runs) => {
|
|
249
|
+
const eset = edgesMemo.get(subject);
|
|
250
|
+
if (eset === undefined)
|
|
251
|
+
return runs;
|
|
252
|
+
const inner = [...eset].sort((a, b) => a - b);
|
|
253
|
+
const out = [];
|
|
254
|
+
for (const [rs, re] of runs) {
|
|
255
|
+
let prev = rs;
|
|
256
|
+
for (const o of inner) {
|
|
257
|
+
if (o <= rs || o >= re)
|
|
258
|
+
continue;
|
|
259
|
+
if (o - prev >= W)
|
|
260
|
+
out.push([prev, o]);
|
|
261
|
+
prev = o;
|
|
262
|
+
}
|
|
263
|
+
if (re - prev >= W)
|
|
264
|
+
out.push([prev, re]);
|
|
265
|
+
out.push([rs, re]);
|
|
266
|
+
}
|
|
267
|
+
return out;
|
|
268
|
+
};
|
|
269
|
+
const queryContent = contentRuns(query, cohortOf(query));
|
|
270
|
+
// Corpus rarity of a unit, by how many trained forms contain its first
|
|
271
|
+
// window — the same container-count reading the bridge's anchor picking uses.
|
|
272
|
+
// Memoised per call: the same units recur across every candidate description.
|
|
273
|
+
const rarityMemo = new Map();
|
|
274
|
+
let rarityReads = 0;
|
|
275
|
+
const rarityOf = (from, to) => {
|
|
276
|
+
if (to - from < W)
|
|
277
|
+
return Infinity;
|
|
278
|
+
const key = spanKey(query, from, to);
|
|
279
|
+
const seen = rarityMemo.get(key);
|
|
280
|
+
if (seen !== undefined)
|
|
281
|
+
return seen;
|
|
282
|
+
rarityReads++;
|
|
283
|
+
let value = Infinity;
|
|
284
|
+
const ids = leafIdRun(ctx, query, from, from + W);
|
|
285
|
+
if (ids !== null) {
|
|
286
|
+
const wid = ctx.store.findBranch(ids);
|
|
287
|
+
if (wid !== null)
|
|
288
|
+
value = ctx.store.containers(wid).length;
|
|
289
|
+
}
|
|
290
|
+
rarityMemo.set(key, value);
|
|
291
|
+
return value;
|
|
292
|
+
};
|
|
293
|
+
/** The most discriminative unit of `[from, to)`, as its span, or null when it
|
|
294
|
+
* has none. Rarest first; at EQUAL rarity the LONGER unit wins, because the
|
|
295
|
+
* longer form carries more content at the same corpus frequency. The
|
|
296
|
+
* tie-break is load-bearing on a small corpus, where every unit's container
|
|
297
|
+
* count collapses to 1 and first-wins would pick the query's opening unit
|
|
298
|
+
* ("What") over its subject ("Eiffel") — the subject is what a description
|
|
299
|
+
* must be about. On a large corpus the counts separate on their own (54 vs
|
|
300
|
+
* 1,586 for exactly that pair) and the tie-break never engages. */
|
|
301
|
+
const rarestUnit = (from, to) => {
|
|
302
|
+
let best = null;
|
|
303
|
+
let bestRarity = Infinity;
|
|
304
|
+
let bestAttested = 0;
|
|
305
|
+
let bestLen = 0;
|
|
306
|
+
// A content run is a stretch the cohort does not share; it is not itself a
|
|
307
|
+
// constituent, and taking it whole asks the evidence for a span no exemplar
|
|
308
|
+
// holds (measured: `" Eiffel Tower is?"` qualified NO candidate, so the tier
|
|
309
|
+
// never probed). Cut each run at the boundaries other exemplars attest —
|
|
310
|
+
// the same edge set the descriptions are enumerated over — so the unit is a
|
|
311
|
+
// span the corpus has actually seen begin and end.
|
|
312
|
+
//
|
|
313
|
+
// A unit may span ANY two attested edges, not just adjacent ones. Slicing
|
|
314
|
+
// only between consecutive edges makes the unit set depend on how DENSE the
|
|
315
|
+
// evidence is rather than on what it says: measured, at k = 24 the query
|
|
316
|
+
// carries 17 edges and `"Eiffel Tower"` is one slice (elected, attested),
|
|
317
|
+
// while at k = 571 it carries 55 and every consecutive slice is 1-3 bytes —
|
|
318
|
+
// all under the constituent bar — leaving only the whole unattested run and
|
|
319
|
+
// electing nothing at all. More evidence made the tier blinder. Taking
|
|
320
|
+
// every edge-bounded sub-span removes that dependence: the corpus decides
|
|
321
|
+
// which spans exist, never how finely it happened to mark them.
|
|
322
|
+
const pieces = [];
|
|
323
|
+
for (const [cs0, ce0] of queryContent) {
|
|
324
|
+
const inner = bs.filter((o) => o > cs0 && o < ce0);
|
|
325
|
+
let prev = cs0;
|
|
326
|
+
for (const o of [...inner, ce0]) {
|
|
327
|
+
pieces.push([prev, o]);
|
|
328
|
+
prev = o;
|
|
329
|
+
}
|
|
330
|
+
pieces.push([cs0, ce0]);
|
|
331
|
+
}
|
|
332
|
+
for (const [cs0, ce0] of pieces) {
|
|
333
|
+
const s = Math.max(cs0, from);
|
|
334
|
+
const e = Math.min(ce0, to);
|
|
335
|
+
// ATTESTED, or it is not evidence. Rarity is read over the span's FIRST
|
|
336
|
+
// window, so every longer span sharing that window scores identically —
|
|
337
|
+
// and the longer-wins tie-break below then elects the longest of them,
|
|
338
|
+
// which is a claim about bytes the measurement never looked at. Measured
|
|
339
|
+
// live: the elected unit was `"Eiffel Tower is?"`, held by 0 of the 24
|
|
340
|
+
// candidates, so guard 1 rejected every one and the tier never probed
|
|
341
|
+
// (`descs=14, qualified=0, probes=0`). Requiring the unit to occur in the
|
|
342
|
+
// evidence is not an extra gate — guard 1 demands exactly this — it just
|
|
343
|
+
// has to hold at ELECTION time, or election spends itself on a span no
|
|
344
|
+
// candidate can qualify.
|
|
345
|
+
let attested = 0;
|
|
346
|
+
for (const c of contexts) {
|
|
347
|
+
if (indexOf(c, query.subarray(s, e), 0) >= 0)
|
|
348
|
+
attested++;
|
|
349
|
+
}
|
|
350
|
+
if (attested === 0)
|
|
351
|
+
continue;
|
|
352
|
+
// THE CONSTITUENT BAR — two quanta, the same reading argument binding
|
|
353
|
+
// holds its constituents to. Content runs are not units of a modality's
|
|
354
|
+
// making, so a run may be as short as one window, and a single window is
|
|
355
|
+
// too weak to say what a description is ABOUT: measured, the W-byte
|
|
356
|
+
// fragment `phot` qualified `What is photosynthesis?` against an
|
|
357
|
+
// unrelated "photo-sharing app" and fabricated an answer.
|
|
358
|
+
if (e - s < 2 * W)
|
|
359
|
+
continue;
|
|
360
|
+
const r = rarityOf(s, e);
|
|
361
|
+
if (r === Infinity)
|
|
362
|
+
continue;
|
|
363
|
+
// Rarity is read over the FIRST window, so spans sharing it are
|
|
364
|
+
// indistinguishable to the measurement; among them the only measured
|
|
365
|
+
// difference is how much evidence actually holds the span. Preferring the
|
|
366
|
+
// LONGER one asserts bytes never looked at — measured, it elected
|
|
367
|
+
// `"Eiffel Tower "` (trailing space, 1 context) over `"Eiffel Tower"`
|
|
368
|
+
// (2 contexts), so the one candidate that could complete the description
|
|
369
|
+
// never qualified. Length breaks only a genuine tie in both.
|
|
370
|
+
if (r < bestRarity ||
|
|
371
|
+
(r === bestRarity &&
|
|
372
|
+
(attested > bestAttested ||
|
|
373
|
+
(attested === bestAttested && e - s > bestLen)))) {
|
|
374
|
+
bestRarity = r;
|
|
375
|
+
bestAttested = attested;
|
|
376
|
+
bestLen = e - s;
|
|
377
|
+
best = [s, e];
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return best;
|
|
381
|
+
};
|
|
382
|
+
const qEdges = edgesMemo.get(query) ?? new Set([0, query.length]);
|
|
383
|
+
for (const [cs0, ce0] of queryContent) {
|
|
384
|
+
qEdges.add(cs0);
|
|
385
|
+
qEdges.add(ce0);
|
|
386
|
+
}
|
|
387
|
+
const bs = [...qEdges].sort((a, b) => a - b);
|
|
388
|
+
// The query's own discriminative content — every candidate description must
|
|
389
|
+
// contain it, or the substitution is not about what the query is asking.
|
|
390
|
+
const queryRare = rarestUnit(0, query.length);
|
|
391
|
+
if (queryRare === null) {
|
|
392
|
+
return done(null, "no corpus-attested unit in the query — nothing to describe", {
|
|
393
|
+
contexts: contexts.length,
|
|
394
|
+
qc: queryContent.length,
|
|
395
|
+
edges: bs.length,
|
|
396
|
+
runs: queryContent.map(([a, b]) => spanKey(query, a, b)),
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
let probes = 0;
|
|
400
|
+
let descs = 0, qualified = 0, fillers = 0;
|
|
401
|
+
const chosen = new Map();
|
|
402
|
+
const fillerSeen = new Set();
|
|
403
|
+
const budget = hubBound(ctx);
|
|
404
|
+
// resolved form id -> the description span and filler that reached it
|
|
405
|
+
const found = new Map();
|
|
406
|
+
// Candidate description edges: every offset some exemplar's shared material
|
|
407
|
+
// begins or ends at, plus the query's own extremes.
|
|
408
|
+
for (let i = 0; i < bs.length; i++) {
|
|
409
|
+
for (let j = i + 1; j < bs.length; j++) {
|
|
410
|
+
const [dStart, dEnd] = [bs[i], bs[j]];
|
|
411
|
+
// GUARD 2 — the frame must be non-empty: replacing the whole query is
|
|
412
|
+
// not substitution.
|
|
413
|
+
if (dStart === 0 && dEnd === query.length)
|
|
414
|
+
continue;
|
|
415
|
+
if (dEnd - dStart < W)
|
|
416
|
+
continue;
|
|
417
|
+
// The description must carry the query's discriminative content.
|
|
418
|
+
if (dStart > queryRare[0] || dEnd < queryRare[1])
|
|
419
|
+
continue;
|
|
420
|
+
const dRare = rarestUnit(dStart, dEnd);
|
|
421
|
+
if (dRare === null)
|
|
422
|
+
continue;
|
|
423
|
+
descs++;
|
|
424
|
+
{
|
|
425
|
+
const u = spanKey(query, dRare[0], dRare[1]);
|
|
426
|
+
chosen.set(u, (chosen.get(u) ?? 0) + 1);
|
|
427
|
+
}
|
|
428
|
+
const rareUnit = query.subarray(dRare[0], dRare[1]);
|
|
429
|
+
for (const sid of ranked) {
|
|
430
|
+
// PHRASE SCALE — the same bound the bridge puts on a candidate's bytes
|
|
431
|
+
// (`capBytes = query.length * W`): a form an order of magnitude longer
|
|
432
|
+
// than the question is not a candidate for BEING that question with one
|
|
433
|
+
// span replaced. Reading candidates in full instead was measured at
|
|
434
|
+
// +650 ms on the winning query, since a stored span can run to hundreds
|
|
435
|
+
// of kilobytes.
|
|
436
|
+
const hit = hitBytes(sid);
|
|
437
|
+
if (hit === null)
|
|
438
|
+
continue;
|
|
439
|
+
// GUARD 1 — this hit must literally hold the description's rarest unit.
|
|
440
|
+
if (indexOf(hit, rareUnit, 0) < 0)
|
|
441
|
+
continue;
|
|
442
|
+
qualified++;
|
|
443
|
+
// Fillers: this hit's CONTENT — the spans its fellow candidates do not
|
|
444
|
+
// share. What the exemplars have in common is the frame they are all
|
|
445
|
+
// instances of; what is left distinguishes THIS one, and that is the
|
|
446
|
+
// constituent standing where the query's description stands.
|
|
447
|
+
const runs = cutAtEdges(hit, contentRuns(hit, cohortOf(hit)));
|
|
448
|
+
for (const [fs, fe] of runs) {
|
|
449
|
+
if (fe - fs < W)
|
|
450
|
+
continue;
|
|
451
|
+
fillers++;
|
|
452
|
+
if (fillerSeen.size < 40)
|
|
453
|
+
fillerSeen.add(spanKey(hit, fs, fe));
|
|
454
|
+
const filler = hit.subarray(fs, fe);
|
|
455
|
+
if (indexOf(query, filler, 0) >= 0)
|
|
456
|
+
continue;
|
|
457
|
+
if (probes >= budget) {
|
|
458
|
+
// Uniqueness cannot be established on a truncated search, and an
|
|
459
|
+
// unestablished uniqueness claim is the ambiguity guard 4 exists
|
|
460
|
+
// to refuse.
|
|
461
|
+
return done(null, `probe budget (${budget}) exhausted — uniqueness unestablished`, {
|
|
462
|
+
version: 1,
|
|
463
|
+
probes,
|
|
464
|
+
rarityReads,
|
|
465
|
+
resolved: found.size,
|
|
466
|
+
contexts: contexts.length,
|
|
467
|
+
edges: bs.length,
|
|
468
|
+
qc: queryContent.length,
|
|
469
|
+
descs,
|
|
470
|
+
qualified,
|
|
471
|
+
fillers,
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
probes++;
|
|
475
|
+
// The KEY: the query with this filler in the described span's place.
|
|
476
|
+
const key = new Uint8Array(dStart + filler.length + (query.length - dEnd));
|
|
477
|
+
key.set(query.subarray(0, dStart), 0);
|
|
478
|
+
key.set(filler, dStart);
|
|
479
|
+
key.set(query.subarray(dEnd), dStart + filler.length);
|
|
480
|
+
// GUARD 3 — the store must already hold it, and it must lead
|
|
481
|
+
// somewhere. EXACT content address only, deliberately not
|
|
482
|
+
// resolve(): that falls through to canonResolve on a miss, and a
|
|
483
|
+
// constructed key misses by design — 451 of the 452 probes on the
|
|
484
|
+
// winning query do — so each miss would pay a canon index query.
|
|
485
|
+
// Measured: +650 ms on that query, for a claim WEAKER than the one
|
|
486
|
+
// this guard wants. A canon hit would mean the store holds a
|
|
487
|
+
// case/width variant of a key we invented; guard 3 asks for the key
|
|
488
|
+
// itself.
|
|
489
|
+
const id = foldTree(ctx, perceive(ctx, key), 0).node;
|
|
490
|
+
if (id === null || !ctx.store.hasNext(id))
|
|
491
|
+
continue;
|
|
492
|
+
if (!found.has(id)) {
|
|
493
|
+
found.set(id, {
|
|
494
|
+
id,
|
|
495
|
+
described: [dStart, dEnd],
|
|
496
|
+
filler: filler.slice(),
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
const data = {
|
|
504
|
+
version: 1,
|
|
505
|
+
probes,
|
|
506
|
+
rarityReads,
|
|
507
|
+
resolved: found.size,
|
|
508
|
+
budget,
|
|
509
|
+
contexts: contexts.length,
|
|
510
|
+
edges: bs.length,
|
|
511
|
+
qc: queryContent.length,
|
|
512
|
+
descs,
|
|
513
|
+
qualified,
|
|
514
|
+
fillers,
|
|
515
|
+
fillerSeen: [...fillerSeen],
|
|
516
|
+
chosen: [...chosen.entries()].map(([u, n]) => {
|
|
517
|
+
const b = new Uint8Array(u.length);
|
|
518
|
+
for (let i = 0; i < u.length; i++)
|
|
519
|
+
b[i] = u.charCodeAt(i);
|
|
520
|
+
const inAny = contexts.filter((c) => indexOf(c, b, 0) >= 0).length;
|
|
521
|
+
return `${JSON.stringify(u)} x${n} inContexts=${inAny}`;
|
|
522
|
+
}),
|
|
523
|
+
ms: Date.now() - _t0,
|
|
524
|
+
};
|
|
525
|
+
// GUARD 4 — exactly one trained form, or the query is ambiguous about its own
|
|
526
|
+
// subject and neither is licensed.
|
|
527
|
+
if (found.size === 1) {
|
|
528
|
+
const hit = [...found.values()][0];
|
|
529
|
+
return done(hit, "frame-filler substitution — the store holds this query with a " +
|
|
530
|
+
"corroborated filler in the described span's place", data);
|
|
531
|
+
}
|
|
532
|
+
return done(null, found.size === 0
|
|
533
|
+
? "no filler makes this query a stored form"
|
|
534
|
+
: `${found.size} fillers make this query a stored form — ambiguous subject`, data);
|
|
535
|
+
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// node. A fact is an EDGE between node ids; recall traverses edges.
|
|
5
5
|
import { bindSeat, companySignature, isChunk } from "../sema.js";
|
|
6
6
|
import { changedNodes } from "./types.js";
|
|
7
|
-
import { inputBytes,
|
|
7
|
+
import { inputBytes, perceiveDeposit, resolve, } from "./primitives.js";
|
|
8
8
|
import { canonicalWindows, leafIdPrefix } from "./canonical.js";
|
|
9
9
|
import { fold as foldVecs } from "../sema.js";
|
|
10
10
|
/** Intern a perceived tree into node ids, bottom-up, sharing equal subtrees.
|
|
@@ -195,16 +195,11 @@ export async function ingestPair(ctx, ctxInput, cont) {
|
|
|
195
195
|
const c = await deposit(ctx, ctxInput, true, true);
|
|
196
196
|
const cont_ = await deposit(ctx, cont, false);
|
|
197
197
|
const ctxId = c.rootId, contId = cont_.rootId;
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
-
|
|
203
|
-
const ctxBytes = inputBytes(ctx, ctxInput);
|
|
204
|
-
const entry = ctx._depositTrees.get(latin1Key(ctxBytes));
|
|
205
|
-
if (entry !== undefined)
|
|
206
|
-
entry.nextBytes = inputBytes(ctx, cont);
|
|
207
|
-
}
|
|
198
|
+
// NO CONTINUATION STAMP. A longer ctxInput reusing this one's folded
|
|
199
|
+
// segments needs no proof that it is "really" the next turn: the deposit
|
|
200
|
+
// fold imposes no boundaries, so reuse is bit-identical to refolding and a
|
|
201
|
+
// coincidental byte prefix gets the tree it would have got anyway. The
|
|
202
|
+
// stamp existed only to gate a boundary guess that no longer happens.
|
|
208
203
|
await ctx.store.link(ctxId, contId);
|
|
209
204
|
await propagateSuffixes(ctx, ctxId, contId);
|
|
210
205
|
// Halos pour company SIGNATURES (identity), not gists (content) — see
|
|
@@ -1033,12 +1033,82 @@ export async function counterfactualTransfer(ctx, query, pre) {
|
|
|
1033
1033
|
// [...] context will be the seat") — its own bytes ARE that seat
|
|
1034
1034
|
// directly, with no predecessor to even check (it was found by a
|
|
1035
1035
|
// forward edge, not matched in the query).
|
|
1036
|
-
|
|
1036
|
+
let b = seats !== undefined
|
|
1037
1037
|
? seats[1]
|
|
1038
1038
|
: bestAnalog.point !== null
|
|
1039
1039
|
? await seatOf(bestAnalog.point, false)
|
|
1040
1040
|
: read(ctx, bestAnalog.anchor);
|
|
1041
|
-
|
|
1041
|
+
// AN ECHO IS NOT A VOICE. `allowForward: false` above leaves seatOfNode
|
|
1042
|
+
// with one last resort — the point's OWN BYTES — and when the aligned
|
|
1043
|
+
// anchor is a QUESTION node those bytes are the question itself. The
|
|
1044
|
+
// comparison then hands the asker their own words back: "What is the
|
|
1045
|
+
// capital of France? And what is the largest planet?" answered "The
|
|
1046
|
+
// capital of France is Paris.What is the largest planet?", one topic
|
|
1047
|
+
// answered and the other merely repeated. (The same corpus answered BOTH
|
|
1048
|
+
// when asked in the opposite order — the echo was never about the topic,
|
|
1049
|
+
// only about whether the climb happened to land on the question node or
|
|
1050
|
+
// the answer node.)
|
|
1051
|
+
//
|
|
1052
|
+
// The fix is NOT to allow the forward edge for every directly aligned
|
|
1053
|
+
// analog. "Directly aligned" does not mean "the query named it": a point
|
|
1054
|
+
// can be aligned by HALO similarity with no literal overlap at all, and
|
|
1055
|
+
// test/43 pins exactly that case — an analog whose own bytes are already a
|
|
1056
|
+
// complete Q+A unit, cited structurally, whose forward edge is an
|
|
1057
|
+
// unrelated next quiz question. There, stopping at its own bytes is
|
|
1058
|
+
// right, because those bytes are an answer and nothing was echoed.
|
|
1059
|
+
//
|
|
1060
|
+
// What separates the two is the RESTATEMENT, which is directly testable:
|
|
1061
|
+
// a seat whose bytes already occur in the query says nothing the asker did
|
|
1062
|
+
// not just say, so it cannot be this analog's contribution — and only then
|
|
1063
|
+
// is the continuation the query literally asked for worth following. Same
|
|
1064
|
+
// `restatesQuery` primitive the substitution schema above already gates
|
|
1065
|
+
// its own forward step on; no new constant and no new notion of "named".
|
|
1066
|
+
// Read the restatement UNDER THE RESPONSE'S OWN EQUIVALENCE. Byte-exact
|
|
1067
|
+
// containment misses the case that actually occurs: the trained node is
|
|
1068
|
+
// "What is the largest planet?" while the query asks "And what is the
|
|
1069
|
+
// largest planet?" — the same words, one capital letter apart, so
|
|
1070
|
+
// `indexOf` finds nothing and the echo sails through. `ctx.canon` is the
|
|
1071
|
+
// response's injected notion of "the same text" (case, width, whitespace);
|
|
1072
|
+
// consulting it here is the same fallback `resolve` already makes when an
|
|
1073
|
+
// exact content lookup misses, and it keeps this mechanism from carrying
|
|
1074
|
+
// any idea of its own about what a character is.
|
|
1075
|
+
const echoesQuery = (x) => {
|
|
1076
|
+
if (restatesQuery(query, x))
|
|
1077
|
+
return true;
|
|
1078
|
+
const canon = ctx.canon;
|
|
1079
|
+
if (canon === null)
|
|
1080
|
+
return false;
|
|
1081
|
+
const cq = canon(query), cx = canon(x);
|
|
1082
|
+
return cx.length < cq.length && indexOf(cq, cx, 0) >= 0;
|
|
1083
|
+
};
|
|
1084
|
+
if (echoesQuery(b)) {
|
|
1085
|
+
const fwd = await follow(ctx, bestAnalog.anchor, qv);
|
|
1086
|
+
if (fwd !== null && fwd.length > 0 && !echoesQuery(fwd))
|
|
1087
|
+
b = fwd;
|
|
1088
|
+
}
|
|
1089
|
+
// VOICED IN THE ORDER THE QUERY POSED THEM. `a` is the DOMINANT point
|
|
1090
|
+
// and `b` the analog, which is a ranking by consensus strength — not by
|
|
1091
|
+
// where either was asked about. Reading the pair out in that ranking
|
|
1092
|
+
// makes a two-topic answer's order depend on which topic resonated
|
|
1093
|
+
// harder, so the same two questions asked in the opposite order produce
|
|
1094
|
+
// the same sentence: measured on test/57, "What is the largest planet?
|
|
1095
|
+
// And what is the capital of France?" answered "The capital of France is
|
|
1096
|
+
// Paris.The largest planet is Jupiter." — both halves right, the order
|
|
1097
|
+
// backwards, because France was the dominant point (accounted [[33,62],
|
|
1098
|
+
// [0,27]] — the runs are literally in reverse query order).
|
|
1099
|
+
//
|
|
1100
|
+
// This is the SAME rule fuseAttention already applies one layer up ("a
|
|
1101
|
+
// multi-topic answer should read in the order the question posed its
|
|
1102
|
+
// topics"), applied to the pair a single comparison voices itself. Each
|
|
1103
|
+
// point's position is the earliest query byte its own aligned runs stand
|
|
1104
|
+
// on — the same runs `cmpAccounted` prices the schema by, so order and
|
|
1105
|
+
// cost read one source.
|
|
1106
|
+
const earliest = (p) => runSpans(p).reduce((m, [s]) => Math.min(m, s), Infinity);
|
|
1107
|
+
const analogPoint = bestAnalog.point ?? bestAnalog.src;
|
|
1108
|
+
const swap = earliest(analogPoint) < earliest(dominant);
|
|
1109
|
+
const answer = swap
|
|
1110
|
+
? await joinWithBridge(ctx, b, a)
|
|
1111
|
+
: await joinWithBridge(ctx, a, b);
|
|
1042
1112
|
record(answer, "analogical comparison — each analog voiced by the context that establishes its role", new Set([dominant.anchor, bestAnalog.anchor]),
|
|
1043
1113
|
// A halo-mediated act (the analogy gate) plus two seat projections.
|
|
1044
1114
|
CONCEPT + STEP + STEP,
|