@hviana/sema 0.4.6 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +290 -77
- package/HOW_IT_WORKS.md +2170 -735
- package/dist/example/train_base.d.ts +9 -3
- package/dist/example/train_base.js +21 -4
- package/dist/src/canon.d.ts +19 -0
- package/dist/src/canon.js +28 -0
- package/dist/src/geometry.d.ts +52 -0
- package/dist/src/geometry.js +87 -1
- package/dist/src/mind/bridge.js +27 -1
- package/dist/src/mind/frame-filler.d.ts +15 -0
- package/dist/src/mind/frame-filler.js +535 -0
- package/dist/src/mind/learning.js +6 -11
- package/dist/src/mind/mechanisms/cast.js +72 -2
- package/dist/src/mind/mechanisms/cover.js +6 -1
- package/dist/src/mind/mechanisms/extraction.js +27 -0
- package/dist/src/mind/mechanisms/recall.js +214 -34
- package/dist/src/mind/mind.d.ts +49 -1
- package/dist/src/mind/mind.js +137 -10
- package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
- package/dist/src/mind/pipeline.js +29 -1
- package/dist/src/mind/prefix-completion.d.ts +59 -0
- package/dist/src/mind/prefix-completion.js +270 -0
- package/dist/src/mind/primitives.d.ts +29 -10
- package/dist/src/mind/primitives.js +52 -61
- package/dist/src/mind/recognition.js +119 -9
- package/dist/src/mind/traverse.d.ts +32 -0
- package/dist/src/mind/traverse.js +52 -0
- package/dist/src/mind/types.d.ts +55 -16
- package/dist/src/mind/types.js +68 -19
- package/dist/src/rabitq-ivf/src/rabitq.js +31 -1
- package/dist/src/store.d.ts +21 -0
- package/dist/src/store.js +21 -0
- package/example/train_base.ts +21 -4
- package/package.json +1 -1
- package/src/canon.ts +28 -0
- package/src/geometry.ts +100 -1
- package/src/mind/bridge.ts +34 -0
- package/src/mind/frame-filler.ts +604 -0
- package/src/mind/learning.ts +5 -9
- package/src/mind/mechanisms/cast.ts +70 -2
- package/src/mind/mechanisms/cover.ts +6 -1
- package/src/mind/mechanisms/extraction.ts +27 -0
- package/src/mind/mechanisms/recall.ts +236 -37
- package/src/mind/mind.ts +154 -14
- package/src/mind/pipeline-mechanism.ts +7 -0
- package/src/mind/pipeline.ts +33 -1
- package/src/mind/prefix-completion.ts +314 -0
- package/src/mind/primitives.ts +59 -70
- package/src/mind/recognition.ts +117 -6
- package/src/mind/traverse.ts +52 -0
- package/src/mind/types.ts +98 -42
- package/src/rabitq-ivf/src/rabitq.ts +31 -1
- package/src/store.ts +25 -0
- package/test/13-conversation.test.mjs +13 -0
- package/test/57-fusion-order.test.mjs +65 -0
- package/test/65-ann-recall.test.mjs +331 -0
- package/test/66-query-edge-whitespace.test.mjs +99 -0
- package/test/67-climb-anchor-breadth.test.mjs +113 -0
- package/test/68-extraction-unanchored.test.mjs +79 -0
- package/test/69-frame-filler.test.mjs +115 -0
- package/test/70-prefix-completion.test.mjs +170 -0
- package/test/71-embedded-canon-equivalence.test.mjs +121 -0
- package/test/72-prefix-candidate-supply.test.mjs +114 -0
- package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
- package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
- package/test/75-multiturn-context-optimisation.test.mjs +1082 -0
|
@@ -108,7 +108,34 @@ export async function think(ctx, query, mechs) {
|
|
|
108
108
|
if (ctx.meter)
|
|
109
109
|
ctx.meter.candidates++;
|
|
110
110
|
candidates.push(c);
|
|
111
|
-
if (best === null
|
|
111
|
+
if (best === null) {
|
|
112
|
+
best = c;
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const g = grade(c.weight), gb = grade(best.weight);
|
|
116
|
+
if (g < gb) {
|
|
117
|
+
best = c;
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
// TIE-BREAK: AT EQUAL GRADE, PREFER THE ANSWER THAT INVENTS LESS.
|
|
121
|
+
//
|
|
122
|
+
// The ladder prices what a candidate leaves UNACCOUNTED, which is the
|
|
123
|
+
// right primary question but cannot separate two candidates that leave
|
|
124
|
+
// the same bytes unaccounted — and then the winner is whichever mechanism
|
|
125
|
+
// happened to be considered first, which is not a reason.
|
|
126
|
+
//
|
|
127
|
+
// What still separates them is what they DID with those bytes. A
|
|
128
|
+
// candidate that carries an unexplained span into its answer is passing
|
|
129
|
+
// the asker's own words back as if they were derived; one that leaves
|
|
130
|
+
// them out has made a smaller, honest claim. Measured on test/22's
|
|
131
|
+
// two-fact chain: cover and recall both graded 11001 over 11 unexplained
|
|
132
|
+
// bytes, cover answering "The capital of France is Paris famous for" (11
|
|
133
|
+
// bytes of scaffolding) against recall's crossing of the hop (0). Order
|
|
134
|
+
// alone decided it, and the shallower reading won.
|
|
135
|
+
//
|
|
136
|
+
// This never overrides the ladder — it only orders within one grade, so
|
|
137
|
+
// coverage and moves still dominate exactly as before.
|
|
138
|
+
if (g === gb && (c.scaffolding ?? 0) < (best.scaffolding ?? 0))
|
|
112
139
|
best = c;
|
|
113
140
|
};
|
|
114
141
|
const worthRunning = (floor) => best === null || grade(floor) < grade(best.weight);
|
|
@@ -150,6 +177,7 @@ export async function think(ctx, query, mechs) {
|
|
|
150
177
|
accounted: r.accounted,
|
|
151
178
|
unexplained: r.unexplained,
|
|
152
179
|
complete: r.complete,
|
|
180
|
+
scaffolding: r.scaffolding,
|
|
153
181
|
});
|
|
154
182
|
}
|
|
155
183
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { MindContext } from "./types.js";
|
|
2
|
+
/** Trained forms the query may OPEN, proposed from the write side's own
|
|
3
|
+
* leaf-id window index — the supply of last resort for {@link
|
|
4
|
+
* prefixCompletion}.
|
|
5
|
+
*
|
|
6
|
+
* WHY A SECOND SUPPLY EXISTS. The ranked list this mechanism normally reads
|
|
7
|
+
* is a resonance list, and resonance cannot rank a proper prefix: measured on
|
|
8
|
+
* the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
|
|
9
|
+
* truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
|
|
10
|
+
* Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
|
|
11
|
+
* bug, so no k and no re-ranking recovers it.
|
|
12
|
+
*
|
|
13
|
+
* WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT. A query's own fold is
|
|
14
|
+
* useless here: content addressing is not phrase-position-invariant, so a
|
|
15
|
+
* standalone prefix folds to a DIFFERENT node than the same bytes sitting
|
|
16
|
+
* inside a longer deposit, and neither the prefix's own node nor its
|
|
17
|
+
* ancestors lead to the deposit (measured: the 22-byte prefix of the
|
|
18
|
+
* photosynthesis form resolves, is shared by 6 contexts, and does not have
|
|
19
|
+
* the form among its ancestors). Leaf ids ARE position-invariant — they are
|
|
20
|
+
* content-addressed on single bytes — and `indexSubSpans` already interns a
|
|
21
|
+
* flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
|
|
22
|
+
* containment edges to the chunks that window spans. A query that is a
|
|
23
|
+
* prefix therefore shares those window nodes exactly, and reaches the deposit
|
|
24
|
+
* by climbing containment then parents. Nothing is added to the write side;
|
|
25
|
+
* this reads an index training already built.
|
|
26
|
+
*
|
|
27
|
+
* BOUNDED (§2.8), AND WITH NO NEW THRESHOLD. The window whose containment is
|
|
28
|
+
* SMALLEST carries the most evidence, and one saturated at `hubBound` carries
|
|
29
|
+
* none — that is the same √N reading of "hub" the rest of the mind uses, not
|
|
30
|
+
* a tuned knob. The upward walk spends a budget of `hubBound` nodes and
|
|
31
|
+
* fans out by W, so a hub query enumerates nothing and the caller stays
|
|
32
|
+
* silent rather than guessing (§2.13). Measured on the trained store: the
|
|
33
|
+
* photosynthesis form at a one-byte truncation picks a window with 52
|
|
34
|
+
* containers, visits 446 nodes, and yields exactly ONE candidate that
|
|
35
|
+
* survives the caller's byte compare — the form itself.
|
|
36
|
+
*
|
|
37
|
+
* These are PROPOSALS only. Every candidate still faces the byte-exact
|
|
38
|
+
* prefix compare and all three guards below, so a wrong proposal costs one
|
|
39
|
+
* bounded read and can never be voiced (§2.3). */
|
|
40
|
+
export declare function prefixCandidates(ctx: MindContext, query: Uint8Array): number[];
|
|
41
|
+
/** A trained form the query opens, and the bytes by which it continues. */
|
|
42
|
+
export interface PrefixCompletion {
|
|
43
|
+
/** The trained form whose opening the query is — the answer, voiced whole. */
|
|
44
|
+
id: number;
|
|
45
|
+
/** The form's own bytes. The mechanism grounds a FORM, never a slice of
|
|
46
|
+
* one: slicing at the query's end would cut at an offset the geometry has
|
|
47
|
+
* no reason to treat as a boundary. */
|
|
48
|
+
form: Uint8Array;
|
|
49
|
+
/** The bytes past the query — carried for the rationale and for the
|
|
50
|
+
* uniqueness comparison, not voiced on its own. */
|
|
51
|
+
continuation: Uint8Array;
|
|
52
|
+
}
|
|
53
|
+
/** The sole trained form the query opens — or null when no candidate opens with
|
|
54
|
+
* it, when the continuation is sub-quantum, when a candidate's continuation
|
|
55
|
+
* cannot be read through, or when the candidates disagree.
|
|
56
|
+
*
|
|
57
|
+
* `ranked` must be a list the caller has ALREADY fetched; this mechanism never
|
|
58
|
+
* resonates on its own (see the header's cost note). */
|
|
59
|
+
export declare function prefixCompletion(ctx: MindContext, query: Uint8Array, ranked: ReadonlyArray<number>): PrefixCompletion | null;
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// prefix-completion.ts — Grounding a query that IS the opening of a trained
|
|
2
|
+
// form.
|
|
3
|
+
//
|
|
4
|
+
// THE SHAPE. `The capital of France is` grounds nothing, while
|
|
5
|
+
// `The capital of France is Paris.` is trained and reads back byte-exact. The
|
|
6
|
+
// query is not SIMILAR to that form, it is a PROPER PREFIX of it: every query
|
|
7
|
+
// byte is a literal match, in order, from offset zero. That is the strongest
|
|
8
|
+
// grounding relation in the store — stronger than the bridge's corroborated
|
|
9
|
+
// substitution, which pays a CONCEPT per substituted span, and stronger than
|
|
10
|
+
// resonance, which only claims an angle. Nothing is invented: the answer IS a
|
|
11
|
+
// trained form, voiced whole.
|
|
12
|
+
//
|
|
13
|
+
// NO NOTION OF TEXT. This mechanism reads bytes and geometry only. It has no
|
|
14
|
+
// separator, no character class, no "word": the only structural quantity it
|
|
15
|
+
// uses is W, the river's grouping window, which is the same capacity the
|
|
16
|
+
// perception tree groups by and the same bar the argument-binding tier holds
|
|
17
|
+
// its constituents to. A completion shorter than one grouping window carries
|
|
18
|
+
// no structure the geometry can perceive, whatever the modality — that is a
|
|
19
|
+
// statement about the fold, not about punctuation. Presentation (what is
|
|
20
|
+
// "spacing", what is "case") belongs to the injected canon and to the modality
|
|
21
|
+
// entry point, never here; see src/canon.ts.
|
|
22
|
+
//
|
|
23
|
+
// WHY THE EARLIER TIERS CANNOT DO IT. Two independent reasons, both measured:
|
|
24
|
+
//
|
|
25
|
+
// 1. `resolve(prefix)` is null. A proper prefix of a deposited stream has no
|
|
26
|
+
// branch of its own unless it was itself deposited, so the exact tiers
|
|
27
|
+
// have nothing to find.
|
|
28
|
+
// 2. The form is not among the resonance candidates AT ALL. Measured on the
|
|
29
|
+
// trained store: cos(query, that form) = 0.5752, yet the form is absent
|
|
30
|
+
// from `resonate(k)` at k = 24, 256 AND 2048 — while forms scoring LOWER
|
|
31
|
+
// (Germany 0.5670, Yemen 0.5591) are returned. `k` only reorders WITHIN
|
|
32
|
+
// the IVF clusters already probed, exactly as Store.resonate's doc warns,
|
|
33
|
+
// so no k recovers it. With `exhaustive` it ranks 8.
|
|
34
|
+
//
|
|
35
|
+
// So this is a RETRIEVABILITY gap, not a semantic one, and it is repaired by
|
|
36
|
+
// reading the candidate list recall's refusal path has ALREADY fetched
|
|
37
|
+
// exhaustively for the substitution bridge — never by resonating on its own.
|
|
38
|
+
// Measured cost of the scan over those 570 candidates: 2.9 ms warm, 20.4 ms
|
|
39
|
+
// cold, against a ~700 ms refusal path. Issuing a FRESH exhaustive call would
|
|
40
|
+
// cost 490 ms median against 13 ms non-exhaustive (36×), which is why this tier
|
|
41
|
+
// takes the candidate list as an argument and adds nothing to it.
|
|
42
|
+
//
|
|
43
|
+
// THREE GUARDS, each falsified into existence by measurement — do not drop any:
|
|
44
|
+
//
|
|
45
|
+
// 1. AN UNREADABLE CONTINUATION VETOES. Reads are bounded (a stored span can
|
|
46
|
+
// run to hundreds of kilobytes), so a candidate that opens with the query
|
|
47
|
+
// but SATURATES the read continues in a way nobody can see. It is a
|
|
48
|
+
// standing disagreement: if any such candidate exists, nothing is grounded.
|
|
49
|
+
// It must NOT be quietly skipped, and that is not a stylistic point — the
|
|
50
|
+
// skip is what MANUFACTURES a fragment. Measured on a one-deposit fixture
|
|
51
|
+
// whose form exceeds the cap: the query matched BOTH the whole 138-byte
|
|
52
|
+
// form (saturating) AND an interior fold node of 34 bytes (unsaturated,
|
|
53
|
+
// continuing `" Paris, an"`). Skipping the saturated candidate removed the
|
|
54
|
+
// only evidence that disagreed, uniqueness then passed on the interior
|
|
55
|
+
// node, and a mid-form slice was voiced as an answer. Suppressing the
|
|
56
|
+
// disagreement is what created the fabrication.
|
|
57
|
+
// (Testing instead whether a candidate is a "complete form" via the fold
|
|
58
|
+
// does NOT work and was measured: content addressing makes an interior
|
|
59
|
+
// node resolve to ITSELF, so self-resolution says nothing about
|
|
60
|
+
// completeness.)
|
|
61
|
+
// 2. THE CONTINUATION MUST REACH ONE GROUPING WINDOW. A trained
|
|
62
|
+
// `What is the capital of France??` opens with `What is the capital of
|
|
63
|
+
// France?` and continues by a single byte. Below W the continuation is
|
|
64
|
+
// sub-quantum — the fold groups nothing from it — and voicing it produces
|
|
65
|
+
// the degenerate reply that is a known failure smell.
|
|
66
|
+
// 3. UNIQUENESS. Several trained forms may open with the query and continue
|
|
67
|
+
// differently, and then the corpus does not say which continuation the
|
|
68
|
+
// asker means. Distinct continuations ⇒ refuse. This is the documented
|
|
69
|
+
// PREFIX TRAP, and it is real — just not for every prefix. Measured: of
|
|
70
|
+
// 15 battery probes exactly ONE yields a unique continuation, and all
|
|
71
|
+
// three honest-silence probes yield none (including `What is the capital
|
|
72
|
+
// of Zamunda?`, whose top hit scores 0.83).
|
|
73
|
+
//
|
|
74
|
+
// Uniqueness is judged on the continuation BYTES, not on the candidate id: the
|
|
75
|
+
// same continuation reached through two trained forms is one answer, not an
|
|
76
|
+
// ambiguity.
|
|
77
|
+
import { bytesEqual } from "../bytes.js";
|
|
78
|
+
import { rItem } from "./trace.js";
|
|
79
|
+
import { canonicalWindows, leafIdPrefix } from "./canonical.js";
|
|
80
|
+
import { hubBound } from "./traverse.js";
|
|
81
|
+
/** Trained forms the query may OPEN, proposed from the write side's own
|
|
82
|
+
* leaf-id window index — the supply of last resort for {@link
|
|
83
|
+
* prefixCompletion}.
|
|
84
|
+
*
|
|
85
|
+
* WHY A SECOND SUPPLY EXISTS. The ranked list this mechanism normally reads
|
|
86
|
+
* is a resonance list, and resonance cannot rank a proper prefix: measured on
|
|
87
|
+
* the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
|
|
88
|
+
* truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
|
|
89
|
+
* Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
|
|
90
|
+
* bug, so no k and no re-ranking recovers it.
|
|
91
|
+
*
|
|
92
|
+
* WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT. A query's own fold is
|
|
93
|
+
* useless here: content addressing is not phrase-position-invariant, so a
|
|
94
|
+
* standalone prefix folds to a DIFFERENT node than the same bytes sitting
|
|
95
|
+
* inside a longer deposit, and neither the prefix's own node nor its
|
|
96
|
+
* ancestors lead to the deposit (measured: the 22-byte prefix of the
|
|
97
|
+
* photosynthesis form resolves, is shared by 6 contexts, and does not have
|
|
98
|
+
* the form among its ancestors). Leaf ids ARE position-invariant — they are
|
|
99
|
+
* content-addressed on single bytes — and `indexSubSpans` already interns a
|
|
100
|
+
* flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
|
|
101
|
+
* containment edges to the chunks that window spans. A query that is a
|
|
102
|
+
* prefix therefore shares those window nodes exactly, and reaches the deposit
|
|
103
|
+
* by climbing containment then parents. Nothing is added to the write side;
|
|
104
|
+
* this reads an index training already built.
|
|
105
|
+
*
|
|
106
|
+
* BOUNDED (§2.8), AND WITH NO NEW THRESHOLD. The window whose containment is
|
|
107
|
+
* SMALLEST carries the most evidence, and one saturated at `hubBound` carries
|
|
108
|
+
* none — that is the same √N reading of "hub" the rest of the mind uses, not
|
|
109
|
+
* a tuned knob. The upward walk spends a budget of `hubBound` nodes and
|
|
110
|
+
* fans out by W, so a hub query enumerates nothing and the caller stays
|
|
111
|
+
* silent rather than guessing (§2.13). Measured on the trained store: the
|
|
112
|
+
* photosynthesis form at a one-byte truncation picks a window with 52
|
|
113
|
+
* containers, visits 446 nodes, and yields exactly ONE candidate that
|
|
114
|
+
* survives the caller's byte compare — the form itself.
|
|
115
|
+
*
|
|
116
|
+
* These are PROPOSALS only. Every candidate still faces the byte-exact
|
|
117
|
+
* prefix compare and all three guards below, so a wrong proposal costs one
|
|
118
|
+
* bounded read and can never be voiced (§2.3). */
|
|
119
|
+
export function prefixCandidates(ctx, query) {
|
|
120
|
+
const store = ctx.store;
|
|
121
|
+
const W = ctx.space.maxGroup;
|
|
122
|
+
const run = leafIdPrefix(ctx, query);
|
|
123
|
+
// The widest canonical window is the most discriminative one the write side
|
|
124
|
+
// ever interned; a query too short to spell one carries no window evidence.
|
|
125
|
+
const len = canonicalWindows(W)[1];
|
|
126
|
+
if (run.length < len)
|
|
127
|
+
return [];
|
|
128
|
+
const bound = hubBound(ctx);
|
|
129
|
+
let best = null;
|
|
130
|
+
let bestN = 0;
|
|
131
|
+
for (let off = 0; off + len <= run.length; off++) {
|
|
132
|
+
const wid = store.findBranch(run.slice(off, off + len));
|
|
133
|
+
if (wid === null)
|
|
134
|
+
continue;
|
|
135
|
+
const n = store.containersSlice(wid, 0, bound).length;
|
|
136
|
+
// Empty says the window spans no chunk; saturated says it is a hub, whose
|
|
137
|
+
// containment discriminates nothing. Neither is evidence.
|
|
138
|
+
if (n === 0 || n >= bound)
|
|
139
|
+
continue;
|
|
140
|
+
if (best === null || n < bestN) {
|
|
141
|
+
best = wid;
|
|
142
|
+
bestN = n;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (best === null)
|
|
146
|
+
return [];
|
|
147
|
+
let frontier = store.containersSlice(best, 0, bound);
|
|
148
|
+
const seen = new Set(frontier);
|
|
149
|
+
let budget = bound;
|
|
150
|
+
while (frontier.length > 0 && budget > 0) {
|
|
151
|
+
const next = [];
|
|
152
|
+
for (const f of frontier) {
|
|
153
|
+
if (budget-- <= 0)
|
|
154
|
+
break;
|
|
155
|
+
for (const p of store.parentsFirst(f, W)) {
|
|
156
|
+
if (seen.has(p))
|
|
157
|
+
continue;
|
|
158
|
+
seen.add(p);
|
|
159
|
+
next.push(p);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
frontier = next;
|
|
163
|
+
}
|
|
164
|
+
return [...seen];
|
|
165
|
+
}
|
|
166
|
+
/** The sole trained form the query opens — or null when no candidate opens with
|
|
167
|
+
* it, when the continuation is sub-quantum, when a candidate's continuation
|
|
168
|
+
* cannot be read through, or when the candidates disagree.
|
|
169
|
+
*
|
|
170
|
+
* `ranked` must be a list the caller has ALREADY fetched; this mechanism never
|
|
171
|
+
* resonates on its own (see the header's cost note). */
|
|
172
|
+
export function prefixCompletion(ctx, query, ranked) {
|
|
173
|
+
const W = ctx.space.maxGroup;
|
|
174
|
+
const t = ctx.trace?.enter("prefixCompletion", [rItem(query, "query")]);
|
|
175
|
+
const done = (hit, note, data) => {
|
|
176
|
+
t?.done(hit === null ? [] : [rItem(hit.continuation, "continuation", hit.id)], note, data);
|
|
177
|
+
return hit;
|
|
178
|
+
};
|
|
179
|
+
// Reads are bounded to phrase scale, the same bound the frame filler uses.
|
|
180
|
+
// A query with no room for a whole grouping window past its own length
|
|
181
|
+
// cannot clear guard 2, so it is not worth a single read.
|
|
182
|
+
const cap = query.length * W;
|
|
183
|
+
if (query.length === 0 || cap < query.length + W) {
|
|
184
|
+
return done(null, "no room for a perceivable continuation within the cap");
|
|
185
|
+
}
|
|
186
|
+
// Distinct continuations, each with the first form that offered it. Held as
|
|
187
|
+
// a list, not a byte-keyed map: candidates that open with the query are few
|
|
188
|
+
// (measured: 1 on the trained store's winning query), and a linear byte
|
|
189
|
+
// compare needs no string encoding of content. Uniqueness (guard 3) is
|
|
190
|
+
// decided over this list, so the scan cannot stop early — a second
|
|
191
|
+
// continuation IS the refusal, and finding it is the point.
|
|
192
|
+
const found = [];
|
|
193
|
+
let opened = 0;
|
|
194
|
+
let unreadable = 0;
|
|
195
|
+
let subQuantum = 0;
|
|
196
|
+
for (const id of ranked) {
|
|
197
|
+
const form = ctx.store.bytesPrefix(id, cap);
|
|
198
|
+
if (form.length <= query.length)
|
|
199
|
+
continue;
|
|
200
|
+
let opens = true;
|
|
201
|
+
for (let i = 0; i < query.length; i++) {
|
|
202
|
+
if (form[i] !== query[i]) {
|
|
203
|
+
opens = false;
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (!opens)
|
|
208
|
+
continue;
|
|
209
|
+
opened++;
|
|
210
|
+
// Guard 1: a saturated read continues out of sight — a disagreement that
|
|
211
|
+
// cannot be resolved, so it ends the search rather than being skipped.
|
|
212
|
+
if (form.length >= cap) {
|
|
213
|
+
unreadable++;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
const rest = form.subarray(query.length);
|
|
217
|
+
// Guard 2: below one grouping window there is no structure to voice.
|
|
218
|
+
if (rest.length < W) {
|
|
219
|
+
subQuantum++;
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
if (!found.some((f) => bytesEqual(f.continuation, rest))) {
|
|
223
|
+
found.push({ id, form, continuation: rest });
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const data = {
|
|
227
|
+
candidates: ranked.length,
|
|
228
|
+
opened,
|
|
229
|
+
unreadable,
|
|
230
|
+
subQuantum,
|
|
231
|
+
distinctContinuations: found.length,
|
|
232
|
+
};
|
|
233
|
+
if (unreadable > 0 && found.length > 0) {
|
|
234
|
+
return done(null, "a form opens with this query but continues past the read bound — " +
|
|
235
|
+
"its continuation cannot be read, so none is licensed", data);
|
|
236
|
+
}
|
|
237
|
+
// Guard 2b: A SUB-QUANTUM CONTINUATION IS STILL A DISAGREEMENT. Guard 2
|
|
238
|
+
// refuses to VOICE a below-window continuation, and rightly — there is no
|
|
239
|
+
// structure there to speak. But dropping such a candidate from the
|
|
240
|
+
// uniqueness tally silently converts "the corpus offers many continuations,
|
|
241
|
+
// most of them unvoiceable" into "the corpus offers exactly one", and
|
|
242
|
+
// guard 3 then passes VACUOUSLY on the sole survivor. That is precisely
|
|
243
|
+
// the failure guard 1 documents for unreadable continuations — suppressing
|
|
244
|
+
// the disagreement is what manufactures the answer — so it is answered the
|
|
245
|
+
// same way, and for the same reason.
|
|
246
|
+
//
|
|
247
|
+
// Measured on a 4,300-fact fixture of "what is the value of <i>?": the
|
|
248
|
+
// query "what is the value of" drew candidates continuing " 0?", " 4?",
|
|
249
|
+
// " 8?" (3 bytes, sub-quantum at W=4) and " 10?" (4 bytes). The first
|
|
250
|
+
// three were dropped, leaving one survivor, and the mechanism reported
|
|
251
|
+
// "exactly one trained form" and voiced "the value of 10 is 20" — an
|
|
252
|
+
// arbitrary pick from thousands of equally-good readings, with the
|
|
253
|
+
// evidence of ambiguity discarded on the way.
|
|
254
|
+
//
|
|
255
|
+
// Note this can only ever cause SILENCE, never a different answer: it
|
|
256
|
+
// withholds a completion the corpus does not uniquely license.
|
|
257
|
+
if (subQuantum > 0 && found.length > 0) {
|
|
258
|
+
return done(null, "other trained forms open with this query but continue below one " +
|
|
259
|
+
"grouping window — the corpus offers competing readings, so no " +
|
|
260
|
+
"single completion is licensed", data);
|
|
261
|
+
}
|
|
262
|
+
// Guard 3: the corpus must agree on ONE continuation.
|
|
263
|
+
if (found.length !== 1) {
|
|
264
|
+
return done(null, found.length === 0
|
|
265
|
+
? "no trained form opens with this query and continues perceivably"
|
|
266
|
+
: "trained forms open with this query but continue differently — " +
|
|
267
|
+
"the corpus does not say which continuation is meant", data);
|
|
268
|
+
}
|
|
269
|
+
return done(found[0], "one trained form opens with this query, and continues perceivably", data);
|
|
270
|
+
}
|
|
@@ -6,6 +6,17 @@ import type { Input, MindContext } from "./types.js";
|
|
|
6
6
|
* (windows, regions, candidate spans), so key construction is far cheaper
|
|
7
7
|
* than the river fold it deduplicates. */
|
|
8
8
|
export declare function latin1Key(bytes: Uint8Array): string;
|
|
9
|
+
/** The {@link perceive} memo key: the span's content PLUS the boundary set it
|
|
10
|
+
* was folded under. The tree is a function of BOTH — the same bytes fold
|
|
11
|
+
* plainly with no boundaries and into a left-nested stable-prefix shape with
|
|
12
|
+
* them — so a content-only key returns whichever shape was computed first.
|
|
13
|
+
* That is exactly what happened: a conversation seeded its cumulative context
|
|
14
|
+
* under the content key, and every later plain `perceive` of those bytes was
|
|
15
|
+
* served the boundary tree instead (measured: respondTurn answered where
|
|
16
|
+
* respond() on byte-identical input did not). NUL separates the two parts —
|
|
17
|
+
* the boundary rendering is digits and commas, so no content byte can forge
|
|
18
|
+
* the split. */
|
|
19
|
+
export declare function perceiveKey(bytes: Uint8Array, boundaries?: readonly number[]): string;
|
|
9
20
|
/** Perceive input into a content-defined tree (the river fold).
|
|
10
21
|
* Deterministic — identical bytes always produce an identical tree.
|
|
11
22
|
*
|
|
@@ -16,16 +27,24 @@ export declare function latin1Key(bytes: Uint8Array): string;
|
|
|
16
27
|
* boundaries are; the geometry never guesses them from the bytes. */
|
|
17
28
|
export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
|
|
18
29
|
/** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
|
|
19
|
-
* bit-identical to what inference computes for the same bytes
|
|
20
|
-
* train/inference agreement is
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
30
|
+
* bit-identical to what inference computes for the same bytes. That
|
|
31
|
+
* train/inference agreement is the whole contract: the trained context node
|
|
32
|
+
* and the node `resolve(query)` reaches must be the SAME node, and the only
|
|
33
|
+
* way to guarantee it is to give this function nothing extra to say. It
|
|
34
|
+
* imposes no boundaries, knows nothing about turns, and reads no convention
|
|
35
|
+
* out of the bytes.
|
|
36
|
+
*
|
|
37
|
+
* An input that EXTENDS a previously deposited one — a conversation context
|
|
38
|
+
* grown by a turn, or a resumed replay — reuses that deposit's already-folded
|
|
39
|
+
* content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
|
|
40
|
+
* instead of O(context). The reuse is TRANSPARENT by construction: a segment
|
|
41
|
+
* is a pure function of its own bytes, so a reused one is bit-identical to a
|
|
42
|
+
* refolded one. Nothing has to prove that the extending deposit is "really"
|
|
43
|
+
* a next turn — a coincidental byte prefix reuses the same segments and gets
|
|
44
|
+
* the same tree it would have got anyway. (It used to matter: while this
|
|
45
|
+
* path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
|
|
46
|
+
* needed a continuation-bytes proof to gate it. Nothing is imposed now, so
|
|
47
|
+
* there is nothing to gate.) */
|
|
29
48
|
export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
|
|
30
49
|
/** The raw bytes of an input — modality-neutral conversion. */
|
|
31
50
|
export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Address — bytes → node (perceive, foldTree, resolve)
|
|
4
4
|
// Read — node → bytes (read)
|
|
5
|
-
import { bytesToTree, gridToTree, hilbertBytes,
|
|
5
|
+
import { bytesToTree, contentFoldIncremental, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
|
|
6
6
|
import { canonHash } from "../canon.js";
|
|
7
7
|
import { bytesEqual } from "../bytes.js";
|
|
8
8
|
import { ALL } from "./types.js";
|
|
@@ -21,6 +21,22 @@ export function latin1Key(bytes) {
|
|
|
21
21
|
}
|
|
22
22
|
return s;
|
|
23
23
|
}
|
|
24
|
+
/** The {@link perceive} memo key: the span's content PLUS the boundary set it
|
|
25
|
+
* was folded under. The tree is a function of BOTH — the same bytes fold
|
|
26
|
+
* plainly with no boundaries and into a left-nested stable-prefix shape with
|
|
27
|
+
* them — so a content-only key returns whichever shape was computed first.
|
|
28
|
+
* That is exactly what happened: a conversation seeded its cumulative context
|
|
29
|
+
* under the content key, and every later plain `perceive` of those bytes was
|
|
30
|
+
* served the boundary tree instead (measured: respondTurn answered where
|
|
31
|
+
* respond() on byte-identical input did not). NUL separates the two parts —
|
|
32
|
+
* the boundary rendering is digits and commas, so no content byte can forge
|
|
33
|
+
* the split. */
|
|
34
|
+
export function perceiveKey(bytes, boundaries) {
|
|
35
|
+
const k = latin1Key(bytes);
|
|
36
|
+
return boundaries === undefined || boundaries.length === 0
|
|
37
|
+
? k
|
|
38
|
+
: k + "\u0000" + boundaries.join(",");
|
|
39
|
+
}
|
|
24
40
|
/** Perceive input into a content-defined tree (the river fold).
|
|
25
41
|
* Deterministic — identical bytes always produce an identical tree.
|
|
26
42
|
*
|
|
@@ -41,7 +57,7 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
|
|
|
41
57
|
// The tree is shared by reference; Sema nodes are never mutated.
|
|
42
58
|
const memo = ctx.perceiveMemo;
|
|
43
59
|
if (memo) {
|
|
44
|
-
const key =
|
|
60
|
+
const key = perceiveKey(bytes, boundaries);
|
|
45
61
|
const hit = memo.get(key);
|
|
46
62
|
if (hit !== undefined) {
|
|
47
63
|
if (ctx.meter)
|
|
@@ -70,67 +86,42 @@ export function perceive(ctx, input, leafAt, lookup, boundaries) {
|
|
|
70
86
|
return gridToTree(ctx.space, ctx.alphabet, input);
|
|
71
87
|
}
|
|
72
88
|
/** The DEPOSIT-shaped perceive. Folds over the stream's own content cuts —
|
|
73
|
-
* bit-identical to what inference computes for the same bytes
|
|
74
|
-
* train/inference agreement is
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
89
|
+
* bit-identical to what inference computes for the same bytes. That
|
|
90
|
+
* train/inference agreement is the whole contract: the trained context node
|
|
91
|
+
* and the node `resolve(query)` reaches must be the SAME node, and the only
|
|
92
|
+
* way to guarantee it is to give this function nothing extra to say. It
|
|
93
|
+
* imposes no boundaries, knows nothing about turns, and reads no convention
|
|
94
|
+
* out of the bytes.
|
|
95
|
+
*
|
|
96
|
+
* An input that EXTENDS a previously deposited one — a conversation context
|
|
97
|
+
* grown by a turn, or a resumed replay — reuses that deposit's already-folded
|
|
98
|
+
* content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
|
|
99
|
+
* instead of O(context). The reuse is TRANSPARENT by construction: a segment
|
|
100
|
+
* is a pure function of its own bytes, so a reused one is bit-identical to a
|
|
101
|
+
* refolded one. Nothing has to prove that the extending deposit is "really"
|
|
102
|
+
* a next turn — a coincidental byte prefix reuses the same segments and gets
|
|
103
|
+
* the same tree it would have got anyway. (It used to matter: while this
|
|
104
|
+
* path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
|
|
105
|
+
* needed a continuation-bytes proof to gate it. Nothing is imposed now, so
|
|
106
|
+
* there is nothing to gate.) */
|
|
83
107
|
export function perceiveDeposit(ctx, bytes, conversational = false) {
|
|
108
|
+
// Longest cached PROPER prefix first — the most segments to reuse.
|
|
84
109
|
let prev;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const lens = [...ctx._depositLens]
|
|
94
|
-
.filter((L) => L >= 2 && L < bytes.length)
|
|
95
|
-
.sort((a, b) => b - a);
|
|
96
|
-
for (const L of lens) {
|
|
97
|
-
const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
|
|
98
|
-
// The suffix must bytes-equal the hit's OWN recorded continuation —
|
|
99
|
-
// proof this deposit is that turn's actual next turn, not a fact
|
|
100
|
-
// that coincidentally shares its byte prefix.
|
|
101
|
-
if (hit !== undefined && hit.nextBytes !== undefined &&
|
|
102
|
-
bytesEqual(hit.nextBytes, bytes.subarray(L))) {
|
|
103
|
-
prev = hit;
|
|
104
|
-
prefixLen = L;
|
|
105
|
-
break;
|
|
106
|
-
}
|
|
110
|
+
const lens = [...ctx._depositLens]
|
|
111
|
+
.filter((L) => L >= 2 && L < bytes.length)
|
|
112
|
+
.sort((a, b) => b - a);
|
|
113
|
+
for (const L of lens) {
|
|
114
|
+
const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
|
|
115
|
+
if (hit !== undefined) {
|
|
116
|
+
prev = hit.content;
|
|
117
|
+
break;
|
|
107
118
|
}
|
|
108
119
|
}
|
|
109
|
-
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
// happened, a deposit's context root and `resolve(question)` were different
|
|
115
|
-
// nodes, so the trained edge hung off a node inference never reached and
|
|
116
|
-
// recall went silent (test/44 caught it as a site that could not be emitted
|
|
117
|
-
// because the resolved node led nowhere). Train and infer must fold
|
|
118
|
-
// identically; the way to guarantee that is to give this function nothing
|
|
119
|
-
// extra to say.
|
|
120
|
-
const cuts = new Set();
|
|
121
|
-
if (prev !== undefined) {
|
|
122
|
-
for (const b of prev.boundaries)
|
|
123
|
-
cuts.add(b);
|
|
124
|
-
cuts.add(prefixLen);
|
|
125
|
-
}
|
|
126
|
-
const boundaries = [...cuts].sort((a, b) => a - b);
|
|
127
|
-
const folded = stablePrefixFoldIncremental(ctx.space, ctx.alphabet, bytes, boundaries, prev?.stable);
|
|
128
|
-
const tree = folded.tree;
|
|
129
|
-
const entry = { boundaries, stable: folded.fold };
|
|
130
|
-
// Only a conversational deposit writes the cache too — otherwise a bare
|
|
131
|
-
// fact's plain fold could later be misread as a conversation's turn-zero
|
|
132
|
-
// boundary by an unrelated conversational deposit that happens to extend
|
|
133
|
-
// its bytes.
|
|
120
|
+
const folded = contentFoldIncremental(ctx.space, ctx.alphabet, bytes, prev);
|
|
121
|
+
// Only a CONVERSATIONAL deposit writes the cache: reuse is sound for any
|
|
122
|
+
// deposit, but the budget is 8 entries and a corpus of unrelated facts would
|
|
123
|
+
// evict the live chains for nothing. Purely a cost decision now, not a
|
|
124
|
+
// correctness one.
|
|
134
125
|
if (conversational && bytes.length >= 2) {
|
|
135
126
|
// The lengths set drifts as the map evicts; past the probe budget the
|
|
136
127
|
// drift itself becomes the cost (each stale length is an O(len) key
|
|
@@ -139,10 +130,10 @@ export function perceiveDeposit(ctx, bytes, conversational = false) {
|
|
|
139
130
|
ctx._depositLens.clear();
|
|
140
131
|
ctx._depositTrees.clear();
|
|
141
132
|
}
|
|
142
|
-
ctx._depositTrees.set(latin1Key(bytes),
|
|
133
|
+
ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
|
|
143
134
|
ctx._depositLens.add(bytes.length);
|
|
144
135
|
}
|
|
145
|
-
return tree;
|
|
136
|
+
return folded.tree;
|
|
146
137
|
}
|
|
147
138
|
/** The raw bytes of an input — modality-neutral conversion. */
|
|
148
139
|
export function inputBytes(ctx, input) {
|