@hviana/sema 0.5.9 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +80 -0
- package/AGENTS.md +73 -13
- package/DATASETS.md +12 -11
- package/dist/example/train_base/cache.d.ts +35 -0
- package/dist/example/train_base/cache.js +211 -0
- package/dist/example/train_base/config.d.ts +21 -0
- package/dist/example/train_base/config.js +94 -0
- package/dist/example/train_base/corpora/aya.d.ts +19 -0
- package/dist/example/train_base/corpora/aya.js +76 -0
- package/dist/example/train_base/corpora/converted-parquet.d.ts +14 -0
- package/dist/example/train_base/corpora/converted-parquet.js +44 -0
- package/dist/example/train_base/corpora/genknow.d.ts +14 -0
- package/dist/example/train_base/corpora/genknow.js +83 -0
- package/dist/example/train_base/corpora/index.d.ts +29 -0
- package/dist/example/train_base/corpora/index.js +81 -0
- package/dist/example/train_base/corpora/massive.d.ts +7 -0
- package/dist/example/train_base/corpora/massive.js +98 -0
- package/dist/example/train_base/corpora/oasst2.d.ts +52 -0
- package/dist/example/train_base/corpora/oasst2.js +120 -0
- package/dist/example/train_base/corpora/smolsent.d.ts +23 -0
- package/dist/example/train_base/corpora/smolsent.js +156 -0
- package/dist/example/train_base/corpora/soda.d.ts +12 -0
- package/dist/example/train_base/corpora/soda.js +113 -0
- package/dist/example/train_base/corpora/taskmaster.d.ts +15 -0
- package/dist/example/train_base/corpora/taskmaster.js +144 -0
- package/dist/example/train_base/corpora/wiki2.d.ts +23 -0
- package/dist/example/train_base/corpora/wiki2.js +132 -0
- package/dist/example/train_base/corpus.d.ts +88 -0
- package/dist/example/train_base/corpus.js +65 -0
- package/dist/example/train_base/discovery.d.ts +48 -0
- package/dist/example/train_base/discovery.js +143 -0
- package/dist/example/train_base/http.d.ts +82 -0
- package/dist/example/train_base/http.js +219 -0
- package/dist/example/train_base/items.d.ts +46 -0
- package/dist/example/train_base/items.js +98 -0
- package/dist/example/train_base/main.d.ts +4 -0
- package/dist/example/train_base/main.js +207 -0
- package/dist/example/train_base/progress.d.ts +34 -0
- package/dist/example/train_base/progress.js +114 -0
- package/dist/example/train_base/readers.d.ts +125 -0
- package/dist/example/train_base/readers.js +391 -0
- package/dist/example/train_base/runtime.d.ts +115 -0
- package/dist/example/train_base/runtime.js +637 -0
- package/dist/example/train_base/stage.d.ts +3 -0
- package/dist/example/train_base/stage.js +246 -0
- package/dist/example/train_base/ui.d.ts +88 -0
- package/dist/example/train_base/ui.js +272 -0
- package/dist/src/meter.d.ts +1 -4
- package/dist/src/meter.js +0 -3
- package/dist/src/mind/attention.js +22 -20
- package/dist/src/mind/graph-search.d.ts +43 -9
- package/dist/src/mind/graph-search.js +82 -15
- package/dist/src/mind/junction.d.ts +13 -0
- package/dist/src/mind/junction.js +13 -0
- package/dist/src/mind/mechanisms/cover.js +23 -2
- package/dist/src/mind/mechanisms/prefix-completion.js +13 -11
- package/dist/src/mind/mechanisms/recall.js +8 -4
- package/dist/src/mind/mind.d.ts +1 -1
- package/dist/src/mind/mind.js +1 -1
- package/dist/src/mind/pipeline-mechanism.d.ts +0 -24
- package/dist/src/mind/pipeline-mechanism.js +13 -36
- package/dist/src/mind/pipeline.d.ts +23 -0
- package/dist/src/mind/pipeline.js +51 -3
- package/dist/src/mind/recognition.d.ts +6 -1
- package/dist/src/mind/recognition.js +11 -6
- package/dist/src/mind/resonance.js +48 -13
- package/dist/src/store.js +22 -1
- package/example/train_base/cache.ts +251 -0
- package/example/train_base/config.ts +128 -0
- package/example/train_base/corpora/aya.ts +106 -0
- package/example/train_base/corpora/converted-parquet.ts +64 -0
- package/example/train_base/corpora/genknow.ts +114 -0
- package/example/train_base/corpora/index.ts +88 -0
- package/example/train_base/corpora/massive.ts +111 -0
- package/example/train_base/corpora/oasst2.ts +163 -0
- package/example/train_base/corpora/smolsent.ts +203 -0
- package/example/train_base/corpora/soda.ts +130 -0
- package/example/train_base/corpora/taskmaster.ts +217 -0
- package/example/train_base/corpora/wiki2.ts +190 -0
- package/example/train_base/corpus.ts +150 -0
- package/example/train_base/discovery.ts +203 -0
- package/example/train_base/http.ts +284 -0
- package/example/train_base/items.ts +118 -0
- package/example/train_base/main.ts +240 -0
- package/example/train_base/progress.ts +149 -0
- package/example/train_base/readers.ts +505 -0
- package/example/train_base/runtime.ts +894 -0
- package/example/train_base/stage.ts +276 -0
- package/example/train_base/ui.ts +333 -0
- package/jsr.json +1 -1
- package/package.json +8 -5
- package/src/meter.ts +1 -4
- package/src/mind/attention.ts +22 -19
- package/src/mind/graph-search.ts +93 -16
- package/src/mind/junction.ts +13 -0
- package/src/mind/mechanisms/cover.ts +23 -4
- package/src/mind/mechanisms/prefix-completion.ts +13 -11
- package/src/mind/mechanisms/recall.ts +8 -4
- package/src/mind/mind.ts +1 -1
- package/src/mind/pipeline-mechanism.ts +13 -42
- package/src/mind/pipeline.ts +87 -3
- package/src/mind/recognition.ts +19 -6
- package/src/mind/resonance.ts +79 -50
- package/src/store.ts +21 -1
- package/test/13-conversation.test.mjs +1 -1
- package/test/84-composed-answer-honesty.test.mjs +2 -1
- package/test/88-dependency-footprint.test.mjs +99 -0
- package/test/89-completion-recursion.test.mjs +230 -0
- package/test/90-connector-read-cap.test.mjs +130 -0
- package/test/91-branch-bytes-cache.test.mjs +152 -0
- package/test/93-regime-prediction.test.mjs +148 -0
- package/test/94-cross-region-budget.test.mjs +67 -0
- package/test/95-wide-resonance-removed.test.mjs +109 -0
- package/dist/example/train_base.d.ts +0 -163
- package/dist/example/train_base.js +0 -3220
- package/example/train_base.ts +0 -3882
|
@@ -12,7 +12,7 @@ import { composeStructuralGist, consensusFloor, dominates, estimatorNoise, } fro
|
|
|
12
12
|
import { foldTree, gistOf, latin1Key, perceive, read, resolve, } from "./primitives.js";
|
|
13
13
|
import { recognise } from "./recognition.js";
|
|
14
14
|
import { leafIdRun } from "./canonical.js";
|
|
15
|
-
import { corpusN, edgeAncestors, hubBound, sharedReachMemo, } from "./traverse.js";
|
|
15
|
+
import { atomIsHub, corpusN, edgeAncestors, hubBound, sharedReachMemo, } from "./traverse.js";
|
|
16
16
|
import { cachedRead, junctionContainersFrom, junctionSeeds, junctionSynonyms, loadJunctionSynonymSides, walkCache, } from "./junction.js";
|
|
17
17
|
import { indexOf } from "../bytes.js";
|
|
18
18
|
import { rItem, rNode, traceDerivation } from "./trace.js";
|
|
@@ -1848,26 +1848,28 @@ async function crossRegionVotes(ctx, query, regions, rvs, k, N, reachMemo, td) {
|
|
|
1848
1848
|
// the same container (or a sub-container of it) twice.
|
|
1849
1849
|
const consumed = new Set();
|
|
1850
1850
|
let probes = 0;
|
|
1851
|
-
//
|
|
1852
|
-
//
|
|
1853
|
-
//
|
|
1854
|
-
//
|
|
1855
|
-
//
|
|
1856
|
-
//
|
|
1851
|
+
// When atoms themselves are hubs (atomIsHub — a single byte reaches ≥ √N
|
|
1852
|
+
// contexts, §2.8's own predicate), the corpus is large enough that the
|
|
1853
|
+
// cross-region junction walks are dominated by the drift through common
|
|
1854
|
+
// content's ancestry. Each of k candidate pairs otherwise spends its own
|
|
1855
|
+
// √N·W budget (profiled: 160,210 junction pops, 31% of think at
|
|
1856
|
+
// N = 325,608), and a cumulative dialogue multiplies bounded work into tens
|
|
1857
|
+
// of seconds. The structural walk is therefore given ONE k·W allowance per
|
|
1858
|
+
// evidence tier, shared across every pair — k pairs × W phrase-scale levels,
|
|
1859
|
+
// the minimal exact check; a pair whose container is not reached within it
|
|
1860
|
+
// falls through to the resonance tier (the ANN proposes what the shallow
|
|
1861
|
+
// walk no longer exhaustively scans, §2.3).
|
|
1857
1862
|
//
|
|
1858
|
-
//
|
|
1859
|
-
//
|
|
1860
|
-
//
|
|
1861
|
-
//
|
|
1862
|
-
//
|
|
1863
|
-
//
|
|
1864
|
-
// too tight below ~10³ contexts
|
|
1865
|
-
//
|
|
1866
|
-
//
|
|
1867
|
-
|
|
1868
|
-
// already makes — pricing the gate on the synthetic corpora.
|
|
1869
|
-
const marketScale = k * ctx.space.maxGroup;
|
|
1870
|
-
const corpusScale = N > marketScale ** 3;
|
|
1863
|
+
// Below atomIsHub the store is small and atoms still discriminate, so the
|
|
1864
|
+
// walks keep exhaustive exact traversal (per-walk √N·W) — the shared budget
|
|
1865
|
+
// would otherwise be smaller than the structures the tests deliberately
|
|
1866
|
+
// build. The gate is the SAME derived predicate the climb already uses for
|
|
1867
|
+
// byte atoms, not a separate corpus-size knob: an earlier `N > (k·W)³` cube
|
|
1868
|
+
// never engaged at real scale (96³ = 884,736 > 325,608), and a "share one
|
|
1869
|
+
// √N·W" experiment was too tight below ~10³ contexts (test/36, test/14) —
|
|
1870
|
+
// both are the same mistake of pricing the gate on corpus size instead of on
|
|
1871
|
+
// the atom-hub scale.
|
|
1872
|
+
const corpusScale = atomIsHub(ctx, N);
|
|
1871
1873
|
const exactBudget = corpusScale ? { n: k * ctx.space.maxGroup } : undefined;
|
|
1872
1874
|
const synonymBudget = corpusScale ? { n: k * ctx.space.maxGroup } : undefined;
|
|
1873
1875
|
for (let a = 0; a < cand.length && probes < k; a++) {
|
|
@@ -66,6 +66,13 @@ export type GItem = {
|
|
|
66
66
|
* because subtraction is more "the point" of its query). See
|
|
67
67
|
* {@link liftAnswer}. */
|
|
68
68
|
computed?: boolean;
|
|
69
|
+
/** Set on the out emitted at a chain's GENUINE FIXPOINT — the one span kind a
|
|
70
|
+
* recursive re-cover ({@link GraphSearch.recompleteNode}) may deepen. The
|
|
71
|
+
* re-cover does NOT run here: this marks the span as eligible, and
|
|
72
|
+
* {@link GraphSearch.deepen} runs it afterwards on the spans the lightest
|
|
73
|
+
* derivation actually CHOSE. Part of {@link key}, because it decides
|
|
74
|
+
* whether the span's final bytes may still change. */
|
|
75
|
+
fix?: boolean;
|
|
69
76
|
};
|
|
70
77
|
export declare const STEP = 1;
|
|
71
78
|
export declare const CONCEPT = 10;
|
|
@@ -87,6 +94,10 @@ export interface Seg {
|
|
|
87
94
|
/** See the `computed` field of the "out" {@link GItem} — set only for an
|
|
88
95
|
* extension's derived value, never a genuinely recognised learned form. */
|
|
89
96
|
computed?: boolean;
|
|
97
|
+
/** See the `fix` field of the "out" {@link GItem} — this span ended a chain at
|
|
98
|
+
* a genuine fixpoint, so it is the one span kind a recursive re-cover may
|
|
99
|
+
* still deepen. Consumed by {@link GraphSearch.deepen}. */
|
|
100
|
+
fix?: boolean;
|
|
90
101
|
}
|
|
91
102
|
/** One rule application inside the cover's lightest derivation — the FINEST
|
|
92
103
|
* grain of Sema's core reasoning, one node of the adapted A*LD proof tree. `move`
|
|
@@ -192,6 +203,20 @@ export declare class GraphSearch {
|
|
|
192
203
|
* decomposes, two recomposes, any mix — and stops only when it reaches a node
|
|
193
204
|
* that leads nowhere new, never at an arbitrary count. */
|
|
194
205
|
private solve;
|
|
206
|
+
/** Re-cover the CHOSEN fixpoint spans, in place.
|
|
207
|
+
*
|
|
208
|
+
* Completion is still "cover, recursively" — it just runs on the answer
|
|
209
|
+
* instead of on the exploration. A cover chooses O(segs) spans, so a level
|
|
210
|
+
* pays O(answer) re-covers however densely the corpus interconnects the forms
|
|
211
|
+
* the search passed through on the way. That is the bound
|
|
212
|
+
* {@link recompleteNode}'s contract always claimed and, running per fixpoint
|
|
213
|
+
* REACHED, never had.
|
|
214
|
+
*
|
|
215
|
+
* Deepening cannot change which cover won: the derivation is already final
|
|
216
|
+
* and every span keeps its i..j and its cost. It only replaces a chosen
|
|
217
|
+
* span's bytes with the deeper learnt form they rewrite to — what the
|
|
218
|
+
* recursion was always for. */
|
|
219
|
+
private deepen;
|
|
195
220
|
/** The weighted deduction system the graph exploration solves (the four
|
|
196
221
|
* reductions of adapted A*LD live in {@link lightestDerivation}; this only states the
|
|
197
222
|
* items, axioms, goal, and rules — see {@link GItem} for the item kinds).
|
|
@@ -253,20 +278,29 @@ export declare class GraphSearch {
|
|
|
253
278
|
* ({@link resolve}) — the graph itself gates against re-expanding a contained
|
|
254
279
|
* form ("ice is cold" ⊅→ "ice is cold is cold").
|
|
255
280
|
*
|
|
256
|
-
* Termination is
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
281
|
+
* Termination is STRUCTURAL: a produced node is re-covered once, never inside
|
|
282
|
+
* another re-cover (see the guard below), so one cover pays at most one
|
|
283
|
+
* nested {@link solve} per distinct produced node it actually reaches, and
|
|
284
|
+
* {@link recompleteMemo} collapses a repeat to nothing.
|
|
285
|
+
*
|
|
286
|
+
* This comment used to argue termination from "distinct node ids are finite
|
|
287
|
+
* and each finished completion is memoised". That is a bound of N — the one
|
|
288
|
+
* AGENTS §2.8 forbids — and it was load-bearing, not pedantic: nested, the
|
|
289
|
+
* recursion reached depth 331 and 9.1 GB on an 18.9M-node store for a 2-byte
|
|
290
|
+
* query and did not terminate, which is what killed a 5 h training run at its
|
|
291
|
+
* checkpoint recall. Guard: test/89-completion-recursion.test.mjs. */
|
|
262
292
|
private recompleteNode;
|
|
263
293
|
/** Per-cover memo of each produced node's completion (so the many terminal
|
|
264
294
|
* outs of a long query re-cover each distinct node at most once); reset at the
|
|
265
295
|
* top of {@link cover}. */
|
|
266
296
|
private recompleteMemo;
|
|
267
|
-
/** The
|
|
268
|
-
*
|
|
269
|
-
* (
|
|
297
|
+
/** The node currently being re-completed — the recursion stack, and so also
|
|
298
|
+
* the nesting depth. {@link recompleteNode} refuses to start while it is
|
|
299
|
+
* non-empty (one re-cover per produced node, never one inside another), which
|
|
300
|
+
* is what keeps a query's cost proportional to its answer rather than to the
|
|
301
|
+
* corpus; it therefore holds at most one id. A Set, not a flag, because it
|
|
302
|
+
* states WHICH node is open — the invariant a reader needs to check the
|
|
303
|
+
* guard, and what makes the old cycle-guard reading still hold. */
|
|
270
304
|
private recompleteOpen;
|
|
271
305
|
/** out(i,j,bytes,…): index it for the binary rules, then offer splicing a
|
|
272
306
|
* learnt connector (the in-search bridge), splitting (at a sub-leaf form
|
|
@@ -64,6 +64,7 @@ function readCover(derivation) {
|
|
|
64
64
|
rec: out.rec,
|
|
65
65
|
node: out.node,
|
|
66
66
|
computed: out.computed,
|
|
67
|
+
fix: out.fix,
|
|
67
68
|
});
|
|
68
69
|
}
|
|
69
70
|
node = node.premises[0];
|
|
@@ -269,9 +270,32 @@ export class GraphSearch {
|
|
|
269
270
|
onDerivation(readDerivation(derivation, substitutions !== undefined));
|
|
270
271
|
}
|
|
271
272
|
return derivation
|
|
272
|
-
? { segs: readCover(derivation), cost: derivation.cost }
|
|
273
|
+
? { segs: this.deepen(readCover(derivation)), cost: derivation.cost }
|
|
273
274
|
: null;
|
|
274
275
|
}
|
|
276
|
+
/** Re-cover the CHOSEN fixpoint spans, in place.
|
|
277
|
+
*
|
|
278
|
+
* Completion is still "cover, recursively" — it just runs on the answer
|
|
279
|
+
* instead of on the exploration. A cover chooses O(segs) spans, so a level
|
|
280
|
+
* pays O(answer) re-covers however densely the corpus interconnects the forms
|
|
281
|
+
* the search passed through on the way. That is the bound
|
|
282
|
+
* {@link recompleteNode}'s contract always claimed and, running per fixpoint
|
|
283
|
+
* REACHED, never had.
|
|
284
|
+
*
|
|
285
|
+
* Deepening cannot change which cover won: the derivation is already final
|
|
286
|
+
* and every span keeps its i..j and its cost. It only replaces a chosen
|
|
287
|
+
* span's bytes with the deeper learnt form they rewrite to — what the
|
|
288
|
+
* recursion was always for. */
|
|
289
|
+
deepen(segs) {
|
|
290
|
+
for (const s of segs) {
|
|
291
|
+
if (!s.fix || s.node === undefined)
|
|
292
|
+
continue;
|
|
293
|
+
const deeper = this.recompleteNode(s.node);
|
|
294
|
+
if (deeper !== null)
|
|
295
|
+
s.bytes = deeper;
|
|
296
|
+
}
|
|
297
|
+
return segs;
|
|
298
|
+
}
|
|
275
299
|
/** The weighted deduction system the graph exploration solves (the four
|
|
276
300
|
* reductions of adapted A*LD live in {@link lightestDerivation}; this only states the
|
|
277
301
|
* items, axioms, goal, and rules — see {@link GItem} for the item kinds).
|
|
@@ -319,7 +343,7 @@ export class GraphSearch {
|
|
|
319
343
|
if (it.kind === "form") {
|
|
320
344
|
return `f${it.i}.${it.j}.${it.node}.${it.via ? 1 : 0}.${it.rcmp ? 1 : 0}`;
|
|
321
345
|
}
|
|
322
|
-
return `o${it.i}.${it.j}.${it.cover ? 1 : 0}.${it.rec ? 1 : 0}.${it.node ?? -1}.${latin1(it.bytes)}`;
|
|
346
|
+
return `o${it.i}.${it.j}.${it.cover ? 1 : 0}.${it.rec ? 1 : 0}.${it.fix ? 1 : 0}.${it.node ?? -1}.${latin1(it.bytes)}`;
|
|
323
347
|
},
|
|
324
348
|
*axioms() {
|
|
325
349
|
yield { item: { kind: "cover", p: 0 }, cost: 0 };
|
|
@@ -614,17 +638,29 @@ export class GraphSearch {
|
|
|
614
638
|
// actual end, never per intermediate stop — so its cost tracks the
|
|
615
639
|
// ANSWER's own structure, not how densely the corpus interconnects
|
|
616
640
|
// the nodes passed through on the way there.
|
|
617
|
-
|
|
641
|
+
// MARK the fixpoint; do not re-cover it here. Re-covering at this point
|
|
642
|
+
// pays a full {@link recompleteNode} — a recognition of the node's whole
|
|
643
|
+
// bytes — for every fixpoint the exploration REACHES, and how many it
|
|
644
|
+
// reaches is set by how densely the corpus interconnects the forms passed
|
|
645
|
+
// through. Measured on an 18.9M-node store, a 2-byte query: 12,000
|
|
646
|
+
// re-covers inside ONE cover, folding 95,258 distinct spans, ~2.4 GB and
|
|
647
|
+
// climbing to a V8 fatal. The answer needs a handful.
|
|
648
|
+
//
|
|
649
|
+
// {@link deepen} runs it instead on the spans the lightest derivation
|
|
650
|
+
// CHOSE. The ladder is untouched — this out still costs 0, so a genuine
|
|
651
|
+
// fixpoint still beats any premature stop at the same depth, exactly as
|
|
652
|
+
// the ordering above states.
|
|
618
653
|
yield {
|
|
619
654
|
premises: [it],
|
|
620
655
|
conclusion: {
|
|
621
656
|
kind: "out",
|
|
622
657
|
i: it.i,
|
|
623
658
|
j: it.j,
|
|
624
|
-
bytes:
|
|
659
|
+
bytes: nodeBytes(it.node),
|
|
625
660
|
cover: true,
|
|
626
661
|
rec: true,
|
|
627
662
|
node: it.node,
|
|
663
|
+
fix: true,
|
|
628
664
|
},
|
|
629
665
|
cost: 0,
|
|
630
666
|
};
|
|
@@ -694,20 +730,47 @@ export class GraphSearch {
|
|
|
694
730
|
* ({@link resolve}) — the graph itself gates against re-expanding a contained
|
|
695
731
|
* form ("ice is cold" ⊅→ "ice is cold is cold").
|
|
696
732
|
*
|
|
697
|
-
* Termination is
|
|
698
|
-
*
|
|
699
|
-
*
|
|
700
|
-
*
|
|
701
|
-
*
|
|
702
|
-
*
|
|
733
|
+
* Termination is STRUCTURAL: a produced node is re-covered once, never inside
|
|
734
|
+
* another re-cover (see the guard below), so one cover pays at most one
|
|
735
|
+
* nested {@link solve} per distinct produced node it actually reaches, and
|
|
736
|
+
* {@link recompleteMemo} collapses a repeat to nothing.
|
|
737
|
+
*
|
|
738
|
+
* This comment used to argue termination from "distinct node ids are finite
|
|
739
|
+
* and each finished completion is memoised". That is a bound of N — the one
|
|
740
|
+
* AGENTS §2.8 forbids — and it was load-bearing, not pedantic: nested, the
|
|
741
|
+
* recursion reached depth 331 and 9.1 GB on an 18.9M-node store for a 2-byte
|
|
742
|
+
* query and did not terminate, which is what killed a 5 h training run at its
|
|
743
|
+
* checkpoint recall. Guard: test/89-completion-recursion.test.mjs. */
|
|
703
744
|
recompleteNode(node) {
|
|
704
745
|
if (!this.host.recogniseSpan)
|
|
705
746
|
return null;
|
|
706
747
|
const memo = this.recompleteMemo;
|
|
707
748
|
if (memo.has(node))
|
|
708
749
|
return memo.get(node) ?? null;
|
|
709
|
-
//
|
|
710
|
-
|
|
750
|
+
// ONE re-cover per produced node — never a re-cover inside a re-cover.
|
|
751
|
+
//
|
|
752
|
+
// Re-covering is how a PRODUCED node's bytes enter the search at all: the
|
|
753
|
+
// cover machinery otherwise only ever sees the QUERY's spans. That is
|
|
754
|
+
// needed once. The alternation of decomposition and recomposition that
|
|
755
|
+
// follows — parts rewriting several times, siblings fusing, a recomposition
|
|
756
|
+
// feeding another — is the main search's own fuse/`rcmp` work, not this
|
|
757
|
+
// recursion's: 15-decomposition-gap §9–§12 all pass with this method
|
|
758
|
+
// disabled outright, and only §6 (the produced composite "p1 p2", whose
|
|
759
|
+
// bytes nothing else brings in) needs it.
|
|
760
|
+
//
|
|
761
|
+
// Nesting it was the defect. Each level is a full {@link solve} with its
|
|
762
|
+
// own agenda and chart, exploring from a node the answer never asked about,
|
|
763
|
+
// so per-query cost tracked how densely the corpus interconnects the forms
|
|
764
|
+
// passed through — the growth AGENTS §2.8 forbids. Measured on an
|
|
765
|
+
// 18.9M-node store: depth 331 and 9.1 GB for a 2-byte query, not
|
|
766
|
+
// terminating; and on the guard corpus every one of 125 nested re-covers
|
|
767
|
+
// was REJECTED by the resolve() gate below, expanding a 70-byte node into a
|
|
768
|
+
// 374-byte concatenation that names nothing. All of it was waste.
|
|
769
|
+
//
|
|
770
|
+
// `recompleteOpen` is that stack, so a non-empty stack means we are already
|
|
771
|
+
// inside one. This subsumes the old cycle guard: a node cannot recurse
|
|
772
|
+
// back into itself when nothing recurses at all.
|
|
773
|
+
if (this.recompleteOpen.size > 0)
|
|
711
774
|
return null;
|
|
712
775
|
// A leaf or single-child node has no parts to recompose; skip before the
|
|
713
776
|
// costly recognition so a plain terminal answer pays nothing.
|
|
@@ -740,9 +803,13 @@ export class GraphSearch {
|
|
|
740
803
|
* outs of a long query re-cover each distinct node at most once); reset at the
|
|
741
804
|
* top of {@link cover}. */
|
|
742
805
|
recompleteMemo = new Map();
|
|
743
|
-
/** The
|
|
744
|
-
*
|
|
745
|
-
* (
|
|
806
|
+
/** The node currently being re-completed — the recursion stack, and so also
|
|
807
|
+
* the nesting depth. {@link recompleteNode} refuses to start while it is
|
|
808
|
+
* non-empty (one re-cover per produced node, never one inside another), which
|
|
809
|
+
* is what keeps a query's cost proportional to its answer rather than to the
|
|
810
|
+
* corpus; it therefore holds at most one id. A Set, not a flag, because it
|
|
811
|
+
* states WHICH node is open — the invariant a reader needs to check the
|
|
812
|
+
* guard, and what makes the old cycle-guard reading still hold. */
|
|
746
813
|
recompleteOpen = new Set();
|
|
747
814
|
/** out(i,j,bytes,…): index it for the binary rules, then offer splicing a
|
|
748
815
|
* learnt connector (the in-search bridge), splitting (at a sub-leaf form
|
|
@@ -96,6 +96,19 @@ export declare function cachedRead(ctx: MindContext, cache: WalkCache | null, id
|
|
|
96
96
|
* edgeAncestors' question and wrong for this one: a junction container
|
|
97
97
|
* is legitimately reached across many containing structures. Half the
|
|
98
98
|
* successful junctions would be lost.
|
|
99
|
+
*
|
|
100
|
+
* REFUTED EARLY-STOP (side-cone exhaustion, §2.17's "real saturation"):
|
|
101
|
+
* stopping the walk the moment ONE side's upward cone is emptied is wrong,
|
|
102
|
+
* in both a hub-guarded form and a hub-flagged form. The junction test is
|
|
103
|
+
* a BYTE containment over the UNION of the two cones, and a junction can be
|
|
104
|
+
* structurally reachable from only ONE side — the side whose seed is a
|
|
105
|
+
* FOLD sub-node of the container (test/16: "cold or hot" is reached from
|
|
106
|
+
* the window "cold", but the 3-byte answer "hot" is not a 4-byte window of
|
|
107
|
+
* it, so "hot"'s cone is empty while the junction still lies ahead in
|
|
108
|
+
* "cold"'s cone; test/34's n-ary binding fails the hub-guarded form the
|
|
109
|
+
* same way). "One cone exhausted" therefore never proves "no junction
|
|
110
|
+
* left", and the walk must keep the √N·W budget as its NET after the
|
|
111
|
+
* per-node saturations below.
|
|
99
112
|
* • per-node hub guards — parent fan-outs beyond √N are hubs (not
|
|
100
113
|
* expanded); each node contributes at most one √N page of containers;
|
|
101
114
|
* √N collected candidates decide. */
|
|
@@ -134,6 +134,19 @@ function cachedContainers(ctx, cache, id, limit) {
|
|
|
134
134
|
* edgeAncestors' question and wrong for this one: a junction container
|
|
135
135
|
* is legitimately reached across many containing structures. Half the
|
|
136
136
|
* successful junctions would be lost.
|
|
137
|
+
*
|
|
138
|
+
* REFUTED EARLY-STOP (side-cone exhaustion, §2.17's "real saturation"):
|
|
139
|
+
* stopping the walk the moment ONE side's upward cone is emptied is wrong,
|
|
140
|
+
* in both a hub-guarded form and a hub-flagged form. The junction test is
|
|
141
|
+
* a BYTE containment over the UNION of the two cones, and a junction can be
|
|
142
|
+
* structurally reachable from only ONE side — the side whose seed is a
|
|
143
|
+
* FOLD sub-node of the container (test/16: "cold or hot" is reached from
|
|
144
|
+
* the window "cold", but the 3-byte answer "hot" is not a 4-byte window of
|
|
145
|
+
* it, so "hot"'s cone is empty while the junction still lies ahead in
|
|
146
|
+
* "cold"'s cone; test/34's n-ary binding fails the hub-guarded form the
|
|
147
|
+
* same way). "One cone exhausted" therefore never proves "no junction
|
|
148
|
+
* left", and the walk must keep the √N·W budget as its NET after the
|
|
149
|
+
* per-node saturations below.
|
|
137
150
|
* • per-node hub guards — parent fan-outs beyond √N are hubs (not
|
|
138
151
|
* expanded); each node contributes at most one √N page of containers;
|
|
139
152
|
* √N collected candidates decide. */
|
|
@@ -42,7 +42,9 @@ export async function resolveConnectors(ctx, sites, query) {
|
|
|
42
42
|
// transcript evidence: cover still needs the site for structural context,
|
|
43
43
|
// but liftAnswer will trim that continuation as already answered. Building
|
|
44
44
|
// pairwise/n-ary bridges for it can only create connectors that are later
|
|
45
|
-
// discarded
|
|
45
|
+
// discarded — a semantically neutral gate (it removes work whose product
|
|
46
|
+
// liftAnswer throws away), and a cumulative (multi-turn) query is exactly
|
|
47
|
+
// where such already-answered continuations recur.
|
|
46
48
|
let answered = 0;
|
|
47
49
|
const ordered = [...sites]
|
|
48
50
|
.sort((a, b) => a.start - b.start)
|
|
@@ -56,7 +58,26 @@ export async function resolveConnectors(ctx, sites, query) {
|
|
|
56
58
|
if (query === undefined || ctx.answeredSpans.length === 0)
|
|
57
59
|
return true;
|
|
58
60
|
const continuations = ctx.store.nextFirst(s.payload, hubBound(ctx));
|
|
59
|
-
return !continuations.some((answer) =>
|
|
61
|
+
return !continuations.some((answer) => {
|
|
62
|
+
// PREFIX-CAPPED (AGENTS §2.8): a candidate longer than the query cannot
|
|
63
|
+
// occur INSIDE it, so read one byte past the query's length — enough to
|
|
64
|
+
// detect the overflow — and reject without reconstructing the rest.
|
|
65
|
+
// The `+ 1` is what makes the test exact rather than a truncation: a
|
|
66
|
+
// result of exactly `query.length + 1` bytes is known to be too long,
|
|
67
|
+
// and anything shorter is the candidate's COMPLETE content, so the
|
|
68
|
+
// substring test below is the same test as before. (The same overflow
|
|
69
|
+
// probe bridge.ts:256 already uses.)
|
|
70
|
+
//
|
|
71
|
+
// This loop runs up to hubBound(ctx) = √N reads PER SITE, and only on a
|
|
72
|
+
// multi-turn response — `answeredSpans` is empty for a plain respond(),
|
|
73
|
+
// so the probe does not execute there. The cap cannot reduce the read
|
|
74
|
+
// COUNT — only a semantic change to the "already answered" test could —
|
|
75
|
+
// but it bounds each read by the query instead of by the corpus, which
|
|
76
|
+
// is what §2.8 asks for and what rescues a SHORT query: at 3 bytes this
|
|
77
|
+
// reads 4 bytes per candidate instead of the ~231 it averaged before.
|
|
78
|
+
const bytes = read(ctx, answer, query.length + 1);
|
|
79
|
+
return bytes.length <= query.length && indexOf(query, bytes, 0) >= 0;
|
|
80
|
+
});
|
|
60
81
|
});
|
|
61
82
|
const bridgePair = async (l, r) => {
|
|
62
83
|
if (l === r || links.has(l + "," + r))
|
|
@@ -204,9 +204,9 @@ export const prefixMechanism = {
|
|
|
204
204
|
provenance: "prefix",
|
|
205
205
|
async floor(ctx, query, _pre, worthRunning) {
|
|
206
206
|
// One projection: the form is voiced whole, nothing is substituted.
|
|
207
|
-
// INVESTMENT DISCIPLINE — the supplies below are
|
|
208
|
-
//
|
|
209
|
-
// bound can still beat the incumbent.
|
|
207
|
+
// INVESTMENT DISCIPLINE — the supplies below are a bounded √N window walk
|
|
208
|
+
// and the response's memoised top-k resonance read, so neither is touched
|
|
209
|
+
// until the bound can still beat the incumbent.
|
|
210
210
|
if (!worthRunning(STEP))
|
|
211
211
|
return STEP;
|
|
212
212
|
// A query with no room for a perceivable continuation inside the phrase
|
|
@@ -218,14 +218,16 @@ export const prefixMechanism = {
|
|
|
218
218
|
return STEP;
|
|
219
219
|
},
|
|
220
220
|
async run(ctx, query, pre) {
|
|
221
|
-
// The
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
|
|
228
|
-
|
|
221
|
+
// The write side's window index proposes FIRST: a proper prefix's gist
|
|
222
|
+
// cannot rank its own continuation (cos falls below reachThreshold at a
|
|
223
|
+
// few bytes of truncation), so the content-addressed window walk is the
|
|
224
|
+
// correct measure for this question (§2.3), and it is a bounded √N walk —
|
|
225
|
+
// cheaper than an exhaustive ANN. The top-k resonance list is the SECOND
|
|
226
|
+
// supply, for prefixes long enough that the gist still ranks the form. A
|
|
227
|
+
// second SUPPLY, not a second mechanism — the same three guards decide
|
|
228
|
+
// either way.
|
|
229
|
+
const completed = prefixCompletion(ctx, query, formsOpenedBy(ctx, query)) ??
|
|
230
|
+
prefixCompletion(ctx, query, (await pre.resonance()).map((h) => h.id));
|
|
229
231
|
if (completed === null)
|
|
230
232
|
return [];
|
|
231
233
|
return [{
|
|
@@ -306,10 +306,14 @@ export async function recallByResonance(ctx, query, pre) {
|
|
|
306
306
|
}
|
|
307
307
|
}
|
|
308
308
|
// 3b. Corroborated-substitution bridge — refusal-path only (bridge.ts).
|
|
309
|
-
// The
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
309
|
+
// The bridge's proposal source is the response's ONE top-k read — the same
|
|
310
|
+
// list recall already ranked above — never an exhaustive √N scan. The
|
|
311
|
+
// bridge's own candidate cap is 2·recallQueryK, so top-k proposals are
|
|
312
|
+
// exactly the budget it can consume, and every proposal is byte-verified
|
|
313
|
+
// downstream (§2.3). Reuse the memoised `resonance()`; scanning every IVF
|
|
314
|
+
// cluster here once made every honest refusal cost hundreds of ms regardless
|
|
315
|
+
// of k.
|
|
316
|
+
const wideIds = async () => (await pre.resonance()).map((h) => h.id);
|
|
313
317
|
// Every gist-based tier has failed; before refusing, align the query
|
|
314
318
|
// byte-for-byte against the trained contexts its own stored windows
|
|
315
319
|
// anchor, accepting mismatches only as corpus-attested, concept-bar
|
package/dist/src/mind/mind.d.ts
CHANGED
|
@@ -282,7 +282,7 @@ export declare class Mind implements MindContext {
|
|
|
282
282
|
* "what byte separates two turns?" because nothing downstream finds
|
|
283
283
|
* boundaries by looking at content at all.
|
|
284
284
|
* 2. A separator in a CORPUS is ordinary content. If a trainer joins
|
|
285
|
-
* turns with "\n" (example/train_base
|
|
285
|
+
* turns with "\n" (example/train_base does), those newlines are
|
|
286
286
|
* simply bytes inside the stream, folded like every other byte. They
|
|
287
287
|
* are a property of that corpus, not of this API and not of the fold.
|
|
288
288
|
* 3. This API can therefore reproduce ANY corpus exactly, with no
|
package/dist/src/mind/mind.js
CHANGED
|
@@ -479,7 +479,7 @@ export class Mind {
|
|
|
479
479
|
* "what byte separates two turns?" because nothing downstream finds
|
|
480
480
|
* boundaries by looking at content at all.
|
|
481
481
|
* 2. A separator in a CORPUS is ordinary content. If a trainer joins
|
|
482
|
-
* turns with "\n" (example/train_base
|
|
482
|
+
* turns with "\n" (example/train_base does), those newlines are
|
|
483
483
|
* simply bytes inside the stream, folded like every other byte. They
|
|
484
484
|
* are a property of that corpus, not of this API and not of the fold.
|
|
485
485
|
* 3. This API can therefore reproduce ANY corpus exactly, with no
|
|
@@ -63,30 +63,6 @@ export declare class Precomputed {
|
|
|
63
63
|
* duplication a profile shows as doubled `annVectorReads` with nothing to
|
|
64
64
|
* account for it. Cached BY PROMISE, so a second caller awaits the first. */
|
|
65
65
|
resonance(): Promise<ReadonlyArray<Hit>>;
|
|
66
|
-
private _wide?;
|
|
67
|
-
/** The response's WIDE candidate list — the top-k when the query's gist has
|
|
68
|
-
* no concept-level match anywhere, and an exhaustive √N read when it does.
|
|
69
|
-
*
|
|
70
|
-
* Every mechanism that has to look PAST the top-k reads this one list: the
|
|
71
|
-
* substitution bridge, prefix completion and the frame filler all did, and
|
|
72
|
-
* it was memoised inside recall for exactly that reason (measured: 490 ms
|
|
73
|
-
* median re-issued against 13 ms non-exhaustive, 36x). A memo inside one
|
|
74
|
-
* mechanism only serves that mechanism's own tiers, so it lives here now —
|
|
75
|
-
* the same move `resonance` made for the top-k.
|
|
76
|
-
*
|
|
77
|
-
* THE CONDITION IS THE TOP HIT'S SCORE, NOT THE CORPUS SIZE. When nothing
|
|
78
|
-
* ranks at concept level, an exhaustive ANN only scores more vectors below
|
|
79
|
-
* the bar (profiled at 38K–40K annVectorReads per refusing query on a 325K-
|
|
80
|
-
* context store); the structural channels — junction walks, anchor climbs,
|
|
81
|
-
* the write side's window index — are the correct proposal source there,
|
|
82
|
-
* because the ANN cannot propose what the gist cannot rank. This was once
|
|
83
|
-
* spelled `corpusN(ctx) <= (k · W)³`, which asks a different question and
|
|
84
|
-
* answers it wrongly at exactly the scale it was written from: at N =
|
|
85
|
-
* 325,608 with k = 24 and W = 4 the cube is 884,736, so that store took the
|
|
86
|
-
* exhaustive branch — the very branch measured above. Measured cost of the
|
|
87
|
-
* mismatch: substitutionBridge 8,544 ms of a 19,548 ms think (44%), against
|
|
88
|
-
* 1,248 ms and 14,218 ms without it, every answer byte-identical. */
|
|
89
|
-
wideResonance(): Promise<ReadonlyArray<number>>;
|
|
90
66
|
private _frames?;
|
|
91
67
|
/** THE FRAME INVENTORY — every ranked candidate that reads as an instance of
|
|
92
68
|
* the same frame as the query, each with the query spans it leaves VARIABLE
|
|
@@ -13,12 +13,12 @@
|
|
|
13
13
|
// 4. TRAVELING EVIDENCE — run() returns MechanismResult with accounted, moves,
|
|
14
14
|
// and unexplained. The pipeline computes the weight.
|
|
15
15
|
import { indexOf } from "../bytes.js";
|
|
16
|
-
import {
|
|
16
|
+
import { dominates } from "../geometry.js";
|
|
17
17
|
import { windowIds } from "./canonical.js";
|
|
18
18
|
import { read, resolve } from "./primitives.js";
|
|
19
19
|
import { alignGraded, frameSlots, skillExemplar, } from "./match.js";
|
|
20
20
|
import { climbAttentionAll } from "./attention.js";
|
|
21
|
-
import {
|
|
21
|
+
import { sharedReachMemo } from "./traverse.js";
|
|
22
22
|
// ── Precomputed ──────────────────────────────────────────────────────────────
|
|
23
23
|
//
|
|
24
24
|
// Precomputed is a LAZY container for structural analyses of the query — the
|
|
@@ -128,40 +128,17 @@ export class Precomputed {
|
|
|
128
128
|
resonance() {
|
|
129
129
|
return this._resonance ??= this.shared("resonance", () => this.ctx.store.resonate(this.guide, this.k));
|
|
130
130
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
* THE CONDITION IS THE TOP HIT'S SCORE, NOT THE CORPUS SIZE. When nothing
|
|
143
|
-
* ranks at concept level, an exhaustive ANN only scores more vectors below
|
|
144
|
-
* the bar (profiled at 38K–40K annVectorReads per refusing query on a 325K-
|
|
145
|
-
* context store); the structural channels — junction walks, anchor climbs,
|
|
146
|
-
* the write side's window index — are the correct proposal source there,
|
|
147
|
-
* because the ANN cannot propose what the gist cannot rank. This was once
|
|
148
|
-
* spelled `corpusN(ctx) <= (k · W)³`, which asks a different question and
|
|
149
|
-
* answers it wrongly at exactly the scale it was written from: at N =
|
|
150
|
-
* 325,608 with k = 24 and W = 4 the cube is 884,736, so that store took the
|
|
151
|
-
* exhaustive branch — the very branch measured above. Measured cost of the
|
|
152
|
-
* mismatch: substitutionBridge 8,544 ms of a 19,548 ms think (44%), against
|
|
153
|
-
* 1,248 ms and 14,218 ms without it, every answer byte-identical. */
|
|
154
|
-
wideResonance() {
|
|
155
|
-
return this._wide ??= this.shared("wideResonance", async () => {
|
|
156
|
-
const hits = await this.resonance();
|
|
157
|
-
if (hits.length > 0 &&
|
|
158
|
-
hits[0].score >= conceptThreshold(this.ctx.store.D)) {
|
|
159
|
-
const exhaustive = await this.ctx.store.resonate(this.guide, hubBound(this.ctx), true);
|
|
160
|
-
return exhaustive.map((h) => h.id);
|
|
161
|
-
}
|
|
162
|
-
return hits.map((h) => h.id);
|
|
163
|
-
});
|
|
164
|
-
}
|
|
131
|
+
// REMOVED — the WIDE exhaustive-√N resonance list (`wideResonance`). It ran
|
|
132
|
+
// `resonate(guide, √N, exhaustive=true)` whenever the top hit cleared
|
|
133
|
+
// conceptThreshold, so consumers could look "past the top-k". Every consumer
|
|
134
|
+
// only ever needed ≤ 2·recallQueryK proposals (the substitution bridge's own
|
|
135
|
+
// candidate cap) or a content-addressed answer (prefix completion's
|
|
136
|
+
// formsOpenedBy), and every proposal is byte-verified downstream (§2.3), so
|
|
137
|
+
// the exhaustive scan bought recall at O(index) cost for an O(k) need —
|
|
138
|
+
// measured: 244K annVectorReads per refusing query, ~1.5 s, every answer
|
|
139
|
+
// byte-identical to a top-k read. The two consumers now read `resonance()`
|
|
140
|
+
// (the one top-k read) and the write side's window index respectively — see
|
|
141
|
+
// recall.ts and prefix-completion.ts.
|
|
165
142
|
_frames;
|
|
166
143
|
/** THE FRAME INVENTORY — every ranked candidate that reads as an instance of
|
|
167
144
|
* the same frame as the query, each with the query spans it leaves VARIABLE
|
|
@@ -37,6 +37,29 @@ export interface NarrowDecisionData {
|
|
|
37
37
|
version: 1;
|
|
38
38
|
margin: number;
|
|
39
39
|
}
|
|
40
|
+
/** Structured payload of the "regimePrediction" rationale step — the R8
|
|
41
|
+
* observation exposed as data. After the first mechanism (cover, which §2.6
|
|
42
|
+
* runs first) grounds or abstains, the market's whole outcome is already
|
|
43
|
+
* determined by the one cost ladder: the consensus climb runs exactly when
|
|
44
|
+
* `worthRunning(2 * STEP)` is true — CAST (floor 2·STEP) is the cheapest
|
|
45
|
+
* mechanism that first-touches it, and confluence (3·STEP) / extraction
|
|
46
|
+
* (CONCEPT+STEP) are only reached after CAST is. An incumbent at or below
|
|
47
|
+
* that floor prunes CAST and, with it, the climb (retrieval); anything above
|
|
48
|
+
* — or no incumbent — runs the full market and the climb (composition).
|
|
49
|
+
* Purely observational; never read by inference. */
|
|
50
|
+
export interface RegimePredictionData {
|
|
51
|
+
version: 1;
|
|
52
|
+
/** retrieval | composition — the two regimes R1 measured as a ~100× cost
|
|
53
|
+
* step. */
|
|
54
|
+
regime: "retrieval" | "composition";
|
|
55
|
+
/** The incumbent's grade right after the first mechanism ran, or null when
|
|
56
|
+
* it grounded nothing (best === null — composition, with no incumbent). */
|
|
57
|
+
incumbentGrade: number | null;
|
|
58
|
+
/** The cheapest composition floor in grade units (`grade(2 * STEP)` = 2,
|
|
59
|
+
* CAST's floor) — the bar the incumbent must sit at or below for the
|
|
60
|
+
* consensus climb to be skipped. */
|
|
61
|
+
climbFloorGrade: number;
|
|
62
|
+
}
|
|
40
63
|
/** Think: a single lightest-derivation exploration of the Sema graph.
|
|
41
64
|
*
|
|
42
65
|
* Every answer travels the same path:
|