@hviana/sema 0.4.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +290 -77
- package/HOW_IT_WORKS.md +2170 -735
- package/dist/example/train_base.d.ts +9 -3
- package/dist/example/train_base.js +21 -4
- package/dist/src/canon.d.ts +19 -0
- package/dist/src/canon.js +28 -0
- package/dist/src/geometry.d.ts +52 -0
- package/dist/src/geometry.js +87 -1
- package/dist/src/mind/attention.d.ts +15 -10
- package/dist/src/mind/attention.js +15 -10
- package/dist/src/mind/bridge.js +27 -1
- package/dist/src/mind/frame-filler.d.ts +15 -0
- package/dist/src/mind/frame-filler.js +535 -0
- package/dist/src/mind/learning.js +6 -11
- package/dist/src/mind/mechanisms/cast.js +72 -2
- package/dist/src/mind/mechanisms/cover.js +6 -1
- package/dist/src/mind/mechanisms/extraction.js +27 -0
- package/dist/src/mind/mechanisms/recall.js +214 -34
- package/dist/src/mind/mind.d.ts +52 -3
- package/dist/src/mind/mind.js +140 -12
- package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
- package/dist/src/mind/pipeline.js +29 -1
- package/dist/src/mind/prefix-completion.d.ts +59 -0
- package/dist/src/mind/prefix-completion.js +270 -0
- package/dist/src/mind/primitives.d.ts +29 -10
- package/dist/src/mind/primitives.js +98 -71
- package/dist/src/mind/recognition.js +153 -26
- package/dist/src/mind/traverse.d.ts +32 -0
- package/dist/src/mind/traverse.js +52 -0
- package/dist/src/mind/types.d.ts +61 -18
- package/dist/src/mind/types.js +68 -19
- package/dist/src/store.d.ts +21 -0
- package/dist/src/store.js +21 -0
- package/example/train_base.ts +21 -4
- package/package.json +1 -1
- package/src/canon.ts +28 -0
- package/src/geometry.ts +100 -1
- package/src/mind/attention.ts +15 -10
- package/src/mind/bridge.ts +34 -0
- package/src/mind/frame-filler.ts +604 -0
- package/src/mind/learning.ts +5 -9
- package/src/mind/mechanisms/cast.ts +70 -2
- package/src/mind/mechanisms/cover.ts +6 -1
- package/src/mind/mechanisms/extraction.ts +27 -0
- package/src/mind/mechanisms/recall.ts +236 -37
- package/src/mind/mind.ts +166 -18
- package/src/mind/pipeline-mechanism.ts +7 -0
- package/src/mind/pipeline.ts +33 -1
- package/src/mind/prefix-completion.ts +314 -0
- package/src/mind/primitives.ts +105 -80
- package/src/mind/recognition.ts +151 -23
- package/src/mind/traverse.ts +52 -0
- package/src/mind/types.ts +104 -44
- package/src/store.ts +25 -0
- package/test/13-conversation.test.mjs +13 -0
- package/test/57-fusion-order.test.mjs +65 -0
- package/test/66-query-edge-whitespace.test.mjs +99 -0
- package/test/67-climb-anchor-breadth.test.mjs +113 -0
- package/test/68-extraction-unanchored.test.mjs +79 -0
- package/test/69-frame-filler.test.mjs +115 -0
- package/test/70-prefix-completion.test.mjs +170 -0
- package/test/71-embedded-canon-equivalence.test.mjs +121 -0
- package/test/72-prefix-candidate-supply.test.mjs +114 -0
- package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
- package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
- package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
package/dist/src/mind/types.d.ts
CHANGED
|
@@ -7,22 +7,17 @@ import type { MindConfig } from "../config.js";
|
|
|
7
7
|
import type { Meter } from "../meter.js";
|
|
8
8
|
import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
|
|
9
9
|
import type { Rationale } from "./rationale.js";
|
|
10
|
-
import type {
|
|
11
|
-
/** One {@link MindContext._depositTrees} entry — see that field's doc.
|
|
10
|
+
import type { ContentFold, Grid } from "../geometry.js";
|
|
11
|
+
/** One {@link MindContext._depositTrees} entry — see that field's doc.
|
|
12
|
+
*
|
|
13
|
+
* A PURE WORK CACHE. It carries the already-folded content segments of a
|
|
14
|
+
* deposited stream so a longer stream sharing its byte prefix can skip
|
|
15
|
+
* refolding them. It holds no turn boundaries and no continuation proof
|
|
16
|
+
* because the deposit fold imposes nothing: reuse is bit-identical to a cold
|
|
17
|
+
* fold, so a hit can only save time, never change a tree. */
|
|
12
18
|
export interface DepositCacheEntry {
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
* whole-context length. Empty for a first-seen (single-turn) input. */
|
|
16
|
-
boundaries: number[];
|
|
17
|
-
/** Stable-prefix segment folds (grown-context inputs only). */
|
|
18
|
-
stable?: StableFold;
|
|
19
|
-
/** The continuation bytes this ctxInput was paired with in ingestPair, if
|
|
20
|
-
* any — the ONLY thing that makes a later, longer ctxInput a genuine next
|
|
21
|
-
* TURN of the same conversation rather than an unrelated fact that
|
|
22
|
-
* happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5"). A
|
|
23
|
-
* later deposit only takes this entry as its stable-prefix `prev` when
|
|
24
|
-
* its own suffix bytes-equal this exactly. */
|
|
25
|
-
nextBytes?: Uint8Array;
|
|
19
|
+
/** The plain content fold's reusable segment state. */
|
|
20
|
+
content: ContentFold;
|
|
26
21
|
}
|
|
27
22
|
export type Input = string | Uint8Array | Grid | Grid[];
|
|
28
23
|
/** The host capabilities GraphSearch consults during a cover. MindContext
|
|
@@ -289,9 +284,23 @@ export interface MindContext extends GraphSearchHost {
|
|
|
289
284
|
/** Subtree-resolution cache: Sema node → its store id and byte length.
|
|
290
285
|
* Populated by {@link foldTree} during inference; checked before
|
|
291
286
|
* walking children. When a conversation's pyramid reuses prefix
|
|
292
|
-
* subtrees, this cache
|
|
293
|
-
*
|
|
294
|
-
*
|
|
287
|
+
* subtrees, this cache names them without a store probe. It does NOT let
|
|
288
|
+
* {@link recognise} skip them: recognise walks with a `visit` callback and
|
|
289
|
+
* emits its sites from it, so a skipped descent would mean fewer sites on
|
|
290
|
+
* a warm cache than a cold one. foldTree short-circuits only for
|
|
291
|
+
* visitor-less walks (O(suffix) there); a visiting walk stays O(context)
|
|
292
|
+
* and banks the elided probes. Mind-lifetime (WeakMap keys are the Sema
|
|
293
|
+
* objects the pyramid keeps alive).
|
|
294
|
+
*
|
|
295
|
+
* THAT REUSE IS A PRECONDITION, NOT A GIVEN: the keys are node IDENTITIES,
|
|
296
|
+
* so it hits only while the conversation's fold hands back the SAME Sema
|
|
297
|
+
* objects for the unchanged prefix. `_growContext` rebuilt the whole tree
|
|
298
|
+
* with `bytesToTree` on every turn, so every key was fresh and this cache
|
|
299
|
+
* could not hit even once — the O(suffix) claim above described an
|
|
300
|
+
* intention rather than the code. It now grows the context through
|
|
301
|
+
* {@link stablePrefixFoldIncremental}, which reuses each already-folded
|
|
302
|
+
* segment: measured over four turns, turn 4 shared 69 of its 95 nodes with
|
|
303
|
+
* turn 3 (26 new ≈ the new turn's own size). */
|
|
295
304
|
_resolvedSubtrees: WeakMap<Sema, {
|
|
296
305
|
id: number;
|
|
297
306
|
len: number;
|
|
@@ -369,6 +378,40 @@ export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: nu
|
|
|
369
378
|
* (lo/hi) decision and the final concatenation: it is stale, not a second
|
|
370
379
|
* answer, but the OTHER spans a derivation chose are independent evidence
|
|
371
380
|
* and must not be discarded along with it. */
|
|
381
|
+
/** The spans {@link liftAnswer} actually concatenates, in order — the answer
|
|
382
|
+
* before it is joined. Exposed so a caller can ask what the lifted answer is
|
|
383
|
+
* MADE OF without re-deriving the selection: in particular how much of it is
|
|
384
|
+
* SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
|
|
385
|
+
* because nothing explained them, the same spans the liftAnswer trace labels
|
|
386
|
+
* "scaffolding" rather than "chosen").
|
|
387
|
+
*
|
|
388
|
+
* That quantity is load-bearing for the grounding decision. Two candidates
|
|
389
|
+
* can leave the SAME number of query bytes unaccounted and therefore grade
|
|
390
|
+
* identically, while one of them pads its answer with those bytes and the
|
|
391
|
+
* other does not — measured on test/22's two-fact chain, cover and recall
|
|
392
|
+
* both graded 11001 with 11 bytes unexplained, and cover won the tie only on
|
|
393
|
+
* consideration order, answering "The capital of France is Paris famous for"
|
|
394
|
+
* where recall had crossed the hop. Carrying an unexplained span into the
|
|
395
|
+
* answer is strictly weaker than not explaining it: it manufactures fluency
|
|
396
|
+
* out of the asker's own words. See the tie-break in pipeline.ts. */
|
|
397
|
+
export declare function liftAnswerParts(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Seg[];
|
|
398
|
+
/** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
|
|
399
|
+
* from spans nothing recognised (see {@link liftAnswerParts}).
|
|
400
|
+
*
|
|
401
|
+
* ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT. Not all carried-through
|
|
402
|
+
* bytes are a failure to explain: a period, a question mark, the space
|
|
403
|
+
* between two fused topics are GLUE — they belong to the answer's surface,
|
|
404
|
+
* and dropping them to look better-derived would be a worse answer, not a
|
|
405
|
+
* more honest one. A substantive phrase the derivation never explained
|
|
406
|
+
* ("famous for") is a different claim entirely.
|
|
407
|
+
*
|
|
408
|
+
* W is the line between them, and it is the same line the rest of the mind
|
|
409
|
+
* already draws: below one river window byte overlap is chance, not evidence
|
|
410
|
+
* (see identityBar, the bridge's attestedQ, and recognition's site floor).
|
|
411
|
+
* Counting every scaffolding byte instead — which is what this did first —
|
|
412
|
+
* made punctuation preservation lose a tie it should win, and test/00's
|
|
413
|
+
* "period preserved" / "question mark preserved" caught it immediately. */
|
|
414
|
+
export declare function liftedScaffolding(segs: Seg[], queryLen: number, query: Uint8Array, W: number): number;
|
|
372
415
|
export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
|
|
373
416
|
/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
|
|
374
417
|
* tracked deposit interned (`prevSeen`). */
|
package/dist/src/mind/types.js
CHANGED
|
@@ -46,7 +46,23 @@ export function segRestatesQuery(s, query, queryLen, W) {
|
|
|
46
46
|
* (lo/hi) decision and the final concatenation: it is stale, not a second
|
|
47
47
|
* answer, but the OTHER spans a derivation chose are independent evidence
|
|
48
48
|
* and must not be discarded along with it. */
|
|
49
|
-
|
|
49
|
+
/** The spans {@link liftAnswer} actually concatenates, in order — the answer
|
|
50
|
+
* before it is joined. Exposed so a caller can ask what the lifted answer is
|
|
51
|
+
* MADE OF without re-deriving the selection: in particular how much of it is
|
|
52
|
+
* SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
|
|
53
|
+
* because nothing explained them, the same spans the liftAnswer trace labels
|
|
54
|
+
* "scaffolding" rather than "chosen").
|
|
55
|
+
*
|
|
56
|
+
* That quantity is load-bearing for the grounding decision. Two candidates
|
|
57
|
+
* can leave the SAME number of query bytes unaccounted and therefore grade
|
|
58
|
+
* identically, while one of them pads its answer with those bytes and the
|
|
59
|
+
* other does not — measured on test/22's two-fact chain, cover and recall
|
|
60
|
+
* both graded 11001 with 11 bytes unexplained, and cover won the tie only on
|
|
61
|
+
* consideration order, answering "The capital of France is Paris famous for"
|
|
62
|
+
* where recall had crossed the hop. Carrying an unexplained span into the
|
|
63
|
+
* answer is strictly weaker than not explaining it: it manufactures fluency
|
|
64
|
+
* out of the asker's own words. See the tie-break in pipeline.ts. */
|
|
65
|
+
export function liftAnswerParts(segs, queryLen, query, W) {
|
|
50
66
|
const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
|
|
51
67
|
const recognised = [];
|
|
52
68
|
for (let k = 0; k < segs.length; k++) {
|
|
@@ -54,32 +70,65 @@ export function liftAnswer(segs, queryLen, query, W) {
|
|
|
54
70
|
recognised.push(k);
|
|
55
71
|
}
|
|
56
72
|
if (recognised.length === 0)
|
|
57
|
-
return
|
|
73
|
+
return [];
|
|
58
74
|
if (recognised.length === 1) {
|
|
59
75
|
const s = segs[recognised[0]];
|
|
60
|
-
// A COMPUTED span's query-side width is operand digit-count, not
|
|
61
|
-
// evidence of how much of the query's meaning it accounts for — the
|
|
62
|
-
// half-dominance check below (built for a genuinely RECOGNISED learned
|
|
63
|
-
// form) is not a valid framing signal for it (see the `computed` field
|
|
64
|
-
// doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
|
|
65
|
-
// because the operands are big, not because the framing matters less.
|
|
66
|
-
// A LITERAL PREFIX before a computed span is unambiguous framing
|
|
67
|
-
// regardless of width — an arithmetic expression is never itself
|
|
68
|
-
// preceded by more literal computed content, so anything literal before
|
|
69
|
-
// it is question wording ("what is ", "compute ") to lift clear of.
|
|
70
|
-
// With no prefix (s.i === 0) the span is judged by the ordinary
|
|
71
|
-
// half-dominance rule below, which already correctly keeps a short
|
|
72
|
-
// trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
|
|
73
76
|
if (s.computed && s.i > 0)
|
|
74
|
-
return s
|
|
77
|
+
return [s];
|
|
75
78
|
if (dominates(s.j - s.i, queryLen)) {
|
|
76
|
-
return
|
|
79
|
+
return segs.filter((_, k) => !restated[k]);
|
|
77
80
|
}
|
|
78
|
-
return s
|
|
81
|
+
return [s];
|
|
79
82
|
}
|
|
80
83
|
const lo = recognised[0];
|
|
81
84
|
const hi = recognised[recognised.length - 1];
|
|
82
|
-
return
|
|
85
|
+
return segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]);
|
|
86
|
+
}
|
|
87
|
+
/** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
|
|
88
|
+
* from spans nothing recognised (see {@link liftAnswerParts}).
|
|
89
|
+
*
|
|
90
|
+
* ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT. Not all carried-through
|
|
91
|
+
* bytes are a failure to explain: a period, a question mark, the space
|
|
92
|
+
* between two fused topics are GLUE — they belong to the answer's surface,
|
|
93
|
+
* and dropping them to look better-derived would be a worse answer, not a
|
|
94
|
+
* more honest one. A substantive phrase the derivation never explained
|
|
95
|
+
* ("famous for") is a different claim entirely.
|
|
96
|
+
*
|
|
97
|
+
* W is the line between them, and it is the same line the rest of the mind
|
|
98
|
+
* already draws: below one river window byte overlap is chance, not evidence
|
|
99
|
+
* (see identityBar, the bridge's attestedQ, and recognition's site floor).
|
|
100
|
+
* Counting every scaffolding byte instead — which is what this did first —
|
|
101
|
+
* made punctuation preservation lose a tie it should win, and test/00's
|
|
102
|
+
* "period preserved" / "question mark preserved" caught it immediately. */
|
|
103
|
+
export function liftedScaffolding(segs, queryLen, query, W) {
|
|
104
|
+
// MEASURED PER CONTIGUOUS RUN, not per span. A PASS span is one BYTE — the
|
|
105
|
+
// cover charges unrecognised bytes individually — so asking whether a single
|
|
106
|
+
// span reaches W would find no run ever, whatever the query. " famous for"
|
|
107
|
+
// arrives as eleven one-byte spans in a row and is one eleven-byte run.
|
|
108
|
+
let n = 0;
|
|
109
|
+
let run = 0;
|
|
110
|
+
const close = () => {
|
|
111
|
+
if (run >= W)
|
|
112
|
+
n += run;
|
|
113
|
+
run = 0;
|
|
114
|
+
};
|
|
115
|
+
for (const s of liftAnswerParts(segs, queryLen, query, W)) {
|
|
116
|
+
if (s.rec)
|
|
117
|
+
close();
|
|
118
|
+
else
|
|
119
|
+
run += s.bytes.length;
|
|
120
|
+
}
|
|
121
|
+
close();
|
|
122
|
+
return n;
|
|
123
|
+
}
|
|
124
|
+
export function liftAnswer(segs, queryLen, query, W) {
|
|
125
|
+
// ONE selection rule, in {@link liftAnswerParts} — this is its join. The
|
|
126
|
+
// two used to be separate copies of the same lo/hi/restated reasoning, which
|
|
127
|
+
// is exactly how an answer and the accounting OF that answer drift apart.
|
|
128
|
+
const parts = liftAnswerParts(segs, queryLen, query, W);
|
|
129
|
+
if (parts.length === 0)
|
|
130
|
+
return null;
|
|
131
|
+
return concatBytes(parts.map((x) => x.bytes));
|
|
83
132
|
}
|
|
84
133
|
/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
|
|
85
134
|
* tracked deposit interned (`prevSeen`). */
|
package/dist/src/store.d.ts
CHANGED
|
@@ -152,6 +152,10 @@ export interface Store {
|
|
|
152
152
|
contentLen(id: NodeId, cap?: number): number;
|
|
153
153
|
findLeaf(bytes: Uint8Array): NodeId | null;
|
|
154
154
|
findBranch(kids: NodeId[]): NodeId | null;
|
|
155
|
+
/** {@link findBranch} for a run of single-byte leaves, addressed by the raw
|
|
156
|
+
* bytes — the allocation-free probe span scanners use. Optional: a store
|
|
157
|
+
* without it is simply probed through `findBranch`. */
|
|
158
|
+
findFlatBranch?(bytes: Uint8Array): NodeId | null;
|
|
155
159
|
/** The branch nodes that list `id` among their children — the reverse of
|
|
156
160
|
* `get(id).kids`. Lets the structural DAG be climbed upward, from a
|
|
157
161
|
* recognised fragment to the larger learned forms that contain it. */
|
|
@@ -585,6 +589,23 @@ export declare abstract class AbstractStore implements Store {
|
|
|
585
589
|
private _prefix;
|
|
586
590
|
contentLen(id: NodeId, cap?: number): number;
|
|
587
591
|
findLeaf(bytes: Uint8Array): NodeId | null;
|
|
592
|
+
/** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
|
|
593
|
+
* bytes themselves — no kid array, no key string, no copy.
|
|
594
|
+
*
|
|
595
|
+
* A flat branch stores its children as {@link flatKidsBytes}, and that
|
|
596
|
+
* encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
|
|
597
|
+
* So for such a run the kid array and the byte span are the same object in
|
|
598
|
+
* two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
|
|
599
|
+
* the same lookup — except that the array path allocates the slice, then
|
|
600
|
+
* `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
|
|
601
|
+
* answer is usually "no". The bloom filter behind `_dbFindBranchByLeaf`
|
|
602
|
+
* answers most of those with no I/O at all, so the allocations dominated.
|
|
603
|
+
*
|
|
604
|
+
* Pass a subarray: it is a view, so a caller scanning spans of a query
|
|
605
|
+
* allocates nothing per probe. Deliberately NOT memoized — its callers
|
|
606
|
+
* probe many spans that miss, and a key string per probe is the cost this
|
|
607
|
+
* exists to remove. */
|
|
608
|
+
findFlatBranch(bytes: Uint8Array): NodeId | null;
|
|
588
609
|
findBranch(kids: NodeId[]): NodeId | null;
|
|
589
610
|
parents(id: NodeId): NodeId[];
|
|
590
611
|
parentsFirst(id: NodeId, limit: number): NodeId[];
|
package/dist/src/store.js
CHANGED
|
@@ -770,6 +770,27 @@ export class AbstractStore {
|
|
|
770
770
|
this._leafKey.set(key, id);
|
|
771
771
|
return id;
|
|
772
772
|
}
|
|
773
|
+
/** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
|
|
774
|
+
* bytes themselves — no kid array, no key string, no copy.
|
|
775
|
+
*
|
|
776
|
+
* A flat branch stores its children as {@link flatKidsBytes}, and that
|
|
777
|
+
* encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
|
|
778
|
+
* So for such a run the kid array and the byte span are the same object in
|
|
779
|
+
* two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
|
|
780
|
+
* the same lookup — except that the array path allocates the slice, then
|
|
781
|
+
* `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
|
|
782
|
+
* answer is usually "no". The bloom filter behind `_dbFindBranchByLeaf`
|
|
783
|
+
* answers most of those with no I/O at all, so the allocations dominated.
|
|
784
|
+
*
|
|
785
|
+
* Pass a subarray: it is a view, so a caller scanning spans of a query
|
|
786
|
+
* allocates nothing per probe. Deliberately NOT memoized — its callers
|
|
787
|
+
* probe many spans that miss, and a key string per probe is the cost this
|
|
788
|
+
* exists to remove. */
|
|
789
|
+
findFlatBranch(bytes) {
|
|
790
|
+
if (this.meter)
|
|
791
|
+
this.meter.branchLookups++;
|
|
792
|
+
return this._dbFindBranchByLeaf(hashOf(bytes), bytes);
|
|
793
|
+
}
|
|
773
794
|
findBranch(kids) {
|
|
774
795
|
if (this.meter)
|
|
775
796
|
this.meter.branchLookups++;
|
package/example/train_base.ts
CHANGED
|
@@ -709,7 +709,18 @@ const isEpisode = (it: TrainingItem): it is Episode => typeof it !== "string";
|
|
|
709
709
|
/** Build the accumulated-context episodes of a turn sequence: each successive
|
|
710
710
|
* turn is the continuation of ALL the turns before it joined together. This is
|
|
711
711
|
* the same cumulative-context shape a multi-turn conversation deposits, so the
|
|
712
|
-
* store learns to continue a growing context.
|
|
712
|
+
* store learns to continue a growing context.
|
|
713
|
+
*
|
|
714
|
+
* The "\n" below is a CORPUS choice, not a protocol. oasst2 turns are
|
|
715
|
+
* paragraphs, and reading them back with the newlines kept is how this corpus
|
|
716
|
+
* reads naturally; a different corpus may join with nothing, and
|
|
717
|
+
* test/13-conversation.test.mjs does exactly that. Neither has to match the
|
|
718
|
+
* other, because Sema never scans content for turn boundaries — those are
|
|
719
|
+
* offsets the Conversation API carries beside the bytes (see Mind.addTurn's
|
|
720
|
+
* "ON SEPARATORS" note). The newline here is simply part of the text this
|
|
721
|
+
* store learnt, so anything replaying this corpus feeds it back as part of
|
|
722
|
+
* the turn: `addTurn(conv, "\n" + turnText)`. It is not a convention the
|
|
723
|
+
* engine, the API, or the tests have to agree on. */
|
|
713
724
|
function accumulate(turns: string[]): Episode[] {
|
|
714
725
|
const out: Episode[] = [];
|
|
715
726
|
for (let i = 1; i < turns.length; i++) {
|
|
@@ -894,9 +905,15 @@ export function bestOasstPath(root: OasstNode): OasstTurn[] {
|
|
|
894
905
|
* turn experiences and local adjacent-pair facts are NOT emitted (they are
|
|
895
906
|
* subsumed by it and would merely replicate the content).
|
|
896
907
|
*
|
|
897
|
-
* The walk is
|
|
898
|
-
* ("teachConversation"): each turn is the continuation of all prior turns
|
|
899
|
-
*
|
|
908
|
+
* The walk is the pattern proven in test/13-conversation.test.mjs
|
|
909
|
+
* ("teachConversation"): each turn is the continuation of all prior turns,
|
|
910
|
+
* with BARE turn text — NO "User:/Assistant:" labels. The SHAPE is identical
|
|
911
|
+
* (cumulative context → next turn); the join string is not, and does not need
|
|
912
|
+
* to be — that file joins with nothing and this corpus joins with "\n" (see
|
|
913
|
+
* `accumulate`). Saying "byte-for-byte", as this comment used to, invites the
|
|
914
|
+
* reading that the two must agree on a separator. They must not agree,
|
|
915
|
+
* because there is nothing to agree about: turn boundaries are offsets, and
|
|
916
|
+
* the join string is just corpus text. Roles already
|
|
900
917
|
* alternate by position in an oasst2 best-path (the root is a prompter), so a
|
|
901
918
|
* label adds nothing the position does not, while a clean continuation matches
|
|
902
919
|
* the test's recall (predictNext queries bare prior turns) and lets a turn share
|
package/package.json
CHANGED
package/src/canon.ts
CHANGED
|
@@ -63,3 +63,31 @@ export function canonHash(key: Uint8Array): number {
|
|
|
63
63
|
}
|
|
64
64
|
return h >>> 0;
|
|
65
65
|
}
|
|
66
|
+
|
|
67
|
+
/** The span of `bytes` between its first and last non-whitespace byte — the
|
|
68
|
+
* QUESTION, with the caller's edge spacing dropped. Returns a subarray (no
|
|
69
|
+
* copy), and the original when there is nothing to trim.
|
|
70
|
+
*
|
|
71
|
+
* THIS LIVES HERE, not in the core byte utilities, for the reason stated at
|
|
72
|
+
* the top of this file: "nothing in the store or the mind's core knows what
|
|
73
|
+
* 'case' or 'whitespace' is". Edge spacing is a TEXT fact — for a binary or
|
|
74
|
+
* grid modality 0x20 is content, not presentation — so it belongs beside the
|
|
75
|
+
* text canonicalizer, is injected on the same modality test, and never leaks
|
|
76
|
+
* into a mechanism. A modality that supplies its own canon supplies its own
|
|
77
|
+
* reading of "edge" too, or none.
|
|
78
|
+
*
|
|
79
|
+
* Why trimming is sound HERE when {@link textCanon} deliberately refuses it:
|
|
80
|
+
* canon preserves edge whitespace because the hazard is a recognised SUB-span
|
|
81
|
+
* swallowing the boundary byte that separates it from its neighbour (observed:
|
|
82
|
+
* "ice " matching the stored "ice"). At the outer edges of a WHOLE input
|
|
83
|
+
* there is no neighbour — nothing precedes byte 0, nothing follows the last
|
|
84
|
+
* byte — so that hazard cannot arise, and only there. */
|
|
85
|
+
export function textEdgeTrim(bytes: Uint8Array): Uint8Array {
|
|
86
|
+
const space = (b: number) =>
|
|
87
|
+
b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d;
|
|
88
|
+
let from = 0;
|
|
89
|
+
let to = bytes.length;
|
|
90
|
+
while (from < to && space(bytes[from])) from++;
|
|
91
|
+
while (to > from && space(bytes[to - 1])) to--;
|
|
92
|
+
return from === 0 && to === bytes.length ? bytes : bytes.subarray(from, to);
|
|
93
|
+
}
|
package/src/geometry.ts
CHANGED
|
@@ -688,6 +688,97 @@ function contentFoldSpan(
|
|
|
688
688
|
return segs[0];
|
|
689
689
|
}
|
|
690
690
|
|
|
691
|
+
/** A plain content fold's reusable state: the level-0 cut edges over the whole
|
|
692
|
+
* stream and each segment's independently-folded root. See
|
|
693
|
+
* {@link contentFoldIncremental}. */
|
|
694
|
+
export interface ContentFold {
|
|
695
|
+
edges: number[];
|
|
696
|
+
segs: Folded[];
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
|
|
700
|
+
* fold of a byte-identical prefix already produced.
|
|
701
|
+
*
|
|
702
|
+
* WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES. A level-0 segment is a
|
|
703
|
+
* pure function of its own bytes ({@link flatFold} reads nothing else), so
|
|
704
|
+
* reusing one whose [start,end) is unchanged is bit-identical to refolding it
|
|
705
|
+
* — the cache can never change the tree, only skip work. And the cuts
|
|
706
|
+
* themselves are stable under APPEND: {@link contentLevels} decides each cut
|
|
707
|
+
* from a rolling hash over a local window, so bytes added at the right edge
|
|
708
|
+
* cannot move a cut to their left (measured over a growing 12-turn context:
|
|
709
|
+
* 100% of prior cuts survive every append, zero tail churn). Together those
|
|
710
|
+
* two facts are the whole optimisation — a grown stream refolds only the
|
|
711
|
+
* segments at its right edge.
|
|
712
|
+
*
|
|
713
|
+
* This is the reuse the conversation path wants, and it costs NOTHING in
|
|
714
|
+
* structure: the tree is exactly the tree {@link bytesToTree} builds for the
|
|
715
|
+
* same bytes with no boundary set at all. Turn boundaries buy prefix-ROOT
|
|
716
|
+
* identity, which is a different property from incremental reuse; conflating
|
|
717
|
+
* the two is what put an imposed boundary set on the inference path and left
|
|
718
|
+
* it folding differently from the deposits it was querying.
|
|
719
|
+
*
|
|
720
|
+
* `groupByLevel` above the segments is re-run whole. It operates on segment
|
|
721
|
+
* ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
|
|
722
|
+
* and only its right edge actually changes shape — measured at ~40 rebuilt
|
|
723
|
+
* nodes per turn, flat as the context grows sevenfold.
|
|
724
|
+
*
|
|
725
|
+
* PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
|
|
726
|
+
* `bytes`. Reuse is keyed on a segment's [start,end) OFFSETS, which is what
|
|
727
|
+
* makes it O(1) per segment; offsets alone cannot witness that the underlying
|
|
728
|
+
* bytes agree. Hand it a fold of DIFFERENT bytes whose cuts happen to land
|
|
729
|
+
* in the same places and it will splice those foreign segments in — measured,
|
|
730
|
+
* a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
|
|
731
|
+
* streams. Verifying the bytes here would cost O(prefix) and defeat the
|
|
732
|
+
* whole point, so the obligation sits with the caller, and every caller
|
|
733
|
+
* discharges it structurally rather than by care: `perceiveDeposit` looks the
|
|
734
|
+
* entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
|
|
735
|
+
* ARE the cache key — and a conversation's fold state advances only by
|
|
736
|
+
* append. A new caller that cannot make the same structural argument must
|
|
737
|
+
* pass no `prev` at all; the cold path is always correct.
|
|
738
|
+
* ({@link stablePrefixFoldIncremental} carries the identical precondition for
|
|
739
|
+
* the identical reason.) */
|
|
740
|
+
export function contentFoldIncremental(
|
|
741
|
+
space: Space,
|
|
742
|
+
alphabet: Alphabet,
|
|
743
|
+
bytes: Uint8Array,
|
|
744
|
+
prev?: ContentFold,
|
|
745
|
+
): { tree: Sema; fold: ContentFold } {
|
|
746
|
+
if (bytes.length === 0) {
|
|
747
|
+
return {
|
|
748
|
+
tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
|
|
749
|
+
fold: { edges: [0], segs: [] },
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
const { cuts, levels } = contentLevels(space, bytes);
|
|
753
|
+
const edges = [0, ...cuts, bytes.length];
|
|
754
|
+
const segs: Folded[] = [];
|
|
755
|
+
for (let i = 0; i + 1 < edges.length; i++) {
|
|
756
|
+
const hit = prev !== undefined && prev.edges[i] === edges[i] &&
|
|
757
|
+
prev.edges[i + 1] === edges[i + 1]
|
|
758
|
+
? prev.segs[i]
|
|
759
|
+
: undefined;
|
|
760
|
+
segs.push(hit ?? flatFold(space, alphabet, bytes, edges[i], edges[i + 1]));
|
|
761
|
+
}
|
|
762
|
+
const folded = segs.length > 1
|
|
763
|
+
? groupByLevel(space, segs, levels, 1)
|
|
764
|
+
: segs[0];
|
|
765
|
+
// THE ROOT IS NORMALIZED IN PLACE, A CACHED SEGMENT NEVER IS. With one
|
|
766
|
+
// segment — or with a grouping that passes a lone item through — `folded`
|
|
767
|
+
// IS a cached seg, and a later turn will reuse it as an interior node whose
|
|
768
|
+
// magnitude must stay byte-proportional. Copy before normalizing, exactly
|
|
769
|
+
// as the stable-prefix twin does. A single LEAF is copied too: its vector
|
|
770
|
+
// is the shared alphabet entry and must never be written.
|
|
771
|
+
const aliased = segs.some((s) => s.tree === folded.tree);
|
|
772
|
+
let tree = folded.tree;
|
|
773
|
+
if (aliased) {
|
|
774
|
+
tree = tree.kids === null
|
|
775
|
+
? sema(tree.v, tree.leaf, null)
|
|
776
|
+
: sema(Float32Array.from(tree.v), null, tree.kids);
|
|
777
|
+
}
|
|
778
|
+
if (tree.kids !== null) normalize(tree.v);
|
|
779
|
+
return { tree, fold: { edges, segs } };
|
|
780
|
+
}
|
|
781
|
+
|
|
691
782
|
/** Group a row of items by the level of the cut BETWEEN them: items separated
|
|
692
783
|
* by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
|
|
693
784
|
* it. Recurses upward until one root remains, so the shape at every level is
|
|
@@ -923,9 +1014,17 @@ export function stablePrefixFoldIncremental(
|
|
|
923
1014
|
boundaries: readonly number[],
|
|
924
1015
|
prev?: StableFold,
|
|
925
1016
|
): { tree: Sema; fold: StableFold } {
|
|
1017
|
+
// SORTED, like {@link bytesToTree} does before calling the non-incremental
|
|
1018
|
+
// twin. The filter below is sequential (`b > prevB`), so an out-of-order
|
|
1019
|
+
// entry is silently DROPPED rather than rejected — and these two functions
|
|
1020
|
+
// are documented as producing the same cuts, so a caller that hands the
|
|
1021
|
+
// same set to each and gets different trees has hit a trap, not a contract.
|
|
1022
|
+
// Sorting here makes the twins genuinely interchangeable; the set is one
|
|
1023
|
+
// entry per conversation turn, so the cost is nil.
|
|
1024
|
+
const sorted = [...boundaries].sort((a, b) => a - b);
|
|
926
1025
|
const cuts: number[] = [];
|
|
927
1026
|
let prevB = 0;
|
|
928
|
-
for (const b of
|
|
1027
|
+
for (const b of sorted) {
|
|
929
1028
|
if (b > prevB && b < bytes.length) {
|
|
930
1029
|
cuts.push(b);
|
|
931
1030
|
prevB = b;
|
package/src/mind/attention.ts
CHANGED
|
@@ -460,16 +460,21 @@ export async function climbAttention(
|
|
|
460
460
|
|
|
461
461
|
/** Full read-out of one consensus climb: both the roots (dominant points of
|
|
462
462
|
* attention) and the entire ranked list. Cached via ctx.climbMemo, ALWAYS —
|
|
463
|
-
* see {@link recognise} for why this memo (and recognise()'s own)
|
|
464
|
-
*
|
|
465
|
-
* the query's perceived tree
|
|
466
|
-
* fast path
|
|
467
|
-
*
|
|
468
|
-
* prefix subtrees
|
|
469
|
-
*
|
|
470
|
-
*
|
|
471
|
-
*
|
|
472
|
-
*
|
|
463
|
+
* see {@link recognise} for why this memo (and recognise()'s own) is never
|
|
464
|
+
* gated on tracing. The short of it: computeAttention's collectRegions
|
|
465
|
+
* votes over what walking the query's perceived tree EMITS, and foldTree's
|
|
466
|
+
* subtree-resolution fast path used to skip that walk on a warm cache, so a
|
|
467
|
+
* second climb over identical bytes saw less evidence than the first — which
|
|
468
|
+
* a conversation's shared prefix subtrees guaranteed by the second turn.
|
|
469
|
+
* foldTree now takes that fast path only when nothing is watching the walk
|
|
470
|
+
* (see primitives.ts), so the climb is idempotent on its own and this memo
|
|
471
|
+
* is an accelerator again. It stays unconditional anyway: attaching a trace
|
|
472
|
+
* must not change which regions attention weighs.
|
|
473
|
+
*
|
|
474
|
+
* A cache hit still emits a trace step — abbreviated, since the full
|
|
475
|
+
* per-sub-region voting detail {@link traceAttention} builds isn't preserved
|
|
476
|
+
* by the cached read-out — so a traced response is never silently blacked
|
|
477
|
+
* out for a repeated query. */
|
|
473
478
|
export async function climbAttentionAll(
|
|
474
479
|
ctx: MindContext,
|
|
475
480
|
query: Uint8Array,
|
package/src/mind/bridge.ts
CHANGED
|
@@ -97,6 +97,7 @@ import type { MindContext } from "./types.js";
|
|
|
97
97
|
import { foldTree, perceive, read } from "./primitives.js";
|
|
98
98
|
import { chainReach, leafIdRun } from "./canonical.js";
|
|
99
99
|
import {
|
|
100
|
+
allWindowsAreScaffolding,
|
|
100
101
|
corpusN,
|
|
101
102
|
edgeAncestors,
|
|
102
103
|
hubBound,
|
|
@@ -383,6 +384,39 @@ async function bridgeImpl(
|
|
|
383
384
|
);
|
|
384
385
|
return null;
|
|
385
386
|
}
|
|
387
|
+
// NO DISCRIMINATING LITERAL EVIDENCE — abstain (§2.13). A bridge grounds
|
|
388
|
+
// through the literal spans it did NOT substitute; those anchors are the
|
|
389
|
+
// whole of its evidence. When every one of them is SATURATED — containment
|
|
390
|
+
// clamped at the √N hub bound, i.e. the window is corpus-global scaffolding
|
|
391
|
+
// — the query's unsubstituted part discriminates nothing, and the single
|
|
392
|
+
// substituted span is carrying the entire semantic load. That is not a
|
|
393
|
+
// corroborated bridge; it is a template match, and it FABRICATES.
|
|
394
|
+
//
|
|
395
|
+
// Measured on the trained store (hubBound 571). "What is the capital of"
|
|
396
|
+
// has 19 anchors, ALL saturated ("What":572, "hat ":572, "at i":572 …), and
|
|
397
|
+
// bridged to an unrelated trained context about an integral, voiced
|
|
398
|
+
// confidently. Every query the bridge answers CORRECTLY has at least one
|
|
399
|
+
// unsaturated anchor, by a wide margin and with no near miss:
|
|
400
|
+
// "Who is the author of Hamlet?" → "let?":12, "How do you say 'thank you'
|
|
401
|
+
// in French?" → "y 't":3, "…largest planet…" → "tem?":31, "What is the
|
|
402
|
+
// capital of France?" → "f Fr":114. The honest-silence probes sit on the
|
|
403
|
+
// same side as the correct ones ("Zamu":3), so this gate is not what makes
|
|
404
|
+
// them silent and cannot be credited for them.
|
|
405
|
+
//
|
|
406
|
+
// This introduces NO new threshold: `bound` is the same √N reading of "hub"
|
|
407
|
+
// the anchor scan already clamps its own containment read to (§2.2, §2.7).
|
|
408
|
+
if (allWindowsAreScaffolding(ctx, query)) {
|
|
409
|
+
ctx.trace?.step(
|
|
410
|
+
"substitutionBridge",
|
|
411
|
+
[rItem(query, "query")],
|
|
412
|
+
[],
|
|
413
|
+
"every query window that could anchor is corpus-global scaffolding — " +
|
|
414
|
+
"no literal evidence to corroborate a substitution",
|
|
415
|
+
undefined,
|
|
416
|
+
diagnostics!,
|
|
417
|
+
);
|
|
418
|
+
return null;
|
|
419
|
+
}
|
|
386
420
|
// CORROBORATION (see the module-level doc) over the precomputed window
|
|
387
421
|
// facts: the query span [qs,qe) attests when every full W-window inside
|
|
388
422
|
// it is a stored flat form and at least one is reused across ≥ 2
|