@hviana/sema 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/attention.d.ts +15 -10
  10. package/dist/src/mind/attention.js +15 -10
  11. package/dist/src/mind/bridge.js +27 -1
  12. package/dist/src/mind/frame-filler.d.ts +15 -0
  13. package/dist/src/mind/frame-filler.js +535 -0
  14. package/dist/src/mind/learning.js +6 -11
  15. package/dist/src/mind/mechanisms/cast.js +72 -2
  16. package/dist/src/mind/mechanisms/cover.js +6 -1
  17. package/dist/src/mind/mechanisms/extraction.js +27 -0
  18. package/dist/src/mind/mechanisms/recall.js +214 -34
  19. package/dist/src/mind/mind.d.ts +52 -3
  20. package/dist/src/mind/mind.js +140 -12
  21. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  22. package/dist/src/mind/pipeline.js +29 -1
  23. package/dist/src/mind/prefix-completion.d.ts +59 -0
  24. package/dist/src/mind/prefix-completion.js +270 -0
  25. package/dist/src/mind/primitives.d.ts +29 -10
  26. package/dist/src/mind/primitives.js +98 -71
  27. package/dist/src/mind/recognition.js +153 -26
  28. package/dist/src/mind/traverse.d.ts +32 -0
  29. package/dist/src/mind/traverse.js +52 -0
  30. package/dist/src/mind/types.d.ts +61 -18
  31. package/dist/src/mind/types.js +68 -19
  32. package/dist/src/store.d.ts +21 -0
  33. package/dist/src/store.js +21 -0
  34. package/example/train_base.ts +21 -4
  35. package/package.json +1 -1
  36. package/src/canon.ts +28 -0
  37. package/src/geometry.ts +100 -1
  38. package/src/mind/attention.ts +15 -10
  39. package/src/mind/bridge.ts +34 -0
  40. package/src/mind/frame-filler.ts +604 -0
  41. package/src/mind/learning.ts +5 -9
  42. package/src/mind/mechanisms/cast.ts +70 -2
  43. package/src/mind/mechanisms/cover.ts +6 -1
  44. package/src/mind/mechanisms/extraction.ts +27 -0
  45. package/src/mind/mechanisms/recall.ts +236 -37
  46. package/src/mind/mind.ts +166 -18
  47. package/src/mind/pipeline-mechanism.ts +7 -0
  48. package/src/mind/pipeline.ts +33 -1
  49. package/src/mind/prefix-completion.ts +314 -0
  50. package/src/mind/primitives.ts +105 -80
  51. package/src/mind/recognition.ts +151 -23
  52. package/src/mind/traverse.ts +52 -0
  53. package/src/mind/types.ts +104 -44
  54. package/src/store.ts +25 -0
  55. package/test/13-conversation.test.mjs +13 -0
  56. package/test/57-fusion-order.test.mjs +65 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
@@ -7,22 +7,17 @@ import type { MindConfig } from "../config.js";
7
7
  import type { Meter } from "../meter.js";
8
8
  import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
9
9
  import type { Rationale } from "./rationale.js";
10
- import type { Grid, StableFold } from "../geometry.js";
11
- /** One {@link MindContext._depositTrees} entry — see that field's doc. */
10
+ import type { ContentFold, Grid } from "../geometry.js";
11
+ /** One {@link MindContext._depositTrees} entry — see that field's doc.
12
+ *
13
+ * A PURE WORK CACHE. It carries the already-folded content segments of a
14
+ * deposited stream so a longer stream sharing its byte prefix can skip
15
+ * refolding them. It holds no turn boundaries and no continuation proof
16
+ * because the deposit fold imposes nothing: reuse is bit-identical to a cold
17
+ * fold, so a hit can only save time, never change a tree. */
12
18
  export interface DepositCacheEntry {
13
- /** Turn boundaries accumulated over this content's deposit chain
14
- * strictly increasing proper offsets, each a previously-deposited
15
- * whole-context length. Empty for a first-seen (single-turn) input. */
16
- boundaries: number[];
17
- /** Stable-prefix segment folds (grown-context inputs only). */
18
- stable?: StableFold;
19
- /** The continuation bytes this ctxInput was paired with in ingestPair, if
20
- * any — the ONLY thing that makes a later, longer ctxInput a genuine next
21
- * TURN of the same conversation rather than an unrelated fact that
22
- * happens to share this one's byte prefix (e.g. "2+2" vs. "2+2=5"). A
23
- * later deposit only takes this entry as its stable-prefix `prev` when
24
- * its own suffix bytes-equal this exactly. */
25
- nextBytes?: Uint8Array;
19
+ /** The plain content fold's reusable segment state. */
20
+ content: ContentFold;
26
21
  }
27
22
  export type Input = string | Uint8Array | Grid | Grid[];
28
23
  /** The host capabilities GraphSearch consults during a cover. MindContext
@@ -289,9 +284,23 @@ export interface MindContext extends GraphSearchHost {
289
284
  /** Subtree-resolution cache: Sema node → its store id and byte length.
290
285
  * Populated by {@link foldTree} during inference; checked before
291
286
  * walking children. When a conversation's pyramid reuses prefix
292
- * subtrees, this cache lets {@link recognise} skip them entirely
293
- * O(suffix) instead of O(context). Mind-lifetime (WeakMap keys are
294
- * the Sema objects the pyramid keeps alive). */
287
+ * subtrees, this cache names them without a store probe. It does NOT let
288
+ * {@link recognise} skip them: recognise walks with a `visit` callback and
289
+ * emits its sites from it, so a skipped descent would mean fewer sites on
290
+ * a warm cache than a cold one. foldTree short-circuits only for
291
+ * visitor-less walks (O(suffix) there); a visiting walk stays O(context)
292
+ * and banks the elided probes. Mind-lifetime (WeakMap keys are the Sema
293
+ * objects the pyramid keeps alive).
294
+ *
295
+ * THAT REUSE IS A PRECONDITION, NOT A GIVEN: the keys are node IDENTITIES,
296
+ * so it hits only while the conversation's fold hands back the SAME Sema
297
+ * objects for the unchanged prefix. `_growContext` rebuilt the whole tree
298
+ * with `bytesToTree` on every turn, so every key was fresh and this cache
299
+ * could not hit even once — the O(suffix) claim above described an
300
+ * intention rather than the code. It now grows the context through
301
+ * {@link stablePrefixFoldIncremental}, which reuses each already-folded
302
+ * segment: measured over four turns, turn 4 shared 69 of its 95 nodes with
303
+ * turn 3 (26 new ≈ the new turn's own size). */
295
304
  _resolvedSubtrees: WeakMap<Sema, {
296
305
  id: number;
297
306
  len: number;
@@ -369,6 +378,40 @@ export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: nu
369
378
  * (lo/hi) decision and the final concatenation: it is stale, not a second
370
379
  * answer, but the OTHER spans a derivation chose are independent evidence
371
380
  * and must not be discarded along with it. */
381
+ /** The spans {@link liftAnswer} actually concatenates, in order — the answer
382
+ * before it is joined. Exposed so a caller can ask what the lifted answer is
383
+ * MADE OF without re-deriving the selection: in particular how much of it is
384
+ * SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
385
+ * because nothing explained them, the same spans the liftAnswer trace labels
386
+ * "scaffolding" rather than "chosen").
387
+ *
388
+ * That quantity is load-bearing for the grounding decision. Two candidates
389
+ * can leave the SAME number of query bytes unaccounted and therefore grade
390
+ * identically, while one of them pads its answer with those bytes and the
391
+ * other does not — measured on test/22's two-fact chain, cover and recall
392
+ * both graded 11001 with 11 bytes unexplained, and cover won the tie only on
393
+ * consideration order, answering "The capital of France is Paris famous for"
394
+ * where recall had crossed the hop. Carrying an unexplained span into the
395
+ * answer is strictly weaker than not explaining it: it manufactures fluency
396
+ * out of the asker's own words. See the tie-break in pipeline.ts. */
397
+ export declare function liftAnswerParts(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Seg[];
398
+ /** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
399
+ * from spans nothing recognised (see {@link liftAnswerParts}).
400
+ *
401
+ * ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT. Not all carried-through
402
+ * bytes are a failure to explain: a period, a question mark, the space
403
+ * between two fused topics are GLUE — they belong to the answer's surface,
404
+ * and dropping them to look better-derived would be a worse answer, not a
405
+ * more honest one. A substantive phrase the derivation never explained
406
+ * ("famous for") is a different claim entirely.
407
+ *
408
+ * W is the line between them, and it is the same line the rest of the mind
409
+ * already draws: below one river window byte overlap is chance, not evidence
410
+ * (see identityBar, the bridge's attestedQ, and recognition's site floor).
411
+ * Counting every scaffolding byte instead — which is what this did first —
412
+ * made punctuation preservation lose a tie it should win, and test/00's
413
+ * "period preserved" / "question mark preserved" caught it immediately. */
414
+ export declare function liftedScaffolding(segs: Seg[], queryLen: number, query: Uint8Array, W: number): number;
372
415
  export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
373
416
  /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
374
417
  * tracked deposit interned (`prevSeen`). */
@@ -46,7 +46,23 @@ export function segRestatesQuery(s, query, queryLen, W) {
46
46
  * (lo/hi) decision and the final concatenation: it is stale, not a second
47
47
  * answer, but the OTHER spans a derivation chose are independent evidence
48
48
  * and must not be discarded along with it. */
49
- export function liftAnswer(segs, queryLen, query, W) {
49
+ /** The spans {@link liftAnswer} actually concatenates, in order — the answer
50
+ * before it is joined. Exposed so a caller can ask what the lifted answer is
51
+ * MADE OF without re-deriving the selection: in particular how much of it is
52
+ * SCAFFOLDING (a `rec: false` span — query bytes carried through verbatim
53
+ * because nothing explained them, the same spans the liftAnswer trace labels
54
+ * "scaffolding" rather than "chosen").
55
+ *
56
+ * That quantity is load-bearing for the grounding decision. Two candidates
57
+ * can leave the SAME number of query bytes unaccounted and therefore grade
58
+ * identically, while one of them pads its answer with those bytes and the
59
+ * other does not — measured on test/22's two-fact chain, cover and recall
60
+ * both graded 11001 with 11 bytes unexplained, and cover won the tie only on
61
+ * consideration order, answering "The capital of France is Paris famous for"
62
+ * where recall had crossed the hop. Carrying an unexplained span into the
63
+ * answer is strictly weaker than not explaining it: it manufactures fluency
64
+ * out of the asker's own words. See the tie-break in pipeline.ts. */
65
+ export function liftAnswerParts(segs, queryLen, query, W) {
50
66
  const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
51
67
  const recognised = [];
52
68
  for (let k = 0; k < segs.length; k++) {
@@ -54,32 +70,65 @@ export function liftAnswer(segs, queryLen, query, W) {
54
70
  recognised.push(k);
55
71
  }
56
72
  if (recognised.length === 0)
57
- return null;
73
+ return [];
58
74
  if (recognised.length === 1) {
59
75
  const s = segs[recognised[0]];
60
- // A COMPUTED span's query-side width is operand digit-count, not
61
- // evidence of how much of the query's meaning it accounts for — the
62
- // half-dominance check below (built for a genuinely RECOGNISED learned
63
- // form) is not a valid framing signal for it (see the `computed` field
64
- // doc on Seg/GItem): "1000 - 421" outweighs "what is …?" by width only
65
- // because the operands are big, not because the framing matters less.
66
- // A LITERAL PREFIX before a computed span is unambiguous framing
67
- // regardless of width — an arithmetic expression is never itself
68
- // preceded by more literal computed content, so anything literal before
69
- // it is question wording ("what is ", "compute ") to lift clear of.
70
- // With no prefix (s.i === 0) the span is judged by the ordinary
71
- // half-dominance rule below, which already correctly keeps a short
72
- // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
73
76
  if (s.computed && s.i > 0)
74
- return s.bytes;
77
+ return [s];
75
78
  if (dominates(s.j - s.i, queryLen)) {
76
- return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
79
+ return segs.filter((_, k) => !restated[k]);
77
80
  }
78
- return s.bytes;
81
+ return [s];
79
82
  }
80
83
  const lo = recognised[0];
81
84
  const hi = recognised[recognised.length - 1];
82
- return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
85
+ return segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]);
86
+ }
87
+ /** The SCAFFOLDING byte count of a lifted answer: how many of its bytes come
88
+ * from spans nothing recognised (see {@link liftAnswerParts}).
89
+ *
90
+ * ONLY RUNS OF AT LEAST ONE RIVER WINDOW COUNT. Not all carried-through
91
+ * bytes are a failure to explain: a period, a question mark, the space
92
+ * between two fused topics are GLUE — they belong to the answer's surface,
93
+ * and dropping them to look better-derived would be a worse answer, not a
94
+ * more honest one. A substantive phrase the derivation never explained
95
+ * ("famous for") is a different claim entirely.
96
+ *
97
+ * W is the line between them, and it is the same line the rest of the mind
98
+ * already draws: below one river window byte overlap is chance, not evidence
99
+ * (see identityBar, the bridge's attestedQ, and recognition's site floor).
100
+ * Counting every scaffolding byte instead — which is what this did first —
101
+ * made punctuation preservation lose a tie it should win, and test/00's
102
+ * "period preserved" / "question mark preserved" caught it immediately. */
103
+ export function liftedScaffolding(segs, queryLen, query, W) {
104
+ // MEASURED PER CONTIGUOUS RUN, not per span. A PASS span is one BYTE — the
105
+ // cover charges unrecognised bytes individually — so asking whether a single
106
+ // span reaches W would find no run ever, whatever the query. " famous for"
107
+ // arrives as eleven one-byte spans in a row and is one eleven-byte run.
108
+ let n = 0;
109
+ let run = 0;
110
+ const close = () => {
111
+ if (run >= W)
112
+ n += run;
113
+ run = 0;
114
+ };
115
+ for (const s of liftAnswerParts(segs, queryLen, query, W)) {
116
+ if (s.rec)
117
+ close();
118
+ else
119
+ run += s.bytes.length;
120
+ }
121
+ close();
122
+ return n;
123
+ }
124
+ export function liftAnswer(segs, queryLen, query, W) {
125
+ // ONE selection rule, in {@link liftAnswerParts} — this is its join. The
126
+ // two used to be separate copies of the same lo/hi/restated reasoning, which
127
+ // is exactly how an answer and the accounting OF that answer drift apart.
128
+ const parts = liftAnswerParts(segs, queryLen, query, W);
129
+ if (parts.length === 0)
130
+ return null;
131
+ return concatBytes(parts.map((x) => x.bytes));
83
132
  }
84
133
  /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
85
134
  * tracked deposit interned (`prevSeen`). */
@@ -152,6 +152,10 @@ export interface Store {
152
152
  contentLen(id: NodeId, cap?: number): number;
153
153
  findLeaf(bytes: Uint8Array): NodeId | null;
154
154
  findBranch(kids: NodeId[]): NodeId | null;
155
+ /** {@link findBranch} for a run of single-byte leaves, addressed by the raw
156
+ * bytes — the allocation-free probe span scanners use. Optional: a store
157
+ * without it is simply probed through `findBranch`. */
158
+ findFlatBranch?(bytes: Uint8Array): NodeId | null;
155
159
  /** The branch nodes that list `id` among their children — the reverse of
156
160
  * `get(id).kids`. Lets the structural DAG be climbed upward, from a
157
161
  * recognised fragment to the larger learned forms that contain it. */
@@ -585,6 +589,23 @@ export declare abstract class AbstractStore implements Store {
585
589
  private _prefix;
586
590
  contentLen(id: NodeId, cap?: number): number;
587
591
  findLeaf(bytes: Uint8Array): NodeId | null;
592
+ /** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
593
+ * bytes themselves — no kid array, no key string, no copy.
594
+ *
595
+ * A flat branch stores its children as {@link flatKidsBytes}, and that
596
+ * encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
597
+ * So for such a run the kid array and the byte span are the same object in
598
+ * two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
599
+ * the same lookup — except that the array path allocates the slice, then
600
+ * `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
601
+ * answer is usually "no". The bloom filter behind `_dbFindBranchByLeaf`
602
+ * answers most of those with no I/O at all, so the allocations dominated.
603
+ *
604
+ * Pass a subarray: it is a view, so a caller scanning spans of a query
605
+ * allocates nothing per probe. Deliberately NOT memoized — its callers
606
+ * probe many spans that miss, and a key string per probe is the cost this
607
+ * exists to remove. */
608
+ findFlatBranch(bytes: Uint8Array): NodeId | null;
588
609
  findBranch(kids: NodeId[]): NodeId | null;
589
610
  parents(id: NodeId): NodeId[];
590
611
  parentsFirst(id: NodeId, limit: number): NodeId[];
package/dist/src/store.js CHANGED
@@ -770,6 +770,27 @@ export class AbstractStore {
770
770
  this._leafKey.set(key, id);
771
771
  return id;
772
772
  }
773
+ /** {@link findBranch} for a run of SINGLE-BYTE leaves, addressed by the
774
+ * bytes themselves — no kid array, no key string, no copy.
775
+ *
776
+ * A flat branch stores its children as {@link flatKidsBytes}, and that
777
+ * encoding is the identity on single-byte leaves: kid id −(b+1) IS byte b.
778
+ * So for such a run the kid array and the byte span are the same object in
779
+ * two spellings, and `findBranch(leafIds.slice(i, j))` and this call are
780
+ * the same lookup — except that the array path allocates the slice, then
781
+ * `kids.join(",")`, then the flat bytes, all O(span), for a probe whose
782
+ * answer is usually "no". The bloom filter behind `_dbFindBranchByLeaf`
783
+ * answers most of those with no I/O at all, so the allocations dominated.
784
+ *
785
+ * Pass a subarray: it is a view, so a caller scanning spans of a query
786
+ * allocates nothing per probe. Deliberately NOT memoized — its callers
787
+ * probe many spans that miss, and a key string per probe is the cost this
788
+ * exists to remove. */
789
+ findFlatBranch(bytes) {
790
+ if (this.meter)
791
+ this.meter.branchLookups++;
792
+ return this._dbFindBranchByLeaf(hashOf(bytes), bytes);
793
+ }
773
794
  findBranch(kids) {
774
795
  if (this.meter)
775
796
  this.meter.branchLookups++;
@@ -709,7 +709,18 @@ const isEpisode = (it: TrainingItem): it is Episode => typeof it !== "string";
709
709
  /** Build the accumulated-context episodes of a turn sequence: each successive
710
710
  * turn is the continuation of ALL the turns before it joined together. This is
711
711
  * the same cumulative-context shape a multi-turn conversation deposits, so the
712
- * store learns to continue a growing context. */
712
+ * store learns to continue a growing context.
713
+ *
714
+ * The "\n" below is a CORPUS choice, not a protocol. oasst2 turns are
715
+ * paragraphs, and reading them back with the newlines kept is how this corpus
716
+ * reads naturally; a different corpus may join with nothing, and
717
+ * test/13-conversation.test.mjs does exactly that. Neither has to match the
718
+ * other, because Sema never scans content for turn boundaries — those are
719
+ * offsets the Conversation API carries beside the bytes (see Mind.addTurn's
720
+ * "ON SEPARATORS" note). The newline here is simply part of the text this
721
+ * store learnt, so anything replaying this corpus feeds it back as part of
722
+ * the turn: `addTurn(conv, "\n" + turnText)`. It is not a convention the
723
+ * engine, the API, or the tests have to agree on. */
713
724
  function accumulate(turns: string[]): Episode[] {
714
725
  const out: Episode[] = [];
715
726
  for (let i = 1; i < turns.length; i++) {
@@ -894,9 +905,15 @@ export function bestOasstPath(root: OasstNode): OasstTurn[] {
894
905
  * turn experiences and local adjacent-pair facts are NOT emitted (they are
895
906
  * subsumed by it and would merely replicate the content).
896
907
  *
897
- * The walk is byte-for-byte the pattern proven in test/13-conversation.test.mjs
898
- * ("teachConversation"): each turn is the continuation of all prior turns joined
899
- * by "\n", with BARE turn text — NO "User:/Assistant:" labels. Roles already
908
+ * The walk is the pattern proven in test/13-conversation.test.mjs
909
+ * ("teachConversation"): each turn is the continuation of all prior turns,
910
+ * with BARE turn text — NO "User:/Assistant:" labels. The SHAPE is identical
911
+ * (cumulative context → next turn); the join string is not, and does not need
912
+ * to be — that file joins with nothing and this corpus joins with "\n" (see
913
+ * `accumulate`). Saying "byte-for-byte", as this comment used to, invites the
914
+ * reading that the two must agree on a separator. They must not agree,
915
+ * because there is nothing to agree about: turn boundaries are offsets, and
916
+ * the join string is just corpus text. Roles already
900
917
  * alternate by position in an oasst2 best-path (the root is a prompter), so a
901
918
  * label adds nothing the position does not, while a clean continuation matches
902
919
  * the test's recall (predictNext queries bare prior turns) and lets a turn share
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hviana/sema",
3
- "version": "0.4.7",
3
+ "version": "0.5.1",
4
4
  "description": "Sema: a non-parametric, instance-based reasoning system.",
5
5
  "type": "module",
6
6
  "main": "dist/src/index.js",
package/src/canon.ts CHANGED
@@ -63,3 +63,31 @@ export function canonHash(key: Uint8Array): number {
63
63
  }
64
64
  return h >>> 0;
65
65
  }
66
+
67
+ /** The span of `bytes` between its first and last non-whitespace byte — the
68
+ * QUESTION, with the caller's edge spacing dropped. Returns a subarray (no
69
+ * copy), and the original when there is nothing to trim.
70
+ *
71
+ * THIS LIVES HERE, not in the core byte utilities, for the reason stated at
72
+ * the top of this file: "nothing in the store or the mind's core knows what
73
+ * 'case' or 'whitespace' is". Edge spacing is a TEXT fact — for a binary or
74
+ * grid modality 0x20 is content, not presentation — so it belongs beside the
75
+ * text canonicalizer, is injected on the same modality test, and never leaks
76
+ * into a mechanism. A modality that supplies its own canon supplies its own
77
+ * reading of "edge" too, or none.
78
+ *
79
+ * Why trimming is sound HERE when {@link textCanon} deliberately refuses it:
80
+ * canon preserves edge whitespace because the hazard is a recognised SUB-span
81
+ * swallowing the boundary byte that separates it from its neighbour (observed:
82
+ * "ice " matching the stored "ice"). At the outer edges of a WHOLE input
83
+ * there is no neighbour — nothing precedes byte 0, nothing follows the last
84
+ * byte — so that hazard cannot arise, and only there. */
85
+ export function textEdgeTrim(bytes: Uint8Array): Uint8Array {
86
+ const space = (b: number) =>
87
+ b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d;
88
+ let from = 0;
89
+ let to = bytes.length;
90
+ while (from < to && space(bytes[from])) from++;
91
+ while (to > from && space(bytes[to - 1])) to--;
92
+ return from === 0 && to === bytes.length ? bytes : bytes.subarray(from, to);
93
+ }
package/src/geometry.ts CHANGED
@@ -688,6 +688,97 @@ function contentFoldSpan(
688
688
  return segs[0];
689
689
  }
690
690
 
691
+ /** A plain content fold's reusable state: the level-0 cut edges over the whole
692
+ * stream and each segment's independently-folded root. See
693
+ * {@link contentFoldIncremental}. */
694
+ export interface ContentFold {
695
+ edges: number[];
696
+ segs: Folded[];
697
+ }
698
+
699
+ /** {@link contentFoldSpan} over a WHOLE stream, reusing the segments a previous
700
+ * fold of a byte-identical prefix already produced.
701
+ *
702
+ * WHY THIS IS SOUND, AND WHY IT NEEDS NO BOUNDARIES. A level-0 segment is a
703
+ * pure function of its own bytes ({@link flatFold} reads nothing else), so
704
+ * reusing one whose [start,end) is unchanged is bit-identical to refolding it
705
+ * — the cache can never change the tree, only skip work. And the cuts
706
+ * themselves are stable under APPEND: {@link contentLevels} decides each cut
707
+ * from a rolling hash over a local window, so bytes added at the right edge
708
+ * cannot move a cut to their left (measured over a growing 12-turn context:
709
+ * 100% of prior cuts survive every append, zero tail churn). Together those
710
+ * two facts are the whole optimisation — a grown stream refolds only the
711
+ * segments at its right edge.
712
+ *
713
+ * This is the reuse the conversation path wants, and it costs NOTHING in
714
+ * structure: the tree is exactly the tree {@link bytesToTree} builds for the
715
+ * same bytes with no boundary set at all. Turn boundaries buy prefix-ROOT
716
+ * identity, which is a different property from incremental reuse; conflating
717
+ * the two is what put an imposed boundary set on the inference path and left
718
+ * it folding differently from the deposits it was querying.
719
+ *
720
+ * `groupByLevel` above the segments is re-run whole. It operates on segment
721
+ * ROOTS (a few dozen items for a several-hundred-byte context), not on bytes,
722
+ * and only its right edge actually changes shape — measured at ~40 rebuilt
723
+ * nodes per turn, flat as the context grows sevenfold.
724
+ *
725
+ * PRECONDITION — `prev` MUST have been folded over a BYTE-IDENTICAL PREFIX of
726
+ * `bytes`. Reuse is keyed on a segment's [start,end) OFFSETS, which is what
727
+ * makes it O(1) per segment; offsets alone cannot witness that the underlying
728
+ * bytes agree. Hand it a fold of DIFFERENT bytes whose cuts happen to land
729
+ * in the same places and it will splice those foreign segments in — measured,
730
+ * a deliberately mismatched `prev` produced a wrong tree on 336 of 400 random
731
+ * streams. Verifying the bytes here would cost O(prefix) and defeat the
732
+ * whole point, so the obligation sits with the caller, and every caller
733
+ * discharges it structurally rather than by care: `perceiveDeposit` looks the
734
+ * entry up under `latin1Key(bytes.subarray(0, L))` — the prefix's own bytes
735
+ * ARE the cache key — and a conversation's fold state advances only by
736
+ * append. A new caller that cannot make the same structural argument must
737
+ * pass no `prev` at all; the cold path is always correct.
738
+ * ({@link stablePrefixFoldIncremental} carries the identical precondition for
739
+ * the identical reason.) */
740
+ export function contentFoldIncremental(
741
+ space: Space,
742
+ alphabet: Alphabet,
743
+ bytes: Uint8Array,
744
+ prev?: ContentFold,
745
+ ): { tree: Sema; fold: ContentFold } {
746
+ if (bytes.length === 0) {
747
+ return {
748
+ tree: sema(alphabet.vecs[0], new Uint8Array(0), null),
749
+ fold: { edges: [0], segs: [] },
750
+ };
751
+ }
752
+ const { cuts, levels } = contentLevels(space, bytes);
753
+ const edges = [0, ...cuts, bytes.length];
754
+ const segs: Folded[] = [];
755
+ for (let i = 0; i + 1 < edges.length; i++) {
756
+ const hit = prev !== undefined && prev.edges[i] === edges[i] &&
757
+ prev.edges[i + 1] === edges[i + 1]
758
+ ? prev.segs[i]
759
+ : undefined;
760
+ segs.push(hit ?? flatFold(space, alphabet, bytes, edges[i], edges[i + 1]));
761
+ }
762
+ const folded = segs.length > 1
763
+ ? groupByLevel(space, segs, levels, 1)
764
+ : segs[0];
765
+ // THE ROOT IS NORMALIZED IN PLACE, A CACHED SEGMENT NEVER IS. With one
766
+ // segment — or with a grouping that passes a lone item through — `folded`
767
+ // IS a cached seg, and a later turn will reuse it as an interior node whose
768
+ // magnitude must stay byte-proportional. Copy before normalizing, exactly
769
+ // as the stable-prefix twin does. A single LEAF is copied too: its vector
770
+ // is the shared alphabet entry and must never be written.
771
+ const aliased = segs.some((s) => s.tree === folded.tree);
772
+ let tree = folded.tree;
773
+ if (aliased) {
774
+ tree = tree.kids === null
775
+ ? sema(tree.v, tree.leaf, null)
776
+ : sema(Float32Array.from(tree.v), null, tree.kids);
777
+ }
778
+ if (tree.kids !== null) normalize(tree.v);
779
+ return { tree, fold: { edges, segs } };
780
+ }
781
+
691
782
  /** Group a row of items by the level of the cut BETWEEN them: items separated
692
783
  * by a cut of level < L belong to the same parent, and a cut of level ≥ L ends
693
784
  * it. Recurses upward until one root remains, so the shape at every level is
@@ -923,9 +1014,17 @@ export function stablePrefixFoldIncremental(
923
1014
  boundaries: readonly number[],
924
1015
  prev?: StableFold,
925
1016
  ): { tree: Sema; fold: StableFold } {
1017
+ // SORTED, like {@link bytesToTree} does before calling the non-incremental
1018
+ // twin. The filter below is sequential (`b > prevB`), so an out-of-order
1019
+ // entry is silently DROPPED rather than rejected — and these two functions
1020
+ // are documented as producing the same cuts, so a caller that hands the
1021
+ // same set to each and gets different trees has hit a trap, not a contract.
1022
+ // Sorting here makes the twins genuinely interchangeable; the set is one
1023
+ // entry per conversation turn, so the cost is nil.
1024
+ const sorted = [...boundaries].sort((a, b) => a - b);
926
1025
  const cuts: number[] = [];
927
1026
  let prevB = 0;
928
- for (const b of boundaries) {
1027
+ for (const b of sorted) {
929
1028
  if (b > prevB && b < bytes.length) {
930
1029
  cuts.push(b);
931
1030
  prevB = b;
@@ -460,16 +460,21 @@ export async function climbAttention(
460
460
 
461
461
  /** Full read-out of one consensus climb: both the roots (dominant points of
462
462
  * attention) and the entire ranked list. Cached via ctx.climbMemo, ALWAYS —
463
- * see {@link recognise} for why this memo (and recognise()'s own) must
464
- * never be skipped while tracing: computeAttention's collectRegions walks
465
- * the query's perceived tree via the same foldTree whose subtree-resolution
466
- * fast path makes a second call on identical bytes non-idempotent once
467
- * ctx._resolvedSubtrees is warm (which a multi-turn conversation's shared
468
- * prefix subtrees guarantee by the second turn). A cache hit still emits
469
- * a trace step abbreviated, since the full per-sub-region voting detail
470
- * {@link traceAttention} builds isn't preserved by the cached read-out
471
- * so a traced response is never silently blacked out for a repeated
472
- * query. */
463
+ * see {@link recognise} for why this memo (and recognise()'s own) is never
464
+ * gated on tracing. The short of it: computeAttention's collectRegions
465
+ * votes over what walking the query's perceived tree EMITS, and foldTree's
466
+ * subtree-resolution fast path used to skip that walk on a warm cache, so a
467
+ * second climb over identical bytes saw less evidence than the first — which
468
+ * a conversation's shared prefix subtrees guaranteed by the second turn.
469
+ * foldTree now takes that fast path only when nothing is watching the walk
470
+ * (see primitives.ts), so the climb is idempotent on its own and this memo
471
+ * is an accelerator again. It stays unconditional anyway: attaching a trace
472
+ * must not change which regions attention weighs.
473
+ *
474
+ * A cache hit still emits a trace step — abbreviated, since the full
475
+ * per-sub-region voting detail {@link traceAttention} builds isn't preserved
476
+ * by the cached read-out — so a traced response is never silently blacked
477
+ * out for a repeated query. */
473
478
  export async function climbAttentionAll(
474
479
  ctx: MindContext,
475
480
  query: Uint8Array,
@@ -97,6 +97,7 @@ import type { MindContext } from "./types.js";
97
97
  import { foldTree, perceive, read } from "./primitives.js";
98
98
  import { chainReach, leafIdRun } from "./canonical.js";
99
99
  import {
100
+ allWindowsAreScaffolding,
100
101
  corpusN,
101
102
  edgeAncestors,
102
103
  hubBound,
@@ -383,6 +384,39 @@ async function bridgeImpl(
383
384
  );
384
385
  return null;
385
386
  }
387
+ // NO DISCRIMINATING LITERAL EVIDENCE — abstain (§2.13). A bridge grounds
388
+ // through the literal spans it did NOT substitute; those anchors are the
389
+ // whole of its evidence. When every one of them is SATURATED — containment
390
+ // clamped at the √N hub bound, i.e. the window is corpus-global scaffolding
391
+ // — the query's unsubstituted part discriminates nothing, and the single
392
+ // substituted span is carrying the entire semantic load. That is not a
393
+ // corroborated bridge; it is a template match, and it FABRICATES.
394
+ //
395
+ // Measured on the trained store (hubBound 571). "What is the capital of"
396
+ // has 19 anchors, ALL saturated ("What":572, "hat ":572, "at i":572 …), and
397
+ // bridged to an unrelated trained context about an integral, voiced
398
+ // confidently. Every query the bridge answers CORRECTLY has at least one
399
+ // unsaturated anchor, by a wide margin and with no near miss:
400
+ // "Who is the author of Hamlet?" → "let?":12, "How do you say 'thank you'
401
+ // in French?" → "y 't":3, "…largest planet…" → "tem?":31, "What is the
402
+ // capital of France?" → "f Fr":114. The honest-silence probes sit on the
403
+ // same side as the correct ones ("Zamu":3), so this gate is not what makes
404
+ // them silent and cannot be credited for them.
405
+ //
406
+ // This introduces NO new threshold: `bound` is the same √N reading of "hub"
407
+ // the anchor scan already clamps its own containment read to (§2.2, §2.7).
408
+ if (allWindowsAreScaffolding(ctx, query)) {
409
+ ctx.trace?.step(
410
+ "substitutionBridge",
411
+ [rItem(query, "query")],
412
+ [],
413
+ "every query window that could anchor is corpus-global scaffolding — " +
414
+ "no literal evidence to corroborate a substitution",
415
+ undefined,
416
+ diagnostics!,
417
+ );
418
+ return null;
419
+ }
386
420
  // CORROBORATION (see the module-level doc) over the precomputed window
387
421
  // facts: the query span [qs,qe) attests when every full W-window inside
388
422
  // it is a stored flat form and at least one is reused across ≥ 2