@hviana/sema 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/AGENTS.md +290 -77
  2. package/HOW_IT_WORKS.md +2170 -735
  3. package/dist/example/train_base.d.ts +9 -3
  4. package/dist/example/train_base.js +21 -4
  5. package/dist/src/canon.d.ts +19 -0
  6. package/dist/src/canon.js +28 -0
  7. package/dist/src/geometry.d.ts +52 -0
  8. package/dist/src/geometry.js +87 -1
  9. package/dist/src/mind/attention.d.ts +15 -10
  10. package/dist/src/mind/attention.js +15 -10
  11. package/dist/src/mind/bridge.js +27 -1
  12. package/dist/src/mind/frame-filler.d.ts +15 -0
  13. package/dist/src/mind/frame-filler.js +535 -0
  14. package/dist/src/mind/learning.js +6 -11
  15. package/dist/src/mind/mechanisms/cast.js +72 -2
  16. package/dist/src/mind/mechanisms/cover.js +6 -1
  17. package/dist/src/mind/mechanisms/extraction.js +27 -0
  18. package/dist/src/mind/mechanisms/recall.js +214 -34
  19. package/dist/src/mind/mind.d.ts +52 -3
  20. package/dist/src/mind/mind.js +140 -12
  21. package/dist/src/mind/pipeline-mechanism.d.ts +7 -0
  22. package/dist/src/mind/pipeline.js +29 -1
  23. package/dist/src/mind/prefix-completion.d.ts +59 -0
  24. package/dist/src/mind/prefix-completion.js +270 -0
  25. package/dist/src/mind/primitives.d.ts +29 -10
  26. package/dist/src/mind/primitives.js +98 -71
  27. package/dist/src/mind/recognition.js +153 -26
  28. package/dist/src/mind/traverse.d.ts +32 -0
  29. package/dist/src/mind/traverse.js +52 -0
  30. package/dist/src/mind/types.d.ts +61 -18
  31. package/dist/src/mind/types.js +68 -19
  32. package/dist/src/store.d.ts +21 -0
  33. package/dist/src/store.js +21 -0
  34. package/example/train_base.ts +21 -4
  35. package/package.json +1 -1
  36. package/src/canon.ts +28 -0
  37. package/src/geometry.ts +100 -1
  38. package/src/mind/attention.ts +15 -10
  39. package/src/mind/bridge.ts +34 -0
  40. package/src/mind/frame-filler.ts +604 -0
  41. package/src/mind/learning.ts +5 -9
  42. package/src/mind/mechanisms/cast.ts +70 -2
  43. package/src/mind/mechanisms/cover.ts +6 -1
  44. package/src/mind/mechanisms/extraction.ts +27 -0
  45. package/src/mind/mechanisms/recall.ts +236 -37
  46. package/src/mind/mind.ts +166 -18
  47. package/src/mind/pipeline-mechanism.ts +7 -0
  48. package/src/mind/pipeline.ts +33 -1
  49. package/src/mind/prefix-completion.ts +314 -0
  50. package/src/mind/primitives.ts +105 -80
  51. package/src/mind/recognition.ts +151 -23
  52. package/src/mind/traverse.ts +52 -0
  53. package/src/mind/types.ts +104 -44
  54. package/src/store.ts +25 -0
  55. package/test/13-conversation.test.mjs +13 -0
  56. package/test/57-fusion-order.test.mjs +65 -0
  57. package/test/66-query-edge-whitespace.test.mjs +99 -0
  58. package/test/67-climb-anchor-breadth.test.mjs +113 -0
  59. package/test/68-extraction-unanchored.test.mjs +79 -0
  60. package/test/69-frame-filler.test.mjs +115 -0
  61. package/test/70-prefix-completion.test.mjs +170 -0
  62. package/test/71-embedded-canon-equivalence.test.mjs +121 -0
  63. package/test/72-prefix-candidate-supply.test.mjs +114 -0
  64. package/test/73-scaffolding-only-bridge-abstains.test.mjs +178 -0
  65. package/test/74-prefix-trap-not-sprung-early.test.mjs +114 -0
  66. package/test/75-multiturn-context-optimisation.test.mjs +1334 -0
package/src/mind/mind.ts CHANGED
@@ -14,16 +14,18 @@ import { bindSeat, fold, Sema, Space } from "../sema.js";
14
14
  import { Alphabet } from "../alphabet.js";
15
15
  import {
16
16
  bytesToTree,
17
+ contentFoldIncremental,
17
18
  Grid,
18
19
  gridToTree,
19
20
  hilbertBytes,
20
21
  reachThreshold,
21
22
  stackGrids,
22
23
  } from "../geometry.js";
24
+ import type { ContentFold } from "../geometry.js";
23
25
  import { BoundedMap, type Store } from "../store.js";
24
26
  import { SQliteStore } from "../store-sqlite.js";
25
27
  import { type MindConfig, resolveConfig } from "../config.js";
26
- import { type Canon, canonHash, textCanon } from "../canon.js";
28
+ import { type Canon, canonHash, textCanon, textEdgeTrim } from "../canon.js";
27
29
  import {
28
30
  type CandidateSpan,
29
31
  coverSequence,
@@ -108,12 +110,31 @@ export interface Conversation {
108
110
  *
109
111
  * {@link resolvedSubtrees} caches foldTree resolutions at the Sema-node
110
112
  * level. When the pyramid reuses prefix subtrees (identical objects),
111
- * foldTree returns their ids immediately O(suffix) instead of
112
- * O(context) for every tree walk. */
113
+ * foldTree recovers their ids without touching the store. A walk that
114
+ * passes no `visit` callback can stop at a cached subtree outright and is
115
+ * O(suffix); a walk that DOES pass one — recognition and attention both do —
116
+ * still descends in full and spends O(context), banking the elided store
117
+ * probes rather than an elided traversal. That asymmetry is deliberate and
118
+ * load-bearing: see foldTree in primitives.ts. */
113
119
  interface ConversationData {
114
120
  tree: Sema;
115
121
  bytes: Uint8Array;
116
122
  boundaries: number[];
123
+ /** The plain fold's reusable segment state (see {@link ContentFold}). A
124
+ * grown context reuses every content segment it already folded and folds
125
+ * only the new turn — O(turn) instead of O(context) — and, because the
126
+ * reused segments are the SAME Sema objects, `resolvedSubtrees` (keyed by
127
+ * node identity) hits across turns, so recognition recovers the prefix's
128
+ * ids without re-probing the store for any of them. It still WALKS the
129
+ * prefix — it must, or it would emit fewer sites on a warm cache than a
130
+ * cold one (see foldTree) — so the saving is in probes, not in traversal.
131
+ * Undefined until the first grow.
132
+ *
133
+ * No turn boundaries are involved: reuse comes from content cuts being
134
+ * stable under append, and imposing boundaries would only change the tree
135
+ * away from what the deposit path folded. `boundaries` beside this field
136
+ * is API metadata, not an input to the fold. */
137
+ content?: ContentFold;
117
138
  answeredSpans: Array<[number, number]>;
118
139
  perceiveMemo: Map<string, Sema>;
119
140
  recogniseMemo: Map<string, Recognition>;
@@ -130,6 +151,7 @@ import {
130
151
  inputBytes,
131
152
  latin1Key,
132
153
  perceive as perceiveImpl,
154
+ perceiveKey,
133
155
  read,
134
156
  resolve as resolveImpl,
135
157
  } from "./primitives.js";
@@ -465,8 +487,9 @@ export class Mind implements MindContext {
465
487
  * serves BOTH entry points: `respond` takes fresh per-response memos,
466
488
  * `respondTurn` passes its conversation, whose memos persist across turns
467
489
  * (content-keyed, so the previous turn's results are found by this turn's
468
- * sub-span calls) and whose `resolvedSubtrees` makes foldTree O(suffix)
469
- * instead of O(context). respondTurn used to inline its own copy of this
490
+ * sub-span calls) and whose `resolvedSubtrees` spares foldTree the store
491
+ * probes for every prefix subtree and, for walks that pass no visitor,
492
+ * the descent as well. respondTurn used to inline its own copy of this
470
493
  * and of {@link endResponse}; the two drifted (a memo added to one was
471
494
  * silently absent from the other), so there is exactly one pair now. */
472
495
  private beginResponse(
@@ -605,6 +628,20 @@ export class Mind implements MindContext {
605
628
  };
606
629
  }
607
630
 
631
+ /** Answer ONE self-contained input.
632
+ *
633
+ * A MULTI-TURN context is not that, and this is the wrong entry point for
634
+ * it. `respond` folds the bytes it is handed with no boundary set, because
635
+ * nothing in a flat byte string says where one turn ended — only the caller
636
+ * who assembled it knows, which is the whole reason `boundaries` is a
637
+ * parameter of {@link perceiveImpl} and never inferred from content. A
638
+ * conversation deposited through {@link ingest} folds its contexts over
639
+ * those turn boundaries, so a hand-concatenated transcript passed here
640
+ * folds differently from the way it was learnt and reaches the trained
641
+ * context node only by luck (measured on a 7-turn conversation: 5/7 here
642
+ * against 7/7 through {@link respondTurn}, same bytes). Use
643
+ * {@link beginConversation} + {@link respondTurn}, or {@link addTurn} to
644
+ * replay turns the Mind should hear but not answer. */
608
645
  async respond(
609
646
  input: Input,
610
647
  inspectRationale?: InspectRationale,
@@ -613,8 +650,43 @@ export class Mind implements MindContext {
613
650
  // through the generic entry point. Raw bytes / grids carry only the
614
651
  // Mind-level canon option, if any.
615
652
  const canon = this._canonFor(typeof input === "string" ? textCanon : null);
653
+ // EDGE WHITESPACE IS NOT PART OF THE QUESTION — trim it once, here, so
654
+ // every mechanism downstream sees the same question regardless of how the
655
+ // caller spaced it. See canon.ts's textEdgeTrim for why the outer edges of a
656
+ // whole input are exactly where canon.ts's no-trimming hazard cannot arise.
657
+ // Gated on the SAME modality test as the canonicalizer above: for bytes and
658
+ // grids 0x20 is content, and nothing is trimmed.
659
+ //
660
+ // Measured on the 15.7M-node store: without this, one leading space took
661
+ // `Who wrote Romeo and Juliet?` and `What is the chemical symbol for
662
+ // water?` from answered to silent, because a shift re-seats every fold
663
+ // boundary — the whole of analyze_training.ts's K2 phase-robustness gap.
664
+ // The caller's EXACT bytes are tried first and the trim is a RETRY, not a
665
+ // pre-filter. Trimming up front is asymmetric — it normalises the query but
666
+ // not the stored forms — so it breaks byte-exact identity for a form trained
667
+ // WITH edge whitespace: test/04 deposits [" ice ", "cold"] and asks
668
+ // " ice ", which must keep answering. Retrying preserves that (the raw
669
+ // query resolves on the first pass) while still reaching the padded case
670
+ // (the raw query grounds nothing, the trimmed one does).
671
+ //
672
+ // COST: nothing on any answering path. The retry needs BOTH silence AND
673
+ // edge whitespace on the query, the same "only on the already-failed path"
674
+ // discipline test/44 and the bridge's own trim retry use. The conversation
675
+ // entry point (respondTurn) is deliberately NOT trimmed — it tracks
676
+ // turn-boundary offsets into its accumulated context, and shifting the bytes
677
+ // under those offsets would desync them.
678
+ const bytes = inputBytes(this, input);
679
+ const first = await this._respondImpl(
680
+ bytes,
681
+ inspectRationale,
682
+ "respond",
683
+ canon,
684
+ );
685
+ if (first.bytes.length > 0 || typeof input !== "string") return first;
686
+ const trimmed = textEdgeTrim(bytes);
687
+ if (trimmed.length === bytes.length || trimmed.length === 0) return first;
616
688
  return this._respondImpl(
617
- inputBytes(this, input),
689
+ trimmed,
618
690
  inspectRationale,
619
691
  "respond",
620
692
  canon,
@@ -651,7 +723,22 @@ export class Mind implements MindContext {
651
723
  beginConversation(state?: ConversationState): Conversation {
652
724
  const id = this._nextConvId++;
653
725
  const initBytes = state?.context ?? new Uint8Array(0);
654
- const initBoundaries = state?.boundaries ? [...state.boundaries] : [];
726
+ // NORMALISE CALLER-SUPPLIED BOUNDARIES. `boundaries` is documented
727
+ // strictly increasing and every boundary this class produces is (they are
728
+ // appended as the context grows), but a restored {@link ConversationState}
729
+ // comes from OUTSIDE — hand-built, migrated, or round-tripped through a
730
+ // store that did not preserve order. The folds consume boundaries with a
731
+ // sequential `b > prev` filter, so an out-of-order entry is silently
732
+ // DROPPED rather than rejected, and the conversation would then fold over
733
+ // a different cut set than the one the caller believes it restored.
734
+ // `bytesToTree` used to sort on the way in and absorbed this; the
735
+ // incremental fold this now calls does not, so the normalisation belongs
736
+ // here, at the one public door untrusted boundaries come through.
737
+ const initBoundaries = state?.boundaries
738
+ ? [...new Set(state.boundaries)]
739
+ .filter((b) => b > 0 && b < initBytes.length)
740
+ .sort((a, b) => a - b)
741
+ : [];
655
742
  const initAnswered = state?.answeredSpans
656
743
  ? state.answeredSpans.map(([start, end]) =>
657
744
  [start, end] as [number, number]
@@ -661,16 +748,18 @@ export class Mind implements MindContext {
661
748
  ? [[start, cuts[i + 1]] as [number, number]]
662
749
  : []
663
750
  );
664
- const tree = bytesToTree(
751
+ // The same incremental fold `_growContext` uses, so a RESTORED
752
+ // conversation starts with segment state its next turn can reuse — a
753
+ // resumed conversation is otherwise identical to a live one and must not
754
+ // pay a full re-fold on every turn for the rest of its life.
755
+ const restored = contentFoldIncremental(
665
756
  this.space,
666
757
  this.alphabet,
667
758
  initBytes,
668
- undefined,
669
- undefined,
670
- initBoundaries.length > 0 ? initBoundaries : undefined,
671
759
  );
672
760
  this._conversations.set(id, {
673
- tree,
761
+ tree: restored.tree,
762
+ content: restored.fold,
674
763
  bytes: initBytes,
675
764
  boundaries: initBoundaries,
676
765
  answeredSpans: initAnswered,
@@ -707,7 +796,41 @@ export class Mind implements MindContext {
707
796
  * This is the primitive for turns the Mind should hear but not answer:
708
797
  * replaying a transcript, feeding the OTHER speaker's line in a
709
798
  * prediction harness, or restoring context piecewise. {@link
710
- * respondTurn} = addTurn + think + its own reply appended the same way. */
799
+ * respondTurn} = addTurn + think + its own reply appended the same way.
800
+ *
801
+ * ── ON SEPARATORS: THERE IS NO SEPARATOR QUESTION ────────────────────
802
+ *
803
+ * "Never a separator" above says what this method DOES — it appends the
804
+ * bytes you give it and records an OFFSET — not that separator bytes are
805
+ * forbidden, unsupported, or something the engine must be taught about.
806
+ * Sema is agnostic to them, and reviewers keep mistaking that agnosticism
807
+ * for a constraint. To be explicit, because the mistake is easy:
808
+ *
809
+ * 1. A turn boundary is an OFFSET, held here, in `boundaries`. It is
810
+ * never a character the geometry scans for. Nothing downstream asks
811
+ * "what byte separates two turns?" because nothing downstream finds
812
+ * boundaries by looking at content at all.
813
+ * 2. A separator in a CORPUS is ordinary content. If a trainer joins
814
+ * turns with "\n" (example/train_base.ts does), those newlines are
815
+ * simply bytes inside the stream, folded like every other byte. They
816
+ * are a property of that corpus, not of this API and not of the fold.
817
+ * 3. This API can therefore reproduce ANY corpus exactly, with no
818
+ * convention to agree on: replaying a "\n"-joined corpus means passing
819
+ * `"\n" + turnText` as the turn. The separator rides along IN the
820
+ * turn bytes, where it belongs. There is nothing to configure and no
821
+ * mode to select.
822
+ * 4. Inference is not exact-match anyway. Recognition works over
823
+ * sub-spans, canonical equivalence and resonance, so a query that
824
+ * differs from the trained bytes by punctuation or whitespace still
825
+ * reaches the trained forms; it degrades, it does not fail closed.
826
+ *
827
+ * What follows from 1–4: differing separator bytes between a corpus and a
828
+ * query is an ordinary CONTENT difference — the same kind as any other
829
+ * wording difference — and it is measured the same way. It is NOT an
830
+ * incompatibility between the trainer and this API, and it does NOT
831
+ * require choosing a project-wide separator convention. A review that
832
+ * concludes otherwise (this one did, before being corrected) has mistaken
833
+ * its own harness feeding untrained bytes for an architectural defect. */
711
834
  addTurn(conv: Conversation, turn: Input): ConversationState {
712
835
  const data = this._conversations.get(conv.id);
713
836
  if (!data) throw new Error(`Conversation ${conv.id} not found`);
@@ -730,17 +853,42 @@ export class Mind implements MindContext {
730
853
  if (!grow) return data.tree;
731
854
  const grown = prevLen > 0 ? concat2(data.bytes, turnBytes) : turnBytes;
732
855
  if (prevLen > 0) data.boundaries.push(prevLen);
733
- const tree = bytesToTree(
856
+ // THE PLAIN FOLD, INCREMENTALLY. No boundary set is imposed here: the
857
+ // tree is exactly the tree `perceive(grown)` builds for these bytes, which
858
+ // is exactly the tree the DEPOSIT path folded when it learnt them. That
859
+ // agreement is the whole point — it is what lets a cumulative context
860
+ // resolve to its trained node, and when it was absent the alignment family
861
+ // went quadratic (measured: 5.2M cells on a 476-byte context, against 0
862
+ // when the two sides agree).
863
+ //
864
+ // The optimisation is unaffected by dropping the boundaries, because it
865
+ // never came from them: content cuts are stable under append, so the
866
+ // incremental fold reuses every segment left of the new turn as the SAME
867
+ // object (see contentFoldIncremental). That object identity is what
868
+ // `resolvedSubtrees` — a WeakMap keyed by node identity — needs in order
869
+ // to hit at all. Measured against the stable-prefix fold it replaces:
870
+ // ~40 rebuilt nodes per turn either way, flat as the context grows
871
+ // sevenfold, and ~92% of nodes reused by identity in both.
872
+ //
873
+ // `data.boundaries` is still tracked, and is still exact — it is API
874
+ // metadata (ConversationState, answeredSpans, currentTurnStart), not a
875
+ // fold instruction.
876
+ const folded = contentFoldIncremental(
734
877
  this.space,
735
878
  this.alphabet,
736
879
  grown,
737
- undefined,
738
- undefined,
739
- data.boundaries.length > 0 ? data.boundaries : undefined,
880
+ data.content,
740
881
  );
882
+ const tree = folded.tree;
883
+ data.content = folded.fold;
741
884
  data.tree = tree;
742
885
  data.bytes = grown;
743
- data.perceiveMemo.set(latin1Key(grown), tree);
886
+ // Seeded under the PLAIN content key, and that is now the only key there
887
+ // is: with no boundary set imposed, this tree IS what `perceive(grown)`
888
+ // computes, so the memo entry is an ordinary cache hit rather than the
889
+ // deliberate alias it had to be while the two folds differed. The entry
890
+ // saves the pipeline re-folding the context it was just handed.
891
+ data.perceiveMemo.set(perceiveKey(grown), tree);
744
892
  return tree;
745
893
  }
746
894
 
@@ -587,6 +587,13 @@ export interface MechanismResult {
587
587
  unexplained: string;
588
588
  /** Explicit weight override. When absent, weight = moves + PASS·unaccounted. */
589
589
  weight?: number;
590
+ /** Bytes of `bytes` that came from spans nothing recognised — the asker's
591
+ * own words carried through verbatim rather than derived (see
592
+ * {@link liftedScaffolding}). Reported, not priced: the ladder prices what
593
+ * a candidate leaves UNACCOUNTED, and this orders candidates that tie on
594
+ * exactly that. Omit when a mechanism composes its answer entirely from
595
+ * recognised material, which is the usual case. */
596
+ scaffolding?: number;
590
597
  /** Override the mechanism's default provenance for this result.
591
598
  * When absent, the pipeline uses `mech.provenance`. */
592
599
  provenance?: string;
@@ -189,6 +189,10 @@ export async function think(
189
189
  accounted: ReadonlyArray<[number, number]>;
190
190
  unexplained: string;
191
191
  complete?: boolean;
192
+ /** Bytes of this candidate's ANSWER that came from spans nothing
193
+ * recognised — query words carried through verbatim (see
194
+ * {@link liftedScaffolding}). Absent means none/unreported. */
195
+ scaffolding?: number;
192
196
  }
193
197
  const grade = (w: number) => Math.floor(w / STEP);
194
198
  const unaccounted = (spans: ReadonlyArray<[number, number]>): number =>
@@ -205,7 +209,34 @@ export async function think(
205
209
  if (c.bytes.length === 0) return;
206
210
  if (ctx.meter) ctx.meter.candidates++;
207
211
  candidates.push(c);
208
- if (best === null || grade(c.weight) < grade(best.weight)) best = c;
212
+ if (best === null) {
213
+ best = c;
214
+ return;
215
+ }
216
+ const g = grade(c.weight), gb = grade(best.weight);
217
+ if (g < gb) {
218
+ best = c;
219
+ return;
220
+ }
221
+ // TIE-BREAK: AT EQUAL GRADE, PREFER THE ANSWER THAT INVENTS LESS.
222
+ //
223
+ // The ladder prices what a candidate leaves UNACCOUNTED, which is the
224
+ // right primary question but cannot separate two candidates that leave
225
+ // the same bytes unaccounted — and then the winner is whichever mechanism
226
+ // happened to be considered first, which is not a reason.
227
+ //
228
+ // What still separates them is what they DID with those bytes. A
229
+ // candidate that carries an unexplained span into its answer is passing
230
+ // the asker's own words back as if they were derived; one that leaves
231
+ // them out has made a smaller, honest claim. Measured on test/22's
232
+ // two-fact chain: cover and recall both graded 11001 over 11 unexplained
233
+ // bytes, cover answering "The capital of France is Paris famous for" (11
234
+ // bytes of scaffolding) against recall's crossing of the hop (0). Order
235
+ // alone decided it, and the shallower reading won.
236
+ //
237
+ // This never overrides the ladder — it only orders within one grade, so
238
+ // coverage and moves still dominate exactly as before.
239
+ if (g === gb && (c.scaffolding ?? 0) < (best.scaffolding ?? 0)) best = c;
209
240
  };
210
241
  const worthRunning = (floor: number) =>
211
242
  best === null || grade(floor) < grade(best.weight);
@@ -260,6 +291,7 @@ export async function think(
260
291
  accounted: r.accounted,
261
292
  unexplained: r.unexplained,
262
293
  complete: r.complete,
294
+ scaffolding: r.scaffolding,
263
295
  });
264
296
  }
265
297
  }
@@ -0,0 +1,314 @@
1
+ // prefix-completion.ts — Grounding a query that IS the opening of a trained
2
+ // form.
3
+ //
4
+ // THE SHAPE. `The capital of France is` grounds nothing, while
5
+ // `The capital of France is Paris.` is trained and reads back byte-exact. The
6
+ // query is not SIMILAR to that form, it is a PROPER PREFIX of it: every query
7
+ // byte is a literal match, in order, from offset zero. That is the strongest
8
+ // grounding relation in the store — stronger than the bridge's corroborated
9
+ // substitution, which pays a CONCEPT per substituted span, and stronger than
10
+ // resonance, which only claims an angle. Nothing is invented: the answer IS a
11
+ // trained form, voiced whole.
12
+ //
13
+ // NO NOTION OF TEXT. This mechanism reads bytes and geometry only. It has no
14
+ // separator, no character class, no "word": the only structural quantity it
15
+ // uses is W, the river's grouping window, which is the same capacity the
16
+ // perception tree groups by and the same bar the argument-binding tier holds
17
+ // its constituents to. A completion shorter than one grouping window carries
18
+ // no structure the geometry can perceive, whatever the modality — that is a
19
+ // statement about the fold, not about punctuation. Presentation (what is
20
+ // "spacing", what is "case") belongs to the injected canon and to the modality
21
+ // entry point, never here; see src/canon.ts.
22
+ //
23
+ // WHY THE EARLIER TIERS CANNOT DO IT. Two independent reasons, both measured:
24
+ //
25
+ // 1. `resolve(prefix)` is null. A proper prefix of a deposited stream has no
26
+ // branch of its own unless it was itself deposited, so the exact tiers
27
+ // have nothing to find.
28
+ // 2. The form is not among the resonance candidates AT ALL. Measured on the
29
+ // trained store: cos(query, that form) = 0.5752, yet the form is absent
30
+ // from `resonate(k)` at k = 24, 256 AND 2048 — while forms scoring LOWER
31
+ // (Germany 0.5670, Yemen 0.5591) are returned. `k` only reorders WITHIN
32
+ // the IVF clusters already probed, exactly as Store.resonate's doc warns,
33
+ // so no k recovers it. With `exhaustive` it ranks 8.
34
+ //
35
+ // So this is a RETRIEVABILITY gap, not a semantic one, and it is repaired by
36
+ // reading the candidate list recall's refusal path has ALREADY fetched
37
+ // exhaustively for the substitution bridge — never by resonating on its own.
38
+ // Measured cost of the scan over those 570 candidates: 2.9 ms warm, 20.4 ms
39
+ // cold, against a ~700 ms refusal path. Issuing a FRESH exhaustive call would
40
+ // cost 490 ms median against 13 ms non-exhaustive (36×), which is why this tier
41
+ // takes the candidate list as an argument and adds nothing to it.
42
+ //
43
+ // THREE GUARDS, each falsified into existence by measurement — do not drop any:
44
+ //
45
+ // 1. AN UNREADABLE CONTINUATION VETOES. Reads are bounded (a stored span can
46
+ // run to hundreds of kilobytes), so a candidate that opens with the query
47
+ // but SATURATES the read continues in a way nobody can see. It is a
48
+ // standing disagreement: if any such candidate exists, nothing is grounded.
49
+ // It must NOT be quietly skipped, and that is not a stylistic point — the
50
+ // skip is what MANUFACTURES a fragment. Measured on a one-deposit fixture
51
+ // whose form exceeds the cap: the query matched BOTH the whole 138-byte
52
+ // form (saturating) AND an interior fold node of 34 bytes (unsaturated,
53
+ // continuing `" Paris, an"`). Skipping the saturated candidate removed the
54
+ // only evidence that disagreed, uniqueness then passed on the interior
55
+ // node, and a mid-form slice was voiced as an answer. Suppressing the
56
+ // disagreement is what created the fabrication.
57
+ // (Testing instead whether a candidate is a "complete form" via the fold
58
+ // does NOT work and was measured: content addressing makes an interior
59
+ // node resolve to ITSELF, so self-resolution says nothing about
60
+ // completeness.)
61
+ // 2. THE CONTINUATION MUST REACH ONE GROUPING WINDOW. A trained
62
+ // `What is the capital of France??` opens with `What is the capital of
63
+ // France?` and continues by a single byte. Below W the continuation is
64
+ // sub-quantum — the fold groups nothing from it — and voicing it produces
65
+ // the degenerate reply that is a known failure smell.
66
+ // 3. UNIQUENESS. Several trained forms may open with the query and continue
67
+ // differently, and then the corpus does not say which continuation the
68
+ // asker means. Distinct continuations ⇒ refuse. This is the documented
69
+ // PREFIX TRAP, and it is real — just not for every prefix. Measured: of
70
+ // 15 battery probes exactly ONE yields a unique continuation, and all
71
+ // three honest-silence probes yield none (including `What is the capital
72
+ // of Zamunda?`, whose top hit scores 0.83).
73
+ //
74
+ // Uniqueness is judged on the continuation BYTES, not on the candidate id: the
75
+ // same continuation reached through two trained forms is one answer, not an
76
+ // ambiguity.
77
+
78
+ import type { MindContext } from "./types.js";
79
+ import { bytesEqual } from "../bytes.js";
80
+ import { rItem } from "./trace.js";
81
+ import { canonicalWindows, leafIdPrefix } from "./canonical.js";
82
+ import { hubBound } from "./traverse.js";
83
+
84
+ /** Trained forms the query may OPEN, proposed from the write side's own
85
+ * leaf-id window index — the supply of last resort for {@link
86
+ * prefixCompletion}.
87
+ *
88
+ * WHY A SECOND SUPPLY EXISTS. The ranked list this mechanism normally reads
89
+ * is a resonance list, and resonance cannot rank a proper prefix: measured on
90
+ * the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
91
+ * truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
92
+ * Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
93
+ * bug, so no k and no re-ranking recovers it.
94
+ *
95
+ * WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT. A query's own fold is
96
+ * useless here: content addressing is not phrase-position-invariant, so a
97
+ * standalone prefix folds to a DIFFERENT node than the same bytes sitting
98
+ * inside a longer deposit, and neither the prefix's own node nor its
99
+ * ancestors lead to the deposit (measured: the 22-byte prefix of the
100
+ * photosynthesis form resolves, is shared by 6 contexts, and does not have
101
+ * the form among its ancestors). Leaf ids ARE position-invariant — they are
102
+ * content-addressed on single bytes — and `indexSubSpans` already interns a
103
+ * flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
104
+ * containment edges to the chunks that window spans. A query that is a
105
+ * prefix therefore shares those window nodes exactly, and reaches the deposit
106
+ * by climbing containment then parents. Nothing is added to the write side;
107
+ * this reads an index training already built.
108
+ *
109
+ * BOUNDED (§2.8), AND WITH NO NEW THRESHOLD. The window whose containment is
110
+ * SMALLEST carries the most evidence, and one saturated at `hubBound` carries
111
+ * none — that is the same √N reading of "hub" the rest of the mind uses, not
112
+ * a tuned knob. The upward walk spends a budget of `hubBound` nodes and
113
+ * fans out by W, so a hub query enumerates nothing and the caller stays
114
+ * silent rather than guessing (§2.13). Measured on the trained store: the
115
+ * photosynthesis form at a one-byte truncation picks a window with 52
116
+ * containers, visits 446 nodes, and yields exactly ONE candidate that
117
+ * survives the caller's byte compare — the form itself.
118
+ *
119
+ * These are PROPOSALS only. Every candidate still faces the byte-exact
120
+ * prefix compare and all three guards below, so a wrong proposal costs one
121
+ * bounded read and can never be voiced (§2.3). */
122
+ export function prefixCandidates(
123
+ ctx: MindContext,
124
+ query: Uint8Array,
125
+ ): number[] {
126
+ const store = ctx.store;
127
+ const W = ctx.space.maxGroup;
128
+ const run = leafIdPrefix(ctx, query);
129
+ // The widest canonical window is the most discriminative one the write side
130
+ // ever interned; a query too short to spell one carries no window evidence.
131
+ const len = canonicalWindows(W)[1];
132
+ if (run.length < len) return [];
133
+ const bound = hubBound(ctx);
134
+
135
+ let best: number | null = null;
136
+ let bestN = 0;
137
+ for (let off = 0; off + len <= run.length; off++) {
138
+ const wid = store.findBranch(run.slice(off, off + len));
139
+ if (wid === null) continue;
140
+ const n = store.containersSlice(wid, 0, bound).length;
141
+ // Empty says the window spans no chunk; saturated says it is a hub, whose
142
+ // containment discriminates nothing. Neither is evidence.
143
+ if (n === 0 || n >= bound) continue;
144
+ if (best === null || n < bestN) {
145
+ best = wid;
146
+ bestN = n;
147
+ }
148
+ }
149
+ if (best === null) return [];
150
+
151
+ let frontier = store.containersSlice(best, 0, bound);
152
+ const seen = new Set<number>(frontier);
153
+ let budget = bound;
154
+ while (frontier.length > 0 && budget > 0) {
155
+ const next: number[] = [];
156
+ for (const f of frontier) {
157
+ if (budget-- <= 0) break;
158
+ for (const p of store.parentsFirst(f, W)) {
159
+ if (seen.has(p)) continue;
160
+ seen.add(p);
161
+ next.push(p);
162
+ }
163
+ }
164
+ frontier = next;
165
+ }
166
+ return [...seen];
167
+ }
168
+
169
+ /** A trained form the query opens, and the bytes by which it continues. */
170
+ export interface PrefixCompletion {
171
+ /** The trained form whose opening the query is — the answer, voiced whole. */
172
+ id: number;
173
+ /** The form's own bytes. The mechanism grounds a FORM, never a slice of
174
+ * one: slicing at the query's end would cut at an offset the geometry has
175
+ * no reason to treat as a boundary. */
176
+ form: Uint8Array;
177
+ /** The bytes past the query — carried for the rationale and for the
178
+ * uniqueness comparison, not voiced on its own. */
179
+ continuation: Uint8Array;
180
+ }
181
+
182
+ /** The sole trained form the query opens — or null when no candidate opens with
183
+ * it, when the continuation is sub-quantum, when a candidate's continuation
184
+ * cannot be read through, or when the candidates disagree.
185
+ *
186
+ * `ranked` must be a list the caller has ALREADY fetched; this mechanism never
187
+ * resonates on its own (see the header's cost note). */
188
+ export function prefixCompletion(
189
+ ctx: MindContext,
190
+ query: Uint8Array,
191
+ ranked: ReadonlyArray<number>,
192
+ ): PrefixCompletion | null {
193
+ const W = ctx.space.maxGroup;
194
+ const t = ctx.trace?.enter("prefixCompletion", [rItem(query, "query")]);
195
+ const done = (
196
+ hit: PrefixCompletion | null,
197
+ note: string,
198
+ data?: unknown,
199
+ ): PrefixCompletion | null => {
200
+ t?.done(
201
+ hit === null ? [] : [rItem(hit.continuation, "continuation", hit.id)],
202
+ note,
203
+ data,
204
+ );
205
+ return hit;
206
+ };
207
+ // Reads are bounded to phrase scale, the same bound the frame filler uses.
208
+ // A query with no room for a whole grouping window past its own length
209
+ // cannot clear guard 2, so it is not worth a single read.
210
+ const cap = query.length * W;
211
+ if (query.length === 0 || cap < query.length + W) {
212
+ return done(null, "no room for a perceivable continuation within the cap");
213
+ }
214
+
215
+ // Distinct continuations, each with the first form that offered it. Held as
216
+ // a list, not a byte-keyed map: candidates that open with the query are few
217
+ // (measured: 1 on the trained store's winning query), and a linear byte
218
+ // compare needs no string encoding of content. Uniqueness (guard 3) is
219
+ // decided over this list, so the scan cannot stop early — a second
220
+ // continuation IS the refusal, and finding it is the point.
221
+ const found: PrefixCompletion[] = [];
222
+ let opened = 0;
223
+ let unreadable = 0;
224
+ let subQuantum = 0;
225
+ for (const id of ranked) {
226
+ const form = ctx.store.bytesPrefix(id, cap);
227
+ if (form.length <= query.length) continue;
228
+ let opens = true;
229
+ for (let i = 0; i < query.length; i++) {
230
+ if (form[i] !== query[i]) {
231
+ opens = false;
232
+ break;
233
+ }
234
+ }
235
+ if (!opens) continue;
236
+ opened++;
237
+ // Guard 1: a saturated read continues out of sight — a disagreement that
238
+ // cannot be resolved, so it ends the search rather than being skipped.
239
+ if (form.length >= cap) {
240
+ unreadable++;
241
+ continue;
242
+ }
243
+ const rest = form.subarray(query.length);
244
+ // Guard 2: below one grouping window there is no structure to voice.
245
+ if (rest.length < W) {
246
+ subQuantum++;
247
+ continue;
248
+ }
249
+ if (!found.some((f) => bytesEqual(f.continuation, rest))) {
250
+ found.push({ id, form, continuation: rest });
251
+ }
252
+ }
253
+
254
+ const data = {
255
+ candidates: ranked.length,
256
+ opened,
257
+ unreadable,
258
+ subQuantum,
259
+ distinctContinuations: found.length,
260
+ };
261
+ if (unreadable > 0 && found.length > 0) {
262
+ return done(
263
+ null,
264
+ "a form opens with this query but continues past the read bound — " +
265
+ "its continuation cannot be read, so none is licensed",
266
+ data,
267
+ );
268
+ }
269
+ // Guard 2b: A SUB-QUANTUM CONTINUATION IS STILL A DISAGREEMENT. Guard 2
270
+ // refuses to VOICE a below-window continuation, and rightly — there is no
271
+ // structure there to speak. But dropping such a candidate from the
272
+ // uniqueness tally silently converts "the corpus offers many continuations,
273
+ // most of them unvoiceable" into "the corpus offers exactly one", and
274
+ // guard 3 then passes VACUOUSLY on the sole survivor. That is precisely
275
+ // the failure guard 1 documents for unreadable continuations — suppressing
276
+ // the disagreement is what manufactures the answer — so it is answered the
277
+ // same way, and for the same reason.
278
+ //
279
+ // Measured on a 4,300-fact fixture of "what is the value of <i>?": the
280
+ // query "what is the value of" drew candidates continuing " 0?", " 4?",
281
+ // " 8?" (3 bytes, sub-quantum at W=4) and " 10?" (4 bytes). The first
282
+ // three were dropped, leaving one survivor, and the mechanism reported
283
+ // "exactly one trained form" and voiced "the value of 10 is 20" — an
284
+ // arbitrary pick from thousands of equally-good readings, with the
285
+ // evidence of ambiguity discarded on the way.
286
+ //
287
+ // Note this can only ever cause SILENCE, never a different answer: it
288
+ // withholds a completion the corpus does not uniquely license.
289
+ if (subQuantum > 0 && found.length > 0) {
290
+ return done(
291
+ null,
292
+ "other trained forms open with this query but continue below one " +
293
+ "grouping window — the corpus offers competing readings, so no " +
294
+ "single completion is licensed",
295
+ data,
296
+ );
297
+ }
298
+ // Guard 3: the corpus must agree on ONE continuation.
299
+ if (found.length !== 1) {
300
+ return done(
301
+ null,
302
+ found.length === 0
303
+ ? "no trained form opens with this query and continues perceivably"
304
+ : "trained forms open with this query but continue differently — " +
305
+ "the corpus does not say which continuation is meant",
306
+ data,
307
+ );
308
+ }
309
+ return done(
310
+ found[0],
311
+ "one trained form opens with this query, and continues perceivably",
312
+ data,
313
+ );
314
+ }