@hviana/sema 0.1.7 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/example/train_base.ts +29 -4
  2. package/package.json +1 -1
  3. package/src/alu/src/parser.ts +105 -4
  4. package/src/canon.ts +65 -0
  5. package/src/geometry.ts +26 -30
  6. package/src/index.ts +1 -0
  7. package/src/mind/learning.ts +39 -1
  8. package/src/mind/match.ts +29 -6
  9. package/src/mind/mechanisms/cast.ts +33 -13
  10. package/src/mind/mechanisms/confluence.ts +15 -1
  11. package/src/mind/mechanisms/recall.ts +116 -41
  12. package/src/mind/mind.ts +172 -29
  13. package/src/mind/primitives.ts +66 -1
  14. package/src/mind/recognition.ts +10 -0
  15. package/src/mind/types.ts +10 -0
  16. package/src/store-sqlite.ts +68 -0
  17. package/src/store.ts +27 -0
  18. package/test/13-conversation.test.mjs +77 -27
  19. package/test/35-prefix-edge.test.mjs +86 -0
  20. package/dist/example/demo.d.ts +0 -1
  21. package/dist/example/demo.js +0 -39
  22. package/dist/example/train_base.d.ts +0 -87
  23. package/dist/example/train_base.js +0 -2232
  24. package/dist/src/alphabet.d.ts +0 -7
  25. package/dist/src/alphabet.js +0 -33
  26. package/dist/src/alu/src/alu.d.ts +0 -185
  27. package/dist/src/alu/src/alu.js +0 -440
  28. package/dist/src/alu/src/expr.d.ts +0 -61
  29. package/dist/src/alu/src/expr.js +0 -318
  30. package/dist/src/alu/src/index.d.ts +0 -11
  31. package/dist/src/alu/src/index.js +0 -19
  32. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  33. package/dist/src/alu/src/kernel-arith.js +0 -264
  34. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  35. package/dist/src/alu/src/kernel-bits.js +0 -152
  36. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  37. package/dist/src/alu/src/kernel-logic.js +0 -60
  38. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  39. package/dist/src/alu/src/kernel-nd.js +0 -208
  40. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  41. package/dist/src/alu/src/kernel-numeric.js +0 -366
  42. package/dist/src/alu/src/operation.d.ts +0 -168
  43. package/dist/src/alu/src/operation.js +0 -189
  44. package/dist/src/alu/src/parser.d.ts +0 -212
  45. package/dist/src/alu/src/parser.js +0 -469
  46. package/dist/src/alu/src/resonance.d.ts +0 -55
  47. package/dist/src/alu/src/resonance.js +0 -126
  48. package/dist/src/alu/src/text.d.ts +0 -31
  49. package/dist/src/alu/src/text.js +0 -73
  50. package/dist/src/alu/src/value.d.ts +0 -109
  51. package/dist/src/alu/src/value.js +0 -300
  52. package/dist/src/alu/test/alu.test.d.ts +0 -1
  53. package/dist/src/alu/test/alu.test.js +0 -764
  54. package/dist/src/bytes.d.ts +0 -14
  55. package/dist/src/bytes.js +0 -59
  56. package/dist/src/config.d.ts +0 -111
  57. package/dist/src/config.js +0 -91
  58. package/dist/src/derive/src/deduction.d.ts +0 -125
  59. package/dist/src/derive/src/deduction.js +0 -155
  60. package/dist/src/derive/src/index.d.ts +0 -7
  61. package/dist/src/derive/src/index.js +0 -11
  62. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  63. package/dist/src/derive/src/priority-queue.js +0 -73
  64. package/dist/src/derive/src/rewrite.d.ts +0 -56
  65. package/dist/src/derive/src/rewrite.js +0 -100
  66. package/dist/src/derive/src/trie.d.ts +0 -90
  67. package/dist/src/derive/src/trie.js +0 -217
  68. package/dist/src/derive/test/derive.test.d.ts +0 -1
  69. package/dist/src/derive/test/derive.test.js +0 -122
  70. package/dist/src/extension.d.ts +0 -37
  71. package/dist/src/extension.js +0 -7
  72. package/dist/src/geometry.d.ts +0 -148
  73. package/dist/src/geometry.js +0 -517
  74. package/dist/src/index.d.ts +0 -15
  75. package/dist/src/index.js +0 -17
  76. package/dist/src/ingest-cache.d.ts +0 -41
  77. package/dist/src/ingest-cache.js +0 -161
  78. package/dist/src/mind/articulation.d.ts +0 -6
  79. package/dist/src/mind/articulation.js +0 -99
  80. package/dist/src/mind/attention.d.ts +0 -72
  81. package/dist/src/mind/attention.js +0 -898
  82. package/dist/src/mind/canonical.d.ts +0 -29
  83. package/dist/src/mind/canonical.js +0 -88
  84. package/dist/src/mind/graph-search.d.ts +0 -270
  85. package/dist/src/mind/graph-search.js +0 -847
  86. package/dist/src/mind/index.d.ts +0 -5
  87. package/dist/src/mind/index.js +0 -5
  88. package/dist/src/mind/junction.d.ts +0 -95
  89. package/dist/src/mind/junction.js +0 -262
  90. package/dist/src/mind/learning.d.ts +0 -47
  91. package/dist/src/mind/learning.js +0 -201
  92. package/dist/src/mind/match.d.ts +0 -111
  93. package/dist/src/mind/match.js +0 -422
  94. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  95. package/dist/src/mind/mechanisms/alu.js +0 -29
  96. package/dist/src/mind/mechanisms/cast.d.ts +0 -35
  97. package/dist/src/mind/mechanisms/cast.js +0 -447
  98. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  99. package/dist/src/mind/mechanisms/confluence.js +0 -213
  100. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  101. package/dist/src/mind/mechanisms/cover.js +0 -179
  102. package/dist/src/mind/mechanisms/extraction.d.ts +0 -67
  103. package/dist/src/mind/mechanisms/extraction.js +0 -342
  104. package/dist/src/mind/mechanisms/recall.d.ts +0 -13
  105. package/dist/src/mind/mechanisms/recall.js +0 -166
  106. package/dist/src/mind/mind.d.ts +0 -211
  107. package/dist/src/mind/mind.js +0 -433
  108. package/dist/src/mind/pipeline-mechanism.d.ts +0 -142
  109. package/dist/src/mind/pipeline-mechanism.js +0 -213
  110. package/dist/src/mind/pipeline.d.ts +0 -20
  111. package/dist/src/mind/pipeline.js +0 -185
  112. package/dist/src/mind/primitives.d.ts +0 -54
  113. package/dist/src/mind/primitives.js +0 -184
  114. package/dist/src/mind/rationale.d.ts +0 -134
  115. package/dist/src/mind/rationale.js +0 -162
  116. package/dist/src/mind/reasoning.d.ts +0 -15
  117. package/dist/src/mind/reasoning.js +0 -162
  118. package/dist/src/mind/recognition.d.ts +0 -20
  119. package/dist/src/mind/recognition.js +0 -236
  120. package/dist/src/mind/resonance.d.ts +0 -23
  121. package/dist/src/mind/resonance.js +0 -0
  122. package/dist/src/mind/trace.d.ts +0 -15
  123. package/dist/src/mind/trace.js +0 -73
  124. package/dist/src/mind/traverse.d.ts +0 -113
  125. package/dist/src/mind/traverse.js +0 -488
  126. package/dist/src/mind/types.d.ts +0 -176
  127. package/dist/src/mind/types.js +0 -84
  128. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  129. package/dist/src/rabitq-ivf/src/database.js +0 -201
  130. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  131. package/dist/src/rabitq-ivf/src/index.js +0 -4
  132. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  133. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  134. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  135. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  136. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  137. package/dist/src/rabitq-ivf/src/rabitq.js +0 -283
  138. package/dist/src/sema.d.ts +0 -31
  139. package/dist/src/sema.js +0 -63
  140. package/dist/src/store-sqlite.d.ts +0 -176
  141. package/dist/src/store-sqlite.js +0 -890
  142. package/dist/src/store.d.ts +0 -642
  143. package/dist/src/store.js +0 -1612
  144. package/dist/src/vec.d.ts +0 -31
  145. package/dist/src/vec.js +0 -109
@@ -13,8 +13,9 @@ import {
13
13
  } from "../../geometry.js";
14
14
  import type { MindContext } from "../types.js";
15
15
  import { gistOf, read, resolve } from "../primitives.js";
16
+ import { bytesEqual, indexOf } from "../../bytes.js";
16
17
  import { corpusN, hubBound } from "../traverse.js";
17
- import { project, reverseContext } from "../match.js";
18
+ import { follow, project, reverseContext } from "../match.js";
18
19
  import { CONCEPT, STEP } from "../graph-search.js";
19
20
  import { unexplainedLabel } from "../rationale.js";
20
21
  import type { PipelineMechanism, Precomputed } from "../pipeline-mechanism.js";
@@ -81,6 +82,51 @@ export async function recallByResonance(
81
82
  }
82
83
  }
83
84
 
85
+ // 0b. ARGUMENT BINDING (RC8): the query is not itself a stored form, but
86
+ // it CONTAINS a recognised constituent that is an edge SOURCE — a learnt
87
+ // pair's left side carried inside a wrapper ("How do you say 'thank you'
88
+ // in French?"). The wrapper is scaffolding; the argument is the span
89
+ // that LEADS somewhere, so its continuation — guided by the whole query's
90
+ // gist — is the answer. Matching the wrapper while ignoring the argument
91
+ // (the observed "good morning" template failure) is worse than silence,
92
+ // so anything short of ONE unambiguous binding falls through: the
93
+ // constituent bar is the same two-quanta (2W) reading confluence binds
94
+ // under, nested recognitions collapse to their MAXIMAL span, and two
95
+ // distinct maximal arguments mean the query asks about neither alone.
96
+ if (qId === null) {
97
+ const W2 = 2 * ctx.space.maxGroup;
98
+ const args = pre.rec.sites.filter((s) =>
99
+ s.end - s.start >= W2 &&
100
+ s.end - s.start < query.length &&
101
+ ctx.store.hasNext(s.payload)
102
+ );
103
+ // Maximal spans by one sorted sweep (starts ascending, ties longest
104
+ // first): every earlier span starts at or before s, so s is contained
105
+ // exactly when the running max end already covers it. O(m log m) — a
106
+ // long input recognises O(|input|) sites, and a pairwise scan here was
107
+ // quadratic in the input.
108
+ args.sort((a, b) => a.start - b.start || b.end - a.end);
109
+ const maximal: typeof args = [];
110
+ let maxEnd = -1;
111
+ for (const s of args) {
112
+ if (s.end <= maxEnd) continue;
113
+ maximal.push(s);
114
+ maxEnd = s.end;
115
+ }
116
+ if (maximal.length === 1) {
117
+ const arg = maximal[0];
118
+ const g = await follow(ctx, arg.payload, queryGist);
119
+ if (g !== null && g.length > 0) {
120
+ return ground(
121
+ g,
122
+ "argument binding — the query's sole edge-source constituent, continuation followed",
123
+ [[arg.start, arg.end]],
124
+ STEP,
125
+ );
126
+ }
127
+ }
128
+ }
129
+
84
130
  const whole = await ctx.store.resonate(queryGist, k);
85
131
  if (whole.length === 0) {
86
132
  return ground(null, "empty store — nothing to resonate with", [], 0);
@@ -99,11 +145,27 @@ export async function recallByResonance(
99
145
  // into bytes — at most one river window (see {@link identityBar}). A
100
146
  // fixed cosine bar let long queries claim "near-identical" while whole
101
147
  // windows — an answer word — differed.
102
- if (
103
- top.score >= identityBar(ctx.store.D, ctx.space.maxGroup, query.length)
104
- ) {
148
+ // A hit RESTATES the query when its bytes are the query's own — exactly,
149
+ // or under the response's equivalence (a case/width twin). Restating
150
+ // hits may only conclude through disciplined reverse recall: voicing
151
+ // their bytes echoes the query back at itself (never an answer — the
152
+ // same principle that keeps cast from voicing stored questions), and
153
+ // projecting them forward is reverse recall's containment failure in the
154
+ // other direction — "whatever followed these bytes in some document".
155
+ const qKey = ctx.canon ? ctx.canon(query) : query;
156
+ const restates = (b: Uint8Array): boolean =>
157
+ bytesEqual(b, query) ||
158
+ (ctx.canon !== null && bytesEqual(ctx.canon(b), qKey));
159
+ const idBar = identityBar(ctx.store.D, ctx.space.maxGroup, query.length);
160
+ if (top.score >= idBar) {
105
161
  for (const h of whole) {
106
- if (h.id === qId || h.score >= 1.0) {
162
+ // The identity claim is PER HIT, not per tier: hits are ranked
163
+ // nearest-first, and grounding one below the bar under this tier's
164
+ // "near-identical" label would launder byte-overlap noise (observed:
165
+ // "merci" projecting through the unrelated near hit "meraih").
166
+ if (h.score < idBar) break;
167
+ const own = read(ctx, h.id);
168
+ if (h.id === qId || restates(own)) {
107
169
  const rev = ctx.store.prevFirst(h.id, hubBound(ctx));
108
170
  const g = reverseContext(ctx, h.id, queryGist, rev);
109
171
  if (g !== null) {
@@ -116,15 +178,7 @@ export async function recallByResonance(
116
178
  STEP,
117
179
  );
118
180
  }
119
- const own = read(ctx, h.id);
120
- if (own.length > 0) {
121
- return ground(
122
- own,
123
- "perfect self-match — the query IS this node",
124
- nothing,
125
- STEP,
126
- );
127
- }
181
+ continue;
128
182
  }
129
183
  const g = await project(ctx, h.id, queryGist);
130
184
  if (g) {
@@ -138,31 +192,11 @@ export async function recallByResonance(
138
192
  }
139
193
  }
140
194
 
141
- // 2. Scaffolding-dominated.
142
- if (top.score >= significanceBar(ctx.store.D)) {
143
- const N = corpusN(ctx);
144
- const minVote = consensusFloor(N);
145
- // The committed points of attention ARE the shared climb's roots (same
146
- // query, same k, same DF mode) — read them from Precomputed instead of
147
- // re-climbing, so even a traced response pays for the climb once.
148
- const forest = (await pre.attention()).roots;
149
- if (forest.length > 0 && forest[0].vote >= minVote) {
150
- const g = await project(ctx, forest[0].anchor, queryGist);
151
- if (g) {
152
- return ground(
153
- g,
154
- "scaffolding-dominated query — ground the consensus-climb anchor",
155
- [[forest[0].start, forest[0].end]],
156
- CONCEPT,
157
- );
158
- }
159
- }
160
- }
161
-
162
- // 3. Last resort — gated on the FRACTION OF THE QUERY the grounding
163
- // explains, not the raw cosine. Root gists are unit vectors, but their
164
- // magnitudes are recoverable from the byte lengths (‖·‖ = √len under the
165
- // linear fold): cos = shared/√(lenQ·lenG), so shared/lenQ = cos·√(lenG/lenQ).
195
+ // The query-relative grounding fraction, shared by tiers 2–4 — gated on
196
+ // the FRACTION OF THE QUERY the grounding explains, not the raw cosine.
197
+ // Root gists are unit vectors, but their magnitudes are recoverable from
198
+ // the byte lengths (‖·‖ = √len under the linear fold):
199
+ // cos = shared/√(lenQ·lenG), so shared/lenQ = cos·√(lenG/lenQ).
166
200
  // The raw cosine punished honest containment — a query fully inside a
167
201
  // longer grounded answer scored √(lenQ/lenG) and was refused — and let a
168
202
  // long answer sharing only scaffolding pass; the query-relative fraction
@@ -177,18 +211,50 @@ export async function recallByResonance(
177
211
  // subtract the significance bar (3/√D, §8.3) before converting. Derived
178
212
  // from the existing bars; never tuned.
179
213
  const sig = significanceBar(ctx.store.D);
214
+ const reach = reachThreshold(ctx.space.maxGroup);
180
215
  const fracOfQuery = (cos: number, otherLen: number): number =>
181
216
  Math.min(
182
217
  1,
183
218
  Math.max(0, cos - sig) *
184
219
  Math.sqrt(otherLen / Math.max(1, query.length)),
185
220
  );
221
+
222
+ // 2. Scaffolding-dominated.
223
+ if (top.score >= sig) {
224
+ const N = corpusN(ctx);
225
+ const minVote = consensusFloor(N);
226
+ // The committed points of attention ARE the shared climb's roots (same
227
+ // query, same k, same DF mode) — read them from Precomputed instead of
228
+ // re-climbing, so even a traced response pays for the climb once.
229
+ const forest = (await pre.attention()).roots;
230
+ if (forest.length > 0 && forest[0].vote >= minVote) {
231
+ const g = await project(ctx, forest[0].anchor, queryGist);
232
+ // The anchor cleared the consensus floor, but the floor prices the
233
+ // ANCHOR's evidence, not the projection's: a junk attractor can clear
234
+ // it and project a PIECE OF THE QUERY back at it (the observed
235
+ // "buenos días in English" → "English" fragment). A projection that
236
+ // is a proper byte-subspan of the query restates part of the question
237
+ // — never an answer (the same principle as `restates` above, extended
238
+ // to fragments). Genuine anchor groundings — longer than the query,
239
+ // or disjoint from it — pass untouched.
240
+ if (g && !(g.length < query.length && indexOf(query, g, 0) >= 0)) {
241
+ return ground(
242
+ g,
243
+ "scaffolding-dominated query — ground the consensus-climb anchor",
244
+ [[forest[0].start, forest[0].end]],
245
+ CONCEPT,
246
+ );
247
+ }
248
+ }
249
+ }
250
+
251
+ // 3. Last resort — the nearest grounded whole-query hit, same gate.
186
252
  for (const h of whole) {
187
253
  const g = await project(ctx, h.id, queryGist);
188
254
  if (g) {
189
255
  if (
190
256
  fracOfQuery(cosine(queryGist, gistOf(ctx, g)), g.length) >=
191
- reachThreshold(ctx.space.maxGroup)
257
+ reach
192
258
  ) {
193
259
  return ground(
194
260
  g,
@@ -208,7 +274,6 @@ export async function recallByResonance(
208
274
  // anyway to be echoed, so the decision uses their EXACT fold: one river
209
275
  // fold of the top hit, measured in the same query-relative,
210
276
  // chance-corrected units as the tier above.
211
- const reach = reachThreshold(ctx.space.maxGroup);
212
277
  const topBytes = read(ctx, top.id);
213
278
  const exact = topBytes.length > 0
214
279
  ? cosine(queryGist, gistOf(ctx, topBytes))
@@ -221,6 +286,16 @@ export async function recallByResonance(
221
286
  0,
222
287
  );
223
288
  }
289
+ // Echoing the query's own bytes back at it is not an echo of a RELATED
290
+ // form — it is the query restated, which answers nothing.
291
+ if (restates(topBytes)) {
292
+ return ground(
293
+ null,
294
+ "the nearest form IS the query itself — restating it answers nothing",
295
+ [],
296
+ 0,
297
+ );
298
+ }
224
299
  // Honest echo.
225
300
  return ground(
226
301
  topBytes,
package/src/mind/mind.ts CHANGED
@@ -25,6 +25,7 @@ import {
25
25
  import { BoundedMap, type Store } from "../store.js";
26
26
  import { SQliteStore } from "../store-sqlite.js";
27
27
  import { type MindConfig, resolveConfig } from "../config.js";
28
+ import { type Canon, canonHash, textCanon } from "../canon.js";
28
29
  import {
29
30
  type CandidateSpan,
30
31
  coverSequence,
@@ -160,6 +161,12 @@ export interface MindOptions {
160
161
  mechanismFactories?: ((
161
162
  host: import("../extension.js").ExtensionHost,
162
163
  ) => import("./pipeline-mechanism.js").PipelineMechanism)[];
164
+ /** Content canonicalizer applied to EVERY response (any modality) for
165
+ * equivalence-class resolution — see src/canon.ts. Text entry points
166
+ * ({@link Mind.respondText}, {@link Mind.respondTurnText}) inject the
167
+ * Unicode text canonicalizer automatically when this is unset; pass
168
+ * `false` to disable canonical resolution everywhere. */
169
+ canon?: Canon | false;
163
170
  }
164
171
 
165
172
  // ═══════════════════════════════════════════════════════════════════════════
@@ -182,6 +189,19 @@ export class Mind implements MindContext {
182
189
  /** The live rationale tracer for the inference currently in flight, or null. */
183
190
  trace: Rationale | null = null;
184
191
 
192
+ /** The content canonicalizer for the response in flight — see
193
+ * {@link MindContext.canon}. Injected per response by the modality entry
194
+ * point; null when the response carries no equivalence. */
195
+ canon: Canon | null = null;
196
+
197
+ /** Per-response canonical-resolution memo — see {@link MindContext.canonMemo}. */
198
+ canonMemo: Map<string, number | null> | null = null;
199
+
200
+ /** The Mind-level canon option: a canonicalizer to use for EVERY response,
201
+ * `false` to disable canonical resolution, or null to let each entry
202
+ * point decide (text entry points inject {@link textCanon}). */
203
+ private _canonOpt: Canon | false | null = null;
204
+
185
205
  /** Memo of the consensus climb — content-keyed. See {@link MindContext.climbMemo}. */
186
206
  climbMemo: Map<string, Map<string, AttentionRead>> | null = null;
187
207
 
@@ -285,8 +305,10 @@ export class Mind implements MindContext {
285
305
  store: optsStore,
286
306
  mechanisms: userMechs,
287
307
  mechanismFactories: userFacts,
308
+ canon: optsCanon,
288
309
  ...rest
289
310
  } = (optsOrCfg ?? {}) as MindOptions;
311
+ this._canonOpt = optsCanon ?? null;
290
312
  this.cfg = resolveConfig(rest as Partial<MindConfig>);
291
313
  this.store = optsStore ?? new SQliteStore({
292
314
  maxGroup: this.cfg.geometry.maxGroup,
@@ -378,11 +400,24 @@ export class Mind implements MindContext {
378
400
  /** Open one response's transient state — the tracer and the per-response
379
401
  * memos. Paired with {@link endResponse}; the ONE place this state is
380
402
  * created, so adding a memo cannot forget its reset. */
381
- private beginResponse(inspectRationale?: InspectRationale): void {
403
+ private beginResponse(
404
+ inspectRationale?: InspectRationale,
405
+ canon?: Canon | null,
406
+ ): void {
382
407
  this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
383
408
  this.climbMemo = new Map();
384
409
  this.recogniseMemo = new Map();
385
410
  this.perceiveMemo = new Map();
411
+ this.canon = canon ?? null;
412
+ this.canonMemo = canon ? new Map() : null;
413
+ }
414
+
415
+ /** The canonicalizer a response should carry: the Mind-level option when
416
+ * set (or none when explicitly disabled), else the entry point's own
417
+ * default — text entry points pass {@link textCanon}, binary ones null. */
418
+ private _canonFor(entryDefault: Canon | null): Canon | null {
419
+ if (this._canonOpt === false) return null;
420
+ return this._canonOpt ?? entryDefault;
386
421
  }
387
422
 
388
423
  /** Close one response's transient state — every per-response field, incl.
@@ -392,6 +427,8 @@ export class Mind implements MindContext {
392
427
  this.climbMemo = null;
393
428
  this.recogniseMemo = null;
394
429
  this.perceiveMemo = null;
430
+ this.canon = null;
431
+ this.canonMemo = null;
395
432
  this._edgeGuide = null;
396
433
  this._edgeChoice.clear();
397
434
  }
@@ -403,8 +440,9 @@ export class Mind implements MindContext {
403
440
  queryBytes: Uint8Array,
404
441
  inspectRationale?: InspectRationale,
405
442
  traceLabel = "respond",
443
+ canon: Canon | null = null,
406
444
  ): Promise<Response> {
407
- this.beginResponse(inspectRationale);
445
+ this.beginResponse(inspectRationale, canon);
408
446
  try {
409
447
  const top = this.trace?.enter(traceLabel, [
410
448
  rItem(queryBytes, "query"),
@@ -434,13 +472,27 @@ export class Mind implements MindContext {
434
472
  input: Input,
435
473
  inspectRationale?: InspectRationale,
436
474
  ): Promise<Response> {
437
- return this._respondImpl(inputBytes(this, input), inspectRationale);
475
+ // A STRING input is text by nature: it carries the text equivalence even
476
+ // through the generic entry point. Raw bytes / grids carry only the
477
+ // Mind-level canon option, if any.
478
+ const canon = this._canonFor(typeof input === "string" ? textCanon : null);
479
+ return this._respondImpl(
480
+ inputBytes(this, input),
481
+ inspectRationale,
482
+ "respond",
483
+ canon,
484
+ );
438
485
  }
439
486
 
440
487
  /** Text view of {@link respond}. NUL bytes (0x00) are stripped before
441
488
  * decoding — they are structural padding in text answers. LOSSY for a
442
489
  * binary answer that legitimately contains NULs: use {@link respond} and
443
- * read `bytes` directly for binary/grid modalities. */
490
+ * read `bytes` directly for binary/grid modalities.
491
+ *
492
+ * Injects the TEXT canonicalizer (src/canon.ts) so resolution treats
493
+ * every character variation of the same text — case, width, whitespace —
494
+ * as one form, provided the store's canon index is built
495
+ * ({@link buildCanonIndex}). */
444
496
  async respondText(
445
497
  input: string,
446
498
  inspectRationale?: InspectRationale,
@@ -498,6 +550,52 @@ export class Mind implements MindContext {
498
550
  };
499
551
  }
500
552
 
553
+ /** Append a turn to a conversation's accumulated context WITHOUT
554
+ * responding — raw byte append plus a boundary offset, never a
555
+ * separator; the fold pyramid advances by O(turn).
556
+ *
557
+ * This is the primitive for turns the Mind should hear but not answer:
558
+ * replaying a transcript, feeding the OTHER speaker's line in a
559
+ * prediction harness, or restoring context piecewise. {@link
560
+ * respondTurn} = addTurn + think + its own reply appended the same way. */
561
+ addTurn(conv: Conversation, turn: Input): ConversationState {
562
+ const data = this._conversations.get(conv.id);
563
+ if (!data) throw new Error(`Conversation ${conv.id} not found`);
564
+ const turnBytes = inputBytes(this, turn);
565
+ this._growContext(data, turnBytes);
566
+ return this.conversationState(conv)!;
567
+ }
568
+
569
+ /** Grow a conversation's accumulated context by one turn's bytes — raw
570
+ * append plus a boundary offset, pyramid advanced by O(turn), the grown
571
+ * context's tree seeded into the conversation's perceive memo. The ONE
572
+ * place a context grows ({@link addTurn} and {@link respondTurn} both
573
+ * come through here), so the append semantics cannot drift. */
574
+ private _growContext(data: ConversationData, turnBytes: Uint8Array): Sema {
575
+ const prevLen = data.pyramid.bytes;
576
+ // An empty turn neither grows the context nor marks a boundary —
577
+ // boundaries are documented strictly increasing, and a zero-length
578
+ // "turn" is no turn. The pyramid's zero-growth shortcut makes the
579
+ // refold below O(1), so the existing tree is returned unchanged.
580
+ const grow = turnBytes.length > 0;
581
+ const grown = !grow
582
+ ? data.bytes
583
+ : prevLen > 0
584
+ ? concat2(data.bytes, turnBytes)
585
+ : turnBytes;
586
+ if (grow && prevLen > 0) data.boundaries.push(prevLen);
587
+ const { tree, pyramid } = bytesToTreePyramid(
588
+ this.space,
589
+ this.alphabet,
590
+ grown,
591
+ data.pyramid,
592
+ );
593
+ data.pyramid = pyramid;
594
+ data.bytes = grown;
595
+ data.perceiveMemo.set(latin1Key(grown), tree);
596
+ return tree;
597
+ }
598
+
501
599
  /** Process one turn of a conversation.
502
600
  *
503
601
  * `turn` is the raw input for the latest turn — its bytes are appended
@@ -507,7 +605,13 @@ export class Mind implements MindContext {
507
605
  *
508
606
  * Returns the response AND the updated {@link ConversationState} so the
509
607
  * caller can persist it. The conversation handle's internal state is
510
- * updated in place — the returned state is a snapshot for storage. */
608
+ * updated in place — the returned state is a snapshot for storage.
609
+ *
610
+ * SINGLE FLIGHT: at most one respondTurn may be in flight per Mind. The
611
+ * conversation's memo caches are swapped into the Mind-level per-response
612
+ * pointers for the duration of the turn, so a concurrently-running
613
+ * respond()/respondTurn() on the SAME Mind would interleave state.
614
+ * Different Minds (or sequential awaits, as in every test) are safe. */
511
615
  async respondTurn(
512
616
  conv: Conversation,
513
617
  turn: Input,
@@ -517,22 +621,9 @@ export class Mind implements MindContext {
517
621
  if (!data) throw new Error(`Conversation ${conv.id} not found`);
518
622
 
519
623
  const turnBytes = inputBytes(this, turn);
520
- const prevLen = data.pyramid.bytes;
521
-
522
- const prevBytes = data.bytes;
523
- const newContext = prevLen > 0 ? concat2(prevBytes, turnBytes) : turnBytes;
524
- data.bytes = newContext;
525
-
526
- if (prevLen > 0) data.boundaries.push(prevLen);
527
-
528
624
  // Incremental perception — O(turn) instead of O(context).
529
- const { tree, pyramid } = bytesToTreePyramid(
530
- this.space,
531
- this.alphabet,
532
- newContext,
533
- data.pyramid,
534
- );
535
- data.pyramid = pyramid;
625
+ const tree = this._growContext(data, turnBytes);
626
+ const newContext = data.bytes;
536
627
 
537
628
  // Swap in the conversation's persistent state so the inference
538
629
  // pipeline does not re-process the prefix from scratch.
@@ -547,10 +638,12 @@ export class Mind implements MindContext {
547
638
  this.climbMemo = data.climbMemo;
548
639
  this._resolvedSubtrees = data.resolvedSubtrees;
549
640
  this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
641
+ // A string turn is text by nature — carry the text equivalence, same as
642
+ // respond() (see _canonFor).
643
+ this.canon = this._canonFor(typeof turn === "string" ? textCanon : null);
644
+ this.canonMemo = this.canon ? new Map() : null;
550
645
 
551
646
  try {
552
- this.perceiveMemo.set(latin1Key(newContext), tree);
553
-
554
647
  const top = this.trace?.enter("respondTurn", [
555
648
  rItem(newContext, "query"),
556
649
  ]);
@@ -570,6 +663,14 @@ export class Mind implements MindContext {
570
663
  "the answer, re-voiced in the asker's words",
571
664
  );
572
665
 
666
+ // The REPLY joins the accumulated context the same way a turn does
667
+ // ({@link addTurn}): raw byte append plus a boundary offset — never a
668
+ // separator. A conversation's context is the full exchange, exactly
669
+ // the cumulative continuous shape multi-turn training deposits, so a
670
+ // later turn can refer to what was ANSWERED ("which of those two…"),
671
+ // not only to what was asked.
672
+ if (voiced.length > 0) this.addTurn(conv, voiced);
673
+
573
674
  return {
574
675
  response: {
575
676
  v: gistOf(this, voiced),
@@ -579,17 +680,19 @@ export class Mind implements MindContext {
579
680
  state: this.conversationState(conv)!,
580
681
  };
581
682
  } finally {
582
- // Save back to the conversation for the next turn.
583
- data.perceiveMemo = this.perceiveMemo;
584
- data.recogniseMemo = this.recogniseMemo;
585
- data.climbMemo = this.climbMemo;
586
- data.resolvedSubtrees = this._resolvedSubtrees!;
587
- // Clear Mind references — non-conversation respond() calls get
588
- // fresh per-response memos.
683
+ // No save-back: the conversation's memo MAPS were mutated in place —
684
+ // `data.*` still points at them. Re-assigning from the Mind-level
685
+ // pointers here was a no-op in the single-flight case and, under a
686
+ // concurrently-started respond() (which swaps its own fresh maps into
687
+ // those pointers), would inject a FOREIGN response's memos into this
688
+ // conversation. Clear Mind references — non-conversation respond()
689
+ // calls get fresh per-response memos.
589
690
  this.trace = null;
590
691
  this.perceiveMemo = null;
591
692
  this.recogniseMemo = null;
592
693
  this.climbMemo = null;
694
+ this.canon = null;
695
+ this.canonMemo = null;
593
696
  this._resolvedSubtrees = null;
594
697
  this._edgeGuide = null;
595
698
  this._edgeChoice.clear();
@@ -694,6 +797,46 @@ export class Mind implements MindContext {
694
797
  );
695
798
  }
696
799
 
800
+ // ── Canonical-form index ───────────────────────────────────────────────
801
+
802
+ /** Build (or incrementally refresh) the store's canonical-form index: for
803
+ * every content-bearing node, record the hash of its CANONICAL key so
804
+ * resolution can find stored forms across surface variation (case, width,
805
+ * whitespace — whatever `canon` equates; see src/canon.ts).
806
+ *
807
+ * Incremental and idempotent: the last indexed node id is remembered in
808
+ * store meta (`canon.upto`), so a refresh after further training scans
809
+ * only the new rows. Run once after training, and again after ingests —
810
+ * the same operational shape as {@link repairContentIndex}.
811
+ *
812
+ * @param canon the canonicalizer to index under — MUST be the same one
813
+ * queries will carry (text queries carry {@link textCanon}
814
+ * unless the Mind was constructed with its own)
815
+ * @returns number of index rows added */
816
+ async buildCanonIndex(canon?: Canon): Promise<number> {
817
+ const c = canon ?? this._canonFor(textCanon);
818
+ const store = this.store;
819
+ if (c === null || !store.canonAdd || !store.eachContent) return 0;
820
+ const from = Number(await store.getMeta("canon.upto") ?? 0);
821
+ let added = 0;
822
+ let maxId = from - 1;
823
+ store.eachContent((id, bytes) => {
824
+ if (id > maxId) maxId = id;
825
+ const key = c(bytes);
826
+ if (key.length === 0) return;
827
+ // Only index content whose canonical key DIFFERS from its raw bytes —
828
+ // an already-canonical span is found by the exact lookup (and by the
829
+ // fallback's own exact probe of the canonical bytes), so indexing it
830
+ // would only add rows.
831
+ if (bytesEqual(key, bytes)) return;
832
+ store.canonAdd!(canonHash(key), id);
833
+ added++;
834
+ }, from);
835
+ await store.setMeta("canon.upto", String(maxId + 1));
836
+ store.commit();
837
+ return added;
838
+ }
839
+
697
840
  // ── Persistence ──────────────────────────────────────────────────────────
698
841
 
699
842
  async save(): Promise<Uint8Array> {
@@ -14,6 +14,8 @@ import {
14
14
  hilbertBytes,
15
15
  stackGrids,
16
16
  } from "../geometry.js";
17
+ import { canonHash } from "../canon.js";
18
+ import { bytesEqual } from "../bytes.js";
17
19
  import { ALL } from "./types.js";
18
20
  import type { Input, MindContext } from "./types.js";
19
21
 
@@ -199,7 +201,70 @@ export function foldTree(
199
201
  * unknown. */
200
202
  export function resolve(ctx: MindContext, bytes: Uint8Array): number | null {
201
203
  if (bytes.length === 0) return null;
202
- return foldTree(ctx, perceive(ctx, bytes), 0).node;
204
+ const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
205
+ if (exact !== null) return exact;
206
+ return canonResolve(ctx, bytes);
207
+ }
208
+
209
+ /** Equivalence-class resolution: when the exact content-addressed lookup
210
+ * misses, find a stored node whose CANONICAL key equals the span's — the
211
+ * store's canon index proposes candidates by key hash, and each is verified
212
+ * by re-canonicalizing its bytes (hash-then-verify, like every content
213
+ * lookup). Among verified candidates, one that leads somewhere (has a
214
+ * continuation edge) is preferred; ties break to the lowest id — a corpus
215
+ * property, not a seed property. Null when the response carries no
216
+ * canonicalizer, the store has no canon index, or nothing verifies. */
217
+ export function canonResolve(
218
+ ctx: MindContext,
219
+ bytes: Uint8Array,
220
+ ): number | null {
221
+ const canon = ctx.canon;
222
+ const store = ctx.store;
223
+ if (canon === null || !store.canonFind) return null;
224
+ if (bytes.length < 2) return null;
225
+ const memo = ctx.canonMemo;
226
+ const memoKey = memo ? latin1Key(bytes) : "";
227
+ if (memo) {
228
+ const hit = memo.get(memoKey);
229
+ if (hit !== undefined) return hit;
230
+ }
231
+ const set = (v: number | null): number | null => {
232
+ memo?.set(memoKey, v);
233
+ return v;
234
+ };
235
+ const key = canon(bytes);
236
+ if (key.length === 0) return set(null);
237
+ // A stored form that IS canonical is not in the index (buildCanonIndex
238
+ // skips identity rows) — the exact content-addressed lookup of the
239
+ // canonical bytes finds it directly.
240
+ if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
241
+ const direct = foldTree(ctx, perceive(ctx, key), 0).node;
242
+ if (direct !== null) return set(direct);
243
+ }
244
+ const candidates = store.canonFind(canonHash(key));
245
+ if (candidates.length === 0) return set(null);
246
+ let best: number | null = null;
247
+ let bestLeads = false;
248
+ for (const id of candidates) {
249
+ const bytesOf = read(ctx, id);
250
+ const stored = canon(bytesOf);
251
+ if (stored.length !== key.length || !bytesEqual(stored, key)) continue;
252
+ // The index stores FLAT content twins; the id the exact path would have
253
+ // resolved for these bytes is their FOLD — the deposit-shaped node that
254
+ // carries the edges and halos. Re-folding the candidate's bytes lands
255
+ // on exactly the node the canonical-case query would have found.
256
+ const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
257
+ const use = folded ?? id;
258
+ const leads = store.hasNext(use) || store.haloMass(use) > 0;
259
+ if (
260
+ best === null || (leads && !bestLeads) ||
261
+ (leads === bestLeads && use < best)
262
+ ) {
263
+ best = use;
264
+ bestLeads = leads;
265
+ }
266
+ }
267
+ return set(best);
203
268
  }
204
269
 
205
270
  /** Walk a perceived tree in POST-ORDER with byte offsets — children before
@@ -8,6 +8,7 @@ import { rItem } from "./trace.js";
8
8
 
9
9
  import type { MindContext, Recognition, Segment } from "./types.js";
10
10
  import {
11
+ canonResolve,
11
12
  foldTree,
12
13
  gistOf,
13
14
  latin1Key,
@@ -96,6 +97,15 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
96
97
  leaves.push({ start, end, bytes: n.leaf ?? new Uint8Array(0), node });
97
98
  }
98
99
  if (node !== null) emit(start, end, node);
100
+ // Canonical fallback: a subtree whose exact content-addressed lookup
101
+ // missed may still be a stored form under the response's equivalence
102
+ // (case, width, whitespace — whatever the injected canonicalizer says).
103
+ // O(subtree bytes) per miss, memoised per response; a no-op when no
104
+ // canonicalizer was injected or the store has no canon index.
105
+ else if (end - start >= 2) {
106
+ const cid = canonResolve(ctx, bytes.subarray(start, end));
107
+ if (cid !== null) emit(start, end, cid);
108
+ }
99
109
  if (isChunk(n)) {
100
110
  starts.add(start);
101
111
  // Try every sub-span within this leaf-parent.
package/src/mind/types.ts CHANGED
@@ -144,6 +144,16 @@ export interface MindContext extends GraphSearchHost {
144
144
  cfg: MindConfig;
145
145
  search: GraphSearch;
146
146
  trace: Rationale | null;
147
+ /** The content canonicalizer for THIS response, or null — injected by the
148
+ * modality entry point (respondText passes the text canonicalizer; a
149
+ * binary respond passes none). Resolution uses it as a fallback: when
150
+ * the exact content-addressed lookup misses, the span's canonical key is
151
+ * probed against the store's canon index (see src/canon.ts). The core
152
+ * never inspects what the equivalence IS. */
153
+ canon: ((bytes: Uint8Array) => Uint8Array) | null;
154
+ /** Per-response memo of canonical-fallback resolutions, keyed by the
155
+ * span's latin1 content key. Null outside respond(). */
156
+ canonMemo: Map<string, number | null> | null;
147
157
  /** Memo of the consensus climb — content-keyed (latin1) so results
148
158
  * persist across conversation turns where the same byte spans recur.
149
159
  * Null outside respond(); during respondTurn() the conversation's