@hviana/sema 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/example/train_base.js +42 -9
  2. package/dist/src/alu/src/parser.d.ts +9 -0
  3. package/dist/src/alu/src/parser.js +110 -2
  4. package/dist/src/canon.d.ts +26 -0
  5. package/dist/src/canon.js +57 -0
  6. package/dist/src/geometry.d.ts +13 -2
  7. package/dist/src/geometry.js +101 -2
  8. package/dist/src/index.d.ts +1 -0
  9. package/dist/src/index.js +1 -0
  10. package/dist/src/mind/attention.js +11 -7
  11. package/dist/src/mind/learning.js +33 -1
  12. package/dist/src/mind/match.d.ts +4 -2
  13. package/dist/src/mind/match.js +30 -6
  14. package/dist/src/mind/mechanisms/cast.js +18 -4
  15. package/dist/src/mind/mechanisms/confluence.js +13 -1
  16. package/dist/src/mind/mechanisms/recall.js +115 -31
  17. package/dist/src/mind/mind.d.ts +138 -12
  18. package/dist/src/mind/mind.js +284 -22
  19. package/dist/src/mind/primitives.d.ts +22 -2
  20. package/dist/src/mind/primitives.js +95 -6
  21. package/dist/src/mind/recognition.js +31 -8
  22. package/dist/src/mind/traverse.d.ts +13 -0
  23. package/dist/src/mind/traverse.js +41 -0
  24. package/dist/src/mind/types.d.ts +33 -21
  25. package/dist/src/store-sqlite.d.ts +10 -0
  26. package/dist/src/store-sqlite.js +58 -0
  27. package/dist/src/store.d.ts +23 -0
  28. package/dist/src/store.js +13 -2
  29. package/example/train_base.ts +49 -9
  30. package/index.html +65 -0
  31. package/package.json +1 -1
  32. package/src/alu/src/parser.ts +105 -4
  33. package/src/canon.ts +65 -0
  34. package/src/geometry.ts +105 -1
  35. package/src/index.ts +1 -0
  36. package/src/mind/attention.ts +11 -6
  37. package/src/mind/learning.ts +39 -1
  38. package/src/mind/match.ts +29 -6
  39. package/src/mind/mechanisms/cast.ts +21 -6
  40. package/src/mind/mechanisms/confluence.ts +15 -1
  41. package/src/mind/mechanisms/recall.ts +132 -41
  42. package/src/mind/mind.ts +396 -22
  43. package/src/mind/primitives.ts +109 -7
  44. package/src/mind/recognition.ts +36 -8
  45. package/src/mind/traverse.ts +47 -0
  46. package/src/mind/types.ts +30 -21
  47. package/src/store-sqlite.ts +76 -0
  48. package/src/store.ts +46 -2
  49. package/test/13-conversation.test.mjs +240 -20
  50. package/test/35-prefix-edge.test.mjs +86 -0
@@ -10,15 +10,16 @@
10
10
  // Implementation split across src/mind/*.ts — this file assembles the Mind class.
11
11
  import { makeKeyring, rng, setVecConfig } from "../vec.js";
12
12
  import { Alphabet } from "../alphabet.js";
13
- import { reachThreshold, } from "../geometry.js";
13
+ import { bytesToTreePyramid, reachThreshold, } from "../geometry.js";
14
14
  import { BoundedMap } from "../store.js";
15
15
  import { SQliteStore } from "../store-sqlite.js";
16
16
  import { resolveConfig } from "../config.js";
17
- import { bytesEqual } from "../bytes.js";
17
+ import { canonHash, textCanon } from "../canon.js";
18
+ import { bytesEqual, concat2 } from "../bytes.js";
18
19
  import { GraphSearch, } from "./graph-search.js";
19
20
  import { Alu } from "../alu/src/index.js";
20
21
  import { decodeText, Rationale, } from "./rationale.js";
21
- import { gistOf, inputBytes, perceive as perceiveImpl, resolve as resolveImpl, } from "./primitives.js";
22
+ import { gistOf, inputBytes, latin1Key, perceive as perceiveImpl, resolve as resolveImpl, } from "./primitives.js";
22
23
  import { chooseNext, edgeAncestors as edgeAncestorsFn } from "./traverse.js";
23
24
  import { follow } from "./match.js";
24
25
  import { recognise, segment } from "./recognition.js";
@@ -42,16 +43,24 @@ export class Mind {
42
43
  mechanisms = [];
43
44
  /** The live rationale tracer for the inference currently in flight, or null. */
44
45
  trace = null;
45
- /** Per-response memo of the consensus climb. NOTE: this memo and
46
- * {@link recogniseMemo} are BYPASSED while a rationale trace is attached
47
- * (every mechanism must emit its own steps), so a traced respond re-pays
48
- * up to four consensus climbs plus repeat recognitions — that is where the
49
- * traced-vs-untraced latency multiple comes from, by design. */
46
+ /** The content canonicalizer for the response in flight see
47
+ * {@link MindContext.canon}. Injected per response by the modality entry
48
+ * point; null when the response carries no equivalence. */
49
+ canon = null;
50
+ /** Per-response canonical-resolution memo see {@link MindContext.canonMemo}. */
51
+ canonMemo = null;
52
+ /** The Mind-level canon option: a canonicalizer to use for EVERY response,
53
+ * `false` to disable canonical resolution, or null to let each entry
54
+ * point decide (text entry points inject {@link textCanon}). */
55
+ _canonOpt = null;
56
+ /** Memo of the consensus climb — content-keyed. See {@link MindContext.climbMemo}. */
50
57
  climbMemo = null;
51
- /** Per-response memo of recognise() — see {@link MindContext.recogniseMemo}. */
58
+ /** Memo of recognise() — content-keyed. See {@link MindContext.recogniseMemo}. */
52
59
  recogniseMemo = null;
53
- /** Per-response memo of perceive() — see {@link MindContext.perceiveMemo}. */
60
+ /** Memo of perceive() — content-keyed. See {@link MindContext.perceiveMemo}. */
54
61
  perceiveMemo = null;
62
+ /** Subtree-resolution cache. See {@link MindContext._resolvedSubtrees}. */
63
+ _resolvedSubtrees = null;
55
64
  /** The perceived gist of the query currently being answered. Set by `think`
56
65
  * before the graph search runs; `chooseNext` consults it as a gate (a null
57
66
  * guide means no query is in flight, so structural walkers keep plain
@@ -75,6 +84,9 @@ export class Mind {
75
84
  _depositTrees = new BoundedMap(8);
76
85
  _depositLens = new Set();
77
86
  _internIds = new WeakMap();
87
+ // ── Conversation state ──────────────────────────────────────────────────
88
+ _nextConvId = 1;
89
+ _conversations = new Map();
78
90
  // ── GraphSearchHost implementation ─────────────────────────────────────
79
91
  /** Canonical node id of a byte span. Required by GraphSearchHost & MindContext. */
80
92
  resolve(bytes) {
@@ -103,7 +115,8 @@ export class Mind {
103
115
  this.store = storeArg;
104
116
  }
105
117
  else {
106
- const { store: optsStore, mechanisms: userMechs, mechanismFactories: userFacts, ...rest } = (optsOrCfg ?? {});
118
+ const { store: optsStore, mechanisms: userMechs, mechanismFactories: userFacts, canon: optsCanon, ...rest } = (optsOrCfg ?? {});
119
+ this._canonOpt = optsCanon ?? null;
107
120
  this.cfg = resolveConfig(rest);
108
121
  this.store = optsStore ?? new SQliteStore({
109
122
  maxGroup: this.cfg.geometry.maxGroup,
@@ -165,11 +178,21 @@ export class Mind {
165
178
  /** Open one response's transient state — the tracer and the per-response
166
179
  * memos. Paired with {@link endResponse}; the ONE place this state is
167
180
  * created, so adding a memo cannot forget its reset. */
168
- beginResponse(inspectRationale) {
181
+ beginResponse(inspectRationale, canon) {
169
182
  this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
170
- this.climbMemo = new WeakMap();
171
- this.recogniseMemo = new WeakMap();
183
+ this.climbMemo = new Map();
184
+ this.recogniseMemo = new Map();
172
185
  this.perceiveMemo = new Map();
186
+ this.canon = canon ?? null;
187
+ this.canonMemo = canon ? new Map() : null;
188
+ }
189
+ /** The canonicalizer a response should carry: the Mind-level option when
190
+ * set (or none when explicitly disabled), else the entry point's own
191
+ * default — text entry points pass {@link textCanon}, binary ones null. */
192
+ _canonFor(entryDefault) {
193
+ if (this._canonOpt === false)
194
+ return null;
195
+ return this._canonOpt ?? entryDefault;
173
196
  }
174
197
  /** Close one response's transient state — every per-response field, incl.
175
198
  * the edge guide/choices `think` sets mid-flight. */
@@ -178,22 +201,26 @@ export class Mind {
178
201
  this.climbMemo = null;
179
202
  this.recogniseMemo = null;
180
203
  this.perceiveMemo = null;
204
+ this.canon = null;
205
+ this.canonMemo = null;
181
206
  this._edgeGuide = null;
182
207
  this._edgeChoice.clear();
183
208
  }
184
- async respond(input, inspectRationale) {
185
- this.beginResponse(inspectRationale);
209
+ /** Shared response core — the one path from bytes to voiced answer.
210
+ * `respond` calls this directly; `respondTurn` has its own path
211
+ * with conversation-persistent memos and incremental perception. */
212
+ async _respondImpl(queryBytes, inspectRationale, traceLabel = "respond", canon = null) {
213
+ this.beginResponse(inspectRationale, canon);
186
214
  try {
187
- const inBytes = inputBytes(this, input);
188
- const top = this.trace?.enter("respond", [
189
- rItem(inBytes, "query"),
215
+ const top = this.trace?.enter(traceLabel, [
216
+ rItem(queryBytes, "query"),
190
217
  ]);
191
- const thought = await think(this, inBytes, this.mechanisms);
218
+ const thought = await think(this, queryBytes, this.mechanisms);
192
219
  if (thought === null) {
193
220
  top?.done([], "nothing to perceive or an empty store — no answer");
194
221
  return { v: null, bytes: new Uint8Array(0) };
195
222
  }
196
- const voiced = await articulate(this, thought.bytes, inBytes);
223
+ const voiced = await articulate(this, thought.bytes, queryBytes);
197
224
  top?.done([rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)], "the answer, re-voiced in the asker's words");
198
225
  return {
199
226
  v: gistOf(this, voiced),
@@ -205,14 +232,207 @@ export class Mind {
205
232
  this.endResponse();
206
233
  }
207
234
  }
235
+ async respond(input, inspectRationale) {
236
+ // A STRING input is text by nature: it carries the text equivalence even
237
+ // through the generic entry point. Raw bytes / grids carry only the
238
+ // Mind-level canon option, if any.
239
+ const canon = this._canonFor(typeof input === "string" ? textCanon : null);
240
+ return this._respondImpl(inputBytes(this, input), inspectRationale, "respond", canon);
241
+ }
208
242
  /** Text view of {@link respond}. NUL bytes (0x00) are stripped before
209
243
  * decoding — they are structural padding in text answers. LOSSY for a
210
244
  * binary answer that legitimately contains NULs: use {@link respond} and
211
- * read `bytes` directly for binary/grid modalities. */
245
+ * read `bytes` directly for binary/grid modalities.
246
+ *
247
+ * Injects the TEXT canonicalizer (src/canon.ts) so resolution treats
248
+ * every character variation of the same text — case, width, whitespace —
249
+ * as one form, provided the store's canon index is built
250
+ * ({@link buildCanonIndex}). */
212
251
  async respondText(input, inspectRationale) {
213
252
  const r = await this.respond(input, inspectRationale);
214
253
  return decodeText(r.bytes);
215
254
  }
255
+ // ── Conversation API ────────────────────────────────────────────────────
256
+ /** Begin a new conversation, optionally restoring from a previously-saved
257
+ * {@link ConversationState}. The returned handle is required for
258
+ * {@link respondTurn} and {@link endConversation}.
259
+ *
260
+ * Conversations are independent — a Mind can manage several concurrently.
261
+ * Each tracks the fold pyramid (accumulated internal processing) and
262
+ * turn-boundary offsets; the geometry never inspects content to guess
263
+ * where one turn ends and the next begins. */
264
+ beginConversation(state) {
265
+ const id = this._nextConvId++;
266
+ // Build the initial pyramid: from scratch for a new conversation,
267
+ // or from the saved context bytes when restoring.
268
+ const initBytes = state?.context ?? new Uint8Array(0);
269
+ const { pyramid } = bytesToTreePyramid(this.space, this.alphabet, initBytes);
270
+ this._conversations.set(id, {
271
+ pyramid,
272
+ bytes: initBytes,
273
+ boundaries: state?.boundaries ? [...state.boundaries] : [],
274
+ perceiveMemo: new Map(),
275
+ recogniseMemo: new Map(),
276
+ climbMemo: new Map(),
277
+ resolvedSubtrees: new WeakMap(),
278
+ });
279
+ return { id };
280
+ }
281
+ /** End a conversation, releasing its internal resources (accumulated
282
+ * context, boundary offsets, and the fold-pyramid cache). Idempotent. */
283
+ endConversation(conv) {
284
+ this._conversations.delete(conv.id);
285
+ }
286
+ /** The current serialisable state of an active conversation. Save this
287
+ * to resume the conversation later via {@link beginConversation}. */
288
+ conversationState(conv) {
289
+ const data = this._conversations.get(conv.id);
290
+ if (!data)
291
+ return null;
292
+ return {
293
+ context: data.bytes,
294
+ boundaries: [...data.boundaries],
295
+ };
296
+ }
297
+ /** Append a turn to a conversation's accumulated context WITHOUT
298
+ * responding — raw byte append plus a boundary offset, never a
299
+ * separator; the fold pyramid advances by O(turn).
300
+ *
301
+ * This is the primitive for turns the Mind should hear but not answer:
302
+ * replaying a transcript, feeding the OTHER speaker's line in a
303
+ * prediction harness, or restoring context piecewise. {@link
304
+ * respondTurn} = addTurn + think + its own reply appended the same way. */
305
+ addTurn(conv, turn) {
306
+ const data = this._conversations.get(conv.id);
307
+ if (!data)
308
+ throw new Error(`Conversation ${conv.id} not found`);
309
+ const turnBytes = inputBytes(this, turn);
310
+ this._growContext(data, turnBytes);
311
+ return this.conversationState(conv);
312
+ }
313
+ /** Grow a conversation's accumulated context by one turn's bytes — raw
314
+ * append plus a boundary offset, pyramid advanced by O(turn), the grown
315
+ * context's tree seeded into the conversation's perceive memo. The ONE
316
+ * place a context grows ({@link addTurn} and {@link respondTurn} both
317
+ * come through here), so the append semantics cannot drift. */
318
+ _growContext(data, turnBytes) {
319
+ const prevLen = data.pyramid.bytes;
320
+ // An empty turn neither grows the context nor marks a boundary —
321
+ // boundaries are documented strictly increasing, and a zero-length
322
+ // "turn" is no turn. The pyramid's zero-growth shortcut makes the
323
+ // refold below O(1), so the existing tree is returned unchanged.
324
+ const grow = turnBytes.length > 0;
325
+ const grown = !grow
326
+ ? data.bytes
327
+ : prevLen > 0
328
+ ? concat2(data.bytes, turnBytes)
329
+ : turnBytes;
330
+ if (grow && prevLen > 0)
331
+ data.boundaries.push(prevLen);
332
+ const { tree, pyramid } = bytesToTreePyramid(this.space, this.alphabet, grown, data.pyramid);
333
+ data.pyramid = pyramid;
334
+ data.bytes = grown;
335
+ data.perceiveMemo.set(latin1Key(grown), tree);
336
+ return tree;
337
+ }
338
+ /** Process one turn of a conversation.
339
+ *
340
+ * `turn` is the raw input for the latest turn — its bytes are appended
341
+ * to the accumulated context directly (raw concatenation). The Mind
342
+ * tracks the byte offset where each turn ends; no separator is ever
343
+ * inserted or inspected.
344
+ *
345
+ * Returns the response AND the updated {@link ConversationState} so the
346
+ * caller can persist it. The conversation handle's internal state is
347
+ * updated in place — the returned state is a snapshot for storage.
348
+ *
349
+ * SINGLE FLIGHT: at most one respondTurn may be in flight per Mind. The
350
+ * conversation's memo caches are swapped into the Mind-level per-response
351
+ * pointers for the duration of the turn, so a concurrently-running
352
+ * respond()/respondTurn() on the SAME Mind would interleave state.
353
+ * Different Minds (or sequential awaits, as in every test) are safe. */
354
+ async respondTurn(conv, turn, inspectRationale) {
355
+ const data = this._conversations.get(conv.id);
356
+ if (!data)
357
+ throw new Error(`Conversation ${conv.id} not found`);
358
+ const turnBytes = inputBytes(this, turn);
359
+ // Incremental perception — O(turn) instead of O(context).
360
+ const tree = this._growContext(data, turnBytes);
361
+ const newContext = data.bytes;
362
+ // Swap in the conversation's persistent state so the inference
363
+ // pipeline does not re-process the prefix from scratch.
364
+ // perceiveMemo / recogniseMemo / climbMemo — content-keyed;
365
+ // the prefix's results from the previous turn are found by
366
+ // the current turn's sub-span calls.
367
+ // _resolvedSubtrees — Sema-node-keyed; when the pyramid reuses
368
+ // prefix subtrees (identical objects), foldTree returns their
369
+ // ids immediately — O(suffix) instead of O(context).
370
+ this.perceiveMemo = data.perceiveMemo;
371
+ this.recogniseMemo = data.recogniseMemo;
372
+ this.climbMemo = data.climbMemo;
373
+ this._resolvedSubtrees = data.resolvedSubtrees;
374
+ this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
375
+ // A string turn is text by nature — carry the text equivalence, same as
376
+ // respond() (see _canonFor).
377
+ this.canon = this._canonFor(typeof turn === "string" ? textCanon : null);
378
+ this.canonMemo = this.canon ? new Map() : null;
379
+ try {
380
+ const top = this.trace?.enter("respondTurn", [
381
+ rItem(newContext, "query"),
382
+ ]);
383
+ const thought = await think(this, newContext, this.mechanisms);
384
+ if (thought === null) {
385
+ top?.done([], "nothing to perceive or an empty store — no answer");
386
+ return {
387
+ response: { v: null, bytes: new Uint8Array(0) },
388
+ state: this.conversationState(conv),
389
+ };
390
+ }
391
+ const voiced = await articulate(this, thought.bytes, newContext);
392
+ top?.done([rItem(voiced, "answer", resolveImpl(this, voiced) ?? undefined)], "the answer, re-voiced in the asker's words");
393
+ // The REPLY joins the accumulated context the same way a turn does
394
+ // ({@link addTurn}): raw byte append plus a boundary offset — never a
395
+ // separator. A conversation's context is the full exchange, exactly
396
+ // the cumulative continuous shape multi-turn training deposits, so a
397
+ // later turn can refer to what was ANSWERED ("which of those two…"),
398
+ // not only to what was asked.
399
+ if (voiced.length > 0)
400
+ this.addTurn(conv, voiced);
401
+ return {
402
+ response: {
403
+ v: gistOf(this, voiced),
404
+ bytes: voiced,
405
+ provenance: thought.provenance,
406
+ },
407
+ state: this.conversationState(conv),
408
+ };
409
+ }
410
+ finally {
411
+ // No save-back: the conversation's memo MAPS were mutated in place —
412
+ // `data.*` still points at them. Re-assigning from the Mind-level
413
+ // pointers here was a no-op in the single-flight case and, under a
414
+ // concurrently-started respond() (which swaps its own fresh maps into
415
+ // those pointers), would inject a FOREIGN response's memos into this
416
+ // conversation. Clear Mind references — non-conversation respond()
417
+ // calls get fresh per-response memos.
418
+ this.trace = null;
419
+ this.perceiveMemo = null;
420
+ this.recogniseMemo = null;
421
+ this.climbMemo = null;
422
+ this.canon = null;
423
+ this.canonMemo = null;
424
+ this._resolvedSubtrees = null;
425
+ this._edgeGuide = null;
426
+ this._edgeChoice.clear();
427
+ }
428
+ }
429
+ /** Text view of {@link respondTurn}. See {@link respondText} for the
430
+ * NUL-stripping caveat. For binary or grid turns use {@link respondTurn}
431
+ * directly — this is a text-only convenience, like {@link respondText}. */
432
+ async respondTurnText(conv, turn, inspectRationale) {
433
+ const { response, state } = await this.respondTurn(conv, turn, inspectRationale);
434
+ return { response: decodeText(response.bytes), state };
435
+ }
216
436
  async embedding(input) {
217
437
  return (await this.respond(input)).v;
218
438
  }
@@ -281,6 +501,48 @@ export class Mind {
281
501
  return gistOf(this, bytes);
282
502
  }, minParents);
283
503
  }
504
+ // ── Canonical-form index ───────────────────────────────────────────────
505
+ /** Build (or incrementally refresh) the store's canonical-form index: for
506
+ * every content-bearing node, record the hash of its CANONICAL key so
507
+ * resolution can find stored forms across surface variation (case, width,
508
+ * whitespace — whatever `canon` equates; see src/canon.ts).
509
+ *
510
+ * Incremental and idempotent: the last indexed node id is remembered in
511
+ * store meta (`canon.upto`), so a refresh after further training scans
512
+ * only the new rows. Run once after training, and again after ingests —
513
+ * the same operational shape as {@link repairContentIndex}.
514
+ *
515
+ * @param canon the canonicalizer to index under — MUST be the same one
516
+ * queries will carry (text queries carry {@link textCanon}
517
+ * unless the Mind was constructed with its own)
518
+ * @returns number of index rows added */
519
+ async buildCanonIndex(canon) {
520
+ const c = canon ?? this._canonFor(textCanon);
521
+ const store = this.store;
522
+ if (c === null || !store.canonAdd || !store.eachContent)
523
+ return 0;
524
+ const from = Number(await store.getMeta("canon.upto") ?? 0);
525
+ let added = 0;
526
+ let maxId = from - 1;
527
+ store.eachContent((id, bytes) => {
528
+ if (id > maxId)
529
+ maxId = id;
530
+ const key = c(bytes);
531
+ if (key.length === 0)
532
+ return;
533
+ // Only index content whose canonical key DIFFERS from its raw bytes —
534
+ // an already-canonical span is found by the exact lookup (and by the
535
+ // fallback's own exact probe of the canonical bytes), so indexing it
536
+ // would only add rows.
537
+ if (bytesEqual(key, bytes))
538
+ return;
539
+ store.canonAdd(canonHash(key), id);
540
+ added++;
541
+ }, from);
542
+ await store.setMeta("canon.upto", String(maxId + 1));
543
+ store.commit();
544
+ return added;
545
+ }
284
546
  // ── Persistence ──────────────────────────────────────────────────────────
285
547
  async save() {
286
548
  const meta = new TextEncoder().encode(JSON.stringify(this.cfg));
@@ -1,9 +1,20 @@
1
1
  import { Vec } from "../vec.js";
2
2
  import { Sema } from "../sema.js";
3
3
  import type { Input, MindContext } from "./types.js";
4
+ /** The content key of a byte span — one latin1 char per byte, an exact,
5
+ * collision-free encoding. Spans on the perception path are query-scale
6
+ * (windows, regions, candidate spans), so key construction is far cheaper
7
+ * than the river fold it deduplicates. */
8
+ export declare function latin1Key(bytes: Uint8Array): string;
4
9
  /** Perceive input into a content-defined tree (the river fold).
5
- * Deterministic — identical bytes always produce an identical tree. */
6
- export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null): Sema;
10
+ * Deterministic — identical bytes always produce an identical tree.
11
+ *
12
+ * `boundaries` is an optional sorted list of proper byte offsets where the
13
+ * fold must split so that each prefix segment folds identically to how it
14
+ * folded when it was learned (§10.3 stable-prefix contract). Only the
15
+ * CALLER — who assembled the multi-turn context — knows where those
16
+ * boundaries are; the geometry never guesses them from the bytes. */
17
+ export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
7
18
  /** The DEPOSIT-shaped perceive: the PLAIN fold (bit-identical to inference
8
19
  * perception — that structural train/inference agreement is load-bearing
9
20
  * for exact recall), computed INCREMENTALLY via the fold's level pyramid
@@ -30,6 +41,15 @@ export declare function foldTree(ctx: MindContext, n: Sema, start: number, visit
30
41
  * training did — and recover its root bottom-up. Returns null if any part is
31
42
  * unknown. */
32
43
  export declare function resolve(ctx: MindContext, bytes: Uint8Array): number | null;
44
+ /** Equivalence-class resolution: when the exact content-addressed lookup
45
+ * misses, find a stored node whose CANONICAL key equals the span's — the
46
+ * store's canon index proposes candidates by key hash, and each is verified
47
+ * by re-canonicalizing its bytes (hash-then-verify, like every content
48
+ * lookup). Among verified candidates, one that leads somewhere (has a
49
+ * continuation edge) is preferred; ties break to the lowest id — a corpus
50
+ * property, not a seed property. Null when the response carries no
51
+ * canonicalizer, the store has no canon index, or nothing verifies. */
52
+ export declare function canonResolve(ctx: MindContext, bytes: Uint8Array): number | null;
33
53
  /** Walk a perceived tree in POST-ORDER with byte offsets — children before
34
54
  * their parent, `visit(node, start, end)` for every node including leaves.
35
55
  * Returns the byte end. The one shared traversal the offset-carrying tree
@@ -3,13 +3,15 @@
3
3
  // Address — bytes → node (perceive, foldTree, resolve)
4
4
  // Read — node → bytes (read)
5
5
  import { bytesToTree, bytesToTreePyramid, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
6
+ import { canonHash } from "../canon.js";
7
+ import { bytesEqual } from "../bytes.js";
6
8
  import { ALL } from "./types.js";
7
9
  // ── Address: bytes → node ──────────────────────────────────────────────
8
10
  /** The content key of a byte span — one latin1 char per byte, an exact,
9
11
  * collision-free encoding. Spans on the perception path are query-scale
10
12
  * (windows, regions, candidate spans), so key construction is far cheaper
11
13
  * than the river fold it deduplicates. */
12
- function latin1Key(bytes) {
14
+ export function latin1Key(bytes) {
13
15
  // Batched String.fromCharCode — avoids the O(n²) cost of repeated += on
14
16
  // potentially-large query spans, and stays well under the ~65536 arg limit.
15
17
  const n = bytes.length;
@@ -20,8 +22,14 @@ function latin1Key(bytes) {
20
22
  return s;
21
23
  }
22
24
  /** Perceive input into a content-defined tree (the river fold).
23
- * Deterministic — identical bytes always produce an identical tree. */
24
- export function perceive(ctx, input, leafAt, lookup) {
25
+ * Deterministic — identical bytes always produce an identical tree.
26
+ *
27
+ * `boundaries` is an optional sorted list of proper byte offsets where the
28
+ * fold must split so that each prefix segment folds identically to how it
29
+ * folded when it was learned (§10.3 stable-prefix contract). Only the
30
+ * CALLER — who assembled the multi-turn context — knows where those
31
+ * boundaries are; the geometry never guesses them from the bytes. */
32
+ export function perceive(ctx, input, leafAt, lookup, boundaries) {
25
33
  if (typeof input === "string" || input instanceof Uint8Array) {
26
34
  const bytes = typeof input === "string"
27
35
  ? new TextEncoder().encode(input)
@@ -37,11 +45,11 @@ export function perceive(ctx, input, leafAt, lookup) {
37
45
  const hit = memo.get(key);
38
46
  if (hit !== undefined)
39
47
  return hit;
40
- const tree = bytesToTree(ctx.space, ctx.alphabet, bytes);
48
+ const tree = bytesToTree(ctx.space, ctx.alphabet, bytes, undefined, undefined, boundaries);
41
49
  memo.set(key, tree);
42
50
  return tree;
43
51
  }
44
- return bytesToTree(ctx.space, ctx.alphabet, bytes);
52
+ return bytesToTree(ctx.space, ctx.alphabet, bytes, undefined, undefined, boundaries);
45
53
  }
46
54
  return bytesToTree(ctx.space, ctx.alphabet, bytes, leafAt, lookup);
47
55
  }
@@ -105,11 +113,24 @@ export function gistOf(ctx, bytes) {
105
113
  * node with its byte span and resolved id. Returns the node's byte end and
106
114
  * resolved id. */
107
115
  export function foldTree(ctx, n, start, visit) {
116
+ // Fast path: subtree already resolved (from a previous conversation turn
117
+ // or an earlier recognition pass). The pyramid reuses prefix subtrees as
118
+ // identical Sema objects, so this cache turns foldTree into O(suffix)
119
+ // instead of O(context) for multi-turn recognition.
120
+ const cached = ctx._resolvedSubtrees?.get(n);
121
+ if (cached !== undefined) {
122
+ const end = start + cached.len;
123
+ visit?.(n, start, end, cached.id);
124
+ return { end, node: cached.id };
125
+ }
108
126
  if (n.kids === null) {
109
127
  const b = n.leaf ?? new Uint8Array(0);
110
128
  const end = start + b.length;
111
129
  const node = ctx.store.findLeaf(b);
112
130
  visit?.(n, start, end, node);
131
+ if (node !== null && ctx._resolvedSubtrees) {
132
+ ctx._resolvedSubtrees.set(n, { id: node, len: b.length });
133
+ }
113
134
  return { end, node };
114
135
  }
115
136
  let pos = start;
@@ -125,6 +146,9 @@ export function foldTree(ctx, n, start, visit) {
125
146
  }
126
147
  const node = known ? ctx.store.findBranch(kids) : null;
127
148
  visit?.(n, start, pos, node);
149
+ if (node !== null && ctx._resolvedSubtrees) {
150
+ ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
151
+ }
128
152
  return { end: pos, node };
129
153
  }
130
154
  /** The canonical node id of a byte span: perceive it in isolation — the way
@@ -133,7 +157,72 @@ export function foldTree(ctx, n, start, visit) {
133
157
  export function resolve(ctx, bytes) {
134
158
  if (bytes.length === 0)
135
159
  return null;
136
- return foldTree(ctx, perceive(ctx, bytes), 0).node;
160
+ const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
161
+ if (exact !== null)
162
+ return exact;
163
+ return canonResolve(ctx, bytes);
164
+ }
165
+ /** Equivalence-class resolution: when the exact content-addressed lookup
166
+ * misses, find a stored node whose CANONICAL key equals the span's — the
167
+ * store's canon index proposes candidates by key hash, and each is verified
168
+ * by re-canonicalizing its bytes (hash-then-verify, like every content
169
+ * lookup). Among verified candidates, one that leads somewhere (has a
170
+ * continuation edge) is preferred; ties break to the lowest id — a corpus
171
+ * property, not a seed property. Null when the response carries no
172
+ * canonicalizer, the store has no canon index, or nothing verifies. */
173
+ export function canonResolve(ctx, bytes) {
174
+ const canon = ctx.canon;
175
+ const store = ctx.store;
176
+ if (canon === null || !store.canonFind)
177
+ return null;
178
+ if (bytes.length < 2)
179
+ return null;
180
+ const memo = ctx.canonMemo;
181
+ const memoKey = memo ? latin1Key(bytes) : "";
182
+ if (memo) {
183
+ const hit = memo.get(memoKey);
184
+ if (hit !== undefined)
185
+ return hit;
186
+ }
187
+ const set = (v) => {
188
+ memo?.set(memoKey, v);
189
+ return v;
190
+ };
191
+ const key = canon(bytes);
192
+ if (key.length === 0)
193
+ return set(null);
194
+ // A stored form that IS canonical is not in the index (buildCanonIndex
195
+ // skips identity rows) — the exact content-addressed lookup of the
196
+ // canonical bytes finds it directly.
197
+ if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
198
+ const direct = foldTree(ctx, perceive(ctx, key), 0).node;
199
+ if (direct !== null)
200
+ return set(direct);
201
+ }
202
+ const candidates = store.canonFind(canonHash(key));
203
+ if (candidates.length === 0)
204
+ return set(null);
205
+ let best = null;
206
+ let bestLeads = false;
207
+ for (const id of candidates) {
208
+ const bytesOf = read(ctx, id);
209
+ const stored = canon(bytesOf);
210
+ if (stored.length !== key.length || !bytesEqual(stored, key))
211
+ continue;
212
+ // The index stores FLAT content twins; the id the exact path would have
213
+ // resolved for these bytes is their FOLD — the deposit-shaped node that
214
+ // carries the edges and halos. Re-folding the candidate's bytes lands
215
+ // on exactly the node the canonical-case query would have found.
216
+ const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
217
+ const use = folded ?? id;
218
+ const leads = store.hasNext(use) || store.haloMass(use) > 0;
219
+ if (best === null || (leads && !bestLeads) ||
220
+ (leads === bestLeads && use < best)) {
221
+ best = use;
222
+ bestLeads = leads;
223
+ }
224
+ }
225
+ return set(best);
137
226
  }
138
227
  /** Walk a perceived tree in POST-ORDER with byte offsets — children before
139
228
  * their parent, `visit(node, start, end)` for every node including leaves.
@@ -5,8 +5,8 @@
5
5
  // that leads somewhere (has a continuation edge or a halo).
6
6
  // segment — leaf-parent segmentation using the geometry's own groupings.
7
7
  import { rItem } from "./trace.js";
8
- import { foldTree, gistOf, perceive, resolve } from "./primitives.js";
9
- import { leadsSomewhere } from "./traverse.js";
8
+ import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
9
+ import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
10
10
  import { chainReach, leafIdAt } from "./canonical.js";
11
11
  import { isChunk } from "../sema.js";
12
12
  /** Decompose a byte stream into every stored form that leads somewhere
@@ -22,16 +22,16 @@ import { isChunk } from "../sema.js";
22
22
  *
23
23
  * Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
24
24
  export function recognise(ctx, bytes) {
25
- // Per-response memo (see MindContext.recogniseMemo): think, articulate and
26
- // the pre-consume pass all recognise the same byte objects; each repeat is
27
- // O(n·maxGroup²) store probes. Skipped while tracing so every call still
28
- // emits its rationale step.
25
+ // Content-keyed memo works for both single-turn respond() and multi-turn
26
+ // respondTurn() (where the map persists across calls). Skipped while
27
+ // tracing so every call still emits its rationale step.
29
28
  if (ctx.recogniseMemo && !ctx.trace) {
30
- const hit = ctx.recogniseMemo.get(bytes);
29
+ const key = latin1Key(bytes);
30
+ const hit = ctx.recogniseMemo.get(key);
31
31
  if (hit !== undefined)
32
32
  return hit;
33
33
  const fresh = recogniseImpl(ctx, bytes);
34
- ctx.recogniseMemo.set(bytes, fresh);
34
+ ctx.recogniseMemo.set(key, fresh);
35
35
  return fresh;
36
36
  }
37
37
  return recogniseImpl(ctx, bytes);
@@ -59,7 +59,20 @@ function recogniseImpl(ctx, bytes) {
59
59
  }
60
60
  return id;
61
61
  };
62
+ // Byte atoms (implicit negative-id single-byte leaves) are admitted as
63
+ // recognised sites only while atoms can still DISCRIMINATE at this corpus
64
+ // scale (see {@link atomIsHub}). On a small store a single-letter fact
65
+ // ("a" → "A") is genuine learnt content and its site is essential; on a
66
+ // large one every letter of every query would otherwise become a
67
+ // "recognised form" — the bridge then finds junction connectors between
68
+ // bare letters, cover follows edges hanging off them, and pure noise
69
+ // ("qq8f3kz9…") grounds to an arbitrary learnt sentence instead of
70
+ // silence. Atoms stay available as leaves (PASS-carried literals) and
71
+ // through exact tier-0 resolution regardless.
72
+ const atomsAreHubs = atomIsHub(ctx, corpusN(ctx));
62
73
  const emit = (start, end, id) => {
74
+ if (id < 0 && atomsAreHubs)
75
+ return;
63
76
  if (leadsSomewhere(ctx, id)) {
64
77
  sites.push({ start, end, payload: id });
65
78
  }
@@ -73,6 +86,16 @@ function recogniseImpl(ctx, bytes) {
73
86
  }
74
87
  if (node !== null)
75
88
  emit(start, end, node);
89
+ // Canonical fallback: a subtree whose exact content-addressed lookup
90
+ // missed may still be a stored form under the response's equivalence
91
+ // (case, width, whitespace — whatever the injected canonicalizer says).
92
+ // O(subtree bytes) per miss, memoised per response; a no-op when no
93
+ // canonicalizer was injected or the store has no canon index.
94
+ else if (end - start >= 2) {
95
+ const cid = canonResolve(ctx, bytes.subarray(start, end));
96
+ if (cid !== null)
97
+ emit(start, end, cid);
98
+ }
76
99
  if (isChunk(n)) {
77
100
  starts.add(start);
78
101
  // Try every sub-span within this leaf-parent.