@hviana/sema 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/example/train_base.js +24 -4
  2. package/dist/src/alu/src/parser.d.ts +9 -0
  3. package/dist/src/alu/src/parser.js +110 -2
  4. package/dist/src/canon.d.ts +26 -0
  5. package/dist/src/canon.js +57 -0
  6. package/dist/src/geometry.js +12 -0
  7. package/dist/src/index.d.ts +1 -0
  8. package/dist/src/index.js +1 -0
  9. package/dist/src/mind/learning.js +33 -1
  10. package/dist/src/mind/match.d.ts +4 -2
  11. package/dist/src/mind/match.js +30 -6
  12. package/dist/src/mind/mechanisms/cast.js +18 -4
  13. package/dist/src/mind/mechanisms/confluence.js +13 -1
  14. package/dist/src/mind/mechanisms/recall.js +97 -28
  15. package/dist/src/mind/mind.d.ts +64 -2
  16. package/dist/src/mind/mind.js +152 -23
  17. package/dist/src/mind/primitives.d.ts +9 -0
  18. package/dist/src/mind/primitives.js +68 -1
  19. package/dist/src/mind/recognition.js +11 -1
  20. package/dist/src/mind/types.d.ts +10 -0
  21. package/dist/src/store-sqlite.d.ts +8 -0
  22. package/dist/src/store-sqlite.js +52 -0
  23. package/dist/src/store.d.ts +12 -0
  24. package/example/train_base.ts +29 -4
  25. package/package.json +1 -1
  26. package/src/alu/src/parser.ts +105 -4
  27. package/src/canon.ts +65 -0
  28. package/src/geometry.ts +12 -0
  29. package/src/index.ts +1 -0
  30. package/src/mind/learning.ts +39 -1
  31. package/src/mind/match.ts +29 -6
  32. package/src/mind/mechanisms/cast.ts +21 -6
  33. package/src/mind/mechanisms/confluence.ts +15 -1
  34. package/src/mind/mechanisms/recall.ts +116 -41
  35. package/src/mind/mind.ts +172 -29
  36. package/src/mind/primitives.ts +66 -1
  37. package/src/mind/recognition.ts +10 -0
  38. package/src/mind/types.ts +10 -0
  39. package/src/store-sqlite.ts +68 -0
  40. package/src/store.ts +27 -0
  41. package/test/13-conversation.test.mjs +77 -27
  42. package/test/35-prefix-edge.test.mjs +86 -0
@@ -3,6 +3,8 @@
3
3
  // Address — bytes → node (perceive, foldTree, resolve)
4
4
  // Read — node → bytes (read)
5
5
  import { bytesToTree, bytesToTreePyramid, gridToTree, hilbertBytes, stackGrids, } from "../geometry.js";
6
+ import { canonHash } from "../canon.js";
7
+ import { bytesEqual } from "../bytes.js";
6
8
  import { ALL } from "./types.js";
7
9
  // ── Address: bytes → node ──────────────────────────────────────────────
8
10
  /** The content key of a byte span — one latin1 char per byte, an exact,
@@ -155,7 +157,72 @@ export function foldTree(ctx, n, start, visit) {
155
157
  export function resolve(ctx, bytes) {
156
158
  if (bytes.length === 0)
157
159
  return null;
158
- return foldTree(ctx, perceive(ctx, bytes), 0).node;
160
+ const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
161
+ if (exact !== null)
162
+ return exact;
163
+ return canonResolve(ctx, bytes);
164
+ }
165
+ /** Equivalence-class resolution: when the exact content-addressed lookup
166
+ * misses, find a stored node whose CANONICAL key equals the span's — the
167
+ * store's canon index proposes candidates by key hash, and each is verified
168
+ * by re-canonicalizing its bytes (hash-then-verify, like every content
169
+ * lookup). Among verified candidates, one that leads somewhere (has a
170
+ * continuation edge) is preferred; ties break to the lowest id — a corpus
171
+ * property, not a seed property. Null when the response carries no
172
+ * canonicalizer, the store has no canon index, or nothing verifies. */
173
+ export function canonResolve(ctx, bytes) {
174
+ const canon = ctx.canon;
175
+ const store = ctx.store;
176
+ if (canon === null || !store.canonFind)
177
+ return null;
178
+ if (bytes.length < 2)
179
+ return null;
180
+ const memo = ctx.canonMemo;
181
+ const memoKey = memo ? latin1Key(bytes) : "";
182
+ if (memo) {
183
+ const hit = memo.get(memoKey);
184
+ if (hit !== undefined)
185
+ return hit;
186
+ }
187
+ const set = (v) => {
188
+ memo?.set(memoKey, v);
189
+ return v;
190
+ };
191
+ const key = canon(bytes);
192
+ if (key.length === 0)
193
+ return set(null);
194
+ // A stored form that IS canonical is not in the index (buildCanonIndex
195
+ // skips identity rows) — the exact content-addressed lookup of the
196
+ // canonical bytes finds it directly.
197
+ if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
198
+ const direct = foldTree(ctx, perceive(ctx, key), 0).node;
199
+ if (direct !== null)
200
+ return set(direct);
201
+ }
202
+ const candidates = store.canonFind(canonHash(key));
203
+ if (candidates.length === 0)
204
+ return set(null);
205
+ let best = null;
206
+ let bestLeads = false;
207
+ for (const id of candidates) {
208
+ const bytesOf = read(ctx, id);
209
+ const stored = canon(bytesOf);
210
+ if (stored.length !== key.length || !bytesEqual(stored, key))
211
+ continue;
212
+ // The index stores FLAT content twins; the id the exact path would have
213
+ // resolved for these bytes is their FOLD — the deposit-shaped node that
214
+ // carries the edges and halos. Re-folding the candidate's bytes lands
215
+ // on exactly the node the canonical-case query would have found.
216
+ const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
217
+ const use = folded ?? id;
218
+ const leads = store.hasNext(use) || store.haloMass(use) > 0;
219
+ if (best === null || (leads && !bestLeads) ||
220
+ (leads === bestLeads && use < best)) {
221
+ best = use;
222
+ bestLeads = leads;
223
+ }
224
+ }
225
+ return set(best);
159
226
  }
160
227
  /** Walk a perceived tree in POST-ORDER with byte offsets — children before
161
228
  * their parent, `visit(node, start, end)` for every node including leaves.
@@ -5,7 +5,7 @@
5
5
  // that leads somewhere (has a continuation edge or a halo).
6
6
  // segment — leaf-parent segmentation using the geometry's own groupings.
7
7
  import { rItem } from "./trace.js";
8
- import { foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
8
+ import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
9
9
  import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
10
10
  import { chainReach, leafIdAt } from "./canonical.js";
11
11
  import { isChunk } from "../sema.js";
@@ -86,6 +86,16 @@ function recogniseImpl(ctx, bytes) {
86
86
  }
87
87
  if (node !== null)
88
88
  emit(start, end, node);
89
+ // Canonical fallback: a subtree whose exact content-addressed lookup
90
+ // missed may still be a stored form under the response's equivalence
91
+ // (case, width, whitespace — whatever the injected canonicalizer says).
92
+ // O(subtree bytes) per miss, memoised per response; a no-op when no
93
+ // canonicalizer was injected or the store has no canon index.
94
+ else if (end - start >= 2) {
95
+ const cid = canonResolve(ctx, bytes.subarray(start, end));
96
+ if (cid !== null)
97
+ emit(start, end, cid);
98
+ }
89
99
  if (isChunk(n)) {
90
100
  starts.add(start);
91
101
  // Try every sub-span within this leaf-parent.
@@ -105,6 +105,16 @@ export interface MindContext extends GraphSearchHost {
105
105
  cfg: MindConfig;
106
106
  search: GraphSearch;
107
107
  trace: Rationale | null;
108
+ /** The content canonicalizer for THIS response, or null — injected by the
109
+ * modality entry point (respondText passes the text canonicalizer; a
110
+ * binary respond passes none). Resolution uses it as a fallback: when
111
+ * the exact content-addressed lookup misses, the span's canonical key is
112
+ * probed against the store's canon index (see src/canon.ts). The core
113
+ * never inspects what the equivalence IS. */
114
+ canon: ((bytes: Uint8Array) => Uint8Array) | null;
115
+ /** Per-response memo of canonical-fallback resolutions, keyed by the
116
+ * span's latin1 content key. Null outside respond(). */
117
+ canonMemo: Map<string, number | null> | null;
108
118
  /** Memo of the consensus climb — content-keyed (latin1) so results
109
119
  * persist across conversation turns where the same byte spans recur.
110
120
  * Null outside respond(); during respondTurn() the conversation's
@@ -43,6 +43,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
43
43
  private _selPrev;
44
44
  private _setMeta;
45
45
  private _getMeta;
46
+ private _insCanon;
47
+ private _selCanon;
48
+ private _cntCanon;
49
+ private _selContentFrom;
46
50
  private _delMeta;
47
51
  private _insSnapshot;
48
52
  private _selSnapshot;
@@ -130,6 +134,10 @@ export declare class SQliteStore extends AbstractStore implements Store {
130
134
  protected _dbGetMeta(key: string): string | null;
131
135
  protected _dbSetMeta(key: string, val: string): void;
132
136
  protected _dbDeleteMeta(key: string): void;
137
+ canonAdd(h: number, id: number): void;
138
+ canonFind(h: number): number[];
139
+ canonCount(): number;
140
+ eachContent(cb: (id: number, bytes: Uint8Array) => void, fromId?: number): void;
133
141
  protected _dbSaveSnapshot(bytes: Uint8Array): void;
134
142
  protected _dbLoadSnapshot(): Uint8Array | null;
135
143
  protected _vecContentUpsert(entries: Array<{
@@ -112,6 +112,18 @@ CREATE TABLE IF NOT EXISTS contain (
112
112
  id INTEGER PRIMARY KEY,
113
113
  parents BLOB NOT NULL
114
114
  );
115
+ -- CANONICAL-FORM index (Store.canonAdd/canonFind): h is the 32-bit hash of a
116
+ -- node's CANONICAL content key (the modality's canonicalizer output — case-
117
+ -- folded, whitespace-collapsed text, etc.; the store never sees the key
118
+ -- itself, only its hash). Same hash-then-verify discipline as idx_node_h:
119
+ -- the caller re-canonicalizes each candidate's bytes before trusting it, so
120
+ -- a collision costs a read, never a wrong id. WITHOUT ROWID on (h, id):
121
+ -- the composite key IS the lookup index and gives free dedup.
122
+ CREATE TABLE IF NOT EXISTS canon (
123
+ h INTEGER NOT NULL,
124
+ id INTEGER NOT NULL,
125
+ PRIMARY KEY (h, id)
126
+ ) WITHOUT ROWID;
115
127
  CREATE TABLE IF NOT EXISTS snapshot (
116
128
  id INTEGER PRIMARY KEY CHECK (id = 1),
117
129
  data BLOB NOT NULL
@@ -212,6 +224,10 @@ export class SQliteStore extends AbstractStore {
212
224
  _selPrev = null;
213
225
  _setMeta = null;
214
226
  _getMeta = null;
227
+ _insCanon = null;
228
+ _selCanon = null;
229
+ _cntCanon = null;
230
+ _selContentFrom = null;
215
231
  _delMeta = null;
216
232
  _insSnapshot = null;
217
233
  _selSnapshot = null;
@@ -797,6 +813,42 @@ export class SQliteStore extends AbstractStore {
797
813
  }
798
814
  this._delMeta.run(key);
799
815
  }
816
+ // -- Canonical-form index (Store optional capability) --
817
+ canonAdd(h, id) {
818
+ if (!this._insCanon) {
819
+ this._insCanon = this.sqlite.prepare("INSERT OR IGNORE INTO canon (h, id) VALUES (?, ?)");
820
+ }
821
+ // Join the deferred write transaction (committed by flush/commit), so a
822
+ // bulk index build coalesces instead of paying autocommit per row.
823
+ this._dbBeginTx();
824
+ this._insCanon.run(h, id);
825
+ }
826
+ canonFind(h) {
827
+ if (!this._selCanon) {
828
+ this._selCanon = this.sqlite.prepare("SELECT id FROM canon WHERE h = ?");
829
+ }
830
+ return this._selCanon.all(h).map((r) => r.id);
831
+ }
832
+ canonCount() {
833
+ if (!this._cntCanon) {
834
+ this._cntCanon = this.sqlite.prepare("SELECT count(*) AS c FROM canon");
835
+ }
836
+ return this._cntCanon.get().c;
837
+ }
838
+ eachContent(cb, fromId = 0) {
839
+ if (!this._selContentFrom) {
840
+ // Content-bearing nodes are FLAT branches: leaf present, kids an
841
+ // empty (zero-length) blob — the same population PART 2 of the
842
+ // diagnostics calls "distinct content spans".
843
+ this._selContentFrom = this.sqlite.prepare("SELECT id, leaf FROM node " +
844
+ "WHERE id >= ? AND leaf IS NOT NULL " +
845
+ "AND (kids IS NULL OR length(kids) = 0)");
846
+ }
847
+ for (const row of this._selContentFrom.iterate(fromId)) {
848
+ const r = row;
849
+ cb(r.id, new Uint8Array(r.leaf));
850
+ }
851
+ }
800
852
  // -- Snapshot --
801
853
  _dbSaveSnapshot(bytes) {
802
854
  if (!this._insSnapshot) {
@@ -240,6 +240,18 @@ export interface Store {
240
240
  * repetition, but repetition outranks insertion-order accident. 0 when
241
241
  * the node has no halo row. */
242
242
  haloMass(id: NodeId): number;
243
+ /** Record that node `id`'s canonical key hashes to `h`. Idempotent. */
244
+ canonAdd?(h: number, id: NodeId): void;
245
+ /** All candidate node ids whose canonical key hashes to `h` (collisions
246
+ * included — the caller verifies). */
247
+ canonFind?(h: number): NodeId[];
248
+ /** Number of (h, id) rows in the canon index — 0 means never built. */
249
+ canonCount?(): number;
250
+ /** Visit every content-bearing node (flat branch: `leaf` present, no
251
+ * kids) — the population a canonical index is built over. `fromId`
252
+ * restricts the scan to ids ≥ fromId, so an index refresh after further
253
+ * training only visits the new rows. */
254
+ eachContent?(cb: (id: NodeId, bytes: Uint8Array) => void, fromId?: NodeId): void;
243
255
  size(): Promise<number>;
244
256
  saveSnapshot(bytes: Uint8Array): Promise<void>;
245
257
  loadSnapshot(): Promise<Uint8Array | null>;
@@ -177,8 +177,11 @@ const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir of *.zip
177
177
  const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
178
178
  const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
179
179
  const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel refresh cadence
180
- // Index maintenance at checkpoints: compact (remove garbage) then repair (fill
181
- // gaps). Both are idempotent batch operations; INDEX_MAINTENANCE=0 disables.
180
+ // Index maintenance at checkpoints: compact (remove garbage), repair (fill
181
+ // gaps), then refresh the canonical-form index (equivalence-class resolution —
182
+ // src/canon.ts). All three are idempotent batch operations (the canon build is
183
+ // additionally incremental via the store's `canon.upto` cursor);
184
+ // INDEX_MAINTENANCE=0 disables.
182
185
  const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
183
186
  const DOWNLOAD_TRIES = 5;
184
187
  // In-progress downloads are written to a sibling "<dest>.part" and atomically
@@ -1823,8 +1826,9 @@ async function main(): Promise<void> {
1823
1826
 
1824
1827
  const checkpoint = () => mind.save();
1825
1828
 
1826
- /** Run index maintenance: compact (remove garbage) then repair (fill gaps).
1827
- * Both are idempotent running twice produces the same result as once.
1829
+ /** Run index maintenance: compact (remove garbage), repair (fill gaps),
1830
+ * then refresh the canonical-form index (see below). All three are
1831
+ * idempotent — running twice produces the same result as once.
1828
1832
  * Compaction frees index space first; repair then adds back every
1829
1833
  * edge/halo-bearing node whose gist was evicted from the pending cache
1830
1834
  * before it reached the content index, completing the coverage that
@@ -1877,6 +1881,27 @@ async function main(): Promise<void> {
1877
1881
  }`,
1878
1882
  );
1879
1883
  }
1884
+ // Canonical-form index (src/canon.ts): lets resolution find stored forms
1885
+ // across surface variation (case, width, whitespace). Incremental and
1886
+ // idempotent by construction — the `canon.upto` meta cursor scans only
1887
+ // nodes newer than the last pass, and the (h, id) primary key ignores
1888
+ // re-inserted rows — so it composes with the resume model exactly like
1889
+ // compact/repair: every checkpoint (and finish) leaves the index
1890
+ // covering all content trained so far.
1891
+ try {
1892
+ const added = await mind.buildCanonIndex();
1893
+ if (added > 0) {
1894
+ progress.log(
1895
+ ` ${GRN}canon index: added ${int(added)} canonical-form entries${R}`,
1896
+ );
1897
+ }
1898
+ } catch (err) {
1899
+ progress.log(
1900
+ ` ${YEL}⚠ canon index build failed${R}: ${
1901
+ err instanceof Error ? err.message : String(err)
1902
+ }`,
1903
+ );
1904
+ }
1880
1905
  };
1881
1906
 
1882
1907
  // The checkpoint recall is a best-effort diagnostic. It is time-bounded so a
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hviana/sema",
3
- "version": "0.1.7",
3
+ "version": "0.1.8",
4
4
  "description": "Sema: a non-parametric, instance-based reasoning system.",
5
5
  "type": "module",
6
6
  "main": "dist/src/index.js",
@@ -172,10 +172,27 @@ export class QueryParser {
172
172
  j,
173
173
  bytes: this.evalRun(query.subarray(i, j)),
174
174
  }));
175
- const spans: ComputedSpan[] = runs.filter(
176
- (r): r is ComputedSpan => r.bytes !== null,
177
- );
178
- let stream = compose(tokens, runs);
175
+ // BRACKETED runs "(3 + 4) * (10 - 2)" — extend across grouping
176
+ // notation the flat alternation cannot cross. The expression grammar
177
+ // (expr.ts) already nests and parenthesizes; this only hands it the
178
+ // WHOLE bracketed span instead of the fragments between brackets. A
179
+ // bracketed run that evaluates ABSORBS the flat runs inside it (the
180
+ // authority law: contained spans are its material); one that does not
181
+ // evaluate leaves the flat readings untouched.
182
+ const bracketed = this.bracketedRuns(query, tokens)
183
+ .map(([i, j]) => ({ i, j, bytes: this.evalRun(query.subarray(i, j)) }))
184
+ .filter((r): r is ComputedSpan => r.bytes !== null);
185
+ const absorbed = (r: { i: number; j: number }) =>
186
+ bracketed.some((b) => b.i <= r.i && r.j <= b.j);
187
+ const composedRuns = [
188
+ ...runs.filter((r) => !absorbed(r)),
189
+ ...bracketed,
190
+ ].sort((a, b) => a.i - b.i);
191
+ const spans: ComputedSpan[] = [
192
+ ...runs.filter((r): r is ComputedSpan => r.bytes !== null),
193
+ ...bracketed,
194
+ ];
195
+ let stream = compose(tokens, composedRuns);
179
196
  const readings = new Map<Token, readonly string[]>();
180
197
  for (;;) {
181
198
  const fired = await this.operations(query, stream, readings);
@@ -302,6 +319,90 @@ export class QueryParser {
302
319
  return runs;
303
320
  }
304
321
 
322
+ /** The maximal BRACKETED arithmetic runs: like {@link arithmeticRuns} but
323
+ * grouping brackets participate — an all-'(' term opens depth where an
324
+ * operand is expected, a term beginning with ')' closes it where an
325
+ * operator could follow (only its bracket prefix joins the run; anything
326
+ * after — a trailing "?" — ends it). A run is recorded only when it is
327
+ * BALANCED, actually crossed a bracket, and contains an operator — the
328
+ * bracket-free case is {@link arithmeticRuns}' own. Evaluation is the
329
+ * same registry-derived grammar, whose lexer already reads the brackets. */
330
+ private bracketedRuns(
331
+ query: Uint8Array,
332
+ tokens: Token[],
333
+ ): Array<[number, number]> {
334
+ const OPEN = 0x28, CLOSE = 0x29;
335
+ const bracketPrefix = (t: Token, b: number): number => {
336
+ if (t.kind !== "term") return 0;
337
+ let n = 0;
338
+ while (t.i + n < t.j && query[t.i + n] === b) n++;
339
+ return n;
340
+ };
341
+ const allOf = (t: Token, b: number): boolean =>
342
+ bracketPrefix(t, b) === t.j - t.i;
343
+ const bridged = (from: number, to: number): boolean =>
344
+ from >= to || this.host.segment(query.subarray(from, to)).length <= 1;
345
+ const runs: Array<[number, number]> = [];
346
+ let k = 0;
347
+ while (k < tokens.length) {
348
+ const first = tokens[k];
349
+ if (first.kind !== "operand" && !allOf(first, OPEN)) {
350
+ k++;
351
+ continue;
352
+ }
353
+ let depth = 0;
354
+ let ops = 0;
355
+ let brackets = 0;
356
+ let expectOperand = true;
357
+ let end = -1; // byte end of the last VALID run state (balanced, after operand/close)
358
+ let endTok = k; // token index just past the accepted prefix
359
+ let m = k;
360
+ let prevJ = -1;
361
+ while (m < tokens.length) {
362
+ const t = tokens[m];
363
+ if (prevJ >= 0 && !bridged(prevJ, t.i)) break;
364
+ if (expectOperand) {
365
+ if (allOf(t, OPEN)) {
366
+ depth += t.j - t.i;
367
+ brackets++;
368
+ } else if (t.kind === "operand") {
369
+ expectOperand = false;
370
+ if (depth === 0) {
371
+ end = t.j;
372
+ endTok = m + 1;
373
+ }
374
+ } else break;
375
+ } else {
376
+ const c = bracketPrefix(t, CLOSE);
377
+ if (c > 0 && depth > 0) {
378
+ const take = Math.min(c, depth);
379
+ depth -= take;
380
+ brackets++;
381
+ if (depth === 0) {
382
+ end = t.i + take;
383
+ endTok = m + 1;
384
+ }
385
+ // A term with content beyond its usable bracket prefix ("?")
386
+ // ends the run inside the term.
387
+ if (take < t.j - t.i) break;
388
+ } else if (t.kind === "operator") {
389
+ expectOperand = true;
390
+ ops++;
391
+ } else break;
392
+ }
393
+ prevJ = t.j;
394
+ m++;
395
+ }
396
+ if (end >= 0 && ops > 0 && brackets > 0) {
397
+ runs.push([first.i, end]);
398
+ k = endTok;
399
+ } else {
400
+ k++;
401
+ }
402
+ }
403
+ return runs;
404
+ }
405
+
305
406
  /** Evaluate an infix-arithmetic run to its canonical result bytes, through
306
407
  * the kernel's recursive expression evaluator, or null if it does not
307
408
  * evaluate. A whole result stays an exact int; otherwise the canonical
package/src/canon.ts ADDED
@@ -0,0 +1,65 @@
1
+ // canon.ts — content canonicalization for equivalence-class resolution.
2
+ //
3
+ // The store is content-addressed on RAW bytes: "What", "WHAT" and "what" are
4
+ // three different hashes, so a query whose surface form varies from the
5
+ // trained form resolves to nothing even though the CONTENT is the same. A
6
+ // CANONICALIZER maps every surface variant of the same content onto one
7
+ // canonical byte string; the store keeps a small hash index from canonical
8
+ // keys to node ids (see Store.canonAdd/canonFind), and resolution falls back
9
+ // to that index when the exact content-addressed lookup misses.
10
+ //
11
+ // The canonicalizer is MODALITY-SPECIFIC and always INJECTED — nothing in the
12
+ // store or the mind's core knows what "case" or "whitespace" is. The text
13
+ // canonicalizer below is the one `respondText`/`respondTurnText` pass down;
14
+ // a grid or audio modality would supply its own (or none).
15
+ //
16
+ // Canonical keys are equivalence-class LABELS, never content: they are hashed
17
+ // and verified (canon(stored bytes) must equal canon(query bytes) before an
18
+ // id is accepted), so a hash collision costs a read, never a wrong id — the
19
+ // same discipline as the node table's own `h` index.
20
+
21
+ /** A content canonicalizer: maps a byte span to the canonical representative
22
+ * of its equivalence class. Must be pure and deterministic. Returning the
23
+ * input unchanged is always sound (the class is then {input}). */
24
+ export type Canon = (bytes: Uint8Array) => Uint8Array;
25
+
26
+ const dec = new TextDecoder("utf-8", { fatal: false });
27
+ const enc = new TextEncoder();
28
+
29
+ /** The TEXT canonicalizer: Unicode-aware equivalence over every character
30
+ * variation that does not change what the text SAYS —
31
+ *
32
+ * • compatibility normalization (NFKC): full-width forms, ligatures,
33
+ * composed vs decomposed accents collapse to one representation;
34
+ * • case folding (locale-independent lowercase after NFKC — the standard
35
+ * simple fold);
36
+ * • whitespace: every INTERIOR run of Unicode whitespace becomes one plain
37
+ * space. EDGE whitespace is preserved verbatim: a span's leading or
38
+ * trailing separator belongs BETWEEN forms, not to the form — trimming
39
+ * it would let a recognised span swallow the boundary byte that
40
+ * separates it from its neighbour (observed: "ice fire" composing to
41
+ * "coldhot" because the span "ice " matched the stored "ice").
42
+ *
43
+ * "WHAT IS", "What is" and "what is" share one canonical form. This is
44
+ * deliberately conservative: punctuation, digits and word order are content
45
+ * and pass through untouched. */
46
+ export function textCanon(bytes: Uint8Array): Uint8Array {
47
+ const s = dec
48
+ .decode(bytes)
49
+ .normalize("NFKC")
50
+ .toLowerCase()
51
+ .replace(/(\S)\s+(?=\S)/g, "$1 ");
52
+ return enc.encode(s);
53
+ }
54
+
55
+ /** 32-bit FNV-1a over a canonical key — the integer the store's canon index
56
+ * is keyed on. Same construction as the node table's content hash; a
57
+ * collision is resolved by verifying canon(stored) === key, never trusted. */
58
+ export function canonHash(key: Uint8Array): number {
59
+ let h = 0x811c9dc5 >>> 0;
60
+ for (let i = 0; i < key.length; i++) {
61
+ h ^= key[i];
62
+ h = Math.imul(h, 0x01000193) >>> 0;
63
+ }
64
+ return h >>> 0;
65
+ }
package/src/geometry.ts CHANGED
@@ -460,6 +460,18 @@ export function bytesToTreePyramid(
460
460
  pyramid: { levels: [], bytes: 0 },
461
461
  };
462
462
  }
463
+ // ZERO GROWTH: the previous pyramid already covers every byte — its top
464
+ // level holds the finished, normalized root. Refolding here would not
465
+ // only waste the work: when the whole span is one full block, the refold's
466
+ // top item is a REUSED raw interior of `prev`, and the final normalize
467
+ // below would mutate that shared raw block in place, corrupting every
468
+ // later incremental fold built on `prev`.
469
+ if (prev && prev.bytes === bytes.length && prev.levels.length > 0) {
470
+ return {
471
+ tree: prev.levels[prev.levels.length - 1][0].tree,
472
+ pyramid: prev,
473
+ };
474
+ }
463
475
  const mg = space.maxGroup;
464
476
  const reusable = (L: number): ReadonlyArray<Folded> | null => {
465
477
  // prev's TOPMOST level holds its normalized ROOT — reusable blocks must
package/src/index.ts CHANGED
@@ -12,6 +12,7 @@ export * from "./mind/index.js";
12
12
  export * from "./store-sqlite.js";
13
13
  export * from "./config.js";
14
14
  export * from "./extension.js";
15
+ export * from "./canon.js";
15
16
  export * from "./ingest-cache.js";
16
17
  // rabitq-ivf: the partitioned (IVF) vector index over 1-bit RaBitQ codes.
17
18
  export {
@@ -7,7 +7,12 @@ import { Vec } from "../vec.js";
7
7
  import { bindSeat, companySignature, isChunk, Sema } from "../sema.js";
8
8
  import type { Input, MindContext } from "./types.js";
9
9
  import { changedNodes } from "./types.js";
10
- import { inputBytes, perceive, perceiveDeposit } from "./primitives.js";
10
+ import {
11
+ inputBytes,
12
+ perceive,
13
+ perceiveDeposit,
14
+ resolve,
15
+ } from "./primitives.js";
11
16
  import { canonicalWindows, leafIdPrefix } from "./canonical.js";
12
17
  import { fold as foldVecs } from "../sema.js";
13
18
 
@@ -171,6 +176,38 @@ export async function ingestOne(
171
176
  return Object.assign(tree, { id: rootId });
172
177
  }
173
178
 
179
+ /** For each right-edge suffix of the context bytes, resolve it against the
180
+ * store. A suffix whose resolved node is already a known form inherits the
181
+ * continuation edge. Gate: ≥ 2 structural parents (reused across deposits),
182
+ * or (halo > 0 ∧ already an edge source). Pure answers do not qualify. */
183
+ async function propagateSuffixes(
184
+ ctx: MindContext,
185
+ src: number,
186
+ dst: number,
187
+ ): Promise<void> {
188
+ const W = ctx.space.maxGroup;
189
+ const bytes = ctx.store.bytes(src);
190
+ const n = bytes.length;
191
+ if (n < 2 * W) return;
192
+ // Existence prefilter — the write side of the canonical contract: every
193
+ // deposit interns its WHOLE byte stream as a flat branch of per-byte leaf
194
+ // ids (deposit(), canonical.ts). A suffix is a stored form exactly when
195
+ // that flat twin exists, so one content-hash probe per offset decides;
196
+ // only a hit pays for resolve()'s deposit-shaped perception. This keeps
197
+ // the scan free of river folds — O(1) probes over cheap byte hashes
198
+ // instead of O(suffix) vector folds per offset.
199
+ const leafIds = leafIdPrefix(ctx, bytes);
200
+ for (let i = 1; i <= n - W; i++) {
201
+ if (ctx.store.findBranch(leafIds.slice(i)) === null) continue;
202
+ const id = resolve(ctx, bytes.subarray(i));
203
+ if (id === null || id === src) continue;
204
+ const known = ctx.store.parentsFirst(id, 2).length >= 2 ||
205
+ (ctx.store.haloMass(id) > 0 && ctx.store.hasNext(id));
206
+ if (!known) continue;
207
+ await ctx.store.link(id, dst);
208
+ }
209
+ }
210
+
174
211
  /** Ingest a pair (context, continuation) — learn an edge and pour halos. */
175
212
  export async function ingestPair(
176
213
  ctx: MindContext,
@@ -182,6 +219,7 @@ export async function ingestPair(
182
219
  const ctxId = c.rootId, contId = cont_.rootId;
183
220
 
184
221
  await ctx.store.link(ctxId, contId);
222
+ await propagateSuffixes(ctx, ctxId, contId);
185
223
 
186
224
  // Halos pour company SIGNATURES (identity), not gists (content) — see
187
225
  // companySignature in sema.ts.
package/src/mind/match.ts CHANGED
@@ -446,8 +446,10 @@ export async function follow(
446
446
  * `guide` the context whose gist resonates with the query wins (seat
447
447
  * symmetry) — without one, the most-corroborated context wins (poured halo
448
448
  * MASS, the direct measure of how many episodes established it), falling
449
- * back to first-learnt on equal mass. Callers that HAVE a query gist must
450
- * pass it, or they silently change disambiguation regime.
449
+ * back to first-learnt on equal mass. Among many predecessors RECIPROCAL
450
+ * ones (mutual edges) are preferred when any exist (RC5). Callers that
451
+ * HAVE a query gist must pass it, or they silently change disambiguation
452
+ * regime.
451
453
  *
452
454
  * `rev`, when the caller has already materialised prevOf (one read per
453
455
  * relation — a hub's reverse fan-in is corpus-sized), is reused instead of
@@ -466,11 +468,32 @@ export function reverseContext(
466
468
  // keeps the single-predecessor shortcut exact.
467
469
  const candidates = rev ?? ctx.store.prevFirst(id, hubBound(ctx));
468
470
  if (candidates.length === 0) return null;
469
- const pick = candidates.length === 1
470
- ? candidates[0]
471
+ // RECIPROCAL PREFERENCE: among many predecessors, one that `id` also
472
+ // continues TO (cand → id AND id → cand both learnt) is a mutually
473
+ // established pairing — the strongest structural evidence a predecessor
474
+ // can carry (bidirectional training deposits both directions of a genuine
475
+ // pair). A bare predecessor is one episode's adjacency; guide-resonance
476
+ // over bare predecessors favours whichever stored document merely
477
+ // CONTAINS the query's bytes (the linear fold's cosine is byte overlap —
478
+ // the observed "merci → unrelated French document" failure). One capped
479
+ // forward read decides; when no reciprocal exists, behaviour is unchanged
480
+ // — bare predecessors ARE the honest answer for a shared deposited
481
+ // continuation (two questions → one answer; audited by 31-audit C1), and
482
+ // this arm serves every mechanism's reverse projection, so abstaining
483
+ // here starves far more than the one containment failure it would fix.
484
+ let pool: readonly number[] = candidates;
485
+ if (candidates.length > 1) {
486
+ const fwd = new Set(ctx.store.nextFirst(id, hubBound(ctx)));
487
+ if (fwd.size > 0) {
488
+ const mutual = candidates.filter((c) => fwd.has(c));
489
+ if (mutual.length > 0) pool = mutual;
490
+ }
491
+ }
492
+ const pick = pool.length === 1
493
+ ? pool[0]
471
494
  : guide
472
- ? chooseAmong(ctx, candidates, guide).id
473
- : pickByMass(ctx, candidates);
495
+ ? chooseAmong(ctx, pool, guide).id
496
+ : pickByMass(ctx, pool);
474
497
  const g = read(ctx, pick);
475
498
  return g.length > 0 ? g : null;
476
499
  }