@hviana/sema 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/package.json +1 -1
  2. package/src/ingest-cache.ts +4 -0
  3. package/src/meter.ts +5 -0
  4. package/src/mind/attention.ts +18 -1
  5. package/src/mind/bridge.ts +292 -54
  6. package/src/mind/junction.ts +21 -7
  7. package/src/mind/match.ts +92 -1
  8. package/src/mind/mechanisms/cast.ts +12 -0
  9. package/src/mind/mechanisms/confluence.ts +30 -1
  10. package/src/mind/mechanisms/cover.ts +36 -4
  11. package/src/mind/mechanisms/recall.ts +21 -44
  12. package/src/mind/mind.ts +39 -2
  13. package/src/mind/pipeline-mechanism.ts +86 -4
  14. package/src/mind/pipeline.ts +1 -1
  15. package/src/mind/reasoning.ts +15 -8
  16. package/src/mind/recognition.ts +40 -0
  17. package/src/mind/resonance.ts +0 -0
  18. package/src/mind/traverse.ts +17 -15
  19. package/src/mind/types.ts +10 -0
  20. package/test/49-natural-units-synonym-bridge.test.mjs +56 -15
  21. package/dist/example/demo.d.ts +0 -1
  22. package/dist/example/demo.js +0 -39
  23. package/dist/example/train_base.d.ts +0 -87
  24. package/dist/example/train_base.js +0 -2252
  25. package/dist/src/alphabet.d.ts +0 -7
  26. package/dist/src/alphabet.js +0 -33
  27. package/dist/src/alu/src/alu.d.ts +0 -185
  28. package/dist/src/alu/src/alu.js +0 -440
  29. package/dist/src/alu/src/expr.d.ts +0 -61
  30. package/dist/src/alu/src/expr.js +0 -318
  31. package/dist/src/alu/src/index.d.ts +0 -11
  32. package/dist/src/alu/src/index.js +0 -19
  33. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  34. package/dist/src/alu/src/kernel-arith.js +0 -264
  35. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  36. package/dist/src/alu/src/kernel-bits.js +0 -152
  37. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  38. package/dist/src/alu/src/kernel-logic.js +0 -60
  39. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  40. package/dist/src/alu/src/kernel-nd.js +0 -208
  41. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  42. package/dist/src/alu/src/kernel-numeric.js +0 -366
  43. package/dist/src/alu/src/operation.d.ts +0 -168
  44. package/dist/src/alu/src/operation.js +0 -189
  45. package/dist/src/alu/src/parser.d.ts +0 -221
  46. package/dist/src/alu/src/parser.js +0 -577
  47. package/dist/src/alu/src/resonance.d.ts +0 -55
  48. package/dist/src/alu/src/resonance.js +0 -126
  49. package/dist/src/alu/src/text.d.ts +0 -31
  50. package/dist/src/alu/src/text.js +0 -73
  51. package/dist/src/alu/src/value.d.ts +0 -109
  52. package/dist/src/alu/src/value.js +0 -300
  53. package/dist/src/alu/test/alu.test.d.ts +0 -1
  54. package/dist/src/alu/test/alu.test.js +0 -764
  55. package/dist/src/bytes.d.ts +0 -14
  56. package/dist/src/bytes.js +0 -59
  57. package/dist/src/canon.d.ts +0 -26
  58. package/dist/src/canon.js +0 -57
  59. package/dist/src/config.d.ts +0 -111
  60. package/dist/src/config.js +0 -91
  61. package/dist/src/derive/src/deduction.d.ts +0 -136
  62. package/dist/src/derive/src/deduction.js +0 -159
  63. package/dist/src/derive/src/index.d.ts +0 -8
  64. package/dist/src/derive/src/index.js +0 -11
  65. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  66. package/dist/src/derive/src/priority-queue.js +0 -73
  67. package/dist/src/derive/src/rewrite.d.ts +0 -56
  68. package/dist/src/derive/src/rewrite.js +0 -100
  69. package/dist/src/derive/src/trie.d.ts +0 -90
  70. package/dist/src/derive/src/trie.js +0 -217
  71. package/dist/src/derive/test/derive.test.d.ts +0 -1
  72. package/dist/src/derive/test/derive.test.js +0 -122
  73. package/dist/src/extension.d.ts +0 -37
  74. package/dist/src/extension.js +0 -7
  75. package/dist/src/geometry.d.ts +0 -175
  76. package/dist/src/geometry.js +0 -823
  77. package/dist/src/index.d.ts +0 -17
  78. package/dist/src/index.js +0 -19
  79. package/dist/src/ingest-cache.d.ts +0 -41
  80. package/dist/src/ingest-cache.js +0 -161
  81. package/dist/src/meter.d.ts +0 -171
  82. package/dist/src/meter.js +0 -269
  83. package/dist/src/mind/articulation.d.ts +0 -6
  84. package/dist/src/mind/articulation.js +0 -99
  85. package/dist/src/mind/attention.d.ts +0 -414
  86. package/dist/src/mind/attention.js +0 -2065
  87. package/dist/src/mind/bridge.d.ts +0 -39
  88. package/dist/src/mind/bridge.js +0 -738
  89. package/dist/src/mind/canonical.d.ts +0 -34
  90. package/dist/src/mind/canonical.js +0 -93
  91. package/dist/src/mind/graph-search.d.ts +0 -294
  92. package/dist/src/mind/graph-search.js +0 -996
  93. package/dist/src/mind/index.d.ts +0 -9
  94. package/dist/src/mind/index.js +0 -5
  95. package/dist/src/mind/junction.d.ts +0 -134
  96. package/dist/src/mind/junction.js +0 -331
  97. package/dist/src/mind/learning.d.ts +0 -75
  98. package/dist/src/mind/learning.js +0 -270
  99. package/dist/src/mind/match.d.ts +0 -166
  100. package/dist/src/mind/match.js +0 -565
  101. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  102. package/dist/src/mind/mechanisms/alu.js +0 -36
  103. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  104. package/dist/src/mind/mechanisms/cast.js +0 -773
  105. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  106. package/dist/src/mind/mechanisms/confluence.js +0 -225
  107. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  108. package/dist/src/mind/mechanisms/cover.js +0 -203
  109. package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
  110. package/dist/src/mind/mechanisms/extraction.js +0 -300
  111. package/dist/src/mind/mechanisms/recall.d.ts +0 -16
  112. package/dist/src/mind/mechanisms/recall.js +0 -388
  113. package/dist/src/mind/mind.d.ts +0 -330
  114. package/dist/src/mind/mind.js +0 -594
  115. package/dist/src/mind/pipeline-mechanism.d.ts +0 -172
  116. package/dist/src/mind/pipeline-mechanism.js +0 -382
  117. package/dist/src/mind/pipeline.d.ts +0 -49
  118. package/dist/src/mind/pipeline.js +0 -275
  119. package/dist/src/mind/primitives.d.ts +0 -66
  120. package/dist/src/mind/primitives.js +0 -306
  121. package/dist/src/mind/rationale.d.ts +0 -139
  122. package/dist/src/mind/rationale.js +0 -163
  123. package/dist/src/mind/reasoning.d.ts +0 -40
  124. package/dist/src/mind/reasoning.js +0 -272
  125. package/dist/src/mind/recognition.d.ts +0 -20
  126. package/dist/src/mind/recognition.js +0 -463
  127. package/dist/src/mind/resonance.d.ts +0 -23
  128. package/dist/src/mind/resonance.js +0 -0
  129. package/dist/src/mind/trace.d.ts +0 -15
  130. package/dist/src/mind/trace.js +0 -73
  131. package/dist/src/mind/traverse.d.ts +0 -124
  132. package/dist/src/mind/traverse.js +0 -649
  133. package/dist/src/mind/types.d.ts +0 -323
  134. package/dist/src/mind/types.js +0 -130
  135. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  136. package/dist/src/rabitq-ivf/src/database.js +0 -201
  137. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  138. package/dist/src/rabitq-ivf/src/index.js +0 -4
  139. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  140. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  141. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  142. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  143. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  144. package/dist/src/rabitq-ivf/src/rabitq.js +0 -283
  145. package/dist/src/sema.d.ts +0 -31
  146. package/dist/src/sema.js +0 -63
  147. package/dist/src/store-sqlite.d.ts +0 -184
  148. package/dist/src/store-sqlite.js +0 -942
  149. package/dist/src/store.d.ts +0 -678
  150. package/dist/src/store.js +0 -1703
  151. package/dist/src/vec.d.ts +0 -31
  152. package/dist/src/vec.js +0 -109
package/src/mind/match.ts CHANGED
@@ -25,7 +25,7 @@
25
25
  // mechanism file states only its configuration, never its own copy of the
26
26
  // machinery. The gates all live in geometry.ts (derived, never tuned).
27
27
 
28
- import { cosine, Vec } from "../vec.js";
28
+ import { addInto, cosine, dot, normalize, Vec, zeros } from "../vec.js";
29
29
  import type { Hit } from "../store.js";
30
30
  import { conceptThreshold, identityBar, significanceBar } from "../geometry.js";
31
31
  import { indexOf } from "../bytes.js";
@@ -36,9 +36,12 @@ import {
36
36
  argmaxCosine,
37
37
  chooseAmong,
38
38
  chooseNext,
39
+ corpusN,
40
+ edgeAncestors,
39
41
  guidedFirst,
40
42
  hubBound,
41
43
  hubCap,
44
+ sharedReachMemo,
42
45
  } from "./traverse.js";
43
46
  import { recognise, segment } from "./recognition.js";
44
47
  import type { Site } from "./graph-search.js";
@@ -347,6 +350,94 @@ export async function haloSiblings(
347
350
  return out;
348
351
  }
349
352
 
353
+ /** Bundle the distributional company of every addressable W-window in a
354
+ * byte span. This is the query-time counterpart of the write-side halo
355
+ * pours: no lexical unit or storage row is invented; the span is represented
356
+ * by VSA superposition of the window concepts the store already knows.
357
+ *
358
+ * Components are normalized before bundling so repetition mass remains
359
+ * evidence about each stored node, not an accidental weight on one window
360
+ * inside the composed phrase. Returns null when the corpus provides no
361
+ * distributional evidence for the span. */
362
+ export function spanHalo(
363
+ ctx: MindContext,
364
+ bytes: Uint8Array,
365
+ from = 0,
366
+ to = bytes.length,
367
+ ): Vec | null {
368
+ const W = ctx.space.maxGroup;
369
+ if (to - from < W) return null;
370
+ if (ctx.meter) ctx.meter.spanHalos++;
371
+ const out = zeros(ctx.store.D);
372
+ let found = false;
373
+ const added = new Set<number>();
374
+ const episodeRoots: number[][] = [];
375
+ const N = corpusN(ctx);
376
+ const reachMemo = sharedReachMemo(ctx);
377
+ const addHalo = (id: number): void => {
378
+ if (added.has(id)) return;
379
+ const halo = ctx.store.halo(id);
380
+ if (halo === null) return;
381
+ const norm = Math.sqrt(dot(halo, halo));
382
+ if (norm === 0) return;
383
+ added.add(id);
384
+ addInto(out, halo, 1 / norm);
385
+ found = true;
386
+ };
387
+ const windowCount = to - from - W + 1;
388
+ const offsets: number[] = [];
389
+ const samples = Math.min(W, windowCount);
390
+ for (let i = 0; i < samples; i++) {
391
+ const relative = samples === 1
392
+ ? 0
393
+ : Math.floor((i * (windowCount - 1)) / (samples - 1));
394
+ const off = from + relative;
395
+ if (offsets[offsets.length - 1] !== off) offsets.push(off);
396
+ }
397
+ for (const off of offsets) {
398
+ if (ctx.meter) ctx.meter.spanHaloWindows++;
399
+ const ids = leafIdRun(ctx, bytes, off, off + W);
400
+ if (ids === null) continue;
401
+ const id = ctx.store.findBranch(ids);
402
+ if (id === null) continue;
403
+ addHalo(id);
404
+ // Canonical flat windows are retrieval addresses and normally carry no
405
+ // halo themselves. Their bounded structural ascent reaches the learned
406
+ // episode forms that contain them; bundling those forms' company is the
407
+ // distributional meaning of the window, derived entirely from existing
408
+ // containment and halo state.
409
+ if (!added.has(id)) {
410
+ episodeRoots.push(edgeAncestors(ctx, id, N, reachMemo).roots);
411
+ }
412
+ }
413
+ for (let rank = 0; added.size < ctx.cfg.haloQueryK; rank++) {
414
+ let any = false;
415
+ for (const roots of episodeRoots) {
416
+ if (rank >= roots.length) continue;
417
+ any = true;
418
+ addHalo(roots[rank]);
419
+ if (added.size >= ctx.cfg.haloQueryK) break;
420
+ }
421
+ if (!any) break;
422
+ }
423
+ return found ? normalize(out) : null;
424
+ }
425
+
426
+ /** Distributional synonym evidence between arbitrary byte spans. Whole words
427
+ * need not be independently interned: their stored W-window occurrences are
428
+ * lifted to episode halos, bundled, and compared. The caller chooses the
429
+ * derived gate appropriate to its claim (concept identity or analogy). */
430
+ export function spanSynonymStrength(
431
+ ctx: MindContext,
432
+ a: Uint8Array,
433
+ b: Uint8Array,
434
+ ): number {
435
+ const ah = spanHalo(ctx, a);
436
+ const bh = spanHalo(ctx, b);
437
+ if (ah === null || bh === null) return 0;
438
+ return cosine(ah, bh);
439
+ }
440
+
350
441
  /** The DISTRIBUTIONAL matcher between two nodes: mutual-nearest-neighbour
351
442
  * strength, not a pick. Returns the direct halo cosine, or failing that the
352
443
  * highest mutual-halo-sibling min-score (second-order analogy), or failing
@@ -274,6 +274,18 @@ export async function counterfactualTransfer(
274
274
  `${points.length} aligned structure(s), but none is one of the climb's ` +
275
275
  `${roots.length} committed root(s) — CAST refuses to transfer through ` +
276
276
  `content the climb itself never settled on`,
277
+ {
278
+ aligned: points.map((p) => ({
279
+ anchor: p.anchor,
280
+ vote: p.vote,
281
+ runs: p.runs.map((r) => ({ ...r })),
282
+ coveredBytes: p.runs.reduce((n, r) => n + r.qe - r.qs, 0),
283
+ })),
284
+ committedRoots: roots.map((r) => ({
285
+ anchor: r.anchor,
286
+ vote: r.vote,
287
+ })),
288
+ },
277
289
  );
278
290
  return [];
279
291
  }
@@ -91,7 +91,23 @@ export async function confluenceJoin(
91
91
  // with the query is the CONSTRAINT being re-named (or its scaffolding),
92
92
  // never the open seat the question asks for — subtracted by identity,
93
93
  // below.
94
- const queryWin = pre.queryWindows;
94
+ // Completed assistant turns are context the engine already produced, not
95
+ // independent constraints asserted by the asker. Treating their windows as
96
+ // fresh constraints makes a cumulative dialogue's confluence work grow with
97
+ // every prior answer and can join the engine's own prose back into a reply.
98
+ // Recognition and attention still see the full transcript; only this
99
+ // mechanism's constraint population excludes answered spans.
100
+ const queryWin = new Map<number, number>();
101
+ let answered = 0;
102
+ for (const [off, id] of pre.queryWindows) {
103
+ while (
104
+ answered < ctx.answeredSpans.length &&
105
+ ctx.answeredSpans[answered][1] <= off
106
+ ) answered++;
107
+ const span = ctx.answeredSpans[answered];
108
+ if (span && span[0] <= off && off + W <= span[1]) continue;
109
+ queryWin.set(off, id);
110
+ }
95
111
  const queryIds = new Set(queryWin.values());
96
112
 
97
113
  // ── Constraint streams: which query content does each anchor CONTAIN ──
@@ -132,6 +148,17 @@ export async function confluenceJoin(
132
148
 
133
149
  const streams: Stream[] = [];
134
150
  const rankedCapped = ranked.length > pre.k ? ranked.slice(0, pre.k) : ranked;
151
+ // CONJUNCTIVITY EARLY-EXIT: a conjunctive query's top-ranked anchors
152
+ // (largest vote weight) must already form at least two independent
153
+ // constraint streams. When the first W anchors yield fewer than 2, the
154
+ // query has at most one topic — no join to compute. The full pre.k scan
155
+ // would produce the same null result after reading every anchor's bytes
156
+ // and computing window identities (profiled at 18K–50K leaf lookups per
157
+ // refusing query), so cutting the scan short here saves 50–70% of
158
+ // confluence cost on non-conjunctive queries while preserving every
159
+ // genuinely conjunctive case (whose top anchors ARE its constraints).
160
+ const earlyExit = Math.min(rankedCapped.length, ctx.space.maxGroup * 2);
161
+ let exitAfter = earlyExit;
135
162
  for (const cand of rankedCapped) {
136
163
  if (streams.some((s) => s.anchor === cand.anchor)) continue;
137
164
  const ids = new Set(windowsOfAnchor(cand.anchor).values());
@@ -151,6 +178,8 @@ export async function confluenceJoin(
151
178
  if (cover.length > 0 && bindsAConstituent(cover)) {
152
179
  streams.push({ anchor: cand.anchor, vote: cand.vote, ids, cover, held });
153
180
  }
181
+ // Early-exit: after 2W anchors, a non-conjunctive query is decided.
182
+ if (--exitAfter <= 0 && streams.length < 2) return null;
154
183
  }
155
184
  if (streams.length < 2) return null;
156
185
 
@@ -12,11 +12,12 @@
12
12
  import type { MindContext } from "../types.js";
13
13
  import type { ComputedResult, Site } from "../graph-search.js";
14
14
  import { read, resolve } from "../primitives.js";
15
- import { guidedFirst } from "../traverse.js";
15
+ import { guidedFirst, hubBound } from "../traverse.js";
16
16
  import { conceptHop } from "../match.js";
17
17
  import { bridge } from "../resonance.js";
18
18
  import { liftAnswer, segRestatesQuery } from "../types.js";
19
19
  import { decodeText, unexplainedLabel } from "../rationale.js";
20
+ import { indexOf } from "../../bytes.js";
20
21
  import type { RationaleItem } from "../rationale.js";
21
22
  import { rItem, rNode, traceDerivation } from "../trace.js";
22
23
  import type { PipelineMechanism } from "../pipeline-mechanism.js";
@@ -50,10 +51,31 @@ export async function resolveConcepts(
50
51
  export async function resolveConnectors(
51
52
  ctx: MindContext,
52
53
  sites: ReadonlyArray<Site>,
54
+ query?: Uint8Array,
53
55
  ): Promise<Map<string, Uint8Array>> {
54
56
  const links = new Map<string, Uint8Array>();
55
- const ordered = [...sites].sort((a, b) => a.start - b.start);
56
57
  const answerOf = (n: number) => guidedFirst(ctx, n) ?? n;
58
+ // A site's continuation already present elsewhere in the query is stale
59
+ // transcript evidence: cover still needs the site for structural context,
60
+ // but liftAnswer will trim that continuation as already answered. Building
61
+ // pairwise/n-ary bridges for it can only create connectors that are later
62
+ // discarded, and on cumulative dialogue that dominated the whole search.
63
+ let answered = 0;
64
+ const ordered = [...sites]
65
+ .sort((a, b) => a.start - b.start)
66
+ .filter((s) => {
67
+ while (
68
+ answered < ctx.answeredSpans.length &&
69
+ ctx.answeredSpans[answered][1] <= s.start
70
+ ) answered++;
71
+ const span = ctx.answeredSpans[answered];
72
+ if (span && span[0] <= s.start && s.end <= span[1]) return false;
73
+ if (query === undefined || ctx.answeredSpans.length === 0) return true;
74
+ const continuations = ctx.store.nextFirst(s.payload, hubBound(ctx));
75
+ return !continuations.some((answer) =>
76
+ indexOf(query, read(ctx, answer), 0) >= 0
77
+ );
78
+ });
57
79
  const bridgePair = async (l: number, r: number) => {
58
80
  if (l === r || links.has(l + "," + r)) return;
59
81
  const link = await bridge(ctx, read(ctx, l), read(ctx, r));
@@ -156,7 +178,12 @@ export const coverMechanism: PipelineMechanism = {
156
178
 
157
179
  if (sites.length === 0 && computed.length === 0) return [];
158
180
 
159
- const connectors = await resolveConnectors(ctx, sites);
181
+ const connectors = ctx.meter
182
+ ? await ctx.meter.time(
183
+ "cover.resolveConnectors",
184
+ () => resolveConnectors(ctx, sites, query),
185
+ )
186
+ : await resolveConnectors(ctx, sites, query);
160
187
  let splits = rec.splits;
161
188
  let starts = rec.starts;
162
189
  if (computed.length > 0) {
@@ -172,7 +199,12 @@ export const coverMechanism: PipelineMechanism = {
172
199
  starts.add(u.j);
173
200
  }
174
201
  }
175
- const concepts = await resolveConcepts(ctx, sites);
202
+ const concepts = ctx.meter
203
+ ? await ctx.meter.time(
204
+ "cover.resolveConcepts",
205
+ () => resolveConcepts(ctx, sites),
206
+ )
207
+ : await resolveConcepts(ctx, sites);
176
208
 
177
209
  const coverDeps = [
178
210
  ctx.trace?.lastIndex("recognise"),
@@ -316,51 +316,28 @@ export async function recallByResonance(
316
316
  // refusing on the reach bar). Approximate scores propose; the bridge's
317
317
  // byte-exact alignment and attestation gates decide.
318
318
  //
319
- // The proposal breadth here is widened PAST `k` — first by requesting
320
- // hubBound(ctx) candidates instead of `k` (recall's own tiers above
321
- // stay at `k`; this re-resonates only on the refusal path, exactly
322
- // where the bridge itself already runs), AND by asking the index to
323
- // search EXHAUSTIVELY. Both matter: the IVF only ever probes
324
- // ⌈√clusters⌉ of them (store.ts's efFor) REGARDLESS of k — widening k
325
- // alone just returns more hits from the SAME already-probed clusters,
326
- // never a hit whose vector lives in an unprobed one. Measured live:
327
- // "What is the chemical symbol for water?" needs "What is the
328
- // chemical formula for water?", scoring only 0.58 against the
329
- // query's gist (a MIDDLE-of-string word swap perturbs the river-fold
330
- // tree hash far more than a same-length TAIL swap like the "carbon"/
331
- // "oxygen" neighbours that outrank it at 0.87+) — absent from the
332
- // resonance list even at k=5000, present and byte-exact-verified the
333
- // moment it's force-fed to the bridge directly. `exhaustive` is the
334
- // natural, tuning-free ceiling (probe every cluster) for a call that
335
- // is ALREADY refusal-path-only and must not miss a candidate hiding
336
- // behind an unlucky structural distance.
337
- //
338
- // MEASURED COST, AND WHY IT STAYS (17.9M vectors / 325K contexts):
339
- // this one call is ~570 ms and ~45% of all inference time on a refusing
340
- // query. The cost is entirely `exhaustive` (nprobe = every cluster),
341
- // NOT the widened k — timed on that store: k=571 exhaustive 632 ms,
342
- // k=24 exhaustive 536 ms, k=571 NON-exhaustive 12 ms. So narrowing k
343
- // buys nothing and the 50x is the whole-index scan itself.
344
- // It is load-bearing: over an 18-query battery the bridge produced a
345
- // winner 4 times, and ALL FOUR winners came from this proposal channel
346
- // — the anchor-climb channel won nothing on its own. Reordering the
347
- // channels (climb first, resonate only on failure) would therefore pay
348
- // the climb, fail, and pay this anyway. Do not weaken it without
349
- // re-running that measurement.
350
- // Handed to the bridge as a THUNK: this exhaustive whole-index probe is
351
- // the most expensive single act on the refusal path, and the bridge's
352
- // own cheap gates (query length, the O(|query|) stored-window anchor
353
- // scan) can refuse without any proposal at all. See substitutionBridge.
319
+ // Reuse recall's already-ranked proposals. Never scan every IVF cluster:
320
+ // exact co-occurrence and bounded anchor ascent are the bridge's structural
321
+ // proposal channels, while an exhaustive ANN call made every honest
322
+ // refusal cost hundreds of milliseconds regardless of k.
354
323
  const wideIds = async (): Promise<ReadonlyArray<number>> => {
355
- const wide = k >= hubBound(ctx)
356
- ? whole
357
- : ctx.meter
358
- ? await ctx.meter.time(
359
- "recall.exhaustiveResonate",
360
- () => ctx.store.resonate(queryGist, hubBound(ctx), true),
361
- )
362
- : await ctx.store.resonate(queryGist, hubBound(ctx), true);
363
- return wide.map((h) => h.id);
324
+ // When the top resonance hit is below the concept threshold, the query
325
+ // gist has no concept-level match to any stored form — an exhaustive √N
326
+ // ANN would only score more vectors below the bar (profiled at 38K–40K
327
+ // annVectorReads per refusing query on a 325K-context store). The
328
+ // bridge's structural channels (junction walks, anchor climbs) are the
329
+ // correct proposal source for a query whose gist has no clean match;
330
+ // the ANN cannot propose what the gist cannot rank.
331
+ const marketScale = k * ctx.space.maxGroup;
332
+ if (corpusN(ctx) <= marketScale ** 3) {
333
+ const exhaustive = await ctx.store.resonate(
334
+ queryGist,
335
+ hubBound(ctx),
336
+ true,
337
+ );
338
+ return exhaustive.map((h) => h.id);
339
+ }
340
+ return whole.map((h) => h.id);
364
341
  };
365
342
  const bridged = await substitutionBridge(ctx, query, wideIds);
366
343
  if (bridged !== null) {
package/src/mind/mind.ts CHANGED
@@ -83,6 +83,10 @@ export interface ConversationState {
83
83
  * is `boundaries[0]`; the second turn starts at that offset, and so
84
84
  * on. Empty for a single-turn or new conversation. */
85
85
  boundaries: number[];
86
+ /** Byte spans occupied by replies produced by this Mind. Unlike boundary
87
+ * parity, this remains exact when a turn receives an empty reply. Optional
88
+ * so states saved before the field existed remain restorable. */
89
+ answeredSpans?: Array<[number, number]>;
86
90
  }
87
91
 
88
92
  /** An active conversation handle. Opaque — interact through the Mind's
@@ -110,6 +114,7 @@ interface ConversationData {
110
114
  tree: Sema;
111
115
  bytes: Uint8Array;
112
116
  boundaries: number[];
117
+ answeredSpans: Array<[number, number]>;
113
118
  perceiveMemo: Map<string, Sema>;
114
119
  recogniseMemo: Map<string, Recognition>;
115
120
  climbMemo: Map<string, Map<string, AttentionRead>>;
@@ -128,7 +133,12 @@ import {
128
133
  read,
129
134
  resolve as resolveImpl,
130
135
  } from "./primitives.js";
131
- import { chooseNext, edgeAncestors as edgeAncestorsFn } from "./traverse.js";
136
+ import {
137
+ chooseNext,
138
+ edgeAncestors as edgeAncestorsFn,
139
+ invalidateStructuralCaches,
140
+ } from "./traverse.js";
141
+ import { invalidateJunctionCache } from "./junction.js";
132
142
  import { follow } from "./match.js";
133
143
  import { recognise, segment } from "./recognition.js";
134
144
  import { meaningOf } from "./resonance.js";
@@ -245,6 +255,7 @@ export class Mind implements MindContext {
245
255
 
246
256
  /** Memo of the consensus climb — content-keyed. See {@link MindContext.climbMemo}. */
247
257
  climbMemo: Map<string, Map<string, AttentionRead>> | null = null;
258
+ _structMemoKey: object = {};
248
259
 
249
260
  /** Memo of recognise() — content-keyed. See {@link MindContext.recogniseMemo}. */
250
261
  recogniseMemo: Map<string, Recognition> | null = null;
@@ -259,6 +270,8 @@ export class Mind implements MindContext {
259
270
  { id: number; len: number }
260
271
  >
261
272
  | null = null;
273
+ answeredSpans: ReadonlyArray<readonly [number, number]> = [];
274
+ currentTurnStart = 0;
262
275
 
263
276
  /** The perceived gist of the query currently being answered. Set by `think`
264
277
  * before the graph search runs; `chooseNext` consults it as a gate (a null
@@ -466,6 +479,11 @@ export class Mind implements MindContext {
466
479
  this.recogniseMemo = conv ? conv.recogniseMemo : new Map();
467
480
  this.perceiveMemo = conv ? conv.perceiveMemo : new Map();
468
481
  this._resolvedSubtrees = conv ? conv.resolvedSubtrees : null;
482
+ // Inference is a pure function of cumulative bytes. Conversation
483
+ // boundaries remain persistence/API metadata and must not select a
484
+ // different mechanism path than respond() on the identical byte stream.
485
+ this.answeredSpans = [];
486
+ this.currentTurnStart = 0;
469
487
  this.canon = canon ?? null;
470
488
  this.canonMemo = canon ? new Map() : null;
471
489
  this._beginMeter();
@@ -514,6 +532,8 @@ export class Mind implements MindContext {
514
532
  this.recogniseMemo = null;
515
533
  this.perceiveMemo = null;
516
534
  this._resolvedSubtrees = null;
535
+ this.answeredSpans = [];
536
+ this.currentTurnStart = 0;
517
537
  this.canon = null;
518
538
  this.canonMemo = null;
519
539
  this._edgeGuide = null;
@@ -623,6 +643,15 @@ export class Mind implements MindContext {
623
643
  const id = this._nextConvId++;
624
644
  const initBytes = state?.context ?? new Uint8Array(0);
625
645
  const initBoundaries = state?.boundaries ? [...state.boundaries] : [];
646
+ const initAnswered = state?.answeredSpans
647
+ ? state.answeredSpans.map(([start, end]) =>
648
+ [start, end] as [number, number]
649
+ )
650
+ : initBoundaries.flatMap((start, i, cuts) =>
651
+ i % 2 === 0 && i + 1 < cuts.length
652
+ ? [[start, cuts[i + 1]] as [number, number]]
653
+ : []
654
+ );
626
655
  const tree = bytesToTree(
627
656
  this.space,
628
657
  this.alphabet,
@@ -635,6 +664,7 @@ export class Mind implements MindContext {
635
664
  tree,
636
665
  bytes: initBytes,
637
666
  boundaries: initBoundaries,
667
+ answeredSpans: initAnswered,
638
668
  perceiveMemo: new Map(),
639
669
  recogniseMemo: new Map(),
640
670
  climbMemo: new Map(),
@@ -657,6 +687,7 @@ export class Mind implements MindContext {
657
687
  return {
658
688
  context: data.bytes,
659
689
  boundaries: [...data.boundaries],
690
+ answeredSpans: data.answeredSpans.map(([start, end]) => [start, end]),
660
691
  };
661
692
  }
662
693
 
@@ -761,7 +792,11 @@ export class Mind implements MindContext {
761
792
  // the cumulative continuous shape multi-turn training deposits, so a
762
793
  // later turn can refer to what was ANSWERED ("which of those two…"),
763
794
  // not only to what was asked.
764
- if (response.bytes.length > 0) this.addTurn(conv, response.bytes);
795
+ if (response.bytes.length > 0) {
796
+ const start = data.bytes.length;
797
+ this.addTurn(conv, response.bytes);
798
+ data.answeredSpans.push([start, data.bytes.length]);
799
+ }
765
800
 
766
801
  return { response, state: this.conversationState(conv)! };
767
802
  } finally {
@@ -816,6 +851,8 @@ export class Mind implements MindContext {
816
851
  second?: Input,
817
852
  onDeposit?: (report: import("./learning.js").DepositReport) => void,
818
853
  ): Promise<(Sema & { id: number }) | undefined> {
854
+ invalidateStructuralCaches(this);
855
+ invalidateJunctionCache(this);
819
856
  return ingest(this, input, second, onDeposit);
820
857
  }
821
858
 
@@ -100,8 +100,8 @@ export class Precomputed {
100
100
  * serves every mechanism that prices commonality — AND the consensus
101
101
  * climb, which is the largest consumer and used to build its own. The
102
102
  * ONE definition of its lifetime lives in traverse.ts
103
- * ({@link sharedReachMemo}): response-scoped for respond(),
104
- * conversation-scoped across turns, always cold under a trace. */
103
+ * ({@link sharedReachMemo}): session-scoped between writes and always cold
104
+ * under a trace. */
105
105
  private _reach?: Map<number, AncestorReach>;
106
106
  get reachMemo(): Map<number, AncestorReach> {
107
107
  return this._reach ??= sharedReachMemo(this.ctx);
@@ -232,7 +232,9 @@ function computeWeave(
232
232
  // recognising) a corpus-sized deposit: profiled on a 17.7M-node store,
233
233
  // uncapped weaves spent 5–8s per query recognising conversation-length
234
234
  // anchors that could never form a weave point.
235
- const capBytes = query.length * quantum;
235
+ const askerBytes = query.length -
236
+ ctx.answeredSpans.reduce((n, [start, end]) => n + end - start, 0);
237
+ const capBytes = askerBytes * quantum;
236
238
  // EXCLUSIVITY IS ARBITRATED BY THE CLIMB'S VOTE ORDER, DELIBERATELY. A query
237
239
  // byte can only be independent evidence for ONE point, so points are built in
238
240
  // ranked order and each new point's runs are trimmed against every point
@@ -265,10 +267,90 @@ function computeWeave(
265
267
  ctxBytes: Uint8Array;
266
268
  raw: GradedRun[];
267
269
  }> = [];
270
+ const querySegments: Array<[number, number]> = [];
271
+ let segmentStart = 0;
272
+ for (const [start, end] of ctx.answeredSpans) {
273
+ if (segmentStart < start) querySegments.push([segmentStart, start]);
274
+ segmentStart = Math.max(segmentStart, end);
275
+ }
276
+ if (segmentStart < query.length) {
277
+ querySegments.push([segmentStart, query.length]);
278
+ }
279
+ const weaveLength = querySegments.reduce((n, [s, e]) => n + e - s, 0);
280
+ const weaveQuery = new Uint8Array(weaveLength);
281
+ const weaveMap: Array<{
282
+ compactStart: number;
283
+ originalStart: number;
284
+ length: number;
285
+ }> = [];
286
+ let compactStart = 0;
287
+ for (const [start, end] of querySegments) {
288
+ weaveQuery.set(query.subarray(start, end), compactStart);
289
+ weaveMap.push({
290
+ compactStart,
291
+ originalStart: start,
292
+ length: end - start,
293
+ });
294
+ compactStart += end - start;
295
+ }
296
+ const segmentOf = (start: number, end: number) => {
297
+ let lo = 0;
298
+ let hi = weaveMap.length;
299
+ while (lo < hi) {
300
+ const mid = (lo + hi) >>> 1;
301
+ if (weaveMap[mid].originalStart <= start) lo = mid + 1;
302
+ else hi = mid;
303
+ }
304
+ const part = lo > 0 ? weaveMap[lo - 1] : undefined;
305
+ return part && end <= part.originalStart + part.length ? part : undefined;
306
+ };
307
+ const weaveSites = pre.rec.sites.flatMap((s) => {
308
+ const part = segmentOf(s.start, s.end);
309
+ return part
310
+ ? [{
311
+ ...s,
312
+ start: part.compactStart + s.start - part.originalStart,
313
+ end: part.compactStart + s.end - part.originalStart,
314
+ }]
315
+ : [];
316
+ });
268
317
  for (const cand of rankedCapped) {
269
318
  const ctxBytes = read(ctx, cand.anchor, capBytes + 1);
270
319
  if (ctxBytes.length === 0 || ctxBytes.length > capBytes) continue;
271
- const raw = alignGraded(ctx, query, ctxBytes, pre.rec.sites);
320
+ // CAST compares structures stated by the asker. Completed assistant turns
321
+ // remain available to recognition and the climb as conversation context,
322
+ // but aligning every candidate across their full prose makes weave work
323
+ // grow with answer length and lets the engine analogise against its own
324
+ // previous output. The compact asker stream is aligned once (so candidate
325
+ // windows are not rebuilt per turn), then every run is split back across
326
+ // the original turn segments so no evidence crosses an omitted boundary.
327
+ const raw = alignGraded(ctx, weaveQuery, ctxBytes, weaveSites).flatMap(
328
+ (r) => {
329
+ let lo = 0;
330
+ let hi = weaveMap.length;
331
+ while (lo < hi) {
332
+ const mid = (lo + hi) >>> 1;
333
+ if (weaveMap[mid].compactStart <= r.qs) lo = mid + 1;
334
+ else hi = mid;
335
+ }
336
+ const out: GradedRun[] = [];
337
+ for (let pi = Math.max(0, lo - 1); pi < weaveMap.length; pi++) {
338
+ const part = weaveMap[pi];
339
+ if (part.compactStart >= r.qe) break;
340
+ const partEnd = part.compactStart + part.length;
341
+ const start = Math.max(r.qs, part.compactStart);
342
+ const end = Math.min(r.qe, partEnd);
343
+ if (start >= end) continue;
344
+ out.push({
345
+ ...r,
346
+ qs: part.originalStart + start - part.compactStart,
347
+ qe: part.originalStart + end - part.compactStart,
348
+ cs: r.cs + start - r.qs,
349
+ });
350
+ }
351
+ return out;
352
+ },
353
+ );
272
354
  if (raw.length === 0) continue;
273
355
  for (const r of raw) {
274
356
  for (let i = r.qs; i < r.qe; i++) depth[i] += r.weight;
@@ -57,7 +57,7 @@ async function collectComputed(
57
57
  // invests in its own precomputation. CAST's and confluence's floors (2*STEP,
58
58
  // 3*STEP) then fail `worthRunning` and are skipped by the SAME admissible-
59
59
  // floor pruning every mechanism is already subject to — not by asking
60
- // "is this an extension?". Grade TIES keep the earlier candidate, so this
60
+ // "is this an extension?". Grade TIES keep the earlier candidate, so this
61
61
  // order is also the tie-break priority: cover, cast, confluence, extraction,
62
62
  // recall.
63
63
  export const defaultMechanisms: PipelineMechanism[] = [
@@ -74,14 +74,21 @@ export async function reason(
74
74
  // grounding can pre-consume one node per recognised site, O(query length));
75
75
  // nodes past the cap are still consumed directly, they just skip the
76
76
  // synonym expansion.
77
- let haloSweeps = 0;
78
- for (const id of preConsumed) {
79
- consumeNode(id);
80
- if (haloSweeps >= ctx.cfg.haloQueryK) continue;
81
- const h = ctx.store.halo(id);
82
- if (!h) continue;
83
- haloSweeps++;
84
- for (const sib of await haloSiblings(ctx, id, h)) consumeNode(sib.id);
77
+ const preconsume = async () => {
78
+ let haloSweeps = 0;
79
+ for (const id of preConsumed) {
80
+ consumeNode(id);
81
+ if (haloSweeps >= ctx.cfg.haloQueryK) continue;
82
+ const h = ctx.store.halo(id);
83
+ if (!h) continue;
84
+ haloSweeps++;
85
+ for (const sib of await haloSiblings(ctx, id, h)) consumeNode(sib.id);
86
+ }
87
+ };
88
+ if (ctx.meter) {
89
+ await ctx.meter.time("reason.preconsumeHalos", preconsume);
90
+ } else {
91
+ await preconsume();
85
92
  }
86
93
 
87
94
  let cur = answer;
@@ -343,6 +343,46 @@ function recogniseImpl(ctx: MindContext, bytes: Uint8Array): Recognition {
343
343
  return singleLeaf[p];
344
344
  };
345
345
 
346
+ // ── exact query-edge forms beyond the canonical chain reach ─────────
347
+ //
348
+ // At corpus scale off-boundary atom chains are deliberately suppressed,
349
+ // but a whole trained form can be longer than chainReach(W) and sit at a
350
+ // query edge without being a subtree of the query's larger root. Appending
351
+ // another topic demonstrates the failure: the exact 30-byte trained
352
+ // question `What is the capital of France?` ends inside the larger query's
353
+ // [27,33) content segment, so neither the structural walk nor a W² chain
354
+ // can name it.
355
+ //
356
+ // Probe only prefix/suffix endpoints within one maximum segment of the
357
+ // query's own content cuts. The flat-branch lookup is byte-exact and runs
358
+ // before resolveSpan pays for a fold; approximate evidence never enters.
359
+ // This tier is needed only where atom chains are suppressed. Small stores
360
+ // retain their existing decomposition unchanged.
361
+ if (atomsAreHubs) {
362
+ const allLeafIds = singleLeaf.map((x) => x?.id ?? null);
363
+ if (allLeafIds.every((x): x is number => x !== null)) {
364
+ const radius = ctx.space.seats.length;
365
+ const endpoints = new Set<number>([0, bytes.length]);
366
+ for (const cut of startList) {
367
+ for (
368
+ let p = Math.max(0, cut - radius);
369
+ p <= Math.min(bytes.length, cut + radius);
370
+ p++
371
+ ) endpoints.add(p);
372
+ }
373
+ const ordered = [...endpoints].sort((a, b) => a - b);
374
+ const probe = (start: number, end: number): void => {
375
+ if (end - start < W || end - start <= chainReach(W)) return;
376
+ const ids = allLeafIds.slice(start, end);
377
+ if (store.findBranch(ids) === null) return;
378
+ const id = resolveSpan(start, end);
379
+ if (id !== null) emit(start, end, id);
380
+ };
381
+ for (const end of ordered) probe(0, end);
382
+ for (const start of ordered) probe(start, bytes.length);
383
+ }
384
+ }
385
+
346
386
  const chunkEnd = new Uint32Array(bytes.length);
347
387
  const sorted = [...starts].sort((a, b) => a - b);
348
388
  for (let si = 0; si < sorted.length; si++) {
Binary file