@hviana/sema 0.5.8 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/AGENTS.md +43 -4
  2. package/DATASETS.md +160 -0
  3. package/README.md +12 -0
  4. package/dist/example/train_base/cache.d.ts +35 -0
  5. package/dist/example/train_base/cache.js +211 -0
  6. package/dist/example/train_base/config.d.ts +21 -0
  7. package/dist/example/train_base/config.js +94 -0
  8. package/dist/example/train_base/corpora/aya.d.ts +19 -0
  9. package/dist/example/train_base/corpora/aya.js +76 -0
  10. package/dist/example/train_base/corpora/converted-parquet.d.ts +14 -0
  11. package/dist/example/train_base/corpora/converted-parquet.js +44 -0
  12. package/dist/example/train_base/corpora/genknow.d.ts +14 -0
  13. package/dist/example/train_base/corpora/genknow.js +83 -0
  14. package/dist/example/train_base/corpora/index.d.ts +29 -0
  15. package/dist/example/train_base/corpora/index.js +81 -0
  16. package/dist/example/train_base/corpora/massive.d.ts +7 -0
  17. package/dist/example/train_base/corpora/massive.js +98 -0
  18. package/dist/example/train_base/corpora/oasst2.d.ts +52 -0
  19. package/dist/example/train_base/corpora/oasst2.js +120 -0
  20. package/dist/example/train_base/corpora/smolsent.d.ts +23 -0
  21. package/dist/example/train_base/corpora/smolsent.js +156 -0
  22. package/dist/example/train_base/corpora/soda.d.ts +12 -0
  23. package/dist/example/train_base/corpora/soda.js +113 -0
  24. package/dist/example/train_base/corpora/taskmaster.d.ts +15 -0
  25. package/dist/example/train_base/corpora/taskmaster.js +144 -0
  26. package/dist/example/train_base/corpora/wiki2.d.ts +23 -0
  27. package/dist/example/train_base/corpora/wiki2.js +132 -0
  28. package/dist/example/train_base/corpus.d.ts +88 -0
  29. package/dist/example/train_base/corpus.js +65 -0
  30. package/dist/example/train_base/discovery.d.ts +48 -0
  31. package/dist/example/train_base/discovery.js +143 -0
  32. package/dist/example/train_base/http.d.ts +82 -0
  33. package/dist/example/train_base/http.js +219 -0
  34. package/dist/example/train_base/items.d.ts +46 -0
  35. package/dist/example/train_base/items.js +98 -0
  36. package/dist/example/train_base/main.d.ts +4 -0
  37. package/dist/example/train_base/main.js +207 -0
  38. package/dist/example/train_base/progress.d.ts +34 -0
  39. package/dist/example/train_base/progress.js +114 -0
  40. package/dist/example/train_base/readers.d.ts +125 -0
  41. package/dist/example/train_base/readers.js +391 -0
  42. package/dist/example/train_base/runtime.d.ts +115 -0
  43. package/dist/example/train_base/runtime.js +637 -0
  44. package/dist/example/train_base/stage.d.ts +3 -0
  45. package/dist/example/train_base/stage.js +246 -0
  46. package/dist/example/train_base/ui.d.ts +88 -0
  47. package/dist/example/train_base/ui.js +272 -0
  48. package/dist/src/geometry.d.ts +20 -0
  49. package/dist/src/geometry.js +22 -0
  50. package/dist/src/mind/attention.d.ts +6 -0
  51. package/dist/src/mind/attention.js +44 -4
  52. package/dist/src/mind/learning.js +134 -50
  53. package/dist/src/mind/mechanisms/cast.js +45 -1
  54. package/dist/src/mind/mind.d.ts +7 -2
  55. package/dist/src/mind/mind.js +15 -3
  56. package/dist/src/mind/reasoning.js +59 -5
  57. package/dist/src/mind/recognition.js +29 -3
  58. package/dist/src/mind/traverse.d.ts +16 -0
  59. package/dist/src/mind/traverse.js +18 -0
  60. package/dist/src/store-sqlite.d.ts +4 -0
  61. package/dist/src/store-sqlite.js +47 -0
  62. package/dist/src/store.d.ts +7 -0
  63. package/example/train_base/cache.ts +251 -0
  64. package/example/train_base/config.ts +128 -0
  65. package/example/train_base/corpora/aya.ts +106 -0
  66. package/example/train_base/corpora/converted-parquet.ts +64 -0
  67. package/example/train_base/corpora/genknow.ts +114 -0
  68. package/example/train_base/corpora/index.ts +88 -0
  69. package/example/train_base/corpora/massive.ts +111 -0
  70. package/example/train_base/corpora/oasst2.ts +163 -0
  71. package/example/train_base/corpora/smolsent.ts +203 -0
  72. package/example/train_base/corpora/soda.ts +130 -0
  73. package/example/train_base/corpora/taskmaster.ts +217 -0
  74. package/example/train_base/corpora/wiki2.ts +190 -0
  75. package/example/train_base/corpus.ts +150 -0
  76. package/example/train_base/discovery.ts +203 -0
  77. package/example/train_base/http.ts +284 -0
  78. package/example/train_base/items.ts +118 -0
  79. package/example/train_base/main.ts +240 -0
  80. package/example/train_base/progress.ts +149 -0
  81. package/example/train_base/readers.ts +505 -0
  82. package/example/train_base/runtime.ts +894 -0
  83. package/example/train_base/stage.ts +276 -0
  84. package/example/train_base/ui.ts +333 -0
  85. package/jsr.json +1 -1
  86. package/package.json +2 -4
  87. package/src/geometry.ts +23 -0
  88. package/src/mind/attention.ts +54 -1
  89. package/src/mind/learning.ts +137 -43
  90. package/src/mind/mechanisms/cast.ts +48 -1
  91. package/src/mind/mind.ts +13 -2
  92. package/src/mind/reasoning.ts +64 -5
  93. package/src/mind/recognition.ts +29 -3
  94. package/src/mind/traverse.ts +19 -0
  95. package/src/store-sqlite.ts +53 -0
  96. package/src/store.ts +28 -0
  97. package/test/13-conversation.test.mjs +1 -1
  98. package/test/29-counterfactual.test.mjs +43 -6
  99. package/test/77-company-saturation.test.mjs +302 -0
  100. package/test/78-atom-hub-recognition-cliff.test.mjs +135 -0
  101. package/test/84-composed-answer-honesty.test.mjs +137 -0
  102. package/test/85-answered-directly.test.mjs +126 -0
  103. package/test/86-cast-voices-committed.test.mjs +164 -0
  104. package/test/87-codominant-commitment.test.mjs +250 -0
  105. package/test/88-dependency-footprint.test.mjs +99 -0
  106. package/dist/example/train_base.d.ts +0 -93
  107. package/dist/example/train_base.js +0 -2269
  108. package/example/train_base.ts +0 -2735
@@ -1169,7 +1169,7 @@ export function commitVotes(ctx, pooled, sat, regions, regionVoter, N, td, cfg)
1169
1169
  const floor = consensusFloor(N);
1170
1170
  const placed = [];
1171
1171
  const roots = [];
1172
- const recordAnchor = (point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, rejectionReasons) => {
1172
+ const recordAnchor = (point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons) => {
1173
1173
  if (!td)
1174
1174
  return;
1175
1175
  td.anchors.push({
@@ -1189,6 +1189,7 @@ export function commitVotes(ctx, pooled, sat, regions, regionVoter, N, td, cfg)
1189
1189
  passesNaturalBreak,
1190
1190
  passesConsensusFloor,
1191
1191
  pastLeadingSaturation,
1192
+ tiedWithDominant,
1192
1193
  rejectionReasons,
1193
1194
  },
1194
1195
  });
@@ -1204,6 +1205,7 @@ export function commitVotes(ctx, pooled, sat, regions, regionVoter, N, td, cfg)
1204
1205
  let passesNaturalBreak;
1205
1206
  let passesConsensusFloor;
1206
1207
  let pastLeadingSaturation;
1208
+ let tiedWithDominant;
1207
1209
  const rejectionReasons = [];
1208
1210
  if (absorbed) {
1209
1211
  status = "overlap";
@@ -1229,7 +1231,45 @@ export function commitVotes(ctx, pooled, sat, regions, regionVoter, N, td, cfg)
1229
1231
  else {
1230
1232
  passesNaturalBreak = vote >= rootCut;
1231
1233
  passesConsensusFloor = vote >= floor;
1232
- if (passesNaturalBreak && passesConsensusFloor && pastLeading) {
1234
+ // CO-DOMINANT an anchor the estimator cannot separate from the
1235
+ // dominant inherits the dominant's exemption, because that exemption's
1236
+ // only warrant is being TOP, and "top" is not a fact about the corpus
1237
+ // when the ordering moves with the seed.
1238
+ //
1239
+ // The dominant bypasses both vote gates ("it always grounds"); the
1240
+ // runner-up is held to an absolute ln(N)+1/2 floor the dominant never
1241
+ // had to clear. Which of them gets the exemption is then decided by a
1242
+ // sort over ESTIMATED quantities. Measured on test/29 D1's corpus,
1243
+ // 60 seeds per D — true separation 0.54s / 0.75s / 1.04s:
1244
+ //
1245
+ // D s=1/sqrt(D) vote SD (estimated anchor) SD/s flips
1246
+ // 256 0.0625 0.0561 0.90 19/60
1247
+ // 1024 0.0313 0.0268 0.86 12/60
1248
+ // 4096 0.0156 0.0074 0.48 2/60
1249
+ //
1250
+ // The SD tracks 1/sqrt(D) and the flip rate collapses with it, so the
1251
+ // reordering is the ESTIMATOR's, not the corpus's. The loser was then
1252
+ // refused by a floor at 1.599 that neither anchor could ever reach
1253
+ // (corpusN 3) — a coin flip decided which single structure the query
1254
+ // was allowed to have settled on.
1255
+ //
1256
+ // THE BAND IS sqrt(k)*s, NOT s. A vote is a SUM over the anchor's own
1257
+ // contributing regions, so its noise grows as sqrt(k); pricing a summed
1258
+ // margin against one s would be the category error chooseNext's comment
1259
+ // warns about. k is `regionAxioms`, already in hand; s is
1260
+ // `estimatorNoise(D)`, already derived. No constant is introduced.
1261
+ // Verified conservative: measured SD/(sqrt(k)*s) never exceeded 0.72.
1262
+ //
1263
+ // BOUNDED BY CONSTRUCTION: admission requires indistinguishability from
1264
+ // an anchor ALREADY admitted, so it can only admit what the ordinary
1265
+ // rule would have admitted had the noise fallen the other way. It is
1266
+ // N-independent for the same reason — a statement about the estimator,
1267
+ // not about corpus size.
1268
+ const tieBand = Math.sqrt(Math.max(1, regionAxioms.get(point.anchor) ?? 1)) * estimatorNoise(ctx.store.D);
1269
+ const dominantVote = votesIdf.get(roots[0].anchor) ?? 0;
1270
+ tiedWithDominant = dominantVote - vote < tieBand;
1271
+ if (((passesNaturalBreak && passesConsensusFloor) || tiedWithDominant) &&
1272
+ pastLeading) {
1233
1273
  status = "root";
1234
1274
  }
1235
1275
  else {
@@ -1247,11 +1287,11 @@ export function commitVotes(ctx, pooled, sat, regions, regionVoter, N, td, cfg)
1247
1287
  roots.push(point);
1248
1288
  }
1249
1289
  else {
1250
- recordAnchor(point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, rejectionReasons);
1290
+ recordAnchor(point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons);
1251
1291
  continue;
1252
1292
  }
1253
1293
  }
1254
- recordAnchor(point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, rejectionReasons);
1294
+ recordAnchor(point, rank, status, dominant, passesNaturalBreak, passesConsensusFloor, pastLeadingSaturation, tiedWithDominant, rejectionReasons);
1255
1295
  placed.push(point);
1256
1296
  }
1257
1297
  traceAttention(ctx, regions, regionVoter, roots, steps, td, cfg ? { ...cfg, naturalBreak: rootCut, consensusFloor: floor } : undefined, ranked);
@@ -7,8 +7,9 @@ import { bindSeat, companySignature, isChunk } from "../sema.js";
7
7
  import { changedNodes } from "./types.js";
8
8
  import { inputBytes, perceiveDeposit, resolve, } from "./primitives.js";
9
9
  import { canonicalWindows, leafIdPrefix } from "./canonical.js";
10
+ import { rItem, rNode } from "./trace.js";
10
11
  import { hubBound } from "./traverse.js";
11
- import { dominates } from "../geometry.js";
12
+ import { dominates, estimatorNoise, profileCapacity } from "../geometry.js";
12
13
  import { fold as foldVecs } from "../sema.js";
13
14
  /** Intern a perceived tree into node ids, bottom-up, sharing equal subtrees.
14
15
  * Returns the root node id and a map from tree nodes to their ids.
@@ -191,14 +192,98 @@ async function propagateSuffixes(ctx, src, dst) {
191
192
  await ctx.store.link(id, dst);
192
193
  }
193
194
  }
194
- /** How many constituents one profile may VISIT. A partner's constituent tree
195
- * is O(len/W) nodes, so an uncapped descent would make a pour cost grow with
196
- * the partner's LENGTH — and a partner is a whole deposit, which may be a
197
- * paragraph. The budget is what keeps a pour O(1) in the input, the property
198
- * that lets {@link companyProfile} claim no new cost class. It binds only on
199
- * long partners whose constituents are all corpus-unique; the descent's own
200
- * stop rule (below) reaches recurring units far sooner on a trained store. */
201
- const PROFILE_VISITS = 64;
195
+ /** Deterministic priority of a node for bottom-k selection a fixed integer
196
+ * mix of the node id, NOT a function of the config seed.
197
+ *
198
+ * Seed-independence is the point: the sketch is a property of the STORE, so
199
+ * two Minds over one store must agree on it, and a rebuilt sketch must match
200
+ * a stored one. (Contrast {@link companySignature}, which is seeded that is
201
+ * the VECTOR, this is only the CHOICE of which vectors to superpose.)
202
+ *
203
+ * Selecting the k smallest priorities makes the sketch a bottom-k sample keyed
204
+ * on each constituent's own identity, so a unit shared by two partners is kept
205
+ * by BOTH or neither, whatever its depth or position in either fold. That is
206
+ * what removes the traversal-order dependence a visit budget necessarily had. */
207
+ function unitPriority(id) {
208
+ let h = (id ^ 0x9e3779b9) >>> 0;
209
+ h = Math.imul(h ^ (h >>> 16), 0x85ebca6b) >>> 0;
210
+ h = Math.imul(h ^ (h >>> 13), 0xc2b2ae35) >>> 0;
211
+ return (h ^ (h >>> 16)) >>> 0;
212
+ }
213
+ /** Whether `n` is a MINIMAL UNIT: a stored branch, at least one fold window
214
+ * wide, no constituent of its own at or above W. Every clause is INTRINSIC —
215
+ * a property of the node's own subtree — which is what lets a sketch be stored
216
+ * once and stay valid as the corpus grows. The two corpus-relative readings
217
+ * (half-dominance of the PARTNER, and the hub test) are deliberately excluded
218
+ * and applied by {@link companyProfile} at pour time. */
219
+ function isMinimalUnit(ctx, n, W) {
220
+ if (n < 0)
221
+ return false; // byte atom — fan-in is the alphabet's
222
+ const kids = ctx.store.get(n)?.kids;
223
+ if (kids == null)
224
+ return false; // stored kid-less node: also an atom
225
+ if (ctx.store.contentLen(n, W) < W)
226
+ return false;
227
+ for (const kid of kids) {
228
+ if (kid >= 0 && ctx.store.contentLen(kid, W) >= W)
229
+ return false; // composite
230
+ }
231
+ return true;
232
+ }
233
+ /** The BOTTOM-K CONSTITUENT SKETCH of a node: the `k = profileCapacity(D)`
234
+ * minimal units of its subtree with the smallest {@link unitPriority}.
235
+ *
236
+ * COMPOSABLE, WHICH IS WHY IT COSTS NOTHING TWICE. Bottom-k of a union is
237
+ * the bottom-k of the children's bottom-k sets, so a node's sketch is built
238
+ * from its kids' sketches and each recursive result is stored on the way out.
239
+ * A partner met again reads O(k); an accumulated conversation, where turn k's
240
+ * context is a prefix of turn k+1's, reuses every unchanged child and pays
241
+ * O(changed) instead of O(context) — the quadratic that made a visit budget
242
+ * look necessary in the first place.
243
+ *
244
+ * It is DURABLE DERIVED STATE, not a cache (see Store.sketchGet): a miss must
245
+ * cost time only, and this decides which terms enter a halo. A backend
246
+ * without the capability recomputes per pour and loses only the amortisation.
247
+ *
248
+ * Recursion depth is the fold's, O(log_W len), and each level does O(k·arity)
249
+ * work, so construction is one pass over the subtree — the same pass the
250
+ * deposit that interned it already performed. */
251
+ function constituentSketch(ctx, id, k) {
252
+ const stored = ctx.store.sketchGet?.(id);
253
+ if (stored != null)
254
+ return stored; // [] is a real answer; null is "unknown"
255
+ const W = ctx.space.maxGroup;
256
+ const kids = id < 0 ? null : ctx.store.get(id)?.kids;
257
+ let out;
258
+ if (kids == null) {
259
+ out = [];
260
+ }
261
+ else {
262
+ const pool = [];
263
+ for (const kid of kids) {
264
+ if (isMinimalUnit(ctx, kid, W))
265
+ pool.push(kid);
266
+ else if (kid >= 0) {
267
+ for (const g of constituentSketch(ctx, kid, k))
268
+ pool.push(g);
269
+ }
270
+ }
271
+ // Bottom-k by identity, then by id so ties are corpus-determined (§2.1).
272
+ pool.sort((a, b) => (unitPriority(a) - unitPriority(b)) || (a - b));
273
+ const seen = new Set();
274
+ out = [];
275
+ for (const n of pool) {
276
+ if (seen.has(n))
277
+ continue;
278
+ seen.add(n);
279
+ out.push(n);
280
+ if (out.length >= k)
281
+ break;
282
+ }
283
+ }
284
+ ctx.store.sketchPut?.(id, out);
285
+ return out;
286
+ }
202
287
  /** The COMPANY PROFILE of a partner: its own identity signature superposed
203
288
  * with the signatures of its RECURRING content-defined constituents.
204
289
  *
@@ -298,54 +383,53 @@ function companyProfile(ctx, id) {
298
383
  const acc = zeros(ctx.space.D);
299
384
  addInto(acc, companySignature(ctx.space, id));
300
385
  const bound = hubBound(ctx);
301
- const W = ctx.space.maxGroup;
386
+ const k = profileCapacity(ctx.space.D);
302
387
  const whole = Math.max(1, ctx.store.contentLen(id));
303
- const frontier = [];
304
- const seen = new Set([id]);
305
- const descend = (n) => {
306
- const kids = ctx.store.get(n)?.kids;
307
- if (!kids)
308
- return;
309
- for (const kid of kids)
310
- if (!seen.has(kid))
311
- frontier.push(kid);
312
- };
313
- descend(id);
314
- for (let visits = 0; visits < PROFILE_VISITS && frontier.length > 0;) {
315
- const n = frontier.shift();
316
- if (seen.has(n))
317
- continue;
318
- seen.add(n);
319
- visits++;
320
- // Atoms in both representations — negative id, or a stored kid-less node.
321
- if (n < 0 || ctx.store.get(n)?.kids == null)
322
- continue;
323
- descend(n);
388
+ const sketch = constituentSketch(ctx, id, k);
389
+ // The two CORPUS-RELATIVE readings, applied here and never stored: which
390
+ // terms count as scaffolding moves as N grows, which is the drift documented
391
+ // above, while the sketch itself must stay intrinsic to remain valid.
392
+ let accepted = 0, hubDropped = 0, dominating = 0;
393
+ for (const n of sketch) {
324
394
  const len = ctx.store.contentLen(n, whole);
325
- if (len < W || dominates(len, whole))
395
+ if (dominates(len, whole)) {
396
+ dominating++;
326
397
  continue;
327
- // MINIMAL units only: a constituent that still has a constituent of its
328
- // own at or above W is a composite, and superposing it as well as its
329
- // parts would count the same content twice. Nested partners — an
330
- // accumulated conversation, where turn k's context is a prefix of turn
331
- // k+1's — share their large chunks structurally rather than
332
- // distributionally, so those composites are exactly the terms that make
333
- // adjacent turns read as synonyms (measured: consecutive turns at 0.809
334
- // and 0.740 against a 0.516 concept threshold). The smallest units at or
335
- // above the fold's own window are the word-sized types company should be
336
- // keyed at.
337
- const kids = ctx.store.get(n).kids;
338
- let composite = false;
339
- for (const kid of kids) {
340
- if (kid >= 0 && ctx.store.contentLen(kid, W) >= W)
341
- composite = true;
342
398
  }
343
- if (composite)
344
- continue;
345
- if (ctx.store.parentsFirst(n, bound + 1).length > bound)
399
+ if (ctx.store.parentsFirst(n, bound + 1).length > bound) {
400
+ hubDropped++;
346
401
  continue;
402
+ }
347
403
  addInto(acc, companySignature(ctx.space, n));
404
+ accepted++;
348
405
  }
406
+ // FALSIFIABILITY. The claim this function makes is that it stops because the
407
+ // representation is FULL, never because a budget ran out — so the diagnostics
408
+ // report the capacity, the mass actually reached, and what the frontier still
409
+ // held. `residual` is the evidence NOT superposed; `marginal` is what one
410
+ // more term would have contributed to a downstream cosine (1/mass), and
411
+ // `saturated` says whether that had fallen to or below `noiseFloor`. A run
412
+ // that reports `saturated: false` with `residual > 0` is this design being
413
+ // WRONG, not tuning: it would mean readable evidence was dropped.
414
+ const mass = accepted + 1; // the node's own signature counts
415
+ const marginal = 1 / mass;
416
+ const noiseFloor = estimatorNoise(ctx.space.D);
417
+ ctx.trace?.step("companyProfile", [rNode(ctx, id, "partner")], [rItem(new Uint8Array(0), "profile", id)], `superposed ${accepted} of ${sketch.length} sketched constituents ` +
418
+ `(capacity ${k}); marginal ${marginal.toFixed(4)} vs noise floor ` +
419
+ `${noiseFloor.toFixed(4)}`, undefined, {
420
+ capacity: k,
421
+ sketched: sketch.length,
422
+ accepted,
423
+ hubDropped,
424
+ dominating,
425
+ residual: sketch.length - accepted,
426
+ mass,
427
+ marginal,
428
+ noiseFloor,
429
+ saturated: sketch.length >= k,
430
+ stopReason: sketch.length >= k ? "capacity" : "constituents-exhausted",
431
+ wholeLen: whole,
432
+ });
349
433
  return normalize(acc);
350
434
  }
351
435
  /** Ingest a pair (context, continuation) — learn an edge and pour halos.
@@ -301,6 +301,36 @@ export async function counterfactualTransfer(ctx, query, pre) {
301
301
  }
302
302
  }
303
303
  const isRoot = (id) => roots.some((r) => r.anchor === id);
304
+ // VOICEABLE — a structure whose own learnt content a schema may SPEAK.
305
+ //
306
+ // The gate below asks only that the weave TOUCH a committed point. That is
307
+ // the right question for MEMBERSHIP — a weave needs uncommitted structure to
308
+ // compare against; that is what an analogy IS — and the wrong one for
309
+ // VOICING: satisfied by any committed bystander, it lets every OTHER aligned
310
+ // point put its own learnt content into the answer while a root that
311
+ // contributed nothing holds the door open. The refusal note below already
312
+ // states the principle — "CAST refuses to transfer through content the climb
313
+ // itself never settled on" — it was simply never asked of the structure a
314
+ // schema actually transfers THROUGH.
315
+ //
316
+ // Measured on a two-hop question over dialogue filler (N ~ 103,
317
+ // consensusFloor 5.13): the climb committed ONE root at vote 8.13, and
318
+ // substitution then voiced a filler deposit at vote 0.15 together with a
319
+ // second structure at 0.57 — neither committed, both an order of magnitude
320
+ // below the floor, while the licensing root supplied no bytes at all.
321
+ //
322
+ // OR THE QUERY NAMED IT. Commitment is not the only warrant: a structure the
323
+ // asker QUOTED is content the query did ask about, whoever the climb settled
324
+ // on. The naming test is the one redirection's own `named` list uses — an
325
+ // aligned run starting at the structure's OPENING bytes (`cs === 0`) — and
326
+ // NOT merely "has an aligned run", which every weave point has by
327
+ // construction. Without this disjunct the gate refuses test/29 B3 ("what if
328
+ // the capital of France were Lyon?" must answer about Lyon), where the
329
+ // substitute is named outright and the climb never commits it. This mirrors
330
+ // the pairing the comparison gate already makes with
331
+ // `!rootTrusted && !namedByQuery`.
332
+ const namedFromOpening = (p) => p.runs.some((r) => r.cs === 0 && usable(r.qs, r.qe));
333
+ const voiceable = (p) => isRoot(p.anchor) || namedFromOpening(p);
304
334
  // The weave must touch a COMMITTED point of attention: the dominant
305
335
  // structure itself, or another aligned point the climb committed to.
306
336
  if (!points.some((p) => isRoot(p.anchor))) {
@@ -450,6 +480,15 @@ export async function counterfactualTransfer(ctx, query, pre) {
450
480
  if (r.cs < quantum || !usable(r.qs, r.qe)) {
451
481
  return null;
452
482
  }
483
+ // The DISPLACED STRUCTURE is what this schema speaks — the answer is its
484
+ // tail past the seat plus its own continuation — so it must be
485
+ // voiceable. Filtered HERE rather than after the argmax so an eligible
486
+ // structure with less depth still fires the schema, instead of an
487
+ // ineligible deepest candidate suppressing it outright. The SUBJECT is
488
+ // deliberately not gated: `fillerOf` reads the QUERY's own bytes for it,
489
+ // so it contributes what the asker already said, not learnt content.
490
+ if (!voiceable(p))
491
+ return null;
453
492
  const before = beforeOf(p, r);
454
493
  if (before === undefined)
455
494
  return null;
@@ -534,7 +573,12 @@ export async function counterfactualTransfer(ctx, query, pre) {
534
573
  const domNext = ctx.store.nextFirst(dominant.anchor, hubBound(ctx));
535
574
  const displaced = domNext
536
575
  .every((n) => indexOf(query, read(ctx, n), 0) < 0);
537
- if (last !== undefined && last.point !== dominant && displaced) {
576
+ // The SUBSTITUTE is what redirection speaks the answer IS `project(last)`,
577
+ // its own fact — so the same bar applies. The displaced structure is only
578
+ // recognised as the slot being overridden and is never voiced, so it is
579
+ // deliberately not gated here.
580
+ if (last !== undefined && last.point !== dominant && displaced &&
581
+ voiceable(last.point)) {
538
582
  const g = await project(ctx, last.point.anchor, qv);
539
583
  if (g !== null) {
540
584
  ctx.trace?.step("projectCounterfactual", [
@@ -282,7 +282,7 @@ export declare class Mind implements MindContext {
282
282
  * "what byte separates two turns?" because nothing downstream finds
283
283
  * boundaries by looking at content at all.
284
284
  * 2. A separator in a CORPUS is ordinary content. If a trainer joins
285
- * turns with "\n" (example/train_base.ts does), those newlines are
285
+ * turns with "\n" (example/train_base does), those newlines are
286
286
  * simply bytes inside the stream, folded like every other byte. They
287
287
  * are a property of that corpus, not of this API and not of the fold.
288
288
  * 3. This API can therefore reproduce ANY corpus exactly, with no
@@ -345,7 +345,12 @@ export declare class Mind implements MindContext {
345
345
  /** See {@link import("./learning.js").ingest} — `onDeposit`, when given,
346
346
  * reports each ingested item's deposited root node ids
347
347
  * ({@link DepositReport}); purely observational. */
348
- ingest(input: Input | (Input | [Input, Input])[], second?: Input, onDeposit?: (report: import("./learning.js").DepositReport) => void): Promise<(Sema & {
348
+ ingest(input: Input | (Input | [Input, Input])[], second?: Input, onDeposit?: (report: import("./learning.js").DepositReport) => void,
349
+ /** Witness the DEPOSIT path the way {@link respond}'s callback witnesses
350
+ * inference — `companyProfile` reports its saturation diagnostics here.
351
+ * Without it the tracer is never constructed and the emit sites cost
352
+ * nothing (§ rationale.ts), exactly as on the inference path. */
353
+ inspectRationale?: InspectRationale): Promise<(Sema & {
349
354
  id: number;
350
355
  }) | undefined>;
351
356
  private extensionHost;
@@ -479,7 +479,7 @@ export class Mind {
479
479
  * "what byte separates two turns?" because nothing downstream finds
480
480
  * boundaries by looking at content at all.
481
481
  * 2. A separator in a CORPUS is ordinary content. If a trainer joins
482
- * turns with "\n" (example/train_base.ts does), those newlines are
482
+ * turns with "\n" (example/train_base does), those newlines are
483
483
  * simply bytes inside the stream, folded like every other byte. They
484
484
  * are a property of that corpus, not of this API and not of the fold.
485
485
  * 3. This API can therefore reproduce ANY corpus exactly, with no
@@ -644,10 +644,22 @@ export class Mind {
644
644
  /** See {@link import("./learning.js").ingest} — `onDeposit`, when given,
645
645
  * reports each ingested item's deposited root node ids
646
646
  * ({@link DepositReport}); purely observational. */
647
- async ingest(input, second, onDeposit) {
647
+ async ingest(input, second, onDeposit,
648
+ /** Witness the DEPOSIT path the way {@link respond}'s callback witnesses
649
+ * inference — `companyProfile` reports its saturation diagnostics here.
650
+ * Without it the tracer is never constructed and the emit sites cost
651
+ * nothing (§ rationale.ts), exactly as on the inference path. */
652
+ inspectRationale) {
648
653
  invalidateStructuralCaches(this);
649
654
  invalidateJunctionCache(this);
650
- return ingest(this, input, second, onDeposit);
655
+ const prevTrace = this.trace;
656
+ this.trace = inspectRationale ? new Rationale(inspectRationale) : null;
657
+ try {
658
+ return await ingest(this, input, second, onDeposit);
659
+ }
660
+ finally {
661
+ this.trace = prevTrace;
662
+ }
651
663
  }
652
664
  // ── Extension Surface ────────────────────────────────────────────────────
653
665
  extensionHost() {
@@ -39,7 +39,6 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
39
39
  const qId = pre.queryResolved;
40
40
  if (qId !== null && ctx.store.prevCount(qId) > 0)
41
41
  return answer;
42
- const consumed = new Set();
43
42
  // Consume a node and its neighbours for pivot-cycle prevention — CAPPED at
44
43
  // the hub bound, via the store's LIMITed edge reads: a common continuation's
45
44
  // reverse fan-in (and a hub context's forward fan-out) is corpus-sized, and
@@ -48,11 +47,64 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
48
47
  // read order); a pivot suppressed only by a beyond-cap neighbour may now
49
48
  // fire — the same visibility trade chooseNext documents.
50
49
  const bound = hubBound(ctx);
51
- const consumeNode = (id) => {
50
+ // ANSWERED DIRECTLY the echo guard's other half, and the same principle:
51
+ // the QUERY's own position in the graph, not the answer's content, says the
52
+ // read-out is complete. Above: the query is itself a learnt CONTINUATION.
53
+ // Here: the query is a learnt CONTEXT and the grounded answer is one of ITS
54
+ // OWN continuations. Either way the question was answered directly and there
55
+ // is nothing left to chain for.
56
+ //
57
+ // Every stopping condition in the loop below judges the ANSWER (`consumed` /
58
+ // `restatesQuery` / `bytesEqual`); none asks whether the QUESTION was
59
+ // satisfied. So a single-hop question whose answer happens to name another
60
+ // learnt context extends past a correct answer and REPLACES it:
61
+ //
62
+ // asked "<subj> father"
63
+ // hop 1 "The father of <subj> is Ernest I of Anhalt-Dessau." <- correct
64
+ // pivot "Ernest I of Anhalt-Dessau" <- a learnt context too
65
+ // got "The date of death of Ernest I of Anhalt-Dessau is 12 June 1516."
66
+ //
67
+ // Any store holding a bare-entity context alongside a relation fact has that
68
+ // shape; it is not exotic.
69
+ //
70
+ // Checked ONCE, before the loop, and ahead of BOTH extension branches:
71
+ // `absorbForward` extends the answer too, and nothing about the defect is
72
+ // specific to pivoting, so a guard between them would gate one arbitrary half
73
+ // of the same step. Hop 0 is also the only hop at which the question can be
74
+ // answered directly at all — after a hop, `cur` is no longer the query's own
75
+ // continuation, so re-testing per hop could only cost reads.
76
+ //
77
+ // Read from the ANSWER's side (`prevFirst`) rather than the query's
78
+ // (`nextFirst`). Same relation, but a CONTEXT's fan-out is hub-sized while
79
+ // this is one answer's establishing-context fan-in. Both the resolve and the
80
+ // reverse read are exactly what hop 0 of the loop below would perform, so
81
+ // they are computed ONCE here and handed down (`groundedId`, `groundedPrev`)
82
+ // — the guard then costs nothing when it does not fire. Stated because the
83
+ // naive placement does NOT: `resolve` re-folds the answer bytes on every call
84
+ // (no memo) and `prevFirst` is a direct read (no memo), so a guard that
85
+ // recomputed them would add one fold plus one √N-bounded read per ask.
86
+ // The √N cap carries the file-wide visibility trade, and fails SAFE in the
87
+ // direction that matters: a missed guard costs an over-extended answer, never
88
+ // a suppressed chain.
89
+ //
90
+ // A genuine multi-hop query is not a deposited context at all ("What is the
91
+ // capital of the country of Eiffel Tower?" resolves to nothing), so this can
92
+ // never gate a real chain.
93
+ const groundedId = resolve(ctx, answer);
94
+ const groundedPrev = groundedId === null
95
+ ? null
96
+ : ctx.store.prevFirst(groundedId, bound);
97
+ if (qId !== null && groundedPrev !== null && groundedPrev.includes(qId)) {
98
+ return answer;
99
+ }
100
+ const consumed = new Set();
101
+ /** `prev` lets a caller hand in an already-read reverse-edge list — hop 0
102
+ * reuses the guard's, above, instead of re-reading it. */
103
+ const consumeNode = (id, prev) => {
52
104
  if (id === null)
53
105
  return;
54
106
  consumed.add(id);
55
- for (const p of ctx.store.prevFirst(id, bound))
107
+ for (const p of prev ?? ctx.store.prevFirst(id, bound))
56
108
  consumed.add(p);
57
109
  };
58
110
  const consumeAll = (id) => {
@@ -92,8 +144,10 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
92
144
  let t;
93
145
  const startedFrom = answer;
94
146
  for (let hop = 0; hop < ctx.cfg.recallQueryK; hop++) {
95
- const curId = resolve(ctx, cur);
96
- consumeNode(curId);
147
+ // Hop 0's `cur` IS `answer`, so the guard above already resolved it and
148
+ // read its reverse edges — reuse both rather than repeat them.
149
+ const curId = hop === 0 ? groundedId : resolve(ctx, cur);
150
+ consumeNode(curId, hop === 0 ? groundedPrev ?? undefined : undefined);
97
151
  // Forward-absorb: follow only UNCONSUMED continuations. The gate below
98
152
  // checks an unconsumed edge EXISTS, but follow()'s chooseNext knows
99
153
  // nothing of `consumed` and may still walk to a consumed fixpoint —
@@ -6,7 +6,7 @@
6
6
  // segment — leaf-parent segmentation using the geometry's own groupings.
7
7
  import { rItem } from "./trace.js";
8
8
  import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
9
- import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
9
+ import { atomIsHub, bearsEdge, corpusN, leadsSomewhere } from "./traverse.js";
10
10
  import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
11
11
  import { canonHash } from "../canon.js";
12
12
  import { isChunk } from "../sema.js";
@@ -585,6 +585,31 @@ function recogniseImpl(ctx, bytes) {
585
585
  // "Eiffel Tower" site vanished with it). The premise is wrong but the
586
586
  // trust it stood in for is real; a replacement signal is still open work.
587
587
  // See bench/README.md.
588
+ //
589
+ // THE REPLACEMENT SIGNAL (2026-08-13): `leadsSomewhere` on the BYTE-EXACT
590
+ // branch the chain already found. The blanket off-boundary suppression is
591
+ // a decision that CHANGES WITH CORPUS SIZE — `atomsAreHubs` flips at
592
+ // N = 4096 (atomReach = ⌈N·W/256⌉ exceeds √N there) — so a store crossing
593
+ // that point silently loses interior sites it used to have. Measured: with
594
+ // the two-hop chain deposited, `recognise("The country of Eiffel Tower is
595
+ // France.")` yields 4 sites including `France` at N = 3920 and 2 sites
596
+ // without it at N = 4227; the pivot dies with the site and multi-hop goes
597
+ // silent from there up (the trained store is N = 325,615).
598
+ //
599
+ // The honest gate is the one `emit` already applies, moved EARLIER and paid
600
+ // for with existence probes instead of a fold: `findBranch` has already
601
+ // proved these bytes are a stored branch, so the only remaining question is
602
+ // whether that branch is a deposited whole (bears an edge or a halo) or an
603
+ // interned fragment. "hi" out of "W[hi]ch" leads nowhere and is still
604
+ // suppressed; `France` bears both and is admitted. Structural, not scalar
605
+ // — no constant enters and nothing reads N, so the verdict no longer moves
606
+ // when the corpus grows.
607
+ //
608
+ // COST: `bearsEdge` is the response-MEMOISED edge probe, not the full
609
+ // `leadsSomewhere` — its uncached `hasHalo` tier took haloProbes from 922 to
610
+ // 9,144 on a nine-query battery over the trained store, which is not a price
611
+ // this pass may charge. `emit` still applies the full predicate, so this is
612
+ // a pre-filter that never widens what is admitted.
588
613
  const tryChain = (p, maxIds, boundary) => {
589
614
  const first = leafFrom(p);
590
615
  if (!first)
@@ -599,9 +624,10 @@ function recogniseImpl(ctx, bytes) {
599
624
  break;
600
625
  ids.push(nx.id);
601
626
  pos = nx.end;
602
- if (store.findBranch(ids) === null)
627
+ const branch = store.findBranch(ids);
628
+ if (branch === null)
603
629
  continue;
604
- if (!boundary && atomsAreHubs)
630
+ if (!boundary && atomsAreHubs && !bearsEdge(ctx, branch))
605
631
  continue;
606
632
  const id = resolveSpan(p, pos);
607
633
  if (id === null || id === prevId)
@@ -42,6 +42,22 @@ export declare function atomReach(ctx: MindContext, contextCount: number): numbe
42
42
  * atom votes and is recognised exactly as any stored form; above it the
43
43
  * alphabet is scaffolding everywhere and abstains. */
44
44
  export declare function atomIsHub(ctx: MindContext, contextCount: number): boolean;
45
+ /** Cached "does this node bear a continuation edge?" — the CHEAP half of
46
+ * {@link leadsSomewhere}, exported for hot paths that must PRE-FILTER a
47
+ * candidate before paying for a fold and cannot afford the halo tier.
48
+ *
49
+ * `leadsSomewhere`'s second tier (`hasHalo`) is deliberately uncached — one
50
+ * indexed point probe per candidate, which is right where candidates are
51
+ * already few. On recognition's off-boundary chain pass they are not few:
52
+ * using the full predicate there took haloProbes from 922 to 9,144 on a
53
+ * nine-query battery over the trained store. The edge tier alone is memoised
54
+ * for the response, so it is ~free, and a node bearing an edge is exactly the
55
+ * "deposited whole, not an interned fragment" claim that pass needs.
56
+ *
57
+ * Strictly NARROWER than `leadsSomewhere` — a halo-only node reads false — so
58
+ * it is sound as a pre-filter before a consumer that applies the full
59
+ * predicate, and never as a replacement for it. */
60
+ export declare function bearsEdge(ctx: MindContext, id: number): boolean;
45
61
  /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
46
62
  * The admission predicate recognition filters sites with (HOW_IT_WORKS
47
63
  * §15.3): a form that leads nowhere contributes nothing to any derivation.
@@ -407,6 +407,24 @@ export function atomReach(ctx, contextCount) {
407
407
  export function atomIsHub(ctx, contextCount) {
408
408
  return atomReach(ctx, contextCount) > boundFor(contextCount);
409
409
  }
410
+ /** Cached "does this node bear a continuation edge?" — the CHEAP half of
411
+ * {@link leadsSomewhere}, exported for hot paths that must PRE-FILTER a
412
+ * candidate before paying for a fold and cannot afford the halo tier.
413
+ *
414
+ * `leadsSomewhere`'s second tier (`hasHalo`) is deliberately uncached — one
415
+ * indexed point probe per candidate, which is right where candidates are
416
+ * already few. On recognition's off-boundary chain pass they are not few:
417
+ * using the full predicate there took haloProbes from 922 to 9,144 on a
418
+ * nine-query battery over the trained store. The edge tier alone is memoised
419
+ * for the response, so it is ~free, and a node bearing an edge is exactly the
420
+ * "deposited whole, not an interned fragment" claim that pass needs.
421
+ *
422
+ * Strictly NARROWER than `leadsSomewhere` — a halo-only node reads false — so
423
+ * it is sound as a pre-filter before a consumer that applies the full
424
+ * predicate, and never as a replacement for it. */
425
+ export function bearsEdge(ctx, id) {
426
+ return cachedHasNext(ctx, id, getStructCache(ctx));
427
+ }
410
428
  /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
411
429
  * The admission predicate recognition filters sites with (HOW_IT_WORKS
412
430
  * §15.3): a form that leads nowhere contributes nothing to any derivation.
@@ -46,6 +46,8 @@ export declare class SQliteStore extends AbstractStore implements Store {
46
46
  private _insCanon;
47
47
  private _selCanon;
48
48
  private _cntCanon;
49
+ private _insSketch;
50
+ private _selSketch;
49
51
  private _selContentFrom;
50
52
  private _delMeta;
51
53
  private _insSnapshot;
@@ -136,6 +138,8 @@ export declare class SQliteStore extends AbstractStore implements Store {
136
138
  protected _dbDeleteMeta(key: string): void;
137
139
  canonAdd(h: number, id: number): void;
138
140
  canonFind(h: number): number[];
141
+ sketchGet(id: number): number[] | null;
142
+ sketchPut(id: number, ids: readonly number[]): void;
139
143
  canonCount(): number;
140
144
  eachContent(cb: (id: number, bytes: Uint8Array) => void, fromId?: number): void;
141
145
  protected _dbSaveSnapshot(bytes: Uint8Array): void;