@hviana/sema 0.5.9 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/.github/workflows/release.yml +80 -0
  2. package/AGENTS.md +73 -13
  3. package/DATASETS.md +12 -11
  4. package/dist/example/train_base/cache.d.ts +35 -0
  5. package/dist/example/train_base/cache.js +211 -0
  6. package/dist/example/train_base/config.d.ts +21 -0
  7. package/dist/example/train_base/config.js +94 -0
  8. package/dist/example/train_base/corpora/aya.d.ts +19 -0
  9. package/dist/example/train_base/corpora/aya.js +76 -0
  10. package/dist/example/train_base/corpora/converted-parquet.d.ts +14 -0
  11. package/dist/example/train_base/corpora/converted-parquet.js +44 -0
  12. package/dist/example/train_base/corpora/genknow.d.ts +14 -0
  13. package/dist/example/train_base/corpora/genknow.js +83 -0
  14. package/dist/example/train_base/corpora/index.d.ts +29 -0
  15. package/dist/example/train_base/corpora/index.js +81 -0
  16. package/dist/example/train_base/corpora/massive.d.ts +7 -0
  17. package/dist/example/train_base/corpora/massive.js +98 -0
  18. package/dist/example/train_base/corpora/oasst2.d.ts +52 -0
  19. package/dist/example/train_base/corpora/oasst2.js +120 -0
  20. package/dist/example/train_base/corpora/smolsent.d.ts +23 -0
  21. package/dist/example/train_base/corpora/smolsent.js +156 -0
  22. package/dist/example/train_base/corpora/soda.d.ts +12 -0
  23. package/dist/example/train_base/corpora/soda.js +113 -0
  24. package/dist/example/train_base/corpora/taskmaster.d.ts +15 -0
  25. package/dist/example/train_base/corpora/taskmaster.js +144 -0
  26. package/dist/example/train_base/corpora/wiki2.d.ts +23 -0
  27. package/dist/example/train_base/corpora/wiki2.js +132 -0
  28. package/dist/example/train_base/corpus.d.ts +88 -0
  29. package/dist/example/train_base/corpus.js +65 -0
  30. package/dist/example/train_base/discovery.d.ts +48 -0
  31. package/dist/example/train_base/discovery.js +143 -0
  32. package/dist/example/train_base/http.d.ts +82 -0
  33. package/dist/example/train_base/http.js +219 -0
  34. package/dist/example/train_base/items.d.ts +46 -0
  35. package/dist/example/train_base/items.js +98 -0
  36. package/dist/example/train_base/main.d.ts +4 -0
  37. package/dist/example/train_base/main.js +207 -0
  38. package/dist/example/train_base/progress.d.ts +34 -0
  39. package/dist/example/train_base/progress.js +114 -0
  40. package/dist/example/train_base/readers.d.ts +125 -0
  41. package/dist/example/train_base/readers.js +391 -0
  42. package/dist/example/train_base/runtime.d.ts +115 -0
  43. package/dist/example/train_base/runtime.js +637 -0
  44. package/dist/example/train_base/stage.d.ts +3 -0
  45. package/dist/example/train_base/stage.js +246 -0
  46. package/dist/example/train_base/ui.d.ts +88 -0
  47. package/dist/example/train_base/ui.js +272 -0
  48. package/dist/src/meter.d.ts +1 -4
  49. package/dist/src/meter.js +0 -3
  50. package/dist/src/mind/attention.js +22 -20
  51. package/dist/src/mind/graph-search.d.ts +43 -9
  52. package/dist/src/mind/graph-search.js +82 -15
  53. package/dist/src/mind/junction.d.ts +13 -0
  54. package/dist/src/mind/junction.js +13 -0
  55. package/dist/src/mind/mechanisms/cover.js +23 -2
  56. package/dist/src/mind/mechanisms/prefix-completion.js +13 -11
  57. package/dist/src/mind/mechanisms/recall.js +8 -4
  58. package/dist/src/mind/mind.d.ts +1 -1
  59. package/dist/src/mind/mind.js +1 -1
  60. package/dist/src/mind/pipeline-mechanism.d.ts +0 -24
  61. package/dist/src/mind/pipeline-mechanism.js +13 -36
  62. package/dist/src/mind/pipeline.d.ts +23 -0
  63. package/dist/src/mind/pipeline.js +51 -3
  64. package/dist/src/mind/recognition.d.ts +6 -1
  65. package/dist/src/mind/recognition.js +11 -6
  66. package/dist/src/mind/resonance.js +48 -13
  67. package/dist/src/store.js +22 -1
  68. package/example/train_base/cache.ts +251 -0
  69. package/example/train_base/config.ts +128 -0
  70. package/example/train_base/corpora/aya.ts +106 -0
  71. package/example/train_base/corpora/converted-parquet.ts +64 -0
  72. package/example/train_base/corpora/genknow.ts +114 -0
  73. package/example/train_base/corpora/index.ts +88 -0
  74. package/example/train_base/corpora/massive.ts +111 -0
  75. package/example/train_base/corpora/oasst2.ts +163 -0
  76. package/example/train_base/corpora/smolsent.ts +203 -0
  77. package/example/train_base/corpora/soda.ts +130 -0
  78. package/example/train_base/corpora/taskmaster.ts +217 -0
  79. package/example/train_base/corpora/wiki2.ts +190 -0
  80. package/example/train_base/corpus.ts +150 -0
  81. package/example/train_base/discovery.ts +203 -0
  82. package/example/train_base/http.ts +284 -0
  83. package/example/train_base/items.ts +118 -0
  84. package/example/train_base/main.ts +240 -0
  85. package/example/train_base/progress.ts +149 -0
  86. package/example/train_base/readers.ts +505 -0
  87. package/example/train_base/runtime.ts +894 -0
  88. package/example/train_base/stage.ts +276 -0
  89. package/example/train_base/ui.ts +333 -0
  90. package/jsr.json +1 -1
  91. package/package.json +8 -5
  92. package/src/meter.ts +1 -4
  93. package/src/mind/attention.ts +22 -19
  94. package/src/mind/graph-search.ts +93 -16
  95. package/src/mind/junction.ts +13 -0
  96. package/src/mind/mechanisms/cover.ts +23 -4
  97. package/src/mind/mechanisms/prefix-completion.ts +13 -11
  98. package/src/mind/mechanisms/recall.ts +8 -4
  99. package/src/mind/mind.ts +1 -1
  100. package/src/mind/pipeline-mechanism.ts +13 -42
  101. package/src/mind/pipeline.ts +87 -3
  102. package/src/mind/recognition.ts +19 -6
  103. package/src/mind/resonance.ts +79 -50
  104. package/src/store.ts +21 -1
  105. package/test/13-conversation.test.mjs +1 -1
  106. package/test/84-composed-answer-honesty.test.mjs +2 -1
  107. package/test/88-dependency-footprint.test.mjs +99 -0
  108. package/test/89-completion-recursion.test.mjs +230 -0
  109. package/test/90-connector-read-cap.test.mjs +130 -0
  110. package/test/91-branch-bytes-cache.test.mjs +152 -0
  111. package/test/93-regime-prediction.test.mjs +148 -0
  112. package/test/94-cross-region-budget.test.mjs +67 -0
  113. package/test/95-wide-resonance-removed.test.mjs +109 -0
  114. package/dist/example/train_base.d.ts +0 -163
  115. package/dist/example/train_base.js +0 -3220
  116. package/example/train_base.ts +0 -3882
@@ -12,7 +12,7 @@ import { concat2, concatBytes, indexOf } from "../bytes.js";
12
12
  import type { MindContext } from "./types.js";
13
13
  import { gistOf, read, resolve, walkTree } from "./primitives.js";
14
14
  import { perceive } from "./primitives.js";
15
- import { argmaxBy, argmaxCosine, candidateGist, hubBound } from "./traverse.js";
15
+ import { argmaxCosine, candidateGist, hubBound } from "./traverse.js";
16
16
  import {
17
17
  cachedRead,
18
18
  type Junction,
@@ -365,7 +365,13 @@ export async function pivotInto(
365
365
  }
366
366
  for (const c of n.kids) queue.push(c); // breadth-first: larger regions first
367
367
  }
368
- const rec = recognise(ctx, answer);
368
+ // TRIMMED recognition: the pivot's own filter below rejects fragments
369
+ // (`hasParents || hasContainers → -Infinity`), and recognition's edge-trim
370
+ // fallbacks exist to find exactly those misaligned FRAGMENTS. Skipping them
371
+ // (the structural pass + canonResolve still run) is byte-identical for every
372
+ // pivot — the fallbacks' output is discarded by the filter — and halves the
373
+ // O(n·W²) recognition of a long answer (measured: 36KB recognise 4.0s → 2.0s).
374
+ const rec = recognise(ctx, answer, true);
369
375
  for (const s of rec.sites) {
370
376
  if (!consumed.has(s.payload) && ctx.store.hasNext(s.payload)) {
371
377
  scored.set(s.payload, Math.max(scored.get(s.payload) ?? 0, 1));
@@ -373,54 +379,77 @@ export async function pivotInto(
373
379
  }
374
380
  // Byte containment, longest wins — the answer literally contains the
375
381
  // pivot's bytes, and the biggest well-evidenced span is the real pivot.
376
- const found = argmaxBy(
377
- scored.keys(),
378
- (id) => {
379
- // A PIVOT MUST BE A THING THE CORPUS DEPOSITED, NOT A PIECE OF ONE.
380
- // "Longest wins" ranks candidates but never asks whether the winner is
381
- // an entity at all, and by the time a chain reaches here `consumeAll`
382
- // has taken the answer's real contexts so on a corpus of
383
- // near-identical records the field is left to whatever interned
384
- // fragments remain. Measured on a 200-line templated log corpus, query
385
- // "what happened to request_id=1042 and request_id=1077?": CAST
386
- // produced the correct comparison and one `pivotStep` replaced it
387
- // wholesale, pivoting through `s=70` — a four-byte tail of
388
- // `latency_ms=70` — onto an unrelated record (`handled 1130`).
389
- //
390
- // The separator is NOT length. Measured against the multi-hop tests'
391
- // own pivots: `Paris` (5 bytes), `Jupiter` (7), `lithium` (7), `Mona
392
- // Lisa` (9) against junk `s=70` (4) a two-quantum floor, which
393
- // confluence.ts applies to a meet for the same "one window is not an
394
- // entity" reason, discards three of the four legitimate pivots.
395
- // Entities are simply short.
396
- //
397
- // What separates them is STRUCTURAL, and the store already holds it:
398
- //
399
- // s=70 parents 2 containers 1 prevCount 0 halo no
400
- // Paris parents 0 containers 0 prevCount 1 halo yes
401
- // Jupiter parents 0 containers 0 prevCount 1 halo yes
402
- // lithium parents 0 containers 0 prevCount 1 halo yes
403
- // Mona Lisa parents 0 containers 0 prevCount 1 halo yes
404
- //
405
- // A deposited whole a context or an answer is interned in its own
406
- // right and has neither structural parents nor containment links. A
407
- // fragment is addressable ONLY because window interning made its span
408
- // addressable inside something bigger, and that containment is exactly
409
- // what `parents`/`containers` record. Reasoning steps THROUGH a fact;
410
- // a span that was never a fact on its own is not one to step through.
411
- // No constant enters it is a structural predicate, not a threshold.
412
- if (ctx.store.hasParents(id) || ctx.store.hasContainers(id)) {
413
- return -Infinity;
414
- }
415
- const bytes = read(ctx, id);
416
- if (indexOf(answer, bytes, 0) < 0) return -Infinity;
417
- for (const v of voiced) if (indexOf(v, bytes, 0) >= 0) return -Infinity;
418
- return bytes.length;
419
- },
420
- 0,
421
- true,
422
- );
423
- return found?.item ?? null;
382
+ //
383
+ // REAL SATURATION, not a hard cap: the score IS the candidate's byte
384
+ // length, so the scan is DECIDED the moment the first candidate that passes
385
+ // every filter is found in DESCENDING length order a shorter candidate can
386
+ // never outscore it. `contentLen` (the prefix-capped length read, §2.8) is
387
+ // the cheap ordering key, and the first-inserted tie-break is made explicit
388
+ // (`a.index - b.index`) so equal lengths keep `scored`'s insertion order
389
+ // exactly the tie argmaxBy(strict) used to keep. The bytes of at most ONE
390
+ // winning candidate are read; every shorter candidate the probes proposed is
391
+ // skipped without reconstruction, where the old argmax read them all.
392
+ const ranked = [...scored.keys()]
393
+ .map((id, index) => ({
394
+ id,
395
+ index,
396
+ len: ctx.store.contentLen(id, answer.length + 1),
397
+ }))
398
+ .sort((a, b) => b.len - a.len || a.index - b.index);
399
+ let pivotId: number | null = null;
400
+ for (const c of ranked) {
401
+ const id = c.id;
402
+ // A PIVOT MUST BE A THING THE CORPUS DEPOSITED, NOT A PIECE OF ONE.
403
+ // "Longest wins" ranks candidates but never asks whether the winner is
404
+ // an entity at all, and by the time a chain reaches here `consumeAll`
405
+ // has taken the answer's real contexts so on a corpus of
406
+ // near-identical records the field is left to whatever interned
407
+ // fragments remain. Measured on a 200-line templated log corpus, query
408
+ // "what happened to request_id=1042 and request_id=1077?": CAST
409
+ // produced the correct comparison and one `pivotStep` replaced it
410
+ // wholesale, pivoting through `s=70` — a four-byte tail of
411
+ // `latency_ms=70`onto an unrelated record (`handled 1130`).
412
+ //
413
+ // The separator is NOT length. Measured against the multi-hop tests'
414
+ // own pivots: `Paris` (5 bytes), `Jupiter` (7), `lithium` (7), `Mona
415
+ // Lisa` (9) against junk `s=70` (4) a two-quantum floor, which
416
+ // confluence.ts applies to a meet for the same "one window is not an
417
+ // entity" reason, discards three of the four legitimate pivots.
418
+ // Entities are simply short.
419
+ //
420
+ // What separates them is STRUCTURAL, and the store already holds it:
421
+ //
422
+ // s=70 parents 2 containers 1 prevCount 0 halo no
423
+ // Paris parents 0 containers 0 prevCount 1 halo yes
424
+ // Jupiter parents 0 containers 0 prevCount 1 halo yes
425
+ // lithium parents 0 containers 0 prevCount 1 halo yes
426
+ // Mona Lisa parents 0 containers 0 prevCount 1 halo yes
427
+ //
428
+ // A deposited whole — a context or an answer — is interned in its own
429
+ // right and has neither structural parents nor containment links. A
430
+ // fragment is addressable ONLY because window interning made its span
431
+ // addressable inside something bigger, and that containment is exactly
432
+ // what `parents`/`containers` record. Reasoning steps THROUGH a fact;
433
+ // a span that was never a fact on its own is not one to step through.
434
+ // No constant enters — it is a structural predicate, not a threshold.
435
+ if (ctx.store.hasParents(id) || ctx.store.hasContainers(id)) continue;
436
+ // A candidate whose bytes are LONGER than the answer cannot be a
437
+ // substring of it — `indexOf` would return −1 regardless. Prune by
438
+ // length BEFORE reconstructing the bytes: `read` is an UNCAPPED read
439
+ // (AGENTS §2.8), and a resonated context far longer than the answer is
440
+ // exactly the candidate that makes it cost a whole deposit's worth of
441
+ // reconstruction for a containment test that must fail. `contentLen`
442
+ // with the `answer.length + 1` cap is the prefix-capped length read the
443
+ // same contract prescribes; the prune is byte-identical to the old
444
+ // `indexOf` miss (it returns −1 for a needle longer than the haystack).
445
+ if (c.len > answer.length) continue;
446
+ const bytes = read(ctx, id);
447
+ if (indexOf(answer, bytes, 0) < 0) continue;
448
+ if (voiced.some((v) => indexOf(v, bytes, 0) >= 0)) continue;
449
+ pivotId = id;
450
+ break;
451
+ }
452
+ return pivotId;
424
453
  }
425
454
 
426
455
  /** Which of the given labelled forms a span MEANS — generic resonance over
package/src/store.ts CHANGED
@@ -1269,7 +1269,27 @@ export abstract class AbstractStore implements Store {
1269
1269
  parts.push(child);
1270
1270
  got += child.length;
1271
1271
  }
1272
- return concat(parts);
1272
+ const out = concat(parts);
1273
+ // Cache the BRANCH too, not just the leaf above. Reconstruction is a pure
1274
+ // function of the store, so this is a transparent cache in the strict sense
1275
+ // — an eviction costs a re-walk and nothing else — which is exactly what
1276
+ // `_bytesCache`'s "smallest"/"clock" configuration is for.
1277
+ //
1278
+ // Caching only leaves made every branch re-walk its whole subtree on every
1279
+ // request, and the DAG is hash-consed, so the same children recur under many
1280
+ // parents. Measured on the 18.9M-node store, ONE 1,314-byte query:
1281
+ // 20,021,474 `_prefix` calls over 469,083 distinct ids (42.7x reuse) to
1282
+ // produce 87,789 results — 97.7% of the work re-derived bytes it had already
1283
+ // built. One single-byte leaf was reconstructed 2,599,984 times. Measuring
1284
+ // reuse at the TOP level only shows 1.1x and hides all of it.
1285
+ //
1286
+ // Only a COMPLETE reconstruction may be cached: `_prefix` is also called
1287
+ // with a cap, and a truncated prefix stored under `id` would be served as
1288
+ // if it were the node's whole content by the `_bytesCache` hit above.
1289
+ // `got < maxLen` is that proof — the walk ran out of children before it ran
1290
+ // out of budget, so nothing below was truncated either.
1291
+ if (got < maxLen) this._bytesCache.set(id, out);
1292
+ return out;
1273
1293
  }
1274
1294
 
1275
1295
  contentLen(id: NodeId, cap = Infinity): number {
@@ -20,7 +20,7 @@
20
20
  // inspects content to find turn boundaries.
21
21
  //
22
22
  // "NO SEPARATOR IS NEEDED" ≠ "A SEPARATOR IS A PROBLEM". This file joins its
23
- // turns with nothing; example/train_base.ts joins its oasst2 turns with "\n".
23
+ // turns with nothing; example/train_base joins its oasst2 turns with "\n".
24
24
  // Both are correct, and neither is a convention the other has to match: a
25
25
  // separator is CORPUS CONTENT, folded like any other byte, while a turn
26
26
  // boundary is an OFFSET the API carries beside the bytes. A harness replaying
@@ -31,7 +31,8 @@
31
31
  // fabricate at low N, this goes red. It does NOT cover the real-text ceiling.
32
32
  //
33
33
  // TO REPRODUCE THE REAL FAILURE: build the same chain, then ingest ~6,000
34
- // deposits produced by the Taskmaster adapter (example/train_base.ts §6e′) from
34
+ // deposits produced by the Taskmaster adapter (example/train_base/corpora/
35
+ // taskmaster.ts) from
35
36
  // TM-2/TM-3/TM-4, and ask the two-hop question. See FINDINGS.md §A1/§A4.
36
37
 
37
38
  import { test } from "node:test";
@@ -0,0 +1,99 @@
1
+ // 88 — the dependency footprint is a PRODUCT PROPERTY, so it is tested.
2
+ //
3
+ // AGENTS.md §6: "do not add runtime dependencies casually — the near-zero-
4
+ // dependency footprint is a product feature." A feature stated only in prose
5
+ // erodes; this suite pins it at the two places it can actually break.
6
+ //
7
+ // 1. The BUILT LIBRARY imports nothing outside `node:`. Not the source — the
8
+ // built output, because that is what a consumer loads.
9
+ // 2. The PACKAGE declares no runtime `dependencies`. A package can be
10
+ // import-clean and still force installs on every consumer; that is exactly
11
+ // how `hyparquet` — used only by example/train_base, never by src/ — came
12
+ // to be installed by everyone who depended on Sema.
13
+ //
14
+ // The trainer example is deliberately NOT covered: it may use whatever it
15
+ // needs, as a dev dependency, loaded lazily (see example/train_base/readers.ts).
16
+
17
+ import { test } from "node:test";
18
+ import assert from "node:assert/strict";
19
+ import { readdirSync, readFileSync, statSync } from "node:fs";
20
+ import { join } from "node:path";
21
+
22
+ const ROOT = new URL("..", import.meta.url).pathname;
23
+
24
+ /** Every built .js file of the library itself. */
25
+ function libFiles(dir = join(ROOT, "dist/src"), out = []) {
26
+ for (const name of readdirSync(dir)) {
27
+ const p = join(dir, name);
28
+ if (statSync(p).isDirectory()) libFiles(p, out);
29
+ else if (name.endsWith(".js")) out.push(p);
30
+ }
31
+ return out;
32
+ }
33
+
34
+ /** Module specifiers of static imports/exports and dynamic import() calls. */
35
+ function specifiersOf(src) {
36
+ const out = [];
37
+ const patterns = [
38
+ /(?:^|\s)(?:import|export)\s[^;]*?from\s*["']([^"']+)["']/g,
39
+ /(?:^|\s)import\s*["']([^"']+)["']/g,
40
+ /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g,
41
+ /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g,
42
+ ];
43
+ for (const re of patterns) {
44
+ for (const m of src.matchAll(re)) out.push(m[1]);
45
+ }
46
+ return out;
47
+ }
48
+
49
+ test("the built library imports nothing outside node: builtins", () => {
50
+ const files = libFiles();
51
+ assert.ok(files.length > 0, "dist/src is empty — run `npm run build` first");
52
+
53
+ const foreign = [];
54
+ for (const file of files) {
55
+ for (const spec of specifiersOf(readFileSync(file, "utf8"))) {
56
+ const ok = spec.startsWith("./") || spec.startsWith("../") ||
57
+ spec.startsWith("node:");
58
+ if (!ok) foreign.push(`${file.slice(ROOT.length)} → ${spec}`);
59
+ }
60
+ }
61
+ assert.deepEqual(
62
+ foreign,
63
+ [],
64
+ "the library gained a third-party import; keep it in an example instead",
65
+ );
66
+ });
67
+
68
+ test("the package forces no runtime dependencies on consumers", () => {
69
+ const pkg = JSON.parse(
70
+ readFileSync(join(ROOT, "package.json"), "utf8"),
71
+ );
72
+ const deps = Object.keys(pkg.dependencies ?? {});
73
+ assert.deepEqual(
74
+ deps,
75
+ [],
76
+ "a package a consumer installs must not carry dependencies only an " +
77
+ "example needs — declare them under devDependencies",
78
+ );
79
+ });
80
+
81
+ test("the published entry points resolve inside dist/src", () => {
82
+ // A `main`/`exports` that reached outside dist/src would drag the example —
83
+ // and its dev dependencies — into a consumer's module graph.
84
+ const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8"));
85
+ const entries = [
86
+ pkg.main,
87
+ pkg.types,
88
+ pkg.exports?.["."]?.default,
89
+ pkg.exports?.["."]?.types,
90
+ ].filter(Boolean);
91
+ assert.ok(entries.length > 0, "no entry points declared");
92
+ for (const e of entries) {
93
+ assert.match(
94
+ e,
95
+ /^(\.\/)?dist\/src\//,
96
+ `entry point "${e}" escapes dist/src`,
97
+ );
98
+ }
99
+ });
@@ -0,0 +1,230 @@
1
+ // 89-completion-recursion.test.mjs — the completion recursion must be
2
+ // OUTPUT-SENSITIVE.
3
+ //
4
+ // AGENTS §2.8: "No per-query read may grow with the corpus." §2.8 enforces
5
+ // that per READ (nextFirst, bytesPrefix, …), and every one of those caps holds.
6
+ // What no guard covered is the NUMBER of reads: `recompleteNode`
7
+ // (src/mind/graph-search.ts) re-covers a produced node by calling `solve`
8
+ // recursively, and each nested solve builds its own agenda and chart. Its own
9
+ // doc states the intent —
10
+ //
11
+ // "its cost tracks the ANSWER's own structure, not how densely the corpus
12
+ // interconnects the nodes passed through"
13
+ //
14
+ // — but argues termination from "Distinct node ids are finite and each finished
15
+ // completion is memoised". Finite-in-the-corpus is exactly the bound §2.8
16
+ // forbids, and the recursion is emitted at `cost: 0` while the nested cover's
17
+ // own `cost` is computed and discarded, so A* has no gradient against depth.
18
+ //
19
+ // MEASURED on a trained store (18,938,834 nodes, edgeSourceCount 796,528):
20
+ // `respond("Hi")` reached recursion depth 331 and 9.1 GB RSS in 56 s without
21
+ // terminating. Every level was a "Hi…" opener — a 2-byte hub re-entering
22
+ // itself — and the descent visited whole utterances unrelated to the answer
23
+ // ("Who's their goalkeeper?", "Glad I could assist, have a great day."). That
24
+ // is what killed a 5 h training run at a checkpoint recall: the 15 s
25
+ // withTimeout around it is a setTimeout, and a synchronous search never yields
26
+ // to the timer phase, so it cannot fire.
27
+ //
28
+ // WHY THE EXISTING GUARD MISSES IT. 14-scaling.test.mjs asserts this same law
29
+ // ("inference: cost is sublinear in corpus size"), but builds each size point
30
+ // from a DISJOINT salted corpus and queries it with `unknownInput()`, whose
31
+ // "tokens are substrings of no learned form". A query that recognises nothing
32
+ // never enters the graph, so it never reaches the fixpoint that gates the
33
+ // recursion. Both assumptions are load-bearing; this file drops them.
34
+ //
35
+ // THE CORPUS. Real English fragments taken from the repo's own *.md prose and
36
+ // recombined, plus the query deposited as a standalone context with several
37
+ // continuations (what makes a greeting a hub in real dialogue). Eight
38
+ // hand-written generators failed to reproduce this — chains stop after two
39
+ // rungs, dense graphs never reach a fixpoint, and a query the pipeline declines
40
+ // never reaches `cover` at all. Real prose plus a deposited hub does it.
41
+ // Nothing is added to the tree: the corpus is the documentation already here.
42
+ //
43
+ // MEASURED HERE, on the deterministic public counters (mind.lastCost), answer
44
+ // byte-identical at every size:
45
+ //
46
+ // pairs searches pops maxDepth
47
+ // 1508 126 55,208 71
48
+ // 2008 196 88,241 127
49
+ // 3008 309 146,282 207
50
+ // 4008 416 201,174 270 (22 s for a 3-byte query)
51
+ //
52
+ // growth exponent k ≈ 1.25 (searches), 1.32 (pops) — SUPER-linear.
53
+ //
54
+ // With the recursion bounded, the same corpus gives searches 10→13 and pops
55
+ // 3,946→5,826 (k ≈ 0.27 / 0.40) in 0.31 s→0.76 s, and the answer does not
56
+ // change. So this file's thresholds are achievable, and the fix costs nothing
57
+ // in output on this corpus.
58
+
59
+ import { test } from "node:test";
60
+ import assert from "node:assert/strict";
61
+ import { readdirSync, readFileSync } from "node:fs";
62
+ import { dirname, join } from "node:path";
63
+ import { fileURLToPath } from "node:url";
64
+ import { Mind } from "../dist/src/index.js";
65
+ import { SQliteStore } from "../dist/src/store-sqlite.js";
66
+
67
+ const REPO = join(dirname(fileURLToPath(import.meta.url)), "..");
68
+
69
+ // ── corpus ────────────────────────────────────────────────────────────────
70
+ // Deterministic scrambler: the suite forbids Math.random in fixtures, and the
71
+ // whole point of the counters is that two runs are diffable.
72
+ const mix = (x) => {
73
+ x = (x ^ 61) ^ (x >>> 16);
74
+ x = x + (x << 3);
75
+ x = x ^ (x >>> 4);
76
+ x = Math.imul(x, 0x27d4eb2d);
77
+ return (x ^ (x >>> 15)) >>> 0;
78
+ };
79
+
80
+ /** Four-word windows of the repo's own English prose. Code fences, inline
81
+ * code and link targets are stripped so what is left is language, which is
82
+ * where the fragment overlap lives. */
83
+ function fragments() {
84
+ const out = [];
85
+ for (const f of readdirSync(REPO).filter((f) => f.endsWith(".md")).sort()) {
86
+ let t = readFileSync(join(REPO, f), "utf8");
87
+ t = t.replace(/```[\s\S]*?```/g, " ").replace(/`[^`]*`/g, " ");
88
+ t = t.replace(/\[[^\]]*\]\([^)]*\)/g, " ");
89
+ t = t.toLowerCase().replace(/[^a-z ]+/g, " ").replace(/\s+/g, " ");
90
+ const w = t.split(" ").filter(Boolean);
91
+ for (let i = 0; i + 4 < w.length; i += 2) {
92
+ out.push(w.slice(i, i + 4).join(" "));
93
+ }
94
+ }
95
+ return out;
96
+ }
97
+
98
+ const FRAG = fragments();
99
+ const QUERY = "hi.";
100
+
101
+ const utter = (i) =>
102
+ [
103
+ FRAG[mix(i * 3 + 1) % FRAG.length],
104
+ FRAG[mix(i * 5 + 2) % FRAG.length],
105
+ FRAG[mix(i * 7 + 3) % FRAG.length],
106
+ ].join(" ");
107
+
108
+ /** n recombined utterance pairs, plus the query as a standalone context with
109
+ * eight distinct continuations — the hub. Every pair's answer is itself a
110
+ * context (multi-turn), so a completed form has somewhere to continue. */
111
+ function corpus(n) {
112
+ const pairs = [];
113
+ for (let k = 0; k < 8; k++) pairs.push([QUERY, utter(k * 101 + 7)]);
114
+ for (let i = 0; i < n; i++) {
115
+ pairs.push([utter(i), utter(i * 3 + 1)]);
116
+ pairs.push([utter(i * 3 + 1), utter(i * 7 + 2)]);
117
+ }
118
+ return pairs;
119
+ }
120
+
121
+ /** Power-law exponent k in work ≈ c·n^k, by log–log least squares — the same
122
+ * statistic and the same k < 0.6 bar 14-scaling.test.mjs uses. */
123
+ function logLogSlope(sizes, ys) {
124
+ const n = sizes.length;
125
+ const xs = sizes.map(Math.log), ly = ys.map((v) => Math.log(Math.max(v, 1)));
126
+ const mx = xs.reduce((a, b) => a + b, 0) / n;
127
+ const my = ly.reduce((a, b) => a + b, 0) / n;
128
+ let num = 0, den = 0;
129
+ for (let i = 0; i < n; i++) {
130
+ num += (xs[i] - mx) * (ly[i] - my);
131
+ den += (xs[i] - mx) ** 2;
132
+ }
133
+ return num / den;
134
+ }
135
+
136
+ // `corpus(n)` emits two pairs per turn plus the eight hub pairs, so these are
137
+ // the 1508 / 2008 / 3008-pair points of the table above.
138
+ const SIZES = [750, 1000, 1500];
139
+
140
+ test("completion recursion: per-query work does not grow with the corpus", async () => {
141
+ assert.ok(
142
+ FRAG.length > 4000,
143
+ `only ${FRAG.length} prose fragments found in ${REPO}/*.md — this test ` +
144
+ `draws its corpus from the repo's own documentation; with the prose gone ` +
145
+ `it can no longer exercise the completion recursion at all`,
146
+ );
147
+
148
+ const searches = [], pops = [], answers = [], secs = [];
149
+
150
+ for (const n of SIZES) {
151
+ const store = new SQliteStore({ path: ":memory:", D: 1024 });
152
+ const mind = new Mind({ seed: 7, store, profile: true });
153
+ await mind.ingest(corpus(n));
154
+
155
+ const t0 = performance.now();
156
+ const answer = String(await mind.respondText(QUERY));
157
+ secs.push((performance.now() - t0) / 1000);
158
+
159
+ const c = mind.lastCost.counters;
160
+ searches.push(c.searches ?? 0);
161
+ pops.push(c.searchPops ?? 0);
162
+ answers.push(answer);
163
+ await store.close();
164
+ }
165
+
166
+ console.log(" completion recursion vs corpus size (fixed 3-byte query):");
167
+ SIZES.forEach((n, i) =>
168
+ console.log(
169
+ ` pairs=${String(n * 2 + 8).padStart(5)} searches=${
170
+ String(searches[i]).padStart(5)
171
+ } pops=${String(pops[i]).padStart(8)} ${secs[i].toFixed(2)}s`,
172
+ )
173
+ );
174
+
175
+ // NON-VACUITY. If the corpus stopped reaching the recursion, every counter
176
+ // would be flat at zero and the growth assertions below would pass while
177
+ // proving nothing. A test that can go green by not exercising the code is
178
+ // worse than no test, so this fails loudly instead.
179
+ assert.ok(
180
+ searches.every((s) => s > 0),
181
+ `the graph search never ran (searches=${JSON.stringify(searches)}) — the ` +
182
+ `corpus no longer engages cover(), so this file is not testing anything`,
183
+ );
184
+
185
+ // NO OUTPUT CONFOUND. Work is allowed to grow with the ANSWER. Pinning the
186
+ // answer byte-for-byte across every size removes that defence entirely: any
187
+ // growth measured below bought exactly nothing.
188
+ assert.ok(
189
+ answers.every((a) => a === answers[0]),
190
+ `the answer changed across corpus sizes (${
191
+ JSON.stringify(answers.map((a) => a.slice(0, 40)))
192
+ }) — with the output moving, work growth is no longer attributable to the ` +
193
+ `corpus alone`,
194
+ );
195
+
196
+ const kSearches = logLogSlope(SIZES, searches);
197
+ const kPops = logLogSlope(SIZES, pops);
198
+ console.log(
199
+ ` growth exponent k ≈ ${kSearches.toFixed(2)} (nested searches), ${
200
+ kPops.toFixed(2)
201
+ } (agenda pops) — target ≪ 1 (sublinear in the corpus)`,
202
+ );
203
+
204
+ // THE LAW. Same answer, more corpus, so cost must not move. k ≈ 0 is flat,
205
+ // k ≈ 1 is linear in the corpus — the bound §2.8 forbids outright.
206
+ //
207
+ // `searches` counts nested solve() calls, which is the recursion itself and
208
+ // nothing else, so it gets 14-scaling.test.mjs's stricter 0.6 bar. Measured
209
+ // 1.28 unfixed, 0.38 fixed.
210
+ assert.ok(
211
+ kSearches < 0.6,
212
+ `nested searches grew with exponent k=${kSearches.toFixed(2)} in corpus ` +
213
+ `size (${searches.join(" → ")}) for a byte-identical answer — each ` +
214
+ `nested solve() builds its own agenda and chart, so this is the ` +
215
+ `completion recursion doing work the answer never asked for`,
216
+ );
217
+ // A LOOSER BAR, FOR A REASON. `searchPops` aggregates the TOP-LEVEL cover's
218
+ // agenda too, and that one legitimately carries some corpus sensitivity: a
219
+ // bigger store recognises more sites inside the same query, so more items are
220
+ // admissible. Only outright linear growth is the forbidden case (§2.8), so
221
+ // this asserts the law itself, k < 1, rather than the stricter 0.6 that suits
222
+ // a counter the fix governs end to end. Measured 1.40 unfixed, 0.54 fixed.
223
+ assert.ok(
224
+ kPops < 1,
225
+ `agenda pops grew with exponent k=${kPops.toFixed(2)} in corpus size (${
226
+ pops.join(" → ")
227
+ }) for a byte-identical answer — k≈1 is work LINEAR in the corpus, which ` +
228
+ `is the bound §2.8 forbids outright`,
229
+ );
230
+ });