@hviana/sema 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/AGENTS.md +470 -0
  2. package/AUTHORS.md +12 -0
  3. package/COMMERCIAL-LICENSE.md +19 -0
  4. package/CONTRIBUTING.md +15 -0
  5. package/HOW_IT_WORKS.md +4210 -0
  6. package/LICENSE.md +117 -0
  7. package/README.md +290 -0
  8. package/TRADEMARKS.md +19 -0
  9. package/example/demo.ts +46 -0
  10. package/example/train_base.ts +2675 -0
  11. package/index.html +893 -0
  12. package/package.json +25 -0
  13. package/src/alphabet.ts +34 -0
  14. package/src/alu/README.md +332 -0
  15. package/src/alu/src/alu.ts +541 -0
  16. package/src/alu/src/expr.ts +339 -0
  17. package/src/alu/src/index.ts +115 -0
  18. package/src/alu/src/kernel-arith.ts +377 -0
  19. package/src/alu/src/kernel-bits.ts +199 -0
  20. package/src/alu/src/kernel-logic.ts +102 -0
  21. package/src/alu/src/kernel-nd.ts +235 -0
  22. package/src/alu/src/kernel-numeric.ts +466 -0
  23. package/src/alu/src/operation.ts +344 -0
  24. package/src/alu/src/parser.ts +574 -0
  25. package/src/alu/src/resonance.ts +161 -0
  26. package/src/alu/src/text.ts +83 -0
  27. package/src/alu/src/value.ts +327 -0
  28. package/src/alu/test/alu.test.ts +1004 -0
  29. package/src/bytes.ts +62 -0
  30. package/src/config.ts +227 -0
  31. package/src/derive/README.md +295 -0
  32. package/src/derive/src/deduction.ts +263 -0
  33. package/src/derive/src/index.ts +24 -0
  34. package/src/derive/src/priority-queue.ts +70 -0
  35. package/src/derive/src/rewrite.ts +127 -0
  36. package/src/derive/src/trie.ts +242 -0
  37. package/src/derive/test/derive.test.ts +137 -0
  38. package/src/extension.ts +42 -0
  39. package/src/geometry.ts +511 -0
  40. package/src/index.ts +38 -0
  41. package/src/ingest-cache.ts +224 -0
  42. package/src/mind/articulation.ts +137 -0
  43. package/src/mind/attention.ts +1061 -0
  44. package/src/mind/canonical.ts +107 -0
  45. package/src/mind/graph-search.ts +1100 -0
  46. package/src/mind/index.ts +18 -0
  47. package/src/mind/junction.ts +347 -0
  48. package/src/mind/learning.ts +246 -0
  49. package/src/mind/match.ts +507 -0
  50. package/src/mind/mechanisms/alu.ts +33 -0
  51. package/src/mind/mechanisms/cast.ts +568 -0
  52. package/src/mind/mechanisms/confluence.ts +268 -0
  53. package/src/mind/mechanisms/cover.ts +248 -0
  54. package/src/mind/mechanisms/extraction.ts +422 -0
  55. package/src/mind/mechanisms/recall.ts +241 -0
  56. package/src/mind/mind.ts +483 -0
  57. package/src/mind/pipeline-mechanism.ts +326 -0
  58. package/src/mind/pipeline.ts +300 -0
  59. package/src/mind/primitives.ts +201 -0
  60. package/src/mind/rationale.ts +275 -0
  61. package/src/mind/reasoning.ts +198 -0
  62. package/src/mind/recognition.ts +234 -0
  63. package/src/mind/resonance.ts +0 -0
  64. package/src/mind/trace.ts +108 -0
  65. package/src/mind/traverse.ts +556 -0
  66. package/src/mind/types.ts +281 -0
  67. package/src/rabitq-hnsw/README.md +303 -0
  68. package/src/rabitq-hnsw/src/database.ts +492 -0
  69. package/src/rabitq-hnsw/src/heap.ts +90 -0
  70. package/src/rabitq-hnsw/src/hnsw.ts +514 -0
  71. package/src/rabitq-hnsw/src/index.ts +15 -0
  72. package/src/rabitq-hnsw/src/prng.ts +39 -0
  73. package/src/rabitq-hnsw/src/rabitq.ts +308 -0
  74. package/src/rabitq-hnsw/src/store.ts +994 -0
  75. package/src/rabitq-hnsw/test/hnsw.test.ts +1213 -0
  76. package/src/store-sqlite.ts +928 -0
  77. package/src/store.ts +2148 -0
  78. package/src/vec.ts +124 -0
  79. package/test/00-extract.test.mjs +151 -0
  80. package/test/01-floor.test.mjs +20 -0
  81. package/test/02-roundtrip.test.mjs +83 -0
  82. package/test/03-recall.test.mjs +98 -0
  83. package/test/04-think.test.mjs +627 -0
  84. package/test/05-concepts.test.mjs +84 -0
  85. package/test/08-storage.test.mjs +948 -0
  86. package/test/09-edges.test.mjs +65 -0
  87. package/test/11-universality.test.mjs +180 -0
  88. package/test/13-conversation.test.mjs +228 -0
  89. package/test/14-scaling.test.mjs +503 -0
  90. package/test/15-decomposition-gap.test.mjs +0 -0
  91. package/test/16-bridge.test.mjs +250 -0
  92. package/test/17-intelligence.test.mjs +240 -0
  93. package/test/18-alu.test.mjs +209 -0
  94. package/test/19-nd.test.mjs +254 -0
  95. package/test/20-stability.test.mjs +489 -0
  96. package/test/21-partial.test.mjs +158 -0
  97. package/test/22-multihop.test.mjs +130 -0
  98. package/test/23-rationale.test.mjs +316 -0
  99. package/test/24-generalization.test.mjs +416 -0
  100. package/test/25-embedding.test.mjs +75 -0
  101. package/test/26-cooperative.test.mjs +89 -0
  102. package/test/27-saturation-drop.test.mjs +637 -0
  103. package/test/28-unknowable.test.mjs +88 -0
  104. package/test/29-counterfactual.test.mjs +476 -0
  105. package/test/30-conflict-resolution.test.mjs +166 -0
  106. package/test/31-audit.test.mjs +288 -0
  107. package/test/32-confluence.test.mjs +173 -0
  108. package/test/33-multi-candidate.test.mjs +222 -0
  109. package/test/34-cross-region.test.mjs +252 -0
  110. package/tsconfig.json +16 -0
@@ -0,0 +1,326 @@
1
+ // pipeline-mechanism.ts — the uniform grounding-mechanism interface.
2
+ //
3
+ // Every grounding mechanism (CAST, confluence, cover, extraction, recall, ALU,
4
+ // user extensions) implements this ONE interface. The pipeline (think()) sees
5
+ // a list of PipelineMechanism objects — it never imports a mechanism-specific
6
+ // type and never has a special-case branch for any mechanism.
7
+ //
8
+ // The four constraints of the free-will architecture (§14.5):
9
+ // 1. DECOUPLING — mechanisms import nothing from each other or from pipeline.
10
+ // 2. DECLARED COMPETENCE — floor() returns null when impossible, a number when
11
+ // possible. Binary, auditable, no learned scores.
12
+ // 3. VISIBLE BUDGET — every mechanism carries its own caps internally (√N, k).
13
+ // 4. TRAVELING EVIDENCE — run() returns MechanismResult with accounted, moves,
14
+ // and unexplained. The pipeline computes the weight.
15
+
16
+ import type { AncestorReach, MindContext, Recognition } from "./types.js";
17
+ import type { AttentionRead } from "./types.js";
18
+ import type { ComputedSpan } from "../extension.js";
19
+ import type { Vec } from "../vec.js";
20
+ import { windowIds } from "./canonical.js";
21
+ import { read, resolve } from "./primitives.js";
22
+ import { alignGraded, type GradedRun } from "./match.js";
23
+ import { climbAttentionAll } from "./attention.js";
24
+ import { skillExemplar } from "./mechanisms/extraction.js";
25
+
26
+ // ── Precomputed ──────────────────────────────────────────────────────────────
27
+ //
28
+ // Precomputed is a LAZY container for structural analyses of the query — the
29
+ // ONE place a response's shared evidence lives, for inter-mechanism exchange
30
+ // and for analyses future mechanisms will want. Eager fields (rec, computed,
31
+ // guide) are populated by the pipeline before the mechanism loop; everything
32
+ // expensive is a lazily-cached method that computes on first access. A
33
+ // mechanism that never asks for an analysis pays nothing for it; two
34
+ // mechanisms asking for the same analysis pay once.
35
+ //
36
+ // This design serves THREE purposes:
37
+ // 1. SHARING — when two mechanisms need the same analysis, it's computed once
38
+ // (even under trace, where the ctx-level memos are deliberately bypassed).
39
+ // 2. EXTENSIBILITY — a new analysis is one method in one file.
40
+ // 3. DECLARATIVE COST — a mechanism's floor() checks its cheap gates and the
41
+ // pipeline's `worthRunning` predicate BEFORE first-touching an expensive
42
+ // analysis, so lazy analyses are only ever computed for a mechanism that
43
+ // could still win.
44
+
45
+ export class Precomputed {
46
+ /** The response's evidence-breadth constant: how many ranked candidates the
47
+ * resonance probes, the weave alignment, and the climb all consider.
48
+ * Derived once from config; every consumer reads it here. */
49
+ readonly k: number;
50
+
51
+ constructor(
52
+ readonly ctx: MindContext,
53
+ readonly query: Uint8Array,
54
+ /** Recognition result (structural + canonical). */
55
+ readonly rec: Recognition,
56
+ /** Computed spans from mechanisms that implement `parse()` (e.g. ALU). */
57
+ readonly computed: ComputedSpan[],
58
+ /** The query's gist — the response-wide disambiguation guide. */
59
+ readonly guide: Vec,
60
+ ) {
61
+ this.k = ctx.cfg.recallQueryK * 2;
62
+ }
63
+
64
+ // ── Cheap lazy analyses ───────────────────────────────────────────────
65
+
66
+ private _windows?: Map<number, number>;
67
+ /** Content-addressed W-window identities for every position in the query
68
+ * (offset → node id). O(|query|) probes. */
69
+ get queryWindows(): Map<number, number> {
70
+ return this._windows ??= windowIds(this.ctx, this.query);
71
+ }
72
+
73
+ private _resolved?: number | null;
74
+ /** The node id of the query itself, or null when it is not a stored form.
75
+ * O(|query|) probes. */
76
+ get queryResolved(): number | null {
77
+ if (this._resolved === undefined) {
78
+ this._resolved = resolve(this.ctx, this.query);
79
+ }
80
+ return this._resolved;
81
+ }
82
+
83
+ private _anchorWindows = new Map<number, Map<number, number>>();
84
+ /** Content-addressed W-window identities of one anchor's own bytes
85
+ * (offset → node id), memoised per anchor. Confluence intersects these;
86
+ * any future identity-based mechanism reads the same cache. */
87
+ windowsOf(anchor: number): Map<number, number> {
88
+ let w = this._anchorWindows.get(anchor);
89
+ if (w === undefined) {
90
+ w = windowIds(this.ctx, read(this.ctx, anchor));
91
+ this._anchorWindows.set(anchor, w);
92
+ }
93
+ return w;
94
+ }
95
+
96
+ /** Shared memo for {@link reachOf} (structural-IDF reads): a window's
97
+ * ancestor reach is a pure function of the read-only store, so one
98
+ * response-scoped memo serves every mechanism that prices commonality. */
99
+ readonly reachMemo = new Map<number, AncestorReach>();
100
+
101
+ // ── Expensive lazy analyses ───────────────────────────────────────────
102
+ //
103
+ // Async, cached-by-promise: the first caller starts the computation, every
104
+ // later caller (any mechanism, any phase) awaits the same promise. A
105
+ // mechanism MUST check its cheap floor gates and the pipeline's
106
+ // `worthRunning` predicate before first-touching one of these.
107
+
108
+ private _attention?: Promise<AttentionRead>;
109
+ /** The full consensus climb (roots + ranked anchors) — the query-level
110
+ * evidence CAST, confluence, extraction, recall's scaffolding tier, and
111
+ * fusion all share. Computed on first access; a query no mechanism
112
+ * climbs for (e.g. one an extension decided outright) never pays for it. */
113
+ attention(): Promise<AttentionRead> {
114
+ return this._attention ??= climbAttentionAll(
115
+ this.ctx,
116
+ this.query,
117
+ this.k,
118
+ );
119
+ }
120
+
121
+ private _weave?: Promise<WeaveInfo>;
122
+ /** Result of {@link alignGraded} for the first k ranked anchors —
123
+ * O(k · |query| · |ctx|). Consumed by CAST; reusable by any future
124
+ * mechanism doing analogical transfer. */
125
+ weave(): Promise<WeaveInfo> {
126
+ return this._weave ??= this.attention().then((climb) =>
127
+ computeWeave(this.ctx, this.query, this, climb)
128
+ );
129
+ }
130
+
131
+ /** Span-shaped classification of one ranked anchor, memoised per anchor id
132
+ * so repeated calls (extraction's own early-exit scan, any future
133
+ * template-based mechanism) never redo the work. Deliberately NOT an
134
+ * eager all-anchors map: `skillExemplar` is the expensive part of
135
+ * extraction (capped fan-out reads plus an O(|ctx|) scan), and most
136
+ * queries are answered by the FIRST ranked anchor that qualifies — paying
137
+ * for every ranked anchor regardless of where the scan stops would turn
138
+ * an early-exit lookup into full O(k) work on every query. */
139
+ private _spanShaped = new Map<number, Promise<SkillInfo | null>>();
140
+ spanShapedOf(anchor: number): Promise<SkillInfo | null> {
141
+ let p = this._spanShaped.get(anchor);
142
+ if (p === undefined) {
143
+ p = skillExemplar(this.ctx, anchor, this.guide);
144
+ this._spanShaped.set(anchor, p);
145
+ }
146
+ return p;
147
+ }
148
+
149
+ /** Every ranked anchor's classification at once, sharing the same
150
+ * per-anchor cache as {@link spanShapedOf} — for a mechanism that
151
+ * genuinely needs the full picture (not an early-exit scan). Mixing
152
+ * access patterns across mechanisms never duplicates work: whichever
153
+ * anchors an early-exit consumer already asked for are reused here, and
154
+ * whichever this computes first are reused by a later early-exit scan. */
155
+ async spanShapedAll(): Promise<Map<number, SkillInfo | null>> {
156
+ const { ranked } = await this.attention();
157
+ const out = new Map<number, SkillInfo | null>();
158
+ for (const cand of ranked) {
159
+ if (out.has(cand.anchor)) continue;
160
+ out.set(cand.anchor, await this.spanShapedOf(cand.anchor));
161
+ }
162
+ return out;
163
+ }
164
+ }
165
+
166
+ // ── WeaveInfo ────────────────────────────────────────────────────────────────
167
+
168
+ /** The weave-local structural alignment, computed once and consumed by CAST
169
+ * (and any future mechanism doing analogical transfer). */
170
+ export interface WeaveInfo {
171
+ /** Per-anchor alignment: context bytes, vote weight, and graded runs. */
172
+ points: Array<{
173
+ anchor: number;
174
+ vote: number;
175
+ ctx: Uint8Array;
176
+ runs: GradedRun[];
177
+ }>;
178
+ /** Weighted depth at each query byte — sum of alignment weights.
179
+ * `depth[i]` is the total evidence that byte i is shared among the
180
+ * aligned structures. */
181
+ depth: Float64Array;
182
+ }
183
+
184
+ function computeWeave(
185
+ ctx: MindContext,
186
+ query: Uint8Array,
187
+ pre: Precomputed,
188
+ climb: AttentionRead,
189
+ ): WeaveInfo {
190
+ const quantum = ctx.space.maxGroup;
191
+ const { ranked } = climb;
192
+ const rankedCapped = ranked.length > pre.k ? ranked.slice(0, pre.k) : ranked;
193
+ const depth = new Float64Array(query.length);
194
+ const points: WeaveInfo["points"] = [];
195
+
196
+ // WEAVE-SCALE anchors only: CAST transfers structure between things the
197
+ // QUERY weaves together — query-scale structures. A context an order of
198
+ // magnitude beyond the query is not woven BY the query (the query can at
199
+ // most quote a fragment of it, and fragment-level evidence is exactly what
200
+ // recognition and the cover already handle); CAST's own comparison gate
201
+ // demands `ctx.length ≤ query.length` before it fires, and its
202
+ // substitution seats sit within a quantum of a context's start. W is the
203
+ // perceptual quantum — the same scale multiplier the bridge's phrase-scale
204
+ // contract uses. The prefix-capped read makes an oversized anchor cost a
205
+ // bounded read instead of reconstructing (and then canonically
206
+ // recognising) a corpus-sized deposit: profiled on a 17.7M-node store,
207
+ // uncapped weaves spent 5–8s per query recognising conversation-length
208
+ // anchors that could never form a weave point.
209
+ const capBytes = query.length * quantum;
210
+ for (const cand of rankedCapped) {
211
+ const ctxBytes = read(ctx, cand.anchor, capBytes + 1);
212
+ if (ctxBytes.length === 0 || ctxBytes.length > capBytes) continue;
213
+ const raw = alignGraded(ctx, query, ctxBytes, pre.rec.sites);
214
+ if (raw.length === 0) continue;
215
+ for (const r of raw) {
216
+ for (let i = r.qs; i < r.qe; i++) depth[i] += r.weight;
217
+ }
218
+ const free: GradedRun[] = [];
219
+ for (const r of raw) {
220
+ let { qs, qe, cs, weight } = r;
221
+ for (const p of points) {
222
+ for (const o of p.runs) {
223
+ if (qs >= qe) break;
224
+ if (o.qe <= qs || o.qs >= qe) continue;
225
+ const left = Math.max(0, o.qs - qs);
226
+ const right = Math.max(0, qe - o.qe);
227
+ if (left >= right) qe = qs + left;
228
+ else {
229
+ cs += qe - right - qs;
230
+ qs = qe - right;
231
+ }
232
+ }
233
+ }
234
+ if (qe - qs >= Math.min(quantum, ctxBytes.length)) {
235
+ free.push({ qs, qe, cs, weight });
236
+ }
237
+ }
238
+ if (free.length > 0) {
239
+ points.push({
240
+ anchor: cand.anchor,
241
+ vote: cand.vote,
242
+ ctx: ctxBytes,
243
+ runs: free,
244
+ });
245
+ }
246
+ }
247
+ return { points, depth };
248
+ }
249
+
250
+ // ── SkillInfo ────────────────────────────────────────────────────────────────
251
+
252
+ /** Span-shaped classification of one anchor — the structural information
253
+ * extraction uses to decide whether a learned fact can serve as a template
254
+ * for reading an analogous span out of the query. */
255
+ export interface SkillInfo {
256
+ contextBytes: Uint8Array;
257
+ answerBytes: Uint8Array;
258
+ }
259
+
260
+ // ── MechanismResult ──────────────────────────────────────────────────────────
261
+
262
+ /** Raw result from a mechanism's `run()`. The pipeline computes the weight
263
+ * from `moves` + `PASS * unaccounted(accounted)` — the mechanism does not
264
+ * know about the cost ladder.
265
+ *
266
+ * When `weight` is present, the pipeline uses it directly instead of
267
+ * computing `weigh(accounted, moves)`. This is for mechanisms whose cost
268
+ * is derived externally (e.g. cover: the A*LD derivation's g-value). */
269
+ export interface MechanismResult {
270
+ bytes: Uint8Array;
271
+ accounted: Array<[number, number]>;
272
+ moves: number;
273
+ used?: ReadonlySet<number>;
274
+ unexplained: string;
275
+ /** Explicit weight override. When absent, weight = moves + PASS·unaccounted. */
276
+ weight?: number;
277
+ /** Override the mechanism's default provenance for this result.
278
+ * When absent, the pipeline uses `mech.provenance`. */
279
+ provenance?: string;
280
+ }
281
+
282
+ // ── PipelineMechanism ────────────────────────────────────────────────────────
283
+
284
+ export interface PipelineMechanism {
285
+ /** Stable identifier for trace/debug. */
286
+ readonly name: string;
287
+
288
+ /** Which provenance tag the pipeline attaches to this mechanism's answers. */
289
+ readonly provenance: string;
290
+
291
+ /** Parse authoritative spans BEFORE the grounding loop.
292
+ * Only needed by computational mechanisms (e.g. ALU). Results from ALL
293
+ * mechanisms that implement this are collected into `Precomputed.computed`
294
+ * before any `floor()` or `run()` is called. */
295
+ parse?(query: Uint8Array): Promise<ComputedSpan[]>;
296
+
297
+ /** Admissible lower bound on this mechanism's weight.
298
+ * Returns `null` when the mechanism structurally cannot fire.
299
+ *
300
+ * `worthRunning(cheapFloor)` reports whether the CURRENT incumbent
301
+ * (established by mechanisms that already ran this response, cover being
302
+ * first — see `defaultMechanisms`) could still be beaten by a floor no
303
+ * tighter than `cheapFloor`. THE INVESTMENT DISCIPLINE: before
304
+ * first-touching an expensive shared analysis (`pre.attention()`,
305
+ * `pre.weave()`, …), check `worthRunning(bound)` with this mechanism's
306
+ * cheapest possible bound — and when it fails, RETURN THE BOUND rather
307
+ * than null. The bound is still admissible (it never overstates cost),
308
+ * the pipeline's own check then prunes `run()` and records the truthful
309
+ * "cannot beat incumbent" trace note, and no analysis was computed just
310
+ * to be discarded. This is uniform: no mechanism asks what produced the
311
+ * incumbent — a computed extension result and an ordinary cheap cover
312
+ * prune the same way. */
313
+ floor(
314
+ ctx: MindContext,
315
+ query: Uint8Array,
316
+ pre: Precomputed,
317
+ worthRunning: (floor: number) => boolean,
318
+ ): Promise<number | null>;
319
+
320
+ /** Produce candidate answers. */
321
+ run(
322
+ ctx: MindContext,
323
+ query: Uint8Array,
324
+ pre: Precomputed,
325
+ ): Promise<MechanismResult[]>;
326
+ }
@@ -0,0 +1,300 @@
1
+ // pipeline.ts — the think pipeline (Section 5 of the mind).
2
+ //
3
+ // think() is the whole file's job: one lightest-derivation choice among
4
+ // UNIFORM mechanisms. The pipeline sees mechanisms through the
5
+ // PipelineMechanism interface only — it never imports a mechanism-specific
6
+ // type and never has a special-case branch for any mechanism. Adding a
7
+ // mechanism means registering one object; removing one means dropping it
8
+ // from the list. The mechanisms themselves live in mechanisms/ (one file
9
+ // each); the shared pre-computation they exchange lives in Precomputed
10
+ // (pipeline-mechanism.ts).
11
+
12
+ import type { MindContext } from "./types.js";
13
+ import { PASS, STEP } from "./graph-search.js";
14
+ import type { ComputedSpan } from "../extension.js";
15
+ import { gistOf, resolve } from "./primitives.js";
16
+ import { recognise } from "./recognition.js";
17
+ import { fuseAttention, reason } from "./reasoning.js";
18
+ import { unexplainedSpans } from "./rationale.js";
19
+ import { rItem } from "./trace.js";
20
+ import { type PipelineMechanism, Precomputed } from "./pipeline-mechanism.js";
21
+ import { coverMechanism } from "./mechanisms/cover.js";
22
+ import { castMechanism } from "./mechanisms/cast.js";
23
+ import { confluenceMechanism } from "./mechanisms/confluence.js";
24
+ import { extractionMechanism } from "./mechanisms/extraction.js";
25
+ import { recallMechanism } from "./mechanisms/recall.js";
26
+
27
+ // Re-exports: cover's pre-resolution helpers and the ALU adapter kept
28
+ // importable from the pipeline module (their historical home).
29
+ export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
30
+ export { aluToMechanism } from "./mechanisms/alu.js";
31
+
32
+ // ── Extension dispatch (pre-loop parse) ─────────────────────────────────────
33
+
34
+ async function collectComputed(
35
+ mechanisms: readonly PipelineMechanism[],
36
+ query: Uint8Array,
37
+ ): Promise<ComputedSpan[]> {
38
+ const out: ComputedSpan[] = [];
39
+ for (const m of mechanisms) {
40
+ if (m.parse) out.push(...await m.parse(query));
41
+ }
42
+ return out;
43
+ }
44
+
45
+ // ── Built-in mechanisms ─────────────────────────────────────────────────────
46
+
47
+ // ORDER MATTERS, but only through the uniform floor/worthRunning pruning —
48
+ // no mechanism is special-cased. Cover runs FIRST: when a computed
49
+ // extension result (e.g. ALU) exists, cover masks it in at near-zero cost
50
+ // (see mechanisms/cover.ts), which becomes `best` before any other mechanism
51
+ // invests in its own precomputation. CAST's and confluence's floors (2*STEP,
52
+ // 3*STEP) then fail `worthRunning` and are skipped by the SAME admissible-
53
+ // floor pruning every mechanism is already subject to — not by asking
54
+ // "is this an extension?". Grade TIES keep the earlier candidate, so this
55
+ // order is also the tie-break priority: cover, cast, confluence, extraction,
56
+ // recall.
57
+ export const defaultMechanisms: PipelineMechanism[] = [
58
+ coverMechanism,
59
+ castMechanism,
60
+ confluenceMechanism,
61
+ extractionMechanism,
62
+ recallMechanism,
63
+ ];
64
+
65
+ // ── think — the main inference pipeline ─────────────────────────────────────
66
+
67
+ export type Provenance =
68
+ | "cast"
69
+ | "join"
70
+ | "cover"
71
+ | "extract"
72
+ | "recall"
73
+ | "recall-echo";
74
+
75
+ export interface Thought {
76
+ bytes: Uint8Array;
77
+ provenance: Provenance;
78
+ }
79
+
80
+ /** Think: a single lightest-derivation exploration of the Sema graph.
81
+ *
82
+ * Every answer travels the same path:
83
+ * 1. Pre-computation — recognise, extension parse, guide; everything
84
+ * expensive stays lazy on Precomputed until a mechanism asks.
85
+ * 2. Grounding — every mechanism yields candidates weighed in the one
86
+ * cost ladder; the lightest grounding derivation wins.
87
+ * 3. Post-grounding — diagnostics (narrowDecision, thinGrounding),
88
+ * reasoning (multi-hop), fusion (multi-topic). */
89
+ export async function think(
90
+ ctx: MindContext,
91
+ query: Uint8Array,
92
+ mechs?: readonly PipelineMechanism[],
93
+ ): Promise<Thought | null> {
94
+ if (query.length === 0) return null;
95
+
96
+ ctx._edgeGuide = gistOf(ctx, query);
97
+ ctx._edgeChoice.clear();
98
+
99
+ const t = ctx.trace?.enter("think", [rItem(query, "query")]);
100
+ const done = (answer: Uint8Array | null, note: string) => {
101
+ t?.done(
102
+ answer
103
+ ? [rItem(answer, "answer", resolve(ctx, answer) ?? undefined)]
104
+ : [],
105
+ note,
106
+ );
107
+ return answer;
108
+ };
109
+
110
+ // ── Pre-computation ──────────────────────────────────────────────────
111
+ const mechanisms = mechs ?? defaultMechanisms;
112
+ const rec = recognise(ctx, query);
113
+
114
+ // Phase 1: collect computed spans from mechanisms that implement parse()
115
+ const computed = await collectComputed(mechanisms, query);
116
+
117
+ if (computed.length > 0) {
118
+ ctx.trace?.step(
119
+ "computeExtensions",
120
+ [rItem(query, "query")],
121
+ computed.map((u) =>
122
+ rItem(query.subarray(u.i, u.j), "operand", undefined, [u.i, u.j])
123
+ ),
124
+ `extensions recognised and evaluated ${computed.length} computation(s)`,
125
+ );
126
+ for (const u of computed) {
127
+ ctx.trace?.step(
128
+ "evalComputation",
129
+ [rItem(query.subarray(u.i, u.j), "expression", undefined, [u.i, u.j])],
130
+ [rItem(u.bytes, "result", resolve(ctx, u.bytes) ?? undefined)],
131
+ "evaluate the recognised operation to its authoritative result",
132
+ );
133
+ }
134
+ }
135
+
136
+ // Phase 2: the shared pre-computation container. Eager fields only
137
+ // (recognition, computed spans, guide) — every expensive analysis
138
+ // (consensus climb, weave, span-shape classification) is a lazily-cached
139
+ // method on Precomputed, first-touched by whichever mechanism's floor
140
+ // survives its cheap gates and the worthRunning check. A query no
141
+ // mechanism climbs for (e.g. one an extension decided) never climbs.
142
+ const pre = new Precomputed(ctx, query, rec, computed, ctx._edgeGuide);
143
+
144
+ // ── Grounding: ONE lightest-derivation choice among the mechanisms ────
145
+
146
+ interface Candidate {
147
+ bytes: Uint8Array;
148
+ provenance: string;
149
+ weight: number;
150
+ used?: ReadonlySet<number>;
151
+ accounted: ReadonlyArray<[number, number]>;
152
+ unexplained: string;
153
+ }
154
+ const grade = (w: number) => Math.floor(w / STEP);
155
+ const unaccounted = (spans: ReadonlyArray<[number, number]>): number =>
156
+ unexplainedSpans(query.length, spans)
157
+ .reduce((sum, [s, e]) => sum + (e - s), 0);
158
+ const weigh = (
159
+ accounted: ReadonlyArray<[number, number]>,
160
+ moves: number,
161
+ ): number => moves + PASS * unaccounted(accounted);
162
+
163
+ const candidates: Candidate[] = [];
164
+ let best: Candidate | null = null;
165
+ const consider = (c: Candidate) => {
166
+ if (c.bytes.length === 0) return;
167
+ candidates.push(c);
168
+ if (best === null || grade(c.weight) < grade(best.weight)) best = c;
169
+ };
170
+ const worthRunning = (floor: number) =>
171
+ best === null || grade(floor) < grade(best.weight);
172
+
173
+ // Phase 3: grounding loop
174
+ for (const mech of mechanisms) {
175
+ const floor = await mech.floor(ctx, query, pre, worthRunning);
176
+ if (floor === null) {
177
+ ctx.trace?.step(
178
+ "skipMechanism",
179
+ [],
180
+ [],
181
+ `${mech.name} skipped — structural precondition failed`,
182
+ );
183
+ continue;
184
+ }
185
+ if (!worthRunning(floor)) {
186
+ ctx.trace?.step(
187
+ "skipMechanism",
188
+ [],
189
+ [],
190
+ `${mech.name} skipped — floor ${floor} cannot beat incumbent (grade ${
191
+ grade(best!.weight)
192
+ })`,
193
+ );
194
+ continue;
195
+ }
196
+ const results = await mech.run(ctx, query, pre);
197
+ for (const r of results) {
198
+ const weight = r.weight ?? weigh(r.accounted, r.moves);
199
+ consider({
200
+ bytes: r.bytes,
201
+ provenance: r.provenance ?? mech.provenance,
202
+ weight,
203
+ used: r.used,
204
+ accounted: r.accounted,
205
+ unexplained: r.unexplained,
206
+ });
207
+ }
208
+ }
209
+
210
+ // (TS cannot see the closure assignments into `best` and narrows it to its
211
+ // initial null, so the read-back needs the assertion.)
212
+ const decided = best as Candidate | null;
213
+ if (candidates.length > 1) {
214
+ ctx.trace?.step(
215
+ "decideGrounding",
216
+ candidates.map((c) =>
217
+ rItem(
218
+ c.bytes,
219
+ `${c.provenance} (weight ${c.weight.toFixed(3)}${
220
+ c.unexplained ? `, unexplained: "${c.unexplained}"` : ""
221
+ })`,
222
+ )
223
+ ),
224
+ decided ? [rItem(decided.bytes, decided.provenance)] : [],
225
+ "the lightest grounding derivation wins — every mechanism weighed in the one cost ladder",
226
+ );
227
+ if (decided !== null) {
228
+ let runnerUp: Candidate | null = null;
229
+ for (const c of candidates) {
230
+ if (c === decided) continue;
231
+ if (runnerUp === null || grade(c.weight) < grade(runnerUp.weight)) {
232
+ runnerUp = c;
233
+ }
234
+ }
235
+ if (runnerUp !== null) {
236
+ const margin = grade(runnerUp.weight) - grade(decided.weight);
237
+ if (margin <= 1) {
238
+ ctx.trace?.step(
239
+ "narrowDecision",
240
+ [
241
+ rItem(
242
+ decided.bytes,
243
+ `${decided.provenance} (weight ${decided.weight.toFixed(3)})`,
244
+ ),
245
+ ],
246
+ [
247
+ rItem(
248
+ runnerUp.bytes,
249
+ `${runnerUp.provenance} (weight ${runnerUp.weight.toFixed(3)})`,
250
+ ),
251
+ ],
252
+ `margin ${margin} grade-unit(s) — the decision could change with one more training fact`,
253
+ );
254
+ }
255
+ }
256
+ }
257
+ }
258
+
259
+ if (decided === null) {
260
+ done(null, "no mechanism grounded an answer");
261
+ return null;
262
+ }
263
+
264
+ // Honesty density
265
+ {
266
+ const covered = query.length - unaccounted(decided.accounted);
267
+ const density = query.length > 0 ? covered / query.length : 1;
268
+ const thinBar = 1 / ctx.space.maxGroup;
269
+ if (density < thinBar) {
270
+ ctx.trace?.step(
271
+ "thinGrounding",
272
+ [rItem(decided.bytes, decided.provenance)],
273
+ [],
274
+ `grounded but thin — density ${density.toFixed(3)} is below 1/W (${
275
+ thinBar.toFixed(3)
276
+ })`,
277
+ );
278
+ }
279
+ }
280
+ const answer: Uint8Array = decided.bytes;
281
+ const provenance = decided.provenance as Provenance;
282
+ const castUsed: ReadonlySet<number> = decided.used ?? new Set();
283
+
284
+ // ── Post-grounding, gated by provenance ──────────────────────────────
285
+ const preConsumed = provenance === "cast" || provenance === "join"
286
+ ? castUsed
287
+ : provenance === "recall" || provenance === "recall-echo"
288
+ ? new Set<number>()
289
+ : new Set(recognise(ctx, answer).sites.map((s) => s.payload));
290
+ const reasoned = await reason(ctx, query, answer, preConsumed, pre);
291
+ const fused = provenance === "recall" || provenance === "recall-echo"
292
+ ? await fuseAttention(ctx, query, reasoned, pre)
293
+ : reasoned;
294
+
295
+ done(
296
+ fused,
297
+ "grounded, reasoned forward, fused across points of attention",
298
+ );
299
+ return { bytes: fused, provenance };
300
+ }