@hviana/sema 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +470 -0
- package/AUTHORS.md +12 -0
- package/COMMERCIAL-LICENSE.md +19 -0
- package/CONTRIBUTING.md +15 -0
- package/HOW_IT_WORKS.md +4210 -0
- package/LICENSE.md +117 -0
- package/README.md +290 -0
- package/TRADEMARKS.md +19 -0
- package/example/demo.ts +46 -0
- package/example/train_base.ts +2675 -0
- package/index.html +893 -0
- package/package.json +25 -0
- package/src/alphabet.ts +34 -0
- package/src/alu/README.md +332 -0
- package/src/alu/src/alu.ts +541 -0
- package/src/alu/src/expr.ts +339 -0
- package/src/alu/src/index.ts +115 -0
- package/src/alu/src/kernel-arith.ts +377 -0
- package/src/alu/src/kernel-bits.ts +199 -0
- package/src/alu/src/kernel-logic.ts +102 -0
- package/src/alu/src/kernel-nd.ts +235 -0
- package/src/alu/src/kernel-numeric.ts +466 -0
- package/src/alu/src/operation.ts +344 -0
- package/src/alu/src/parser.ts +574 -0
- package/src/alu/src/resonance.ts +161 -0
- package/src/alu/src/text.ts +83 -0
- package/src/alu/src/value.ts +327 -0
- package/src/alu/test/alu.test.ts +1004 -0
- package/src/bytes.ts +62 -0
- package/src/config.ts +227 -0
- package/src/derive/README.md +295 -0
- package/src/derive/src/deduction.ts +263 -0
- package/src/derive/src/index.ts +24 -0
- package/src/derive/src/priority-queue.ts +70 -0
- package/src/derive/src/rewrite.ts +127 -0
- package/src/derive/src/trie.ts +242 -0
- package/src/derive/test/derive.test.ts +137 -0
- package/src/extension.ts +42 -0
- package/src/geometry.ts +511 -0
- package/src/index.ts +38 -0
- package/src/ingest-cache.ts +224 -0
- package/src/mind/articulation.ts +137 -0
- package/src/mind/attention.ts +1061 -0
- package/src/mind/canonical.ts +107 -0
- package/src/mind/graph-search.ts +1100 -0
- package/src/mind/index.ts +18 -0
- package/src/mind/junction.ts +347 -0
- package/src/mind/learning.ts +246 -0
- package/src/mind/match.ts +507 -0
- package/src/mind/mechanisms/alu.ts +33 -0
- package/src/mind/mechanisms/cast.ts +568 -0
- package/src/mind/mechanisms/confluence.ts +268 -0
- package/src/mind/mechanisms/cover.ts +248 -0
- package/src/mind/mechanisms/extraction.ts +422 -0
- package/src/mind/mechanisms/recall.ts +241 -0
- package/src/mind/mind.ts +483 -0
- package/src/mind/pipeline-mechanism.ts +326 -0
- package/src/mind/pipeline.ts +300 -0
- package/src/mind/primitives.ts +201 -0
- package/src/mind/rationale.ts +275 -0
- package/src/mind/reasoning.ts +198 -0
- package/src/mind/recognition.ts +234 -0
- package/src/mind/resonance.ts +0 -0
- package/src/mind/trace.ts +108 -0
- package/src/mind/traverse.ts +556 -0
- package/src/mind/types.ts +281 -0
- package/src/rabitq-hnsw/README.md +303 -0
- package/src/rabitq-hnsw/src/database.ts +492 -0
- package/src/rabitq-hnsw/src/heap.ts +90 -0
- package/src/rabitq-hnsw/src/hnsw.ts +514 -0
- package/src/rabitq-hnsw/src/index.ts +15 -0
- package/src/rabitq-hnsw/src/prng.ts +39 -0
- package/src/rabitq-hnsw/src/rabitq.ts +308 -0
- package/src/rabitq-hnsw/src/store.ts +994 -0
- package/src/rabitq-hnsw/test/hnsw.test.ts +1213 -0
- package/src/store-sqlite.ts +928 -0
- package/src/store.ts +2148 -0
- package/src/vec.ts +124 -0
- package/test/00-extract.test.mjs +151 -0
- package/test/01-floor.test.mjs +20 -0
- package/test/02-roundtrip.test.mjs +83 -0
- package/test/03-recall.test.mjs +98 -0
- package/test/04-think.test.mjs +627 -0
- package/test/05-concepts.test.mjs +84 -0
- package/test/08-storage.test.mjs +948 -0
- package/test/09-edges.test.mjs +65 -0
- package/test/11-universality.test.mjs +180 -0
- package/test/13-conversation.test.mjs +228 -0
- package/test/14-scaling.test.mjs +503 -0
- package/test/15-decomposition-gap.test.mjs +0 -0
- package/test/16-bridge.test.mjs +250 -0
- package/test/17-intelligence.test.mjs +240 -0
- package/test/18-alu.test.mjs +209 -0
- package/test/19-nd.test.mjs +254 -0
- package/test/20-stability.test.mjs +489 -0
- package/test/21-partial.test.mjs +158 -0
- package/test/22-multihop.test.mjs +130 -0
- package/test/23-rationale.test.mjs +316 -0
- package/test/24-generalization.test.mjs +416 -0
- package/test/25-embedding.test.mjs +75 -0
- package/test/26-cooperative.test.mjs +89 -0
- package/test/27-saturation-drop.test.mjs +637 -0
- package/test/28-unknowable.test.mjs +88 -0
- package/test/29-counterfactual.test.mjs +476 -0
- package/test/30-conflict-resolution.test.mjs +166 -0
- package/test/31-audit.test.mjs +288 -0
- package/test/32-confluence.test.mjs +173 -0
- package/test/33-multi-candidate.test.mjs +222 -0
- package/test/34-cross-region.test.mjs +252 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// pipeline-mechanism.ts — the uniform grounding-mechanism interface.
|
|
2
|
+
//
|
|
3
|
+
// Every grounding mechanism (CAST, confluence, cover, extraction, recall, ALU,
|
|
4
|
+
// user extensions) implements this ONE interface. The pipeline (think()) sees
|
|
5
|
+
// a list of PipelineMechanism objects — it never imports a mechanism-specific
|
|
6
|
+
// type and never has a special-case branch for any mechanism.
|
|
7
|
+
//
|
|
8
|
+
// The four constraints of the free-will architecture (§14.5):
|
|
9
|
+
// 1. DECOUPLING — mechanisms import nothing from each other or from pipeline.
|
|
10
|
+
// 2. DECLARED COMPETENCE — floor() returns null when impossible, a number when
|
|
11
|
+
// possible. Binary, auditable, no learned scores.
|
|
12
|
+
// 3. VISIBLE BUDGET — every mechanism carries its own caps internally (√N, k).
|
|
13
|
+
// 4. TRAVELING EVIDENCE — run() returns MechanismResult with accounted, moves,
|
|
14
|
+
// and unexplained. The pipeline computes the weight.
|
|
15
|
+
|
|
16
|
+
import type { AncestorReach, MindContext, Recognition } from "./types.js";
|
|
17
|
+
import type { AttentionRead } from "./types.js";
|
|
18
|
+
import type { ComputedSpan } from "../extension.js";
|
|
19
|
+
import type { Vec } from "../vec.js";
|
|
20
|
+
import { windowIds } from "./canonical.js";
|
|
21
|
+
import { read, resolve } from "./primitives.js";
|
|
22
|
+
import { alignGraded, type GradedRun } from "./match.js";
|
|
23
|
+
import { climbAttentionAll } from "./attention.js";
|
|
24
|
+
import { skillExemplar } from "./mechanisms/extraction.js";
|
|
25
|
+
|
|
26
|
+
// ── Precomputed ──────────────────────────────────────────────────────────────
|
|
27
|
+
//
|
|
28
|
+
// Precomputed is a LAZY container for structural analyses of the query — the
|
|
29
|
+
// ONE place a response's shared evidence lives, for inter-mechanism exchange
|
|
30
|
+
// and for analyses future mechanisms will want. Eager fields (rec, computed,
|
|
31
|
+
// guide) are populated by the pipeline before the mechanism loop; everything
|
|
32
|
+
// expensive is a lazily-cached method that computes on first access. A
|
|
33
|
+
// mechanism that never asks for an analysis pays nothing for it; two
|
|
34
|
+
// mechanisms asking for the same analysis pay once.
|
|
35
|
+
//
|
|
36
|
+
// This design serves THREE purposes:
|
|
37
|
+
// 1. SHARING — when two mechanisms need the same analysis, it's computed once
|
|
38
|
+
// (even under trace, where the ctx-level memos are deliberately bypassed).
|
|
39
|
+
// 2. EXTENSIBILITY — a new analysis is one method in one file.
|
|
40
|
+
// 3. DECLARATIVE COST — a mechanism's floor() checks its cheap gates and the
|
|
41
|
+
// pipeline's `worthRunning` predicate BEFORE first-touching an expensive
|
|
42
|
+
// analysis, so lazy analyses are only ever computed for a mechanism that
|
|
43
|
+
// could still win.
|
|
44
|
+
|
|
45
|
+
export class Precomputed {
|
|
46
|
+
/** The response's evidence-breadth constant: how many ranked candidates the
|
|
47
|
+
* resonance probes, the weave alignment, and the climb all consider.
|
|
48
|
+
* Derived once from config; every consumer reads it here. */
|
|
49
|
+
readonly k: number;
|
|
50
|
+
|
|
51
|
+
constructor(
|
|
52
|
+
readonly ctx: MindContext,
|
|
53
|
+
readonly query: Uint8Array,
|
|
54
|
+
/** Recognition result (structural + canonical). */
|
|
55
|
+
readonly rec: Recognition,
|
|
56
|
+
/** Computed spans from mechanisms that implement `parse()` (e.g. ALU). */
|
|
57
|
+
readonly computed: ComputedSpan[],
|
|
58
|
+
/** The query's gist — the response-wide disambiguation guide. */
|
|
59
|
+
readonly guide: Vec,
|
|
60
|
+
) {
|
|
61
|
+
this.k = ctx.cfg.recallQueryK * 2;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ── Cheap lazy analyses ───────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
private _windows?: Map<number, number>;
|
|
67
|
+
/** Content-addressed W-window identities for every position in the query
|
|
68
|
+
* (offset → node id). O(|query|) probes. */
|
|
69
|
+
get queryWindows(): Map<number, number> {
|
|
70
|
+
return this._windows ??= windowIds(this.ctx, this.query);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private _resolved?: number | null;
|
|
74
|
+
/** The node id of the query itself, or null when it is not a stored form.
|
|
75
|
+
* O(|query|) probes. */
|
|
76
|
+
get queryResolved(): number | null {
|
|
77
|
+
if (this._resolved === undefined) {
|
|
78
|
+
this._resolved = resolve(this.ctx, this.query);
|
|
79
|
+
}
|
|
80
|
+
return this._resolved;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private _anchorWindows = new Map<number, Map<number, number>>();
|
|
84
|
+
/** Content-addressed W-window identities of one anchor's own bytes
|
|
85
|
+
* (offset → node id), memoised per anchor. Confluence intersects these;
|
|
86
|
+
* any future identity-based mechanism reads the same cache. */
|
|
87
|
+
windowsOf(anchor: number): Map<number, number> {
|
|
88
|
+
let w = this._anchorWindows.get(anchor);
|
|
89
|
+
if (w === undefined) {
|
|
90
|
+
w = windowIds(this.ctx, read(this.ctx, anchor));
|
|
91
|
+
this._anchorWindows.set(anchor, w);
|
|
92
|
+
}
|
|
93
|
+
return w;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Shared memo for {@link reachOf} (structural-IDF reads): a window's
|
|
97
|
+
* ancestor reach is a pure function of the read-only store, so one
|
|
98
|
+
* response-scoped memo serves every mechanism that prices commonality. */
|
|
99
|
+
readonly reachMemo = new Map<number, AncestorReach>();
|
|
100
|
+
|
|
101
|
+
// ── Expensive lazy analyses ───────────────────────────────────────────
|
|
102
|
+
//
|
|
103
|
+
// Async, cached-by-promise: the first caller starts the computation, every
|
|
104
|
+
// later caller (any mechanism, any phase) awaits the same promise. A
|
|
105
|
+
// mechanism MUST check its cheap floor gates and the pipeline's
|
|
106
|
+
// `worthRunning` predicate before first-touching one of these.
|
|
107
|
+
|
|
108
|
+
private _attention?: Promise<AttentionRead>;
|
|
109
|
+
/** The full consensus climb (roots + ranked anchors) — the query-level
|
|
110
|
+
* evidence CAST, confluence, extraction, recall's scaffolding tier, and
|
|
111
|
+
* fusion all share. Computed on first access; a query no mechanism
|
|
112
|
+
* climbs for (e.g. one an extension decided outright) never pays for it. */
|
|
113
|
+
attention(): Promise<AttentionRead> {
|
|
114
|
+
return this._attention ??= climbAttentionAll(
|
|
115
|
+
this.ctx,
|
|
116
|
+
this.query,
|
|
117
|
+
this.k,
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private _weave?: Promise<WeaveInfo>;
|
|
122
|
+
/** Result of {@link alignGraded} for the first k ranked anchors —
|
|
123
|
+
* O(k · |query| · |ctx|). Consumed by CAST; reusable by any future
|
|
124
|
+
* mechanism doing analogical transfer. */
|
|
125
|
+
weave(): Promise<WeaveInfo> {
|
|
126
|
+
return this._weave ??= this.attention().then((climb) =>
|
|
127
|
+
computeWeave(this.ctx, this.query, this, climb)
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Span-shaped classification of one ranked anchor, memoised per anchor id
|
|
132
|
+
* so repeated calls (extraction's own early-exit scan, any future
|
|
133
|
+
* template-based mechanism) never redo the work. Deliberately NOT an
|
|
134
|
+
* eager all-anchors map: `skillExemplar` is the expensive part of
|
|
135
|
+
* extraction (capped fan-out reads plus an O(|ctx|) scan), and most
|
|
136
|
+
* queries are answered by the FIRST ranked anchor that qualifies — paying
|
|
137
|
+
* for every ranked anchor regardless of where the scan stops would turn
|
|
138
|
+
* an early-exit lookup into full O(k) work on every query. */
|
|
139
|
+
private _spanShaped = new Map<number, Promise<SkillInfo | null>>();
|
|
140
|
+
spanShapedOf(anchor: number): Promise<SkillInfo | null> {
|
|
141
|
+
let p = this._spanShaped.get(anchor);
|
|
142
|
+
if (p === undefined) {
|
|
143
|
+
p = skillExemplar(this.ctx, anchor, this.guide);
|
|
144
|
+
this._spanShaped.set(anchor, p);
|
|
145
|
+
}
|
|
146
|
+
return p;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Every ranked anchor's classification at once, sharing the same
|
|
150
|
+
* per-anchor cache as {@link spanShapedOf} — for a mechanism that
|
|
151
|
+
* genuinely needs the full picture (not an early-exit scan). Mixing
|
|
152
|
+
* access patterns across mechanisms never duplicates work: whichever
|
|
153
|
+
* anchors an early-exit consumer already asked for are reused here, and
|
|
154
|
+
* whichever this computes first are reused by a later early-exit scan. */
|
|
155
|
+
async spanShapedAll(): Promise<Map<number, SkillInfo | null>> {
|
|
156
|
+
const { ranked } = await this.attention();
|
|
157
|
+
const out = new Map<number, SkillInfo | null>();
|
|
158
|
+
for (const cand of ranked) {
|
|
159
|
+
if (out.has(cand.anchor)) continue;
|
|
160
|
+
out.set(cand.anchor, await this.spanShapedOf(cand.anchor));
|
|
161
|
+
}
|
|
162
|
+
return out;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// ── WeaveInfo ────────────────────────────────────────────────────────────────
|
|
167
|
+
|
|
168
|
+
/** The weave-local structural alignment, computed once and consumed by CAST
|
|
169
|
+
* (and any future mechanism doing analogical transfer). */
|
|
170
|
+
export interface WeaveInfo {
|
|
171
|
+
/** Per-anchor alignment: context bytes, vote weight, and graded runs. */
|
|
172
|
+
points: Array<{
|
|
173
|
+
anchor: number;
|
|
174
|
+
vote: number;
|
|
175
|
+
ctx: Uint8Array;
|
|
176
|
+
runs: GradedRun[];
|
|
177
|
+
}>;
|
|
178
|
+
/** Weighted depth at each query byte — sum of alignment weights.
|
|
179
|
+
* `depth[i]` is the total evidence that byte i is shared among the
|
|
180
|
+
* aligned structures. */
|
|
181
|
+
depth: Float64Array;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function computeWeave(
|
|
185
|
+
ctx: MindContext,
|
|
186
|
+
query: Uint8Array,
|
|
187
|
+
pre: Precomputed,
|
|
188
|
+
climb: AttentionRead,
|
|
189
|
+
): WeaveInfo {
|
|
190
|
+
const quantum = ctx.space.maxGroup;
|
|
191
|
+
const { ranked } = climb;
|
|
192
|
+
const rankedCapped = ranked.length > pre.k ? ranked.slice(0, pre.k) : ranked;
|
|
193
|
+
const depth = new Float64Array(query.length);
|
|
194
|
+
const points: WeaveInfo["points"] = [];
|
|
195
|
+
|
|
196
|
+
// WEAVE-SCALE anchors only: CAST transfers structure between things the
|
|
197
|
+
// QUERY weaves together — query-scale structures. A context an order of
|
|
198
|
+
// magnitude beyond the query is not woven BY the query (the query can at
|
|
199
|
+
// most quote a fragment of it, and fragment-level evidence is exactly what
|
|
200
|
+
// recognition and the cover already handle); CAST's own comparison gate
|
|
201
|
+
// demands `ctx.length ≤ query.length` before it fires, and its
|
|
202
|
+
// substitution seats sit within a quantum of a context's start. W is the
|
|
203
|
+
// perceptual quantum — the same scale multiplier the bridge's phrase-scale
|
|
204
|
+
// contract uses. The prefix-capped read makes an oversized anchor cost a
|
|
205
|
+
// bounded read instead of reconstructing (and then canonically
|
|
206
|
+
// recognising) a corpus-sized deposit: profiled on a 17.7M-node store,
|
|
207
|
+
// uncapped weaves spent 5–8s per query recognising conversation-length
|
|
208
|
+
// anchors that could never form a weave point.
|
|
209
|
+
const capBytes = query.length * quantum;
|
|
210
|
+
for (const cand of rankedCapped) {
|
|
211
|
+
const ctxBytes = read(ctx, cand.anchor, capBytes + 1);
|
|
212
|
+
if (ctxBytes.length === 0 || ctxBytes.length > capBytes) continue;
|
|
213
|
+
const raw = alignGraded(ctx, query, ctxBytes, pre.rec.sites);
|
|
214
|
+
if (raw.length === 0) continue;
|
|
215
|
+
for (const r of raw) {
|
|
216
|
+
for (let i = r.qs; i < r.qe; i++) depth[i] += r.weight;
|
|
217
|
+
}
|
|
218
|
+
const free: GradedRun[] = [];
|
|
219
|
+
for (const r of raw) {
|
|
220
|
+
let { qs, qe, cs, weight } = r;
|
|
221
|
+
for (const p of points) {
|
|
222
|
+
for (const o of p.runs) {
|
|
223
|
+
if (qs >= qe) break;
|
|
224
|
+
if (o.qe <= qs || o.qs >= qe) continue;
|
|
225
|
+
const left = Math.max(0, o.qs - qs);
|
|
226
|
+
const right = Math.max(0, qe - o.qe);
|
|
227
|
+
if (left >= right) qe = qs + left;
|
|
228
|
+
else {
|
|
229
|
+
cs += qe - right - qs;
|
|
230
|
+
qs = qe - right;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
if (qe - qs >= Math.min(quantum, ctxBytes.length)) {
|
|
235
|
+
free.push({ qs, qe, cs, weight });
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (free.length > 0) {
|
|
239
|
+
points.push({
|
|
240
|
+
anchor: cand.anchor,
|
|
241
|
+
vote: cand.vote,
|
|
242
|
+
ctx: ctxBytes,
|
|
243
|
+
runs: free,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return { points, depth };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ── SkillInfo ────────────────────────────────────────────────────────────────
|
|
251
|
+
|
|
252
|
+
/** Span-shaped classification of one anchor — the structural information
|
|
253
|
+
* extraction uses to decide whether a learned fact can serve as a template
|
|
254
|
+
* for reading an analogous span out of the query. */
|
|
255
|
+
export interface SkillInfo {
|
|
256
|
+
contextBytes: Uint8Array;
|
|
257
|
+
answerBytes: Uint8Array;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// ── MechanismResult ──────────────────────────────────────────────────────────
|
|
261
|
+
|
|
262
|
+
/** Raw result from a mechanism's `run()`. The pipeline computes the weight
|
|
263
|
+
* from `moves` + `PASS * unaccounted(accounted)` — the mechanism does not
|
|
264
|
+
* know about the cost ladder.
|
|
265
|
+
*
|
|
266
|
+
* When `weight` is present, the pipeline uses it directly instead of
|
|
267
|
+
* computing `weigh(accounted, moves)`. This is for mechanisms whose cost
|
|
268
|
+
* is derived externally (e.g. cover: the A*LD derivation's g-value). */
|
|
269
|
+
export interface MechanismResult {
|
|
270
|
+
bytes: Uint8Array;
|
|
271
|
+
accounted: Array<[number, number]>;
|
|
272
|
+
moves: number;
|
|
273
|
+
used?: ReadonlySet<number>;
|
|
274
|
+
unexplained: string;
|
|
275
|
+
/** Explicit weight override. When absent, weight = moves + PASS·unaccounted. */
|
|
276
|
+
weight?: number;
|
|
277
|
+
/** Override the mechanism's default provenance for this result.
|
|
278
|
+
* When absent, the pipeline uses `mech.provenance`. */
|
|
279
|
+
provenance?: string;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ── PipelineMechanism ────────────────────────────────────────────────────────
|
|
283
|
+
|
|
284
|
+
export interface PipelineMechanism {
|
|
285
|
+
/** Stable identifier for trace/debug. */
|
|
286
|
+
readonly name: string;
|
|
287
|
+
|
|
288
|
+
/** Which provenance tag the pipeline attaches to this mechanism's answers. */
|
|
289
|
+
readonly provenance: string;
|
|
290
|
+
|
|
291
|
+
/** Parse authoritative spans BEFORE the grounding loop.
|
|
292
|
+
* Only needed by computational mechanisms (e.g. ALU). Results from ALL
|
|
293
|
+
* mechanisms that implement this are collected into `Precomputed.computed`
|
|
294
|
+
* before any `floor()` or `run()` is called. */
|
|
295
|
+
parse?(query: Uint8Array): Promise<ComputedSpan[]>;
|
|
296
|
+
|
|
297
|
+
/** Admissible lower bound on this mechanism's weight.
|
|
298
|
+
* Returns `null` when the mechanism structurally cannot fire.
|
|
299
|
+
*
|
|
300
|
+
* `worthRunning(cheapFloor)` reports whether the CURRENT incumbent
|
|
301
|
+
* (established by mechanisms that already ran this response, cover being
|
|
302
|
+
* first — see `defaultMechanisms`) could still be beaten by a floor no
|
|
303
|
+
* tighter than `cheapFloor`. THE INVESTMENT DISCIPLINE: before
|
|
304
|
+
* first-touching an expensive shared analysis (`pre.attention()`,
|
|
305
|
+
* `pre.weave()`, …), check `worthRunning(bound)` with this mechanism's
|
|
306
|
+
* cheapest possible bound — and when it fails, RETURN THE BOUND rather
|
|
307
|
+
* than null. The bound is still admissible (it never overstates cost),
|
|
308
|
+
* the pipeline's own check then prunes `run()` and records the truthful
|
|
309
|
+
* "cannot beat incumbent" trace note, and no analysis was computed just
|
|
310
|
+
* to be discarded. This is uniform: no mechanism asks what produced the
|
|
311
|
+
* incumbent — a computed extension result and an ordinary cheap cover
|
|
312
|
+
* prune the same way. */
|
|
313
|
+
floor(
|
|
314
|
+
ctx: MindContext,
|
|
315
|
+
query: Uint8Array,
|
|
316
|
+
pre: Precomputed,
|
|
317
|
+
worthRunning: (floor: number) => boolean,
|
|
318
|
+
): Promise<number | null>;
|
|
319
|
+
|
|
320
|
+
/** Produce candidate answers. */
|
|
321
|
+
run(
|
|
322
|
+
ctx: MindContext,
|
|
323
|
+
query: Uint8Array,
|
|
324
|
+
pre: Precomputed,
|
|
325
|
+
): Promise<MechanismResult[]>;
|
|
326
|
+
}
|
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
// pipeline.ts — the think pipeline (Section 5 of the mind).
|
|
2
|
+
//
|
|
3
|
+
// think() is the whole file's job: one lightest-derivation choice among
|
|
4
|
+
// UNIFORM mechanisms. The pipeline sees mechanisms through the
|
|
5
|
+
// PipelineMechanism interface only — it never imports a mechanism-specific
|
|
6
|
+
// type and never has a special-case branch for any mechanism. Adding a
|
|
7
|
+
// mechanism means registering one object; removing one means dropping it
|
|
8
|
+
// from the list. The mechanisms themselves live in mechanisms/ (one file
|
|
9
|
+
// each); the shared pre-computation they exchange lives in Precomputed
|
|
10
|
+
// (pipeline-mechanism.ts).
|
|
11
|
+
|
|
12
|
+
import type { MindContext } from "./types.js";
|
|
13
|
+
import { PASS, STEP } from "./graph-search.js";
|
|
14
|
+
import type { ComputedSpan } from "../extension.js";
|
|
15
|
+
import { gistOf, resolve } from "./primitives.js";
|
|
16
|
+
import { recognise } from "./recognition.js";
|
|
17
|
+
import { fuseAttention, reason } from "./reasoning.js";
|
|
18
|
+
import { unexplainedSpans } from "./rationale.js";
|
|
19
|
+
import { rItem } from "./trace.js";
|
|
20
|
+
import { type PipelineMechanism, Precomputed } from "./pipeline-mechanism.js";
|
|
21
|
+
import { coverMechanism } from "./mechanisms/cover.js";
|
|
22
|
+
import { castMechanism } from "./mechanisms/cast.js";
|
|
23
|
+
import { confluenceMechanism } from "./mechanisms/confluence.js";
|
|
24
|
+
import { extractionMechanism } from "./mechanisms/extraction.js";
|
|
25
|
+
import { recallMechanism } from "./mechanisms/recall.js";
|
|
26
|
+
|
|
27
|
+
// Re-exports: cover's pre-resolution helpers and the ALU adapter kept
|
|
28
|
+
// importable from the pipeline module (their historical home).
|
|
29
|
+
export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
|
|
30
|
+
export { aluToMechanism } from "./mechanisms/alu.js";
|
|
31
|
+
|
|
32
|
+
// ── Extension dispatch (pre-loop parse) ─────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
async function collectComputed(
|
|
35
|
+
mechanisms: readonly PipelineMechanism[],
|
|
36
|
+
query: Uint8Array,
|
|
37
|
+
): Promise<ComputedSpan[]> {
|
|
38
|
+
const out: ComputedSpan[] = [];
|
|
39
|
+
for (const m of mechanisms) {
|
|
40
|
+
if (m.parse) out.push(...await m.parse(query));
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ── Built-in mechanisms ─────────────────────────────────────────────────────
|
|
46
|
+
|
|
47
|
+
// ORDER MATTERS, but only through the uniform floor/worthRunning pruning —
|
|
48
|
+
// no mechanism is special-cased. Cover runs FIRST: when a computed
|
|
49
|
+
// extension result (e.g. ALU) exists, cover masks it in at near-zero cost
|
|
50
|
+
// (see mechanisms/cover.ts), which becomes `best` before any other mechanism
|
|
51
|
+
// invests in its own precomputation. CAST's and confluence's floors (2*STEP,
|
|
52
|
+
// 3*STEP) then fail `worthRunning` and are skipped by the SAME admissible-
|
|
53
|
+
// floor pruning every mechanism is already subject to — not by asking
|
|
54
|
+
// "is this an extension?". Grade TIES keep the earlier candidate, so this
|
|
55
|
+
// order is also the tie-break priority: cover, cast, confluence, extraction,
|
|
56
|
+
// recall.
|
|
57
|
+
export const defaultMechanisms: PipelineMechanism[] = [
|
|
58
|
+
coverMechanism,
|
|
59
|
+
castMechanism,
|
|
60
|
+
confluenceMechanism,
|
|
61
|
+
extractionMechanism,
|
|
62
|
+
recallMechanism,
|
|
63
|
+
];
|
|
64
|
+
|
|
65
|
+
// ── think — the main inference pipeline ─────────────────────────────────────
|
|
66
|
+
|
|
67
|
+
export type Provenance =
|
|
68
|
+
| "cast"
|
|
69
|
+
| "join"
|
|
70
|
+
| "cover"
|
|
71
|
+
| "extract"
|
|
72
|
+
| "recall"
|
|
73
|
+
| "recall-echo";
|
|
74
|
+
|
|
75
|
+
export interface Thought {
|
|
76
|
+
bytes: Uint8Array;
|
|
77
|
+
provenance: Provenance;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Think: a single lightest-derivation exploration of the Sema graph.
|
|
81
|
+
*
|
|
82
|
+
* Every answer travels the same path:
|
|
83
|
+
* 1. Pre-computation — recognise, extension parse, guide; everything
|
|
84
|
+
* expensive stays lazy on Precomputed until a mechanism asks.
|
|
85
|
+
* 2. Grounding — every mechanism yields candidates weighed in the one
|
|
86
|
+
* cost ladder; the lightest grounding derivation wins.
|
|
87
|
+
* 3. Post-grounding — diagnostics (narrowDecision, thinGrounding),
|
|
88
|
+
* reasoning (multi-hop), fusion (multi-topic). */
|
|
89
|
+
export async function think(
|
|
90
|
+
ctx: MindContext,
|
|
91
|
+
query: Uint8Array,
|
|
92
|
+
mechs?: readonly PipelineMechanism[],
|
|
93
|
+
): Promise<Thought | null> {
|
|
94
|
+
if (query.length === 0) return null;
|
|
95
|
+
|
|
96
|
+
ctx._edgeGuide = gistOf(ctx, query);
|
|
97
|
+
ctx._edgeChoice.clear();
|
|
98
|
+
|
|
99
|
+
const t = ctx.trace?.enter("think", [rItem(query, "query")]);
|
|
100
|
+
const done = (answer: Uint8Array | null, note: string) => {
|
|
101
|
+
t?.done(
|
|
102
|
+
answer
|
|
103
|
+
? [rItem(answer, "answer", resolve(ctx, answer) ?? undefined)]
|
|
104
|
+
: [],
|
|
105
|
+
note,
|
|
106
|
+
);
|
|
107
|
+
return answer;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
// ── Pre-computation ──────────────────────────────────────────────────
|
|
111
|
+
const mechanisms = mechs ?? defaultMechanisms;
|
|
112
|
+
const rec = recognise(ctx, query);
|
|
113
|
+
|
|
114
|
+
// Phase 1: collect computed spans from mechanisms that implement parse()
|
|
115
|
+
const computed = await collectComputed(mechanisms, query);
|
|
116
|
+
|
|
117
|
+
if (computed.length > 0) {
|
|
118
|
+
ctx.trace?.step(
|
|
119
|
+
"computeExtensions",
|
|
120
|
+
[rItem(query, "query")],
|
|
121
|
+
computed.map((u) =>
|
|
122
|
+
rItem(query.subarray(u.i, u.j), "operand", undefined, [u.i, u.j])
|
|
123
|
+
),
|
|
124
|
+
`extensions recognised and evaluated ${computed.length} computation(s)`,
|
|
125
|
+
);
|
|
126
|
+
for (const u of computed) {
|
|
127
|
+
ctx.trace?.step(
|
|
128
|
+
"evalComputation",
|
|
129
|
+
[rItem(query.subarray(u.i, u.j), "expression", undefined, [u.i, u.j])],
|
|
130
|
+
[rItem(u.bytes, "result", resolve(ctx, u.bytes) ?? undefined)],
|
|
131
|
+
"evaluate the recognised operation to its authoritative result",
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Phase 2: the shared pre-computation container. Eager fields only
|
|
137
|
+
// (recognition, computed spans, guide) — every expensive analysis
|
|
138
|
+
// (consensus climb, weave, span-shape classification) is a lazily-cached
|
|
139
|
+
// method on Precomputed, first-touched by whichever mechanism's floor
|
|
140
|
+
// survives its cheap gates and the worthRunning check. A query no
|
|
141
|
+
// mechanism climbs for (e.g. one an extension decided) never climbs.
|
|
142
|
+
const pre = new Precomputed(ctx, query, rec, computed, ctx._edgeGuide);
|
|
143
|
+
|
|
144
|
+
// ── Grounding: ONE lightest-derivation choice among the mechanisms ────
|
|
145
|
+
|
|
146
|
+
interface Candidate {
|
|
147
|
+
bytes: Uint8Array;
|
|
148
|
+
provenance: string;
|
|
149
|
+
weight: number;
|
|
150
|
+
used?: ReadonlySet<number>;
|
|
151
|
+
accounted: ReadonlyArray<[number, number]>;
|
|
152
|
+
unexplained: string;
|
|
153
|
+
}
|
|
154
|
+
const grade = (w: number) => Math.floor(w / STEP);
|
|
155
|
+
const unaccounted = (spans: ReadonlyArray<[number, number]>): number =>
|
|
156
|
+
unexplainedSpans(query.length, spans)
|
|
157
|
+
.reduce((sum, [s, e]) => sum + (e - s), 0);
|
|
158
|
+
const weigh = (
|
|
159
|
+
accounted: ReadonlyArray<[number, number]>,
|
|
160
|
+
moves: number,
|
|
161
|
+
): number => moves + PASS * unaccounted(accounted);
|
|
162
|
+
|
|
163
|
+
const candidates: Candidate[] = [];
|
|
164
|
+
let best: Candidate | null = null;
|
|
165
|
+
const consider = (c: Candidate) => {
|
|
166
|
+
if (c.bytes.length === 0) return;
|
|
167
|
+
candidates.push(c);
|
|
168
|
+
if (best === null || grade(c.weight) < grade(best.weight)) best = c;
|
|
169
|
+
};
|
|
170
|
+
const worthRunning = (floor: number) =>
|
|
171
|
+
best === null || grade(floor) < grade(best.weight);
|
|
172
|
+
|
|
173
|
+
// Phase 3: grounding loop
|
|
174
|
+
for (const mech of mechanisms) {
|
|
175
|
+
const floor = await mech.floor(ctx, query, pre, worthRunning);
|
|
176
|
+
if (floor === null) {
|
|
177
|
+
ctx.trace?.step(
|
|
178
|
+
"skipMechanism",
|
|
179
|
+
[],
|
|
180
|
+
[],
|
|
181
|
+
`${mech.name} skipped — structural precondition failed`,
|
|
182
|
+
);
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (!worthRunning(floor)) {
|
|
186
|
+
ctx.trace?.step(
|
|
187
|
+
"skipMechanism",
|
|
188
|
+
[],
|
|
189
|
+
[],
|
|
190
|
+
`${mech.name} skipped — floor ${floor} cannot beat incumbent (grade ${
|
|
191
|
+
grade(best!.weight)
|
|
192
|
+
})`,
|
|
193
|
+
);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const results = await mech.run(ctx, query, pre);
|
|
197
|
+
for (const r of results) {
|
|
198
|
+
const weight = r.weight ?? weigh(r.accounted, r.moves);
|
|
199
|
+
consider({
|
|
200
|
+
bytes: r.bytes,
|
|
201
|
+
provenance: r.provenance ?? mech.provenance,
|
|
202
|
+
weight,
|
|
203
|
+
used: r.used,
|
|
204
|
+
accounted: r.accounted,
|
|
205
|
+
unexplained: r.unexplained,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// (TS cannot see the closure assignments into `best` and narrows it to its
|
|
211
|
+
// initial null, so the read-back needs the assertion.)
|
|
212
|
+
const decided = best as Candidate | null;
|
|
213
|
+
if (candidates.length > 1) {
|
|
214
|
+
ctx.trace?.step(
|
|
215
|
+
"decideGrounding",
|
|
216
|
+
candidates.map((c) =>
|
|
217
|
+
rItem(
|
|
218
|
+
c.bytes,
|
|
219
|
+
`${c.provenance} (weight ${c.weight.toFixed(3)}${
|
|
220
|
+
c.unexplained ? `, unexplained: "${c.unexplained}"` : ""
|
|
221
|
+
})`,
|
|
222
|
+
)
|
|
223
|
+
),
|
|
224
|
+
decided ? [rItem(decided.bytes, decided.provenance)] : [],
|
|
225
|
+
"the lightest grounding derivation wins — every mechanism weighed in the one cost ladder",
|
|
226
|
+
);
|
|
227
|
+
if (decided !== null) {
|
|
228
|
+
let runnerUp: Candidate | null = null;
|
|
229
|
+
for (const c of candidates) {
|
|
230
|
+
if (c === decided) continue;
|
|
231
|
+
if (runnerUp === null || grade(c.weight) < grade(runnerUp.weight)) {
|
|
232
|
+
runnerUp = c;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (runnerUp !== null) {
|
|
236
|
+
const margin = grade(runnerUp.weight) - grade(decided.weight);
|
|
237
|
+
if (margin <= 1) {
|
|
238
|
+
ctx.trace?.step(
|
|
239
|
+
"narrowDecision",
|
|
240
|
+
[
|
|
241
|
+
rItem(
|
|
242
|
+
decided.bytes,
|
|
243
|
+
`${decided.provenance} (weight ${decided.weight.toFixed(3)})`,
|
|
244
|
+
),
|
|
245
|
+
],
|
|
246
|
+
[
|
|
247
|
+
rItem(
|
|
248
|
+
runnerUp.bytes,
|
|
249
|
+
`${runnerUp.provenance} (weight ${runnerUp.weight.toFixed(3)})`,
|
|
250
|
+
),
|
|
251
|
+
],
|
|
252
|
+
`margin ${margin} grade-unit(s) — the decision could change with one more training fact`,
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (decided === null) {
|
|
260
|
+
done(null, "no mechanism grounded an answer");
|
|
261
|
+
return null;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Honesty density
|
|
265
|
+
{
|
|
266
|
+
const covered = query.length - unaccounted(decided.accounted);
|
|
267
|
+
const density = query.length > 0 ? covered / query.length : 1;
|
|
268
|
+
const thinBar = 1 / ctx.space.maxGroup;
|
|
269
|
+
if (density < thinBar) {
|
|
270
|
+
ctx.trace?.step(
|
|
271
|
+
"thinGrounding",
|
|
272
|
+
[rItem(decided.bytes, decided.provenance)],
|
|
273
|
+
[],
|
|
274
|
+
`grounded but thin — density ${density.toFixed(3)} is below 1/W (${
|
|
275
|
+
thinBar.toFixed(3)
|
|
276
|
+
})`,
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
const answer: Uint8Array = decided.bytes;
|
|
281
|
+
const provenance = decided.provenance as Provenance;
|
|
282
|
+
const castUsed: ReadonlySet<number> = decided.used ?? new Set();
|
|
283
|
+
|
|
284
|
+
// ── Post-grounding, gated by provenance ──────────────────────────────
|
|
285
|
+
const preConsumed = provenance === "cast" || provenance === "join"
|
|
286
|
+
? castUsed
|
|
287
|
+
: provenance === "recall" || provenance === "recall-echo"
|
|
288
|
+
? new Set<number>()
|
|
289
|
+
: new Set(recognise(ctx, answer).sites.map((s) => s.payload));
|
|
290
|
+
const reasoned = await reason(ctx, query, answer, preConsumed, pre);
|
|
291
|
+
const fused = provenance === "recall" || provenance === "recall-echo"
|
|
292
|
+
? await fuseAttention(ctx, query, reasoned, pre)
|
|
293
|
+
: reasoned;
|
|
294
|
+
|
|
295
|
+
done(
|
|
296
|
+
fused,
|
|
297
|
+
"grounded, reasoned forward, fused across points of attention",
|
|
298
|
+
);
|
|
299
|
+
return { bytes: fused, provenance };
|
|
300
|
+
}
|