@hviana/sema 0.4.2 → 0.4.4

package/dist/src/mind/pipeline.d.ts

@@ -0,0 +1,49 @@
+import type { MindContext } from "./types.js";
+import { type PipelineMechanism } from "./pipeline-mechanism.js";
+export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
+export { aluToMechanism } from "./mechanisms/alu.js";
+export declare const defaultMechanisms: PipelineMechanism[];
+export type Provenance = "cast" | "join" | "cover" | "extract" | "recall" | "recall-echo";
+export interface Thought {
+    bytes: Uint8Array;
+    provenance: Provenance;
+}
+/** Structured payload of the "decideGrounding" rationale step — the same
+ *  numbers the human-readable candidate labels already carry, exposed as
+ *  data so a downstream tool need not parse free text.  Purely additive
+ *  instrumentation: built only under `ctx.trace?.` (optional chaining
+ *  short-circuits its arguments), never read by inference. */
+export interface DecideGroundingData {
+    version: 1;
+    /** Every grounding candidate weighed, in consideration order. */
+    candidates: Array<{
+        provenance: string;
+        /** The candidate's exact weight in the one cost ladder. */
+        weight: number;
+        /** The DISCRETE grade the decision actually compares (floor(weight/STEP)). */
+        grade: number;
+        /** Query bytes the candidate's accounted spans leave unexplained. */
+        unexplainedBytes: number;
+        /** Whether this candidate won the decision. */
+        decided: boolean;
+    }>;
+    /** Grade margin between the winner and the runner-up, when both exist —
+     *  the same quantity the "narrowDecision" step reports as narrow when
+     *  ≤ 1.  Absent for a single-candidate decision. */
+    runnerUpMargin?: number;
+}
+/** Structured payload of the "narrowDecision" rationale step. */
+export interface NarrowDecisionData {
+    version: 1;
+    margin: number;
+}
+/** Think: a single lightest-derivation exploration of the Sema graph.
+ *
+ *  Every answer travels the same path:
+ *    1. Pre-computation — recognise, extension parse, guide; everything
+ *       expensive stays lazy on Precomputed until a mechanism asks.
+ *    2. Grounding — every mechanism yields candidates weighed in the one
+ *       cost ladder; the lightest grounding derivation wins.
+ *    3. Post-grounding — diagnostics (narrowDecision, thinGrounding),
+ *       reasoning (multi-hop), fusion (multi-topic). */
+export declare function think(ctx: MindContext, query: Uint8Array, mechs?: readonly PipelineMechanism[]): Promise<Thought | null>;

package/dist/src/mind/pipeline.js

@@ -0,0 +1,275 @@
+// pipeline.ts — the think pipeline (Section 5 of the mind).
+//
+// think() is the whole file's job: one lightest-derivation choice among
+// UNIFORM mechanisms.  The pipeline sees mechanisms through the
+// PipelineMechanism interface only — it never imports a mechanism-specific
+// type and never has a special-case branch for any mechanism.  Adding a
+// mechanism means registering one object; removing one means dropping it
+// from the list.  The mechanisms themselves live in mechanisms/ (one file
+// each); the shared pre-computation they exchange lives in Precomputed
+// (pipeline-mechanism.ts).
+import { PASS, STEP } from "./graph-search.js";
+import { gistOf, resolve } from "./primitives.js";
+import { recognise } from "./recognition.js";
+import { fuseAttention, reason } from "./reasoning.js";
+import { unexplainedSpans } from "./rationale.js";
+import { rItem } from "./trace.js";
+import { Precomputed } from "./pipeline-mechanism.js";
+import { coverMechanism } from "./mechanisms/cover.js";
+import { castMechanism } from "./mechanisms/cast.js";
+import { confluenceMechanism } from "./mechanisms/confluence.js";
+import { extractionMechanism } from "./mechanisms/extraction.js";
+import { recallMechanism } from "./mechanisms/recall.js";
+// Re-exports: cover's pre-resolution helpers and the ALU adapter kept
+// importable from the pipeline module (their historical home).
+export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
+export { aluToMechanism } from "./mechanisms/alu.js";
+// ── Extension dispatch (pre-loop parse) ─────────────────────────────────────
+async function collectComputed(ctx, mechanisms, query) {
+    const out = [];
+    const meter = ctx.meter;
+    for (const m of mechanisms) {
+        if (!m.parse)
+            continue;
+        const spans = meter
+            ? await meter.time(`${m.name}.parse`, () => m.parse(query))
+            : await m.parse(query);
+        out.push(...spans);
+    }
+    return out;
+}
+// ── Built-in mechanisms ─────────────────────────────────────────────────────
+// ORDER MATTERS, but only through the uniform floor/worthRunning pruning —
+// no mechanism is special-cased.  Cover runs FIRST: when a computed
+// extension result (e.g. ALU) exists, cover masks it in at near-zero cost
+// (see mechanisms/cover.ts), which becomes `best` before any other mechanism
+// invests in its own precomputation.  CAST's and confluence's floors (2*STEP,
+// 3*STEP) then fail `worthRunning` and are skipped by the SAME admissible-
+// floor pruning every mechanism is already subject to — not by asking
+// "is this an extension?". Grade TIES keep the earlier candidate, so this
+// order is also the tie-break priority: cover, cast, confluence, extraction,
+// recall.
+export const defaultMechanisms = [
+    coverMechanism,
+    castMechanism,
+    confluenceMechanism,
+    extractionMechanism,
+    recallMechanism,
+];
+/** Think: a single lightest-derivation exploration of the Sema graph.
+ *
+ *  Every answer travels the same path:
+ *    1. Pre-computation — recognise, extension parse, guide; everything
+ *       expensive stays lazy on Precomputed until a mechanism asks.
+ *    2. Grounding — every mechanism yields candidates weighed in the one
+ *       cost ladder; the lightest grounding derivation wins.
+ *    3. Post-grounding — diagnostics (narrowDecision, thinGrounding),
+ *       reasoning (multi-hop), fusion (multi-topic). */
+export async function think(ctx, query, mechs) {
+    if (query.length === 0)
+        return null;
+    ctx._edgeGuide = gistOf(ctx, query);
+    ctx._edgeChoice.clear();
+    const t = ctx.trace?.enter("think", [rItem(query, "query")]);
+    const done = (answer, note) => {
+        t?.done(answer
+            ? [rItem(answer, "answer", resolve(ctx, answer) ?? undefined)]
+            : [], note);
+        return answer;
+    };
+    // ── Pre-computation ──────────────────────────────────────────────────
+    const mechanisms = mechs ?? defaultMechanisms;
+    const rec = recognise(ctx, query);
+    // Phase 1: collect computed spans from mechanisms that implement parse()
+    const computed = await collectComputed(ctx, mechanisms, query);
+    if (computed.length > 0) {
+        ctx.trace?.step("computeExtensions", [rItem(query, "query")], computed.map((u) => rItem(query.subarray(u.i, u.j), "operand", undefined, [u.i, u.j])), `extensions recognised and evaluated ${computed.length} computation(s)`);
+        for (const u of computed) {
+            ctx.trace?.step("evalComputation", [rItem(query.subarray(u.i, u.j), "expression", undefined, [u.i, u.j])], [rItem(u.bytes, "result", resolve(ctx, u.bytes) ?? undefined)], "evaluate the recognised operation to its authoritative result");
+        }
+    }
+    // Phase 2: the shared pre-computation container.  Eager fields only
+    // (recognition, computed spans, guide) — every expensive analysis
+    // (consensus climb, weave, span-shape classification) is a lazily-cached
+    // method on Precomputed, first-touched by whichever mechanism's floor
+    // survives its cheap gates and the worthRunning check.  A query no
+    // mechanism climbs for (e.g. one an extension decided) never climbs.
+    const pre = new Precomputed(ctx, query, rec, computed, ctx._edgeGuide);
+    const grade = (w) => Math.floor(w / STEP);
+    const unaccounted = (spans) => unexplainedSpans(query.length, spans)
+        .reduce((sum, [s, e]) => sum + (e - s), 0);
+    const weigh = (accounted, moves) => moves + PASS * unaccounted(accounted);
+    const candidates = [];
+    let best = null;
+    const consider = (c) => {
+        if (c.bytes.length === 0)
+            return;
+        if (ctx.meter)
+            ctx.meter.candidates++;
+        candidates.push(c);
+        if (best === null || grade(c.weight) < grade(best.weight))
+            best = c;
+    };
+    const worthRunning = (floor) => best === null || grade(floor) < grade(best.weight);
+    // Phase 3: grounding loop
+    // Per-mechanism accounting (src/meter.ts).  The market's whole premise is
+    // that mechanisms compete on one cost scale — so the profiling read-out is
+    // also per-mechanism, uniformly: the loop never asks which one it holds.
+    const meter = ctx.meter;
+    for (const mech of mechanisms) {
+        const floor = meter
+            ? await meter.time(`${mech.name}.floor`, () => mech.floor(ctx, query, pre, worthRunning))
+            : await mech.floor(ctx, query, pre, worthRunning);
+        if (meter) {
+            if (floor === null)
+                meter.mechanismSkips++;
+            else
+                meter.mechanismFloors++;
+        }
+        if (floor === null) {
+            ctx.trace?.step("skipMechanism", [], [], `${mech.name} skipped — structural precondition failed`);
+            continue;
+        }
+        if (!worthRunning(floor)) {
+            ctx.trace?.step("skipMechanism", [], [], `${mech.name} skipped — floor ${floor} cannot beat incumbent (grade ${grade(best.weight)})`);
+            continue;
+        }
+        if (meter)
+            meter.mechanismRuns++;
+        const results = meter
+            ? await meter.time(`${mech.name}.run`, () => mech.run(ctx, query, pre))
+            : await mech.run(ctx, query, pre);
+        for (const r of results) {
+            const weight = r.weight ?? weigh(r.accounted, r.moves);
+            consider({
+                bytes: r.bytes,
+                provenance: r.provenance ?? mech.provenance,
+                weight,
+                used: r.used,
+                accounted: r.accounted,
+                unexplained: r.unexplained,
+                complete: r.complete,
+            });
+        }
+    }
+    // (TS cannot see the closure assignments into `best` and narrows it to its
+    // initial null, so the read-back needs the assertion.)
+    const decided = best;
+    if (candidates.length > 1) {
+        // The runner-up is computed BEFORE the decideGrounding step so its grade
+        // margin can ride along in the step's structured data payload; the
+        // computation itself is pure and was always unconditional — only its
+        // position moved.
+        let runnerUp = null;
+        if (decided !== null) {
+            for (const c of candidates) {
+                if (c === decided)
+                    continue;
+                if (runnerUp === null || grade(c.weight) < grade(runnerUp.weight)) {
+                    runnerUp = c;
+                }
+            }
+        }
+        const margin = decided !== null && runnerUp !== null
+            ? grade(runnerUp.weight) - grade(decided.weight)
+            : null;
+        ctx.trace?.step("decideGrounding", candidates.map((c) => rItem(c.bytes, `${c.provenance} (weight ${c.weight.toFixed(3)}${c.unexplained ? `, unexplained: "${c.unexplained}"` : ""})`)), decided ? [rItem(decided.bytes, decided.provenance)] : [], "the lightest grounding derivation wins — every mechanism weighed in the one cost ladder", undefined, {
+            version: 1,
+            candidates: candidates.map((c) => ({
+                provenance: c.provenance,
+                weight: c.weight,
+                grade: grade(c.weight),
+                unexplainedBytes: unaccounted(c.accounted),
+                decided: c === decided,
+            })),
+            ...(margin !== null ? { runnerUpMargin: margin } : {}),
+        });
+        if (decided !== null && runnerUp !== null && margin !== null) {
+            if (margin <= 1) {
+                ctx.trace?.step("narrowDecision", [
+                    rItem(decided.bytes, `${decided.provenance} (weight ${decided.weight.toFixed(3)})`),
+                ], [
+                    rItem(runnerUp.bytes, `${runnerUp.provenance} (weight ${runnerUp.weight.toFixed(3)})`),
+                ], `margin ${margin} grade-unit(s) — the decision could change with one more training fact`, undefined, { version: 1, margin });
+            }
+        }
+    }
+    if (decided === null) {
+        done(null, "no mechanism grounded an answer");
+        return null;
+    }
+    // Honesty density
+    {
+        const covered = query.length - unaccounted(decided.accounted);
+        const density = query.length > 0 ? covered / query.length : 1;
+        const thinBar = 1 / ctx.space.maxGroup;
+        if (density < thinBar) {
+            ctx.trace?.step("thinGrounding", [rItem(decided.bytes, decided.provenance)], [], `grounded but thin — density ${density.toFixed(3)} is below 1/W (${thinBar.toFixed(3)})`);
+        }
+    }
+    const answer = decided.bytes;
+    const provenance = decided.provenance;
+    const castUsed = decided.used ?? new Set();
+    // ── Post-grounding, gated by provenance ──────────────────────────────
+    const preConsumed = provenance === "cast" || provenance === "join"
+        ? castUsed
+        : provenance === "recall" || provenance === "recall-echo"
+            ? new Set()
+            : new Set(recognise(ctx, answer).sites.map((s) => s.payload));
+    // A grounding that DECLARED itself complete is not extended: the answer is
+    // already a trained form's own continuation, reached through an identity
+    // claim about the query, so a multi-hop pivot could only chain past the
+    // fact that produced it (see MechanismResult.complete).
+    const reasoned = decided.complete ? answer : meter
+        ? await meter.time("reason", () => reason(ctx, query, answer, preConsumed, pre))
+        : await reason(ctx, query, answer, preConsumed, pre);
+    // Fuse only when the query has a genuine REMAINDER no mechanism's
+    // structural evidence touched at all.  `decided.accounted` alone
+    // undercounts this: it is a COST-LADDER quantity (cover.ts prices its
+    // masked/computed spans at near-zero and deliberately leaves them out of
+    // `accounted` so PASS-bridged bytes are still charged), not a coverage
+    // one — a query fully explained by one computed span plus bridged
+    // connectors can report `accounted: []` while nothing is actually left
+    // unexplained.  The genuine remainder is what NEITHER the winning
+    // candidate's accounted spans NOR any recognised extension's computed
+    // span (`pre.computed` — every mechanism's parse() output, ALU included)
+    // ever touched.  A remainder under one river-fold quantum (W, the same
+    // floor cover.ts's restatedSpan and the honesty-density bar above both
+    // use) is bridging punctuation/whitespace, never a second topic —
+    // observed: a single space between two fully-computed arithmetic spans
+    // ("2+2 3+3") registered as "unaccounted" and pulled in an unrelated
+    // corpus fact, corrupting "4 6" into "4 63".
+    const explained = [
+        ...decided.accounted,
+        ...pre.computed.map((u) => [u.i, u.j]),
+    ];
+    const remainder = unaccounted(explained);
+    // Whether the winning candidate's entire recognised substance is
+    // COMPUTED — every accounted span exactly a pre.computed span, nothing
+    // from a genuinely recognised/climbed site.  fuseAttention's lone-root
+    // shortcut assumes a single point of attention already IS primary's own
+    // source; that assumption is exactly backwards for a pure computation
+    // (an ALU result has no anchor of its own) — see fuseAttention's
+    // `unclimbed` parameter, gated there by Attention.breadth so a
+    // coincidental echo (which this flag alone cannot distinguish) is still
+    // rejected.
+    const unclimbed = decided.accounted.length > 0 &&
+        decided.accounted.every(([i, j]) => pre.computed.some((u) => u.i === i && u.j === j));
+    // Where the winning grounding stands in the query — fusion places primary
+    // by it (see fuseAttention's `primarySpans`).  `accounted` is the
+    // cost-ladder read and is authoritative when non-empty; when it is empty
+    // the grounding is a pure COMPUTATION, whose evidence is its computed span.
+    // Exactly the cost-ladder-vs-coverage distinction `explained` above draws,
+    // read here for POSITION instead of for coverage — and resolved here, where
+    // both readings are in hand, rather than inside fuseAttention.
+    const primarySpans = decided.accounted.length > 0
+        ? decided.accounted
+        : pre.computed.map((u) => [u.i, u.j]);
+    const fused = remainder < ctx.space.maxGroup
+        ? reasoned
+        : meter
+            ? await meter.time("fuse", () => fuseAttention(ctx, query, reasoned, pre, unclimbed, primarySpans))
+            : await fuseAttention(ctx, query, reasoned, pre, unclimbed, decided.accounted);
+    done(fused, "grounded, reasoned forward, fused across points of attention");
+    return { bytes: fused, provenance };
+}

package/dist/src/mind/primitives.d.ts

@@ -0,0 +1,66 @@
+import { Vec } from "../vec.js";
+import { Sema } from "../sema.js";
+import type { Input, MindContext } from "./types.js";
+/** The content key of a byte span — one latin1 char per byte, an exact,
+ *  collision-free encoding.  Spans on the perception path are query-scale
+ *  (windows, regions, candidate spans), so key construction is far cheaper
+ *  than the river fold it deduplicates. */
+export declare function latin1Key(bytes: Uint8Array): string;
+/** Perceive input into a content-defined tree (the river fold).
+ *  Deterministic — identical bytes always produce an identical tree.
+ *
+ *  `boundaries` is an optional sorted list of proper byte offsets where the
+ *  fold must split so that each prefix segment folds identically to how it
+ *  folded when it was learned (§10.3 stable-prefix contract).  Only the
+ *  CALLER — who assembled the multi-turn context — knows where those
+ *  boundaries are; the geometry never guesses them from the bytes. */
+export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
+/** The DEPOSIT-shaped perceive.  Folds over the stream's own content cuts —
+ *  bit-identical to what inference computes for the same bytes, and that
+ *  train/inference agreement is load-bearing for exact recall.  An input that
+ *  EXTENDS a previously deposited one is a conversation context grown by one
+ *  turn; the cached prefix length IS the turn boundary (derived from the deposit
+ *  sequence itself, never from a content convention) and joins the cut set, so
+ *  the trained context node and the query's context subtree are the SAME node.
+ *  Segment folds reuse across deposits ({@link stablePrefixFoldIncremental}) —
+ *  O(turn) instead of O(context) per turn.  All of it is purely a cache: an
+ *  evicted chain loses only the turn boundaries, and since the content cuts do
+ *  not depend on the cache, the segments themselves are unchanged. */
+export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
+/** The raw bytes of an input — modality-neutral conversion. */
+export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
+/** Convenience: the gist vector of a byte span. */
+export declare function gistOf(ctx: MindContext, bytes: Uint8Array): Vec;
+/** Fold a perceived tree bottom-up against the store's content-addressed maps:
+ *  every leaf is named by findLeaf, every branch by findBranch over its kids'
+ *  ids (null the moment any child is unknown).  `visit`, when given, sees each
+ *  node with its byte span and resolved id.  Returns the node's byte end and
+ *  resolved id. */
+export declare function foldTree(ctx: MindContext, n: Sema, start: number, visit?: (n: Sema, start: number, end: number, node: number | null) => void): {
+    end: number;
+    node: number | null;
+};
+/** The canonical node id of a byte span: perceive it in isolation — the way
+ *  training did — and recover its root bottom-up.  Returns null if any part is
+ *  unknown. */
+export declare function resolve(ctx: MindContext, bytes: Uint8Array): number | null;
+/** Equivalence-class resolution: when the exact content-addressed lookup
+ *  misses, find a stored node whose CANONICAL key equals the span's — the
+ *  store's canon index proposes candidates by key hash, and each is verified
+ *  by re-canonicalizing its bytes (hash-then-verify, like every content
+ *  lookup).  Among verified candidates, one that leads somewhere (has a
+ *  continuation edge) is preferred; ties break to the lowest id — a corpus
+ *  property, not a seed property.  Null when the response carries no
+ *  canonicalizer, the store has no canon index, or nothing verifies. */
+export declare function canonResolve(ctx: MindContext, bytes: Uint8Array): number | null;
+/** Walk a perceived tree in POST-ORDER with byte offsets — children before
+ *  their parent, `visit(node, start, end)` for every node including leaves.
+ *  Returns the byte end.  The one shared traversal the offset-carrying tree
+ *  readers (recognition via foldTree's richer variant, attention's region
+ *  collection, resonance's branch counting) build on, so each does not
+ *  re-derive the offset bookkeeping.  (recognition.segment keeps its own
+ *  walk: its flush semantics need PRE-order decisions at leaf-parents, which
+ *  a post-order visitor cannot express.) */
+export declare function walkTree(n: Sema, start: number, visit: (node: Sema, start: number, end: number) => void): number;
+/** Reconstruct a node's byte content from the DAG, up to `maxLen` bytes. */
+export declare function read(ctx: MindContext, id: number, maxLen?: number): Uint8Array;