@hviana/sema 0.5.3 → 0.5.4

package/dist/src/mind/pipeline.js

@@ -1,343 +0,0 @@
-// pipeline.ts — the think pipeline (Section 5 of the mind).
-//
-// think() is the whole file's job: one lightest-derivation choice among
-// UNIFORM mechanisms.  The pipeline sees mechanisms through the
-// PipelineMechanism interface only — it never imports a mechanism-specific
-// type and never has a special-case branch for any mechanism.  Adding a
-// mechanism means registering one object; removing one means dropping it
-// from the list.  The mechanisms themselves live in mechanisms/ (one file
-// each); the shared pre-computation they exchange lives in Precomputed
-// (pipeline-mechanism.ts).
-import { PASS, STEP } from "./graph-search.js";
-import { gistOf, read, resolve } from "./primitives.js";
-import { recognise } from "./recognition.js";
-import { fuseAttention, reason } from "./reasoning.js";
-import { unexplainedSpans } from "./rationale.js";
-import { rItem } from "./trace.js";
-import { hubBound } from "./traverse.js";
-import { Precomputed } from "./pipeline-mechanism.js";
-import { coverMechanism } from "./mechanisms/cover.js";
-import { castMechanism } from "./mechanisms/cast.js";
-import { confluenceMechanism } from "./mechanisms/confluence.js";
-import { extractionMechanism } from "./mechanisms/extraction.js";
-import { referenceMechanism } from "./mechanisms/reference.js";
-import { prefixMechanism } from "./mechanisms/prefix-completion.js";
-import { recallMechanism } from "./mechanisms/recall.js";
-// Re-exports: cover's pre-resolution helpers and the ALU adapter kept
-// importable from the pipeline module (their historical home).
-export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
-export { aluToMechanism } from "./mechanisms/alu.js";
-// ── Extension dispatch (pre-loop parse) ─────────────────────────────────────
-async function collectComputed(ctx, mechanisms, query) {
-    const out = [];
-    const meter = ctx.meter;
-    for (const m of mechanisms) {
-        if (!m.parse)
-            continue;
-        const spans = meter
-            ? await meter.time(`${m.name}.parse`, () => m.parse(query))
-            : await m.parse(query);
-        out.push(...spans);
-    }
-    return out;
-}
-// ── Built-in mechanisms ─────────────────────────────────────────────────────
-// ORDER MATTERS, but only through the uniform floor/worthRunning pruning —
-// no mechanism is special-cased.  Cover runs FIRST: when a computed
-// extension result (e.g. ALU) exists, cover masks it in at near-zero cost
-// (see mechanisms/cover.ts), which becomes `best` before any other mechanism
-// invests in its own precomputation.  CAST's and confluence's floors (2*STEP,
-// 3*STEP) then fail `worthRunning` and are skipped by the SAME admissible-
-// floor pruning every mechanism is already subject to — not by asking
-// "is this an extension?". Grade TIES keep the earlier candidate, so this
-// order is also the tie-break priority: cover, cast, confluence, extraction,
-// reference, recall.
-//
-// REFERENCE sits after extraction and before recall because that is what its
-// claim is worth: extraction READS a span out of the query (no synthesis),
-// reference voices one through a learnt slot, and recall's tiers degrade
-// toward echo and silence.  It does not PRUNE recall — its floor is two
-// projections, so recall's one-STEP floor still clears `worthRunning` — and it
-// is not meant to: both run, share one resonance read
-// (Precomputed.resonance), and the ladder decides.
-export const defaultMechanisms = [
-    coverMechanism,
-    castMechanism,
-    confluenceMechanism,
-    extractionMechanism,
-    referenceMechanism,
-    recallMechanism,
-    prefixMechanism,
-];
-/** Think: a single lightest-derivation exploration of the Sema graph.
- *
- *  Every answer travels the same path:
- *    1. Pre-computation — recognise, extension parse, guide; everything
- *       expensive stays lazy on Precomputed until a mechanism asks.
- *    2. Grounding — every mechanism yields candidates weighed in the one
- *       cost ladder; the lightest grounding derivation wins.
- *    3. Post-grounding — diagnostics (narrowDecision, thinGrounding),
- *       reasoning (multi-hop), fusion (multi-topic). */
-export async function think(ctx, query, mechs) {
-    if (query.length === 0)
-        return null;
-    ctx._edgeGuide = gistOf(ctx, query);
-    ctx._edgeChoice.clear();
-    const t = ctx.trace?.enter("think", [rItem(query, "query")]);
-    const done = (answer, note) => {
-        t?.done(answer
-            ? [rItem(answer, "answer", resolve(ctx, answer) ?? undefined)]
-            : [], note);
-        return answer;
-    };
-    // ── Pre-computation ──────────────────────────────────────────────────
-    const mechanisms = mechs ?? defaultMechanisms;
-    const rec = recognise(ctx, query);
-    // Phase 1: collect computed spans from mechanisms that implement parse()
-    const computed = await collectComputed(ctx, mechanisms, query);
-    if (computed.length > 0) {
-        ctx.trace?.step("computeExtensions", [rItem(query, "query")], computed.map((u) => rItem(query.subarray(u.i, u.j), "operand", undefined, [u.i, u.j])), `extensions recognised and evaluated ${computed.length} computation(s)`);
-        for (const u of computed) {
-            ctx.trace?.step("evalComputation", [rItem(query.subarray(u.i, u.j), "expression", undefined, [u.i, u.j])], [rItem(u.bytes, "result", resolve(ctx, u.bytes) ?? undefined)], "evaluate the recognised operation to its authoritative result");
-        }
-    }
-    // Phase 2: the shared pre-computation container.  Eager fields only
-    // (recognition, computed spans, guide) — every expensive analysis
-    // (consensus climb, weave, span-shape classification) is a lazily-cached
-    // method on Precomputed, first-touched by whichever mechanism's floor
-    // survives its cheap gates and the worthRunning check.  A query no
-    // mechanism climbs for (e.g. one an extension decided) never climbs.
-    const pre = new Precomputed(ctx, query, rec, computed, ctx._edgeGuide);
-    const grade = (w) => Math.floor(w / STEP);
-    const unaccounted = (spans) => unexplainedSpans(query.length, spans)
-        .reduce((sum, [s, e]) => sum + (e - s), 0);
-    const weigh = (accounted, moves) => moves + PASS * unaccounted(accounted);
-    const candidates = [];
-    let best = null;
-    const consider = (c) => {
-        if (c.bytes.length === 0)
-            return;
-        if (ctx.meter)
-            ctx.meter.candidates++;
-        candidates.push(c);
-        if (best === null) {
-            best = c;
-            return;
-        }
-        const g = grade(c.weight), gb = grade(best.weight);
-        if (g < gb) {
-            best = c;
-            return;
-        }
-        // TIE-BREAK: AT EQUAL GRADE, PREFER THE ANSWER THAT INVENTS LESS.
-        //
-        // The ladder prices what a candidate leaves UNACCOUNTED, which is the
-        // right primary question but cannot separate two candidates that leave
-        // the same bytes unaccounted — and then the winner is whichever mechanism
-        // happened to be considered first, which is not a reason.
-        //
-        // What still separates them is what they DID with those bytes.  A
-        // candidate that carries an unexplained span into its answer is passing
-        // the asker's own words back as if they were derived; one that leaves
-        // them out has made a smaller, honest claim.  Measured on test/22's
-        // two-fact chain: cover and recall both graded 11001 over 11 unexplained
-        // bytes, cover answering "The capital of France is Paris famous for" (11
-        // bytes of scaffolding) against recall's crossing of the hop (0).  Order
-        // alone decided it, and the shallower reading won.
-        //
-        // This never overrides the ladder — it only orders within one grade, so
-        // coverage and moves still dominate exactly as before.
-        if (g === gb && (c.scaffolding ?? 0) < (best.scaffolding ?? 0))
-            best = c;
-    };
-    const worthRunning = (floor) => best === null || grade(floor) < grade(best.weight);
-    // Phase 3: grounding loop
-    // Per-mechanism accounting (src/meter.ts).  The market's whole premise is
-    // that mechanisms compete on one cost scale — so the profiling read-out is
-    // also per-mechanism, uniformly: the loop never asks which one it holds.
-    const meter = ctx.meter;
-    for (const mech of mechanisms) {
-        const floor = meter
-            ? await meter.time(`${mech.name}.floor`, () => mech.floor(ctx, query, pre, worthRunning))
-            : await mech.floor(ctx, query, pre, worthRunning);
-        if (meter) {
-            if (floor === null)
-                meter.mechanismSkips++;
-            else
-                meter.mechanismFloors++;
-        }
-        if (floor === null) {
-            ctx.trace?.step("skipMechanism", [], [], `${mech.name} skipped — structural precondition failed`);
-            continue;
-        }
-        if (!worthRunning(floor)) {
-            ctx.trace?.step("skipMechanism", [], [], `${mech.name} skipped — floor ${floor} cannot beat incumbent (grade ${grade(best.weight)})`);
-            continue;
-        }
-        if (meter)
-            meter.mechanismRuns++;
-        const results = meter
-            ? await meter.time(`${mech.name}.run`, () => mech.run(ctx, query, pre))
-            : await mech.run(ctx, query, pre);
-        for (const r of results) {
-            const weight = r.weight ?? weigh(r.accounted, r.moves);
-            consider({
-                bytes: r.bytes,
-                provenance: r.provenance ?? mech.provenance,
-                weight,
-                used: r.used,
-                accounted: r.accounted,
-                unexplained: r.unexplained,
-                complete: r.complete,
-                scaffolding: r.scaffolding,
-            });
-        }
-    }
-    // (TS cannot see the closure assignments into `best` and narrows it to its
-    // initial null, so the read-back needs the assertion.)
-    const decided = best;
-    if (candidates.length > 1) {
-        // The runner-up is computed BEFORE the decideGrounding step so its grade
-        // margin can ride along in the step's structured data payload; the
-        // computation itself is pure and was always unconditional — only its
-        // position moved.
-        let runnerUp = null;
-        if (decided !== null) {
-            for (const c of candidates) {
-                if (c === decided)
-                    continue;
-                if (runnerUp === null || grade(c.weight) < grade(runnerUp.weight)) {
-                    runnerUp = c;
-                }
-            }
-        }
-        const margin = decided !== null && runnerUp !== null
-            ? grade(runnerUp.weight) - grade(decided.weight)
-            : null;
-        ctx.trace?.step("decideGrounding", candidates.map((c) => rItem(c.bytes, `${c.provenance} (weight ${c.weight.toFixed(3)}${c.unexplained ? `, unexplained: "${c.unexplained}"` : ""})`)), decided ? [rItem(decided.bytes, decided.provenance)] : [], "the lightest grounding derivation wins — every mechanism weighed in the one cost ladder", undefined, {
-            version: 1,
-            candidates: candidates.map((c) => ({
-                provenance: c.provenance,
-                weight: c.weight,
-                grade: grade(c.weight),
-                unexplainedBytes: unaccounted(c.accounted),
-                decided: c === decided,
-            })),
-            ...(margin !== null ? { runnerUpMargin: margin } : {}),
-        });
-        if (decided !== null && runnerUp !== null && margin !== null) {
-            if (margin <= 1) {
-                ctx.trace?.step("narrowDecision", [
-                    rItem(decided.bytes, `${decided.provenance} (weight ${decided.weight.toFixed(3)})`),
-                ], [
-                    rItem(runnerUp.bytes, `${runnerUp.provenance} (weight ${runnerUp.weight.toFixed(3)})`),
-                ], `margin ${margin} grade-unit(s) — the decision could change with one more training fact`, undefined, { version: 1, margin });
-            }
-        }
-    }
-    if (decided === null) {
-        done(null, "no mechanism grounded an answer");
-        return null;
-    }
-    // Honesty density
-    {
-        const covered = query.length - unaccounted(decided.accounted);
-        const density = query.length > 0 ? covered / query.length : 1;
-        const thinBar = 1 / ctx.space.maxGroup;
-        if (density < thinBar) {
-            ctx.trace?.step("thinGrounding", [rItem(decided.bytes, decided.provenance)], [], `grounded but thin — density ${density.toFixed(3)} is below 1/W (${thinBar.toFixed(3)})`);
-        }
-    }
-    const answer = decided.bytes;
-    const provenance = decided.provenance;
-    const castUsed = decided.used ?? new Set();
-    // ── Post-grounding, gated by provenance ──────────────────────────────
-    const preConsumed = provenance === "cast" || provenance === "join"
-        ? castUsed
-        : provenance === "recall" || provenance === "recall-echo"
-            ? new Set()
-            : new Set(recognise(ctx, answer).sites.map((s) => s.payload));
-    // A grounding that DECLARED itself complete is not extended: the answer is
-    // already a trained form's own continuation, reached through an identity
-    // claim about the query, so a multi-hop pivot could only chain past the
-    // fact that produced it (see MechanismResult.complete).
-    // WHAT THE MECHANISM WITHHELD, NOT WHAT IT VOICED.  A pivot must not
-    // re-open content a grounding deliberately kept out: comparison cites two
-    // analogs and refuses their own downstream facts, so pivoting into one is
-    // the mechanism's own refusal undone one step later (test/29 C2 pivoted
-    // through `speare` — a stored fragment of the analog `William Shakespeare`
-    // — into the biography CAST had declined).
-    //
-    // Reading the used anchors' OWN bytes here says something stronger and
-    // wrong: that nothing INSIDE what was voiced may be pivoted through.  A
-    // comparison's seat sentence legitimately contains further terms with
-    // their own unrelated facts, and C3 pins exactly that — `Mona Lisa`, inside
-    // the voiced seat `The Mona Lisa was painted by Leonardo da Vinci.`, leads
-    // on to `Mona Lisa hangs in the Louvre`, which is about neither analog.
-    // The withheld content is the used anchors' CONTINUATIONS, so that is what
-    // the containment rule reads: `speare` is contained in `Shakespeare wrote
-    // 39 plays` and stays refused, while `Mona Lisa` appears in no withheld
-    // continuation and the genuine further hop fires.
-    //
-    // Only a mechanism carrying its own `used` set (cast/join) gets this: there
-    // `preConsumed` is a deliberate, short list of the anchors the answer
-    // speaks for, so the fan-out is bounded.  For every other provenance
-    // `preConsumed` is derived by re-recognising the answer — "everything in
-    // it", not "what it voiced" — and a containment rule over that would
-    // suppress every pivot the answer legitimately contains.
-    const voiced = (provenance === "cast" || provenance === "join")
-        ? [...castUsed].flatMap((id) => ctx.store.nextFirst(id, hubBound(ctx)).map((n) => read(ctx, n)))
-        : [];
-    const reasoned = decided.complete ? answer : meter
-        ? await meter.time("reason", () => reason(ctx, query, answer, preConsumed, pre, voiced))
-        : await reason(ctx, query, answer, preConsumed, pre, voiced);
-    // Fuse only when the query has a genuine REMAINDER no mechanism's
-    // structural evidence touched at all.  `decided.accounted` alone
-    // undercounts this: it is a COST-LADDER quantity (cover.ts prices its
-    // masked/computed spans at near-zero and deliberately leaves them out of
-    // `accounted` so PASS-bridged bytes are still charged), not a coverage
-    // one — a query fully explained by one computed span plus bridged
-    // connectors can report `accounted: []` while nothing is actually left
-    // unexplained.  The genuine remainder is what NEITHER the winning
-    // candidate's accounted spans NOR any recognised extension's computed
-    // span (`pre.computed` — every mechanism's parse() output, ALU included)
-    // ever touched.  A remainder under one river-fold quantum (W, the same
-    // floor cover.ts's restatedSpan and the honesty-density bar above both
-    // use) is bridging punctuation/whitespace, never a second topic —
-    // observed: a single space between two fully-computed arithmetic spans
-    // ("2+2 3+3") registered as "unaccounted" and pulled in an unrelated
-    // corpus fact, corrupting "4 6" into "4 63".
-    const explained = [
-        ...decided.accounted,
-        ...pre.computed.map((u) => [u.i, u.j]),
-    ];
-    const remainder = unaccounted(explained);
-    // Whether the winning candidate's entire recognised substance is
-    // COMPUTED — every accounted span exactly a pre.computed span, nothing
-    // from a genuinely recognised/climbed site.  fuseAttention's lone-root
-    // shortcut assumes a single point of attention already IS primary's own
-    // source; that assumption is exactly backwards for a pure computation
-    // (an ALU result has no anchor of its own) — see fuseAttention's
-    // `unclimbed` parameter, gated there by Attention.breadth so a
-    // coincidental echo (which this flag alone cannot distinguish) is still
-    // rejected.
-    const unclimbed = decided.accounted.length > 0 &&
-        decided.accounted.every(([i, j]) => pre.computed.some((u) => u.i === i && u.j === j));
-    // Where the winning grounding stands in the query — fusion places primary
-    // by it (see fuseAttention's `primarySpans`).  `accounted` is the
-    // cost-ladder read and is authoritative when non-empty; when it is empty
-    // the grounding is a pure COMPUTATION, whose evidence is its computed span.
-    // Exactly the cost-ladder-vs-coverage distinction `explained` above draws,
-    // read here for POSITION instead of for coverage — and resolved here, where
-    // both readings are in hand, rather than inside fuseAttention.
-    const primarySpans = decided.accounted.length > 0
-        ? decided.accounted
-        : pre.computed.map((u) => [u.i, u.j]);
-    const fused = remainder < ctx.space.maxGroup
-        ? reasoned
-        : meter
-            ? await meter.time("fuse", () => fuseAttention(ctx, query, reasoned, pre, unclimbed, primarySpans))
-            : await fuseAttention(ctx, query, reasoned, pre, unclimbed, decided.accounted);
-    done(fused, "grounded, reasoned forward, fused across points of attention");
-    return { bytes: fused, provenance };
-}

package/dist/src/mind/primitives.d.ts

@@ -1,85 +0,0 @@
-import { Vec } from "../vec.js";
-import { Sema } from "../sema.js";
-import type { Input, MindContext } from "./types.js";
-/** The content key of a byte span — one latin1 char per byte, an exact,
- *  collision-free encoding.  Spans on the perception path are query-scale
- *  (windows, regions, candidate spans), so key construction is far cheaper
- *  than the river fold it deduplicates. */
-export declare function latin1Key(bytes: Uint8Array): string;
-/** The {@link perceive} memo key: the span's content PLUS the boundary set it
- *  was folded under.  The tree is a function of BOTH — the same bytes fold
- *  plainly with no boundaries and into a left-nested stable-prefix shape with
- *  them — so a content-only key returns whichever shape was computed first.
- *  That is exactly what happened: a conversation seeded its cumulative context
- *  under the content key, and every later plain `perceive` of those bytes was
- *  served the boundary tree instead (measured: respondTurn answered where
- *  respond() on byte-identical input did not).  NUL separates the two parts —
- *  the boundary rendering is digits and commas, so no content byte can forge
- *  the split. */
-export declare function perceiveKey(bytes: Uint8Array, boundaries?: readonly number[]): string;
-/** Perceive input into a content-defined tree (the river fold).
- *  Deterministic — identical bytes always produce an identical tree.
- *
- *  `boundaries` is an optional sorted list of proper byte offsets where the
- *  fold must split so that each prefix segment folds identically to how it
- *  folded when it was learned (§10.3 stable-prefix contract).  Only the
- *  CALLER — who assembled the multi-turn context — knows where those
- *  boundaries are; the geometry never guesses them from the bytes. */
-export declare function perceive(ctx: MindContext, input: Input, leafAt?: (i: number) => number | null, lookup?: (ids: number[]) => number | null, boundaries?: readonly number[]): Sema;
-/** The DEPOSIT-shaped perceive.  Folds over the stream's own content cuts —
- *  bit-identical to what inference computes for the same bytes.  That
- *  train/inference agreement is the whole contract: the trained context node
- *  and the node `resolve(query)` reaches must be the SAME node, and the only
- *  way to guarantee it is to give this function nothing extra to say.  It
- *  imposes no boundaries, knows nothing about turns, and reads no convention
- *  out of the bytes.
- *
- *  An input that EXTENDS a previously deposited one — a conversation context
- *  grown by a turn, or a resumed replay — reuses that deposit's already-folded
- *  content segments ({@link contentFoldIncremental}), so it costs O(new bytes)
- *  instead of O(context).  The reuse is TRANSPARENT by construction: a segment
- *  is a pure function of its own bytes, so a reused one is bit-identical to a
- *  refolded one.  Nothing has to prove that the extending deposit is "really"
- *  a next turn — a coincidental byte prefix reuses the same segments and gets
- *  the same tree it would have got anyway.  (It used to matter: while this
- *  path imposed turn BOUNDARIES, a wrong guess changed the tree, so the cache
- *  needed a continuation-bytes proof to gate it.  Nothing is imposed now, so
- *  there is nothing to gate.) */
-export declare function perceiveDeposit(ctx: MindContext, bytes: Uint8Array, conversational?: boolean): Sema;
-/** The raw bytes of an input — modality-neutral conversion. */
-export declare function inputBytes(ctx: MindContext, input: Input): Uint8Array;
-/** Convenience: the gist vector of a byte span. */
-export declare function gistOf(ctx: MindContext, bytes: Uint8Array): Vec;
-/** Fold a perceived tree bottom-up against the store's content-addressed maps:
- *  every leaf is named by findLeaf, every branch by findBranch over its kids'
- *  ids (null the moment any child is unknown).  `visit`, when given, sees each
- *  node with its byte span and resolved id.  Returns the node's byte end and
- *  resolved id. */
-export declare function foldTree(ctx: MindContext, n: Sema, start: number, visit?: (n: Sema, start: number, end: number, node: number | null) => void): {
-    end: number;
-    node: number | null;
-};
-/** The canonical node id of a byte span: perceive it in isolation — the way
- *  training did — and recover its root bottom-up.  Returns null if any part is
- *  unknown. */
-export declare function resolve(ctx: MindContext, bytes: Uint8Array): number | null;
-/** Equivalence-class resolution: when the exact content-addressed lookup
- *  misses, find a stored node whose CANONICAL key equals the span's — the
- *  store's canon index proposes candidates by key hash, and each is verified
- *  by re-canonicalizing its bytes (hash-then-verify, like every content
- *  lookup).  Among verified candidates, one that leads somewhere (has a
- *  continuation edge) is preferred; ties break to the lowest id — a corpus
- *  property, not a seed property.  Null when the response carries no
- *  canonicalizer, the store has no canon index, or nothing verifies. */
-export declare function canonResolve(ctx: MindContext, bytes: Uint8Array): number | null;
-/** Walk a perceived tree in POST-ORDER with byte offsets — children before
- *  their parent, `visit(node, start, end)` for every node including leaves.
- *  Returns the byte end.  The one shared traversal the offset-carrying tree
- *  readers (recognition via foldTree's richer variant, attention's region
- *  collection, resonance's branch counting) build on, so each does not
- *  re-derive the offset bookkeeping.  (recognition.segment keeps its own
- *  walk: its flush semantics need PRE-order decisions at leaf-parents, which
- *  a post-order visitor cannot express.) */
-export declare function walkTree(n: Sema, start: number, visit: (node: Sema, start: number, end: number) => void): number;
-/** Reconstruct a node's byte content from the DAG, up to `maxLen` bytes. */
-export declare function read(ctx: MindContext, id: number, maxLen?: number): Uint8Array;