@hviana/sema 0.1.7 → 0.1.9

package/dist/src/mind/mechanisms/extraction.js

@@ -1,342 +0,0 @@
-// extraction.ts — Extraction (Skill) — Section 4 of the mind.
-//
-// Given a query and its consensus-ranked anchors, find the first span-shaped
-// skill exemplar among the ranked anchors and read the analogous span of the
-// query.  A skill exemplar is a learnt fact whose context and answer together
-// form a span-in-context pattern: the answer is a subsequence of the context
-// (or one of its pieces is), and the context is the smallest spanning frame
-// that contains it.
-import { foldTree, perceive, read, resolve } from "../primitives.js";
-import { follow, locate } from "../match.js";
-import { chooseAmong, hubBound } from "../traverse.js";
-import { concatBytes, indexOf } from "../../bytes.js";
-import { decodeText, unexplainedLabel } from "../rationale.js";
-import { CONCEPT, STEP } from "../graph-search.js";
-import { rItem, rNode, traceFail } from "../trace.js";
-// ── Extraction ────────────────────────────────────────────────────────────
-/** Find the first span-shaped skill exemplar among the ranked anchors from
- *  climbAttentionAll and read the analogous span from the query.  Returns
- *  the extracted bytes PLUS the query spans the skill ACCOUNTED FOR — the
- *  located frames AND any read span BOUNDED by located frames on both
- *  sides, the elementary evidence think's grounding decider weighs.  A
- *  bounded read is explained: the skill located both its borders in the
- *  query and emitted exactly what sits between them.  An OPEN-ENDED read
- *  (the exemplar's answer reaches the context's end, so the query is read
- *  to its own end with no located right border) remains a guess about where
- *  the span stops — it stays unaccounted, priced by exclusion like the
- *  cover's bridged bytes.  (Accounting frames only — the earlier convention
- *  — let a CAST juxtaposition that merely echoed the query's exact site
- *  outweigh a correct bounded extraction: the same span counted as
- *  explained for one mechanism and not the other, and the asymmetry, not
- *  the answers' merits, decided the grounding.)  Null when no skill
- *  applies. */
-export async function extractBySkill(ctx, query, pre) {
-    const t = ctx.trace?.enter("extractBySkill", [
-        rItem(query, "query"),
-    ]);
-    const fail = traceFail(t);
-    // Use climbAttentionAll to get the FULL ranked list, not just the
-    // roots that cleared commitVotes' significance floor.  The floor
-    // gates further points of attention for fusion, but extraction only
-    // needs ONE anchor that IS a span-shaped skill exemplar — and on
-    // some seeds the top-voted anchor is not one (e.g. a concept-merge
-    // nickname outvotes the painting exemplars on shared substrings,
-    // while the exemplars' votes fall below the floor).  Iterating the
-    // ranked list instead of just the roots lets extraction reach the
-    // first painting-exemplar anchor regardless of its floor status.
-    //
-    const { ranked } = await pre.attention();
-    if (ranked.length === 0) {
-        return fail("no consensus anchor — no skill to apply");
-    }
-    let exemplar = null;
-    let chosenAnchor = null;
-    let skipped = 0;
-    for (const cand of ranked) {
-        const ex = await pre.spanShapedOf(cand.anchor);
-        if (ex) {
-            exemplar = ex;
-            chosenAnchor = cand.anchor;
-            break;
-        }
-        skipped++;
-    }
-    if (exemplar === null) {
-        ctx.trace?.step("trySkillAnchors", [], [], `none of ${ranked.length} ranked anchor(s) is a span-shaped skill exemplar`);
-        return fail("no consensus root is a span-shaped skill exemplar");
-    }
-    if (skipped > 0) {
-        ctx.trace?.step("trySkillAnchors", [
-            rItem(query.subarray(0, 0), `skipped ${skipped}`),
-            rNode(ctx, chosenAnchor, "chosen"),
-        ], [], `skipped ${skipped} anchor(s) that are not skill exemplars`);
-    }
-    const { contextBytes, answerBytes } = exemplar;
-    const ansCtxRuns = answerRunsInContext(ctx, contextBytes, answerBytes);
-    if (ansCtxRuns === null || ansCtxRuns.length === 0) {
-        return fail("answer is not a subsequence of the context");
-    }
-    if (ansCtxRuns.length > 1) {
-        ctx.trace?.step("decomposeAnswer", [rItem(answerBytes, "multi-piece-answer")], ansCtxRuns.map((r) => rItem(contextBytes.subarray(r.start, r.end), "piece", undefined, [
-            r.start,
-            r.end,
-        ])), `answer splits into ${ansCtxRuns.length} piece(s) within the exemplar context`);
-    }
-    const pieces = [];
-    const accounted = [];
-    for (let ri = 0; ri < ansCtxRuns.length; ri++) {
-        const run = ansCtxRuns[ri];
-        const isLast = ri === ansCtxRuns.length - 1;
-        const framePreLen = Math.min(run.start, ctx.space.maxGroup);
-        const framePre = run.start > 0
-            ? contextBytes.subarray(run.start - framePreLen, run.start)
-            : null;
-        const frames = [];
-        let start = 0;
-        if (framePre) {
-            const prePos = locate(ctx, query, framePre, 0, pre.rec.sites);
-            if (prePos < 0)
-                continue;
-            start = prePos + framePre.length;
-            frames.push([prePos, start]); // the located frame IS matched evidence
-        }
-        let end;
-        if (isLast) {
-            if (run.end < contextBytes.length) {
-                const framePostLen = Math.min(contextBytes.length - run.end, ctx.space.maxGroup);
-                const framePost = contextBytes.subarray(run.end, run.end + framePostLen);
-                const postPos = locate(ctx, query.subarray(start), framePost, 0, pre.rec.sites);
-                if (postPos < 0)
-                    continue;
-                end = start + postPos;
-                frames.push([end, end + framePost.length]); // matched post-frame
-            }
-            else {
-                end = query.length;
-            }
-        }
-        else {
-            const nextRun = ansCtxRuns[ri + 1];
-            const nextPreLen = Math.min(nextRun.start, ctx.space.maxGroup);
-            const nextPre = contextBytes.subarray(nextRun.start - nextPreLen, nextRun.start);
-            const nextPos = locate(ctx, query.subarray(start), nextPre, 0, pre.rec.sites);
-            if (nextPos < 0) {
-                end = start + run.ansLen;
-            }
-            else {
-                end = start + nextPos;
-                frames.push([end, end + nextPre.length]); // matched next-frame
-            }
-        }
-        if (start >= end)
-            continue;
-        pieces.push(query.subarray(start, end));
-        accounted.push(...frames);
-        // Bounded on both sides ⇒ the read span itself is explained (see doc).
-        // frames carries the pre-border (when the answer is not at the context's
-        // start) and the located right border (post-frame or next piece's
-        // pre-frame); only when BOTH borders were located is the read bounded.
-        const preBounded = run.start === 0 || frames.some(([, e]) => e === start);
-        const postBounded = frames.some(([b]) => b === end);
-        if (preBounded && postBounded)
-            accounted.push([start, end]);
-    }
-    if (pieces.length === 0) {
-        return fail("no answer piece's frame located in the query");
-    }
-    const out = pieces.length === 1 ? pieces[0] : concatBytes(pieces);
-    t?.done([rItem(out, "extracted")], pieces.length === 1
-        ? `apply a learnt extraction skill — read the analogous span of the query` +
-            ` framed like "${decodeText(answerBytes)}" sits in its exemplar`
-        : `apply a learnt MULTI-PIECE skill — read ${pieces.length} analogous` +
-            ` pieces of the query and synthesize them like "${decodeText(answerBytes)}"`);
-    return {
-        bytes: out,
-        accounted,
-        unexplained: unexplainedLabel(query, accounted),
-    };
-}
-// ── The two span-shape readings: OPEN acceptance vs. STRONG decomposition ──
-//
-// isSpanShaped and answerRunsInContext read the SAME relation ("the answer is
-// drawn from the context") at two deliberately different strengths, and they
-// are NOT interchangeable:
-//
-//   • isSpanShaped — the OPEN reading: any in-order embedding (a sparse
-//     subsequence, arbitrary gaps).  O(|context|) byte scan.  Used to ACCEPT
-//     an exemplar candidate.
-//   • answerRunsInContext — the STRONG reading: a greedy longest-run
-//     DECOMPOSITION into contiguous pieces.  Greedy-longest is strictly
-//     stronger than subsequence (a long late match can consume context an
-//     earlier shorter choice needed), so an ACCEPTED exemplar can still fail
-//     to decompose — extractBySkill then fails with "answer is not a
-//     subsequence of the context" and think falls through to recall.  That
-//     fall-through is BEHAVIOUR, pinned by the extraction suites: do not
-//     "unify" the two into one machine — replacing the open reading with the
-//     strong one silently rejects exemplars extraction today accepts, and
-//     replacing the strong one with a backtracking embedding changes which
-//     pieces are read out of the query.
-/** Decompose an answer into substrings of its surrounding context, in order —
- *  the STRONG span-shape reading (see the section note above).  Returns null
- *  when no greedy longest-run decomposition exists.  Adjacent runs that
- *  connect contiguously are merged. */
-export function answerRunsInContext(_ctx, context, answer) {
-    const pos = indexOf(context, answer, 0);
-    if (pos >= 0) {
-        return [{ start: pos, end: pos + answer.length, ansLen: answer.length }];
-    }
-    const runs = [];
-    let ai = 0;
-    let ci = 0;
-    while (ai < answer.length) {
-        // Longest match of the remaining answer at any position of the remaining
-        // context: one direct extend per context position — O(|ctx|·match) per
-        // run, replacing the previous longest-first indexOf countdown whose
-        // repeated scans were cubic on long sparse-subsequence answers.
-        let bestLen = 0;
-        let bestPos = -1;
-        for (let p = ci; p < context.length; p++) {
-            let l = 0;
-            const maxL = Math.min(context.length - p, answer.length - ai);
-            if (maxL <= bestLen)
-                break; // no later position can beat the best
-            while (l < maxL && context[p + l] === answer[ai + l])
-                l++;
-            if (l > bestLen) {
-                bestLen = l;
-                bestPos = p;
-                if (ai + l === answer.length)
-                    break; // the whole remainder matched
-            }
-        }
-        if (bestLen === 0)
-            return null;
-        runs.push({ start: bestPos, end: bestPos + bestLen, ansLen: bestLen });
-        ai += bestLen;
-        ci = bestPos + bestLen;
-    }
-    const merged = [];
-    for (const r of runs) {
-        const last = merged[merged.length - 1];
-        if (last && r.start === last.end) {
-            last.end = r.end;
-            last.ansLen += r.ansLen;
-        }
-        else {
-            merged.push({ ...r });
-        }
-    }
-    return merged.length > 0 ? merged : null;
-}
-/** Check whether an anchor is a span-shaped skill exemplar: it represents a
- *  fact whose context and answer together form a span-in-context pattern.
- *  If the anchor has a nextOf continuation, that is the answer and the anchor
- *  itself is the context.  Otherwise the anchor's prevOf parents provide
- *  candidate contexts, and the longest one whose span is span-shaped wins. */
-export async function skillExemplar(ctx, anchor, guide) {
-    if (ctx.store.hasNext(anchor)) {
-        const contextBytes = read(ctx, anchor);
-        const answerBytes = await follow(ctx, anchor, guide);
-        if (answerBytes !== null && isSpanShaped(ctx, contextBytes, answerBytes)) {
-            return { contextBytes, answerBytes };
-        }
-        return null;
-    }
-    const answerBytes = read(ctx, anchor);
-    // Candidate contexts, capped at the hub bound (a common answer's reverse
-    // fan-in is corpus-sized).
-    const capped = ctx.store.prevFirst(anchor, hubBound(ctx));
-    const spanShaped = [];
-    for (const p of capped) {
-        const ctxB = read(ctx, p);
-        if (ctxB.length > 0 && isSpanShaped(ctx, ctxB, answerBytes)) {
-            spanShaped.push({ id: p, bytes: ctxB });
-        }
-    }
-    if (spanShaped.length === 0)
-        return null;
-    // Among span-shaped contexts, the longest wins (the smallest spanning frame
-    // heuristic's dual: more frame to locate in the query); the query gist,
-    // when given, breaks LENGTH TIES via chooseAmong — the same reverse-regime
-    // disambiguator every context pick uses, whose gist cache spares the
-    // re-fold this block once paid per tied candidate.  Same strict first-seen
-    // tie-break as the hand loop it replaces.
-    const maxLen = Math.max(...spanShaped.map((s) => s.bytes.length));
-    const longest = spanShaped.filter((s) => s.bytes.length === maxLen);
-    let contextBytes = longest[0].bytes;
-    if (guide && longest.length > 1) {
-        const pick = chooseAmong(ctx, longest.map((s) => s.id), guide).id;
-        contextBytes = longest.find((s) => s.id === pick).bytes;
-    }
-    return { contextBytes, answerBytes };
-}
-/** Whether the answer is a SPARSE subsequence of the context (bytes in
- *  order, arbitrary gaps) — the OPEN span-shape reading (see the section
- *  note above).  This is what lets extraction validate a MULTI-PIECE
- *  exemplar whose answer is stitched from several context runs — but it is
- *  deliberately permissive, so it must never be used as evidence that one
- *  span was "drawn from" another (see {@link containsSpan} for that).
- *
- *  There is deliberately NO containsSpan pre-check here: strict containment
- *  IMPLIES the subsequence embedding (a contiguous run, or a resolved node —
- *  whose content-addressed identity means its bytes occur contiguously — is
- *  an in-order embedding with zero gaps), so the scan below decides alone,
- *  with the same truth value.  The old pre-check re-perceived the context
- *  (a full river fold) per CANDIDATE in skillExemplar's √N-capped loop —
- *  pure cost, no discrimination. */
-export function isSpanShaped(_ctx, context, answer) {
-    let ai = 0;
-    for (let ci = 0; ci < context.length && ai < answer.length; ci++) {
-        if (context[ci] === answer[ai])
-            ai++;
-    }
-    return ai === answer.length;
-}
-/** STRICT containment: the answer's resolved node appears in the context's
- *  folded tree, or the answer occurs as one CONTIGUOUS byte run of the
- *  context.  This is real evidence the answer was drawn from the context.
- *  Fusion gates on this — the sparse-subsequence reading of
- *  {@link isSpanShaped} is trivially satisfied by short answers over long
- *  queries ("cold" is a gap-tolerant subsequence of most sentences holding
- *  c…o…l…d in order), and gating fusion on it silently starved multi-topic
- *  queries of their further points of attention. */
-export function containsSpan(ctx, context, answer) {
-    const ansId = resolve(ctx, answer);
-    if (ansId !== null) {
-        let found = false;
-        foldTree(ctx, perceive(ctx, context), 0, (_n, _s, _e, node) => {
-            if (node === ansId)
-                found = true;
-        });
-        if (found)
-            return true;
-    }
-    return indexOf(context, answer, 0) >= 0;
-}
-// ── Pipeline mechanism ──────────────────────────────────────────────────────
-export const extractionMechanism = {
-    name: "extraction",
-    provenance: "extract",
-    async floor(_ctx, _query, pre, worthRunning) {
-        // Extraction's floor is always exactly CONCEPT+STEP when it exists —
-        // same investment discipline as CAST's (see cast.ts): when the bound
-        // already cannot beat the incumbent, return it UNINVESTED (never
-        // first-touch the climb just to be pruned).
-        if (!worthRunning(CONCEPT + STEP))
-            return CONCEPT + STEP;
-        if ((await pre.attention()).ranked.length === 0)
-            return null;
-        return CONCEPT + STEP;
-    },
-    async run(ctx, query, pre) {
-        const ex = await extractBySkill(ctx, query, pre);
-        if (!ex)
-            return [];
-        return [{
-                bytes: ex.bytes,
-                accounted: ex.accounted,
-                moves: CONCEPT + STEP * ex.accounted.length,
-                unexplained: ex.unexplained,
-            }];
-    },
-};

package/dist/src/mind/mechanisms/recall.d.ts

@@ -1,13 +0,0 @@
-import type { MindContext } from "../types.js";
-import type { PipelineMechanism, Precomputed } from "../pipeline-mechanism.js";
-/** A recall result. */
-export interface RecallResult {
-    bytes: Uint8Array;
-    echoed: boolean;
-    accounted: Array<[number, number]>;
-    moves: number;
-    unexplained: string;
-}
-/** Recall the answer by resonating the whole query against the content index. */
-export declare function recallByResonance(ctx: MindContext, query: Uint8Array, pre: Precomputed): Promise<RecallResult | null>;
-export declare const recallMechanism: PipelineMechanism;

package/dist/src/mind/mechanisms/recall.js

@@ -1,166 +0,0 @@
-// mechanisms/recall.ts — Recall by resonance (Grounding IV).
-//
-// The recall mechanism resonates the whole query's gist against the content
-// index and grounds the nearest learned form.  Four tiers, orderly degrading
-// from exact self-match to honest echo.
-import { cosine } from "../../vec.js";
-import { consensusFloor, identityBar, reachThreshold, significanceBar, } from "../../geometry.js";
-import { gistOf, read, resolve } from "../primitives.js";
-import { corpusN, hubBound } from "../traverse.js";
-import { project, reverseContext } from "../match.js";
-import { CONCEPT, STEP } from "../graph-search.js";
-import { unexplainedLabel } from "../rationale.js";
-import { rItem, rNode } from "../trace.js";
-/** Recall the answer by resonating the whole query against the content index. */
-export async function recallByResonance(ctx, query, pre) {
-    const t = ctx.trace?.enter("recallByResonance", [
-        rItem(query, "query"),
-    ]);
-    const whole_ = [[0, query.length]];
-    const nothing = [];
-    const ground = (bytes, note, accounted, moves, echoed = false) => {
-        t?.done(bytes === null
-            ? []
-            : [rItem(bytes, "answer", resolve(ctx, bytes) ?? undefined)], note);
-        return bytes === null ? null : {
-            bytes,
-            echoed,
-            accounted,
-            moves,
-            unexplained: unexplainedLabel(query, accounted),
-        };
-    };
-    const k = pre.k;
-    const queryGist = pre.guide;
-    // 0. Exact self-match — content-addressed, deterministic.
-    const qId = pre.queryResolved;
-    if (qId !== null) {
-        const rev = ctx.store.prevFirst(qId, hubBound(ctx));
-        const g = reverseContext(ctx, qId, queryGist, rev);
-        if (g !== null) {
-            return ground(g, rev.length === 1
-                ? "exact self-match — reverse recall to the sole predecessor"
-                : "exact self-match — reverse recall to the best-resonating predecessor", nothing, STEP);
-        }
-    }
-    const whole = await ctx.store.resonate(queryGist, k);
-    if (whole.length === 0) {
-        return ground(null, "empty store — nothing to resonate with", [], 0);
-    }
-    const top = whole[0];
-    ctx.trace?.step("resonate", [rItem(query, "query-gist")], whole.map((h) => rNode(ctx, h.id, "hit", h.score)), `resonate the whole-query gist → ${whole.length} nearest learnt form(s)`);
-    // 1. Clean resonance — the scale-aware identity claim.  The ANGLE
-    // (top.score) carries the shared fraction; the query's MAGNITUDE (√len,
-    // the linear fold's own norm) converts the tolerated foreign fraction
-    // into bytes — at most one river window (see {@link identityBar}).  A
-    // fixed cosine bar let long queries claim "near-identical" while whole
-    // windows — an answer word — differed.
-    if (top.score >= identityBar(ctx.store.D, ctx.space.maxGroup, query.length)) {
-        for (const h of whole) {
-            if (h.id === qId || h.score >= 1.0) {
-                const rev = ctx.store.prevFirst(h.id, hubBound(ctx));
-                const g = reverseContext(ctx, h.id, queryGist, rev);
-                if (g !== null) {
-                    return ground(g, rev.length === 1
-                        ? "perfect self-match — reverse recall to the sole predecessor"
-                        : "perfect self-match — reverse recall to the best-resonating predecessor", nothing, STEP);
-                }
-                const own = read(ctx, h.id);
-                if (own.length > 0) {
-                    return ground(own, "perfect self-match — the query IS this node", nothing, STEP);
-                }
-            }
-            const g = await project(ctx, h.id, queryGist);
-            if (g) {
-                return ground(g, "clean whole-query resonance — ground the nearest hit", whole_, STEP);
-            }
-        }
-    }
-    // 2. Scaffolding-dominated.
-    if (top.score >= significanceBar(ctx.store.D)) {
-        const N = corpusN(ctx);
-        const minVote = consensusFloor(N);
-        // The committed points of attention ARE the shared climb's roots (same
-        // query, same k, same DF mode) — read them from Precomputed instead of
-        // re-climbing, so even a traced response pays for the climb once.
-        const forest = (await pre.attention()).roots;
-        if (forest.length > 0 && forest[0].vote >= minVote) {
-            const g = await project(ctx, forest[0].anchor, queryGist);
-            if (g) {
-                return ground(g, "scaffolding-dominated query — ground the consensus-climb anchor", [[forest[0].start, forest[0].end]], CONCEPT);
-            }
-        }
-    }
-    // 3. Last resort — gated on the FRACTION OF THE QUERY the grounding
-    // explains, not the raw cosine.  Root gists are unit vectors, but their
-    // magnitudes are recoverable from the byte lengths (‖·‖ = √len under the
-    // linear fold): cos = shared/√(lenQ·lenG), so shared/lenQ = cos·√(lenG/lenQ).
-    // The raw cosine punished honest containment — a query fully inside a
-    // longer grounded answer scored √(lenQ/lenG) and was refused — and let a
-    // long answer sharing only scaffolding pass; the query-relative fraction
-    // measures exactly what the reach bar means: how much of THE QUERY the
-    // store accounts for.
-    // Chance similarity survives the length conversion AMPLIFIED: the same
-    // √(lenG/lenQ) factor that converts an honest shared fraction into a
-    // query-relative one multiplies the estimator/chance floor too, so a long
-    // stored form (√(lenG/lenQ) ≈ 10 at 100×) lifted a noise-level cosine past
-    // the reach bar and grounded pure gibberish (observed).  Only the
-    // ABOVE-CHANCE part of the similarity is evidence of shared content —
-    // subtract the significance bar (3/√D, §8.3) before converting.  Derived
-    // from the existing bars; never tuned.
-    const sig = significanceBar(ctx.store.D);
-    const fracOfQuery = (cos, otherLen) => Math.min(1, Math.max(0, cos - sig) *
-        Math.sqrt(otherLen / Math.max(1, query.length)));
-    for (const h of whole) {
-        const g = await project(ctx, h.id, queryGist);
-        if (g) {
-            if (fracOfQuery(cosine(queryGist, gistOf(ctx, g)), g.length) >=
-                reachThreshold(ctx.space.maxGroup)) {
-                return ground(g, "last resort: the nearest grounded whole-query hit", [], STEP);
-            }
-        }
-    }
-    // The refusal/echo decision.  The echo returns a stored form's bytes AS
-    // the answer — a near-identity claim about the query — and identity-grade
-    // decisions are never made on an estimated score ("approximate scores may
-    // rank and propose; they may never decide", §6.2): the RaBitQ estimate
-    // overshooting the reach bar echoed a WRONG-entity neighbour ("capital of
-    // Zamunda?" echoed the Armenia fact, observed).  The bytes are read
-    // anyway to be echoed, so the decision uses their EXACT fold: one river
-    // fold of the top hit, measured in the same query-relative,
-    // chance-corrected units as the tier above.
-    const reach = reachThreshold(ctx.space.maxGroup);
-    const topBytes = read(ctx, top.id);
-    const exact = topBytes.length > 0
-        ? cosine(queryGist, gistOf(ctx, topBytes))
-        : 0;
-    if (fracOfQuery(exact, topBytes.length) < reach) {
-        return ground(null, "below reach threshold — nothing in the store relates to this query", [], 0);
-    }
-    // Honest echo.
-    return ground(topBytes, "last resort: the nearest resonant form's own bytes (echo, not grounded)", [], 0, true);
-}
-// ── Pipeline mechanism ──────────────────────────────────────────────────────
-export const recallMechanism = {
-    name: "recall",
-    provenance: "recall",
-    // Recall's floor is free to state (one STEP-grade projection) and its run
-    // gates its own tiers — no expensive investment happens inside floor, so
-    // there is nothing to guard with worthRunning here: the pipeline's own
-    // check prunes run() against the incumbent.
-    async floor(_ctx, _query, _pre, _worthRunning) {
-        return STEP;
-    },
-    async run(ctx, query, pre) {
-        const r = await recallByResonance(ctx, query, pre);
-        if (!r)
-            return [];
-        return [{
-                bytes: r.bytes,
-                accounted: r.accounted,
-                moves: r.moves,
-                unexplained: r.unexplained,
-                provenance: r.echoed ? "recall-echo" : "recall",
-            }];
-    },
-};