@hviana/sema 0.2.2 → 0.2.4

package/src/mind/mechanisms/extraction.ts

@@ -70,43 +70,118 @@ export async function extractBySkill(
   if (ranked.length === 0) {
     return fail("no consensus anchor — no skill to apply");
   }
-  let exemplar: SkillInfo | null = null;
-  let chosenAnchor: number | null = null;
-  let skipped = 0;
-  for (const cand of ranked) {
-    const ex = await pre.spanShapedOf(cand.anchor);
-    if (ex) {
-      exemplar = ex;
-      chosenAnchor = cand.anchor;
-      break;
+
+  // Try ranked anchors IN ORDER until one yields a USABLE extraction — not
+  // merely a span-shaped exemplar, but one whose extracted span clears the
+  // same one-river-fold quantum (W) cover.ts's restatedSpan gate already
+  // treats as the floor below which byte overlap is chance, not evidence.
+  // isSpanShaped (spanShapedOf) is a deliberately permissive sparse-
+  // subsequence check — see the section note below — so it accepts exemplars
+  // whose relation to the query is coincidental gap-matching, not genuine
+  // structure.  Stopping at the FIRST such exemplar let a coincidental match
+  // early in the ranked list win outright and read out a sub-quantum
+  // fragment (observed: a 3-byte "Hel" pulled from an unrelated exemplar,
+  // while a later ranked anchor would have read the query's own "Hello…"
+  // correctly).  Trying further anchors when one produces nothing usable is
+  // the same idiom this loop already uses for non-exemplars — extended to
+  // cover a bad extraction, not just a structural non-match.
+  //
+  // The retry is bounded at pre.k — the SAME evidence-breadth constant every
+  // other consumer of a ranked list already self-limits to (resonance, the
+  // weave, the climb itself; see Precomputed.k's own doc comment) — not the
+  // full ranked list.  locate()'s frame match has an EXACT-byte tier with no
+  // significance correction of its own (short W-byte frames are cheap to
+  // match by pure chance), so trying every ranked anchor turns that per-
+  // anchor chance into a near-certainty over enough attempts: on a pure-
+  // gibberish query, 170 anchors deep found an unrelated Zulu exemplar whose
+  // short frame happened to byte-match, producing "xyzzy pl" — a coincidence
+  // no different in kind from the "RaBitQ estimate overshot the reach bar
+  // and grounded pure gibberish" failure recall.ts's own significance
+  // correction exists to prevent (see recallByResonance's reach-threshold
+  // comment).  Bounding the search to the ranked list's own top-k restores
+  // the "genuinely relevant but not root-significant" exemplars this loop
+  // was built for, without the unbounded tail's chance collisions.
+  const W = ctx.space.maxGroup;
+  const searched = ranked.slice(0, pre.k);
+  let shapeMisses = 0;
+  let subQuantum = 0;
+  for (const cand of searched) {
+    const exemplar = await pre.spanShapedOf(cand.anchor);
+    if (!exemplar) {
+      shapeMisses++;
+      continue;
+    }
+    const built = buildFromExemplar(ctx, query, pre, exemplar);
+    if (built === null || built.bytes.length < W) {
+      subQuantum++;
+      continue;
+    }
+    if (shapeMisses > 0 || subQuantum > 0) {
+      ctx.trace?.step(
+        "trySkillAnchors",
+        [
+          rItem(
+            query.subarray(0, 0),
+            `skipped ${shapeMisses + subQuantum}`,
+          ),
+          rNode(ctx, cand.anchor, "chosen"),
+        ],
+        [],
+        `skipped ${shapeMisses} non-exemplar and ${subQuantum} sub-quantum ` +
+          `anchor(s) before one yielded a usable extraction`,
+      );
     }
-    skipped++;
+    t?.done(
+      [rItem(built.bytes, "extracted")],
+      built.pieces === 1
+        ? `apply a learnt extraction skill — read the analogous span of the query` +
+          ` framed like "${
+            decodeText(exemplar.answerBytes)
+          }" sits in its exemplar`
+        : `apply a learnt MULTI-PIECE skill — read ${built.pieces} analogous` +
+          ` pieces of the query and synthesize them like "${
+            decodeText(exemplar.answerBytes)
+          }"`,
+    );
+    return {
+      bytes: built.bytes,
+      accounted: built.accounted,
+      unexplained: unexplainedLabel(query, built.accounted),
+    };
   }
-  if (exemplar === null) {
+  if (shapeMisses === searched.length) {
     ctx.trace?.step(
       "trySkillAnchors",
       [],
       [],
-      `none of ${ranked.length} ranked anchor(s) is a span-shaped skill exemplar`,
+      `none of the top ${searched.length} ranked anchor(s) (of ${ranked.length} total) ` +
+        `is a span-shaped skill exemplar`,
     );
     return fail("no consensus root is a span-shaped skill exemplar");
   }
-  if (skipped > 0) {
-    ctx.trace?.step(
-      "trySkillAnchors",
-      [
-        rItem(query.subarray(0, 0), `skipped ${skipped}`),
-        rNode(ctx, chosenAnchor!, "chosen"),
-      ],
-      [],
-      `skipped ${skipped} anchor(s) that are not skill exemplars`,
-    );
-  }
+  return fail(
+    "no ranked anchor yielded an extraction at or above the quantum floor",
+  );
+}
+
+/** Build the extracted bytes for ONE already-accepted span-shaped exemplar —
+ *  factored out of {@link extractBySkill} so its anchor loop can try
+ *  successive ranked candidates instead of committing to the first
+ *  structural match.  Null when the exemplar's answer does not decompose
+ *  against its context, or no piece's frame locates in the query. */
+function buildFromExemplar(
+  ctx: MindContext,
+  query: Uint8Array,
+  pre: Precomputed,
+  exemplar: SkillInfo,
+):
+  | { bytes: Uint8Array; accounted: Array<[number, number]>; pieces: number }
+  | null {
   const { contextBytes, answerBytes } = exemplar;
 
   const ansCtxRuns = answerRunsInContext(ctx, contextBytes, answerBytes);
   if (ansCtxRuns === null || ansCtxRuns.length === 0) {
-    return fail("answer is not a subsequence of the context");
+    return null;
   }
 
   if (ansCtxRuns.length > 1) {
@@ -200,25 +275,11 @@ export async function extractBySkill(
     if (preBounded && postBounded) accounted.push([start, end]);
   }
   if (pieces.length === 0) {
-    return fail("no answer piece's frame located in the query");
+    return null;
   }
 
   const out = pieces.length === 1 ? pieces[0] : concatBytes(pieces);
-  t?.done(
-    [rItem(out, "extracted")],
-    pieces.length === 1
-      ? `apply a learnt extraction skill — read the analogous span of the query` +
-        ` framed like "${decodeText(answerBytes)}" sits in its exemplar`
-      : `apply a learnt MULTI-PIECE skill — read ${pieces.length} analogous` +
-        ` pieces of the query and synthesize them like "${
-          decodeText(answerBytes)
-        }"`,
-  );
-  return {
-    bytes: out,
-    accounted,
-    unexplained: unexplainedLabel(query, accounted),
-  };
+  return { bytes: out, accounted, pieces: pieces.length };
 }
 
 // ── The two span-shape readings: OPEN acceptance vs. STRONG decomposition ──

package/src/mind/mechanisms/recall.ts

@@ -20,6 +20,7 @@ import { CONCEPT, STEP } from "../graph-search.js";
 import { unexplainedLabel } from "../rationale.js";
 import type { PipelineMechanism, Precomputed } from "../pipeline-mechanism.js";
 import { rItem, rNode } from "../trace.js";
+import { substitutionBridge } from "../bridge.js";
 
 /** A recall result. */
 export interface RecallResult {
@@ -293,6 +294,84 @@ export async function recallByResonance(
       }
     }
   }
+  // 3b. Corroborated-substitution bridge — refusal-path only (bridge.ts).
+  // Every gist-based tier has failed; before refusing, align the query
+  // byte-for-byte against the trained contexts its own stored windows
+  // anchor, accepting mismatches only as corpus-attested, concept-bar
+  // substitutions.  A bridged context grounds exactly like any hit —
+  // projected through its learnt edges — under the same restated-fragment
+  // guard tiers 0b/2 apply.  Costs nothing on any answering path.
+  {
+    // The resonance hits already ranked above are handed to the bridge as
+    // PROPOSED candidates alongside its own anchor climbs: on a corpus this
+    // size a W-byte window is far too common for the clamped climb to
+    // single out the right trained context, while the whole-query gist
+    // already ranked it nearest (observed live: "what is the capital of
+    // france" resonating straight to "What is the capital of France?" yet
+    // refusing on the reach bar).  Approximate scores propose; the bridge's
+    // byte-exact alignment and attestation gates decide.
+    //
+    // The proposal breadth here is widened PAST `k` — first by requesting
+    // hubBound(ctx) candidates instead of `k` (recall's own tiers above
+    // stay at `k`; this re-resonates only on the refusal path, exactly
+    // where the bridge itself already runs), AND by asking the index to
+    // search EXHAUSTIVELY.  Both matter: the IVF only ever probes
+    // ⌈√clusters⌉ of them (store.ts's efFor) REGARDLESS of k — widening k
+    // alone just returns more hits from the SAME already-probed clusters,
+    // never a hit whose vector lives in an unprobed one.  Measured live:
+    // "What is the chemical symbol for water?" needs "What is the
+    // chemical formula for water?", scoring only 0.58 against the
+    // query's gist (a MIDDLE-of-string word swap perturbs the river-fold
+    // tree hash far more than a same-length TAIL swap like the "carbon"/
+    // "oxygen" neighbours that outrank it at 0.87+) — absent from the
+    // resonance list even at k=5000, present and byte-exact-verified the
+    // moment it's force-fed to the bridge directly.  `exhaustive` is the
+    // natural, tuning-free ceiling (probe every cluster) for a call that
+    // is ALREADY refusal-path-only and must not miss a candidate hiding
+    // behind an unlucky structural distance.
+    const wide = k < hubBound(ctx)
+      ? await ctx.store.resonate(queryGist, hubBound(ctx), true)
+      : whole;
+    const bridged = await substitutionBridge(
+      ctx,
+      query,
+      wide.map((h) => h.id),
+    );
+    if (bridged !== null) {
+      const g = await project(ctx, bridged.id, queryGist);
+      // A projection contained in a substituted candidate-side span is the
+      // substitution RESTATED as if it were knowledge — the exact failure
+      // observed live: "Darwin was born in England." bridged to the
+      // Einstein fact through " England." → " Germany." and would have
+      // voiced "Germany", an answer the substitution itself manufactured.
+      // The same principle as the restated-fragment guards above, extended
+      // to the bridge's own substitutions.
+      const cBytes = ctx.store.bytes(bridged.id);
+      const manufactured = g !== null &&
+        bridged.subs.some((s) =>
+          indexOf(cBytes.subarray(s.cs, s.ce), g!, 0) >= 0
+        );
+      if (
+        g !== null && g.length > 0 && !restates(g) && !manufactured &&
+        !(g.length < query.length && indexOf(query, g, 0) >= 0)
+      ) {
+        return ground(
+          g,
+          `substitution bridge — a trained context accounts for the query ` +
+            `up to ${bridged.subs.length} corroborated substitution(s)`,
+          // Accounted NOTHING — the same epistemic humility as the echo
+          // tier below: a substitution-bridged grounding is a last resort
+          // that must lose to ANY mechanism that actually explained the
+          // query (observed: pricing the aligned spans here outweighed
+          // extraction's correct answer in the grounding decider), while
+          // still beating silence when everything else refused.
+          [],
+          CONCEPT * bridged.subs.length + STEP,
+        );
+      }
+    }
+  }
+
   // The refusal/echo decision.  The echo returns a stored form's bytes AS
   // the answer — a near-identity claim about the query — and identity-grade
   // decisions are never made on an estimated score ("approximate scores may

package/src/mind/mind.ts

@@ -266,9 +266,15 @@ export class Mind implements MindContext {
     sites: ReadonlyArray<Site>;
     leaves: ReadonlyArray<Leaf>;
     splits: ReadonlySet<number>;
+    starts: ReadonlySet<number>;
   } {
     const r = recognise(this, bytes);
-    return { sites: r.sites, leaves: r.leaves, splits: r.splits };
+    return {
+      sites: r.sites,
+      leaves: r.leaves,
+      splits: r.splits,
+      starts: r.starts,
+    };
   }
 
   /** Disambiguate among multiple learnt continuations of the same context node.

package/src/mind/pipeline.ts

@@ -288,8 +288,43 @@ export async function think(
     ? new Set<number>()
     : new Set(recognise(ctx, answer).sites.map((s) => s.payload));
   const reasoned = await reason(ctx, query, answer, preConsumed, pre);
-  const fused = provenance === "recall" || provenance === "recall-echo"
-    ? await fuseAttention(ctx, query, reasoned, pre)
+
+  // Fuse only when the query has a genuine REMAINDER no mechanism's
+  // structural evidence touched at all.  `decided.accounted` alone
+  // undercounts this: it is a COST-LADDER quantity (cover.ts prices its
+  // masked/computed spans at near-zero and deliberately leaves them out of
+  // `accounted` so PASS-bridged bytes are still charged), not a coverage
+  // one — a query fully explained by one computed span plus bridged
+  // connectors can report `accounted: []` while nothing is actually left
+  // unexplained.  The genuine remainder is what NEITHER the winning
+  // candidate's accounted spans NOR any recognised extension's computed
+  // span (`pre.computed` — every mechanism's parse() output, ALU included)
+  // ever touched.  A remainder under one river-fold quantum (W, the same
+  // floor cover.ts's restatedSpan and the honesty-density bar above both
+  // use) is bridging punctuation/whitespace, never a second topic —
+  // observed: a single space between two fully-computed arithmetic spans
+  // ("2+2 3+3") registered as "unaccounted" and pulled in an unrelated
+  // corpus fact, corrupting "4 6" into "4 63".
+  const explained: Array<[number, number]> = [
+    ...decided.accounted,
+    ...pre.computed.map((u): [number, number] => [u.i, u.j]),
+  ];
+  const remainder = unaccounted(explained);
+  // Whether the winning candidate's entire recognised substance is
+  // COMPUTED — every accounted span exactly a pre.computed span, nothing
+  // from a genuinely recognised/climbed site.  fuseAttention's lone-root
+  // shortcut assumes a single point of attention already IS primary's own
+  // source; that assumption is exactly backwards for a pure computation
+  // (an ALU result has no anchor of its own) — see fuseAttention's
+  // `unclimbed` parameter, gated there by Attention.breadth so a
+  // coincidental echo (which this flag alone cannot distinguish) is still
+  // rejected.
+  const unclimbed = decided.accounted.length > 0 &&
+    decided.accounted.every(([i, j]) =>
+      pre.computed.some((u) => u.i === i && u.j === j)
+    );
+  const fused = remainder >= ctx.space.maxGroup
+    ? await fuseAttention(ctx, query, reasoned, pre, unclimbed)
     : reasoned;
 
   done(

package/src/mind/reasoning.ts

@@ -13,6 +13,20 @@ import { joinWithBridge, pivotInto } from "./resonance.js";
 import type { Precomputed } from "./pipeline-mechanism.js";
 import type { Rationale } from "./rationale.js";
 
+/** Whether `bytes` is a proper byte-subspan of `query` — already present in
+ *  the question, so voicing it back only restates part of what was asked,
+ *  never answers it.  The exact guard recallByResonance already applies to
+ *  its OWN grounding candidates (tier 1's `restates`, tier 2's subspan
+ *  check, tier 0b's argument-binding subspan check) — every mechanism that
+ *  walks a LEARNT CONTINUATION EDGE past an already-vetted grounding
+ *  (reason()'s own hops below, and CAST's `projectCounterfactual` seat
+ *  substitution — see cast.ts) needs the same guard applied to what the
+ *  walk turns up, since `follow()`/`chooseNext`/`pivotInto` know nothing of
+ *  the query at all — only of what structurally continues what. */
+export function restatesQuery(query: Uint8Array, bytes: Uint8Array): boolean {
+  return bytes.length < query.length && indexOf(query, bytes, 0) >= 0;
+}
+
 /** Extend a grounded answer forward across facts (multi-hop reasoning).
  *  Pivots on the longest unconsumed learnt context each answer contains,
  *  then follows the pivot's continuation to the next fact.  Repeats up
@@ -91,7 +105,8 @@ export async function reason(
       const fwdId = fwd !== null ? resolve(ctx, fwd) : null;
       if (
         fwd !== null && !bytesEqual(fwd, cur) &&
-        (fwdId === null || !consumed.has(fwdId))
+        (fwdId === null || !consumed.has(fwdId)) &&
+        !restatesQuery(query, fwd)
       ) {
         consumeAll(curId);
         t ??= ctx.trace?.enter("reason", [
@@ -115,7 +130,7 @@ export async function reason(
 
     const fc = await follow(ctx, pivot, qv);
     consumeAll(pivot);
-    if (fc === null || bytesEqual(fc, cur)) break;
+    if (fc === null || bytesEqual(fc, cur) || restatesQuery(query, fc)) break;
     t ??= ctx.trace?.enter("reason", [rItem(startedFrom, "grounded")]);
     ctx.trace?.step(
       "pivotStep",
@@ -141,6 +156,14 @@ export async function fuseAttention(
   query: Uint8Array,
   primary: Uint8Array,
   pre: Precomputed,
+  /** True when `primary` never touched the consensus climb at all — e.g. a
+   *  pure ALU computation, which has no anchor of its own.  commitVotes
+   *  ALWAYS admits the dominant root regardless of its vote (attention.ts:
+   *  "roots.length === 0 || …") on the assumption a lone root already IS
+   *  primary's own source; that assumption is exactly backwards when
+   *  primary is unclimbed.  Absent or false preserves the original
+   *  behaviour exactly. */
+  unclimbed = false,
 ): Promise<Uint8Array> {
   // When the answer is structurally drawn from the query itself
   // (extraction), it already spans all the query's pieces — fusion
@@ -155,17 +178,86 @@ export async function fuseAttention(
   // query, same k, same DF mode) — read them from Precomputed instead of
   // re-climbing, so even a traced response pays for the climb once.
   const forest = (await pre.attention()).roots;
-  if (forest.length <= 1) return primary;
+  // A LONE root is ordinarily primary's own source — nothing to fuse.  But
+  // when primary is unclimbed, the lone root was never checked against
+  // anything: it is admitted by commitVotes unconditionally, so it may be
+  // genuine consensus (Attention.breadth dominates — most of the query's
+  // OWN regions corroborate it) or a coincidental echo (breadth does not
+  // dominate — see test/35-attention-confidence).  breadth is the SCALE-
+  // INVARIANT read of exactly this question: the raw IDF vote cannot serve
+  // here, since it is an absolute ln(N)-scaled quantity (a genuine root on
+  // a large store can score BELOW its own floor while a coincidental echo
+  // on a small one scores comfortably above its own, smaller, floor).
+  const lonePromotes = unclimbed && forest.length === 1 &&
+    forest[0].breadth > 0.5;
+  if (forest.length === 0 || (forest.length <= 1 && !lonePromotes)) {
+    return primary;
+  }
 
   const pieces: Array<{ start: number; bytes: Uint8Array }> = [
     { start: forest[0].start, bytes: primary },
   ];
   const qv = pre.guide; // once, not per root
+  const rest = lonePromotes ? forest : forest.slice(1);
   const t = ctx.trace?.enter("fuseAttention", [
     rItem(primary, "primary"),
-    ...forest.slice(1).map((r) => rNode(ctx, r.anchor, "point", r.vote)),
+    ...rest.map((r) => rNode(ctx, r.anchor, "point", r.vote)),
   ]);
-  for (const root of forest.slice(1)) {
+  for (const root of rest) {
+    // DISPERSION: this root's contributing regions are confined to a
+    // single cluster (see Attention.clusters) — one local neighbourhood of
+    // the query, not several separate places.  Raw region count already
+    // failed to discriminate a coincidental match from a genuine further
+    // topic (test/24 gap 3.1 vs test/35's echo); dispersion is a different
+    // question — not how MUCH evidence, but how many separate PLACES in the
+    // query corroborate it — and a coincidental match is structurally
+    // confined to one cluster no matter how strongly it resonates.
+    //
+    // EXCEPTION: crossRegionVotes' own joint conclusions (a query naming
+    // two attributes that were only ever learnt TOGETHER — test/34's own
+    // binding corpus) are inherently ONE fused context and are pooled from
+    // a single synthetic region, so they always read as one cluster even
+    // though they already, by construction, account for both original
+    // mentions.  `breadth` (dominates — the same > half-the-query bar used
+    // everywhere else) still correctly recognises these: a genuine joint
+    // binding explains the MAJORITY of the query's regions on its own,
+    // which a coincidental echo never does (verified: test/35's echo tops
+    // out at 0.40).  So a root is trusted when EITHER measure alone
+    // indicates real signal — excluded only when BOTH are weak.  Cheap and
+    // synchronous — checked before the async already-answered walk below.
+    if (root.clusters < 2 && root.breadth <= 0.5) {
+      ctx.trace?.step(
+        "singleCluster",
+        [rNode(ctx, root.anchor, "point", root.vote)],
+        [],
+        "this point's evidence is confined to one local neighbourhood of the query — not trusted as an independent topic",
+      );
+      continue;
+    }
+    // ALREADY ANSWERED: this root's own learnt continuation — the same
+    // content-addressed walk reason()'s echo guard already trusts
+    // (`ctx.store.prevCount(qId) > 0`), here applied per-candidate instead
+    // of to the whole query — is VERBATIM present later in the query.  A
+    // query that embeds both an exchange's ask and its own already-given
+    // reply (a conversation's turn plus its own prior answer, concatenated
+    // raw by addTurn — or any caller pasting a transcript into one
+    // respond() call; the check is Mind-bookkeeping-free, so it treats both
+    // identically) has already spoken this root's answer — fusing it in
+    // again would only restate it.  Deliberately NOT a magnitude measure:
+    // it fires on exact content-addressed recurrence, not on how strongly
+    // the root resonates.
+    const cont = await follow(ctx, root.anchor, qv);
+    if (
+      cont !== null && cont.length > 0 && indexOf(query, cont, root.end) >= 0
+    ) {
+      ctx.trace?.step(
+        "alreadyAnswered",
+        [rNode(ctx, root.anchor, "point", root.vote)],
+        [rItem(cont, "continuation")],
+        "this point's own learnt continuation already appears later in the query — already answered, not fused",
+      );
+      continue;
+    }
     const g = await project(ctx, root.anchor, qv);
     if (g === null || g.length === 0) continue;
     if (pieces.some((p) => indexOf(p.bytes, g, 0) >= 0)) continue;