@hviana/sema 0.2.3 → 0.2.5

package/src/mind/traverse.ts

@@ -7,8 +7,7 @@
 // project) live in match.ts — the elementary match-and-project operation.
 
 import { cosine, Vec } from "../vec.js";
-import { consensusFloor } from "../geometry.js";
-import type { AncestorReach, MindContext } from "./types.js";
+import type { AncestorReach, MindContext, SaturationStop } from "./types.js";
 import { gistOf, read } from "./primitives.js";
 
 // ── Per-response structural memo ────────────────────────────────────────
@@ -135,7 +134,22 @@ export function edgeAncestors(
   // is withdrawn.  On a small store the floor stays ≤ √N and the atom
   // climbs exactly as before, so single-letter facts keep working.
   if (id < 0 && atomIsHub(ctx, contextCount)) {
-    const reach = { roots: [], contextsReached: 0, saturated: true };
+    const bound0 = Math.ceil(Math.sqrt(Math.max(2, contextCount)));
+    const reach: AncestorReach = {
+      roots: [],
+      contextsReached: 0,
+      saturated: true,
+      ...(ctx.trace
+        ? {
+          saturation: {
+            reason: "byte-atom-commonality" as const,
+            node: id,
+            observed: atomReach(ctx, contextCount),
+            limit: bound0,
+          },
+        }
+        : {}),
+    };
     memo?.set(id, reach);
     return reach;
   }
@@ -145,6 +159,10 @@ export function edgeAncestors(
   const seen = new Set<number>([id]);
   const ctxSeen = new Set<number>();
   let saturated = false;
+  // Provenance of the FIRST decision that saturated this climb — allocated
+  // only when a trace is requested (see AncestorReach.saturation's doc); the
+  // climb itself never reads it back.
+  let satStop: SaturationStop | undefined;
 
   // EXPAND-UNTIL-DECIDED: a reach is consumed either as a VOTE (which needs
   // contextsReached exactly, and only while ≤ √N — beyond that the region is
@@ -189,12 +207,45 @@ export function edgeAncestors(
     if (hasNx || pc > 0) {
       roots.push(x);
       if (hasNx) ctxSeen.add(x);
-      if (pc > bound) return false; // decided: ≥ pc > √N distinct contexts
+      if (pc > bound) {
+        // decided: ≥ pc > √N distinct contexts
+        if (ctx.trace) {
+          satStop = {
+            reason: "predecessor-fan-in",
+            node: x,
+            observed: pc,
+            limit: bound,
+          };
+        }
+        return false;
+      }
       for (const p of ctx.store.prevFirst(x, bound)) ctxSeen.add(p);
-      if (ctxSeen.size > bound) return false; // decided
+      if (ctxSeen.size > bound) {
+        // decided
+        if (ctx.trace) {
+          satStop = {
+            reason: "distinct-context-limit",
+            node: x,
+            observed: ctxSeen.size,
+            limit: bound,
+          };
+        }
+        return false;
+      }
     }
     const parents = ctx.store.parentsFirst(x, bound + 1);
-    if (parents.length > bound) return false; // decided: hub
+    if (parents.length > bound) {
+      // decided: hub
+      if (ctx.trace) {
+        satStop = {
+          reason: "parent-fan-out",
+          node: x,
+          observed: parents.length,
+          limit: bound,
+        };
+      }
+      return false;
+    }
     let fresh = 0;
     for (const p of parents) {
       if (!seen.has(p)) {
@@ -205,7 +256,18 @@ export function edgeAncestors(
     }
     if (fresh > 1) {
       lateral += fresh - 1;
-      if (lateral > bound) return false; // decided: cone-wide hub
+      if (lateral > bound) {
+        // decided: cone-wide hub
+        if (ctx.trace) {
+          satStop = {
+            reason: "lateral-cone-limit",
+            node: x,
+            observed: lateral,
+            limit: bound,
+          };
+        }
+        return false;
+      }
     }
     return true;
   };
@@ -267,7 +329,12 @@ export function edgeAncestors(
     }
   }
 
-  const reach = { roots, contextsReached: ctxSeen.size, saturated };
+  const reach: AncestorReach = {
+    roots,
+    contextsReached: ctxSeen.size,
+    saturated,
+    ...(saturated && satStop ? { saturation: satStop } : {}),
+  };
   memo?.set(id, reach);
   return reach;
 }
@@ -528,22 +595,22 @@ export function chooseNext(
     }
   }
 
-  // A pick among GENUINELY competing continuations still needs to clear the
-  // same genuine-corroboration floor {@link consensusFloor} draws for every
-  // other consumer that turns distinct-context support into a vote (the
-  // climb's recallByResonance/commitVotes) — below it, "most corroborated"
-  // is only the least-thin echo among several, not real evidence.  Gated to
-  // corpus scale large enough that this echo is even possible (the same
-  // scale {@link atomIsHub} already switches on for the identical reason:
-  // at small N every edge IS the evidence there is).  A hub id has no
-  // meaningful "distinct context" reading at all (id < 0 atoms are excluded
-  // from this path already, since chooseNext only ever sees real edges).
-  const N = corpusN(ctx);
-  if (
-    capped.length > 1 && atomIsHub(ctx, N) && bestSupport < consensusFloor(N)
-  ) {
-    return undefined;
-  }
+  // NO consensusFloor gate here (tried and reverted — see
+  // test/40-choosenext-scale-guard.test.mjs): that floor is calibrated for
+  // POOLED, IDF-weighted CLIMB VOTES (recallByResonance, commitVotes), where
+  // each corroborating region contributes at most ln N and the floor grows
+  // with N exactly as that per-region ceiling does (HOW_IT_WORKS.md §8.6).
+  // `bestSupport` here is a different kind of quantity — a raw prevCount of
+  // how many training contexts predicted ONE destination, bounded by how
+  // often that specific fact was retold, never by corpus size N.  Gating an
+  // N-invariant count against an N-growing threshold guarantees failure
+  // once N is large enough, discarding genuinely, structurally dominant
+  // edges (observed: a fact corroborated 2-to-1-1-1 refused at N≈325K,
+  // falling back to a noisy concept-hop).  The loop above already IS the
+  // "genuinely competing" test: a tie leaves first-inserted as the pick
+  // (test/30's own pinned behaviour); a strict winner is real evidence
+  // regardless of corpus scale.  Matches HOW_IT_WORKS.md §25's own
+  // chooseNext pseudocode, which has no such floor.
 
   // Trace is built lazily — the filter + map below only execute when a
   // trace listener is attached, so the common (no-trace) path pays only

package/src/mind/types.ts

@@ -39,7 +39,7 @@ export interface DepositCacheEntry {
    *  its own suffix bytes-equal this exactly. */
   nextBytes?: Uint8Array;
 }
-import { concatBytes } from "../bytes.js";
+import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
 import { dominates } from "../geometry.js";
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -111,6 +111,23 @@ export interface Attention {
    *  consensus; one that does not is a coincidental single-region echo —
    *  see test/35-attention-confidence.test.mjs. */
   breadth: number;
+  /** DISPERSION: the number of distinct clusters this point's contributing
+   *  regions form, merging any two whose gap is under one river-fold
+   *  quantum W.  Neither breadth NOR raw region count discriminates a
+   *  genuine further topic from a coincidental echo (both were tried and
+   *  falsified — breadth starves a genuine, evenly-split multi-topic query,
+   *  since no root in a real N-way split can exceed half the vote; raw
+   *  count doesn't separate them either, since a short, structurally simple
+   *  echo racks up as many corroborating regions as a real topic does).
+   *  Dispersion asks a different question: not how MUCH evidence, but how
+   *  many separate PLACES in the query corroborate it.  A coincidental
+   *  match — one local phrase resonating with an unrelated stored form —
+   *  is structurally confined to ONE cluster no matter how strong its vote;
+   *  a genuine further topic is named in its own distinctive wording
+   *  somewhere the query's scaffolding does not reach, always a SEPARATE
+   *  cluster from whatever else corroborates it.  See
+   *  test/37-cluster-dispersion-fusion.test.mjs. */
+  clusters: number;
 }
 
 /** Both read-outs of one consensus climb. */
@@ -158,11 +175,36 @@ export interface RegionVote {
   absorbed?: number;
 }
 
+/** The structural gate that first decided an {@link edgeAncestors} climb was
+ *  saturated (an abstention, not a discriminative conclusion) — pure
+ *  instrumentation for {@link ClimbConsensusData}'s reach trace; it never
+ *  feeds back into the climb itself. */
+export type SaturationReason =
+  | "byte-atom-commonality"
+  | "predecessor-fan-in"
+  | "distinct-context-limit"
+  | "parent-fan-out"
+  | "lateral-cone-limit";
+
+/** One saturation stop's provenance: which reason fired, at which node, the
+ *  observed count against the bound that decided it. */
+export interface SaturationStop {
+  reason: SaturationReason;
+  node: number;
+  observed: number;
+  limit: number;
+}
+
 /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
 export interface AncestorReach {
   roots: number[];
   contextsReached: number;
   saturated: boolean;
+  /** The saturation gate that stopped this climb, when {@link saturated} is
+   *  true and a trace was requested — see {@link edgeAncestors}.  Absent for
+   *  a non-saturated reach, and absent (even when saturated) when no trace
+   *  was requested — instrumentation must not allocate when tracing is off. */
+  saturation?: SaturationStop;
 }
 
 /** Saturated-interval information for the noise-drop gate. */
@@ -273,11 +315,52 @@ export function spliceAll(segs: Seg[]): Uint8Array | null {
   return concatBytes(segs.map((s) => s.bytes));
 }
 
+/** Whether a chosen span RESTATES the query rather than answering it: its
+ *  SUBSTITUTED bytes (an edge followed from a recognised site, not the
+ *  site's own literal text read back) already occur elsewhere in the query
+ *  — the same principle recall.ts's tiers apply to a whole-query projection
+ *  ("a projection that is a proper byte-subspan of the query restates part
+ *  of the question").  A LITERAL span (the site's own bytes, unchanged) is
+ *  exempt: naming what's already there at its OWN position is not a
+ *  substitution.  A recognised site that is itself an entire PRIOR TURN of
+ *  a multi-turn query is exactly this shape: it carries a genuine learnt
+ *  continuation, but that continuation is something the asker already said
+ *  moments later in the SAME query, not a new answer.  Below one river
+ *  window, byte overlap is chance, not evidence — the same floor
+ *  identityBar and reachThreshold hold every other structural-overlap claim
+ *  to. */
+export function segRestatesQuery(
+  s: Seg,
+  query: Uint8Array,
+  queryLen: number,
+  W: number,
+): boolean {
+  if (!s.rec) return false;
+  const literal = s.j - s.i === s.bytes.length &&
+    bytesEqual(s.bytes, query.subarray(s.i, s.j));
+  if (literal) return false;
+  return s.bytes.length >= W && s.bytes.length < queryLen &&
+    indexOf(query, s.bytes, 0) >= 0;
+}
+
 /** Lift the answer out of the cover for think: the recognised region, free of
- *  the asker's surrounding (unrecognised) framing. */
-export function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null {
+ *  the asker's surrounding (unrecognised) framing — and free of any chosen
+ *  span that only RESTATES content the query already contains (see {@link
+ *  segRestatesQuery}).  A restating span is excluded from both the framing
+ *  (lo/hi) decision and the final concatenation: it is stale, not a second
+ *  answer, but the OTHER spans a derivation chose are independent evidence
+ *  and must not be discarded along with it. */
+export function liftAnswer(
+  segs: Seg[],
+  queryLen: number,
+  query: Uint8Array,
+  W: number,
+): Uint8Array | null {
+  const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
   const recognised: number[] = [];
-  for (let k = 0; k < segs.length; k++) if (segs[k].rec) recognised.push(k);
+  for (let k = 0; k < segs.length; k++) {
+    if (segs[k].rec && !restated[k]) recognised.push(k);
+  }
   if (recognised.length === 0) return null;
 
   if (recognised.length === 1) {
@@ -297,13 +380,19 @@ export function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null {
     // trailing glue byte ("2+2." → "4.", the span dominates a 4-byte query).
     if (s.computed && s.i > 0) return s.bytes;
     if (dominates(s.j - s.i, queryLen)) {
-      return concatBytes(segs.map((x) => x.bytes));
+      return concatBytes(
+        segs.filter((_, k) => !restated[k]).map((x) => x.bytes),
+      );
     }
     return s.bytes;
   }
   const lo = recognised[0];
   const hi = recognised[recognised.length - 1];
-  return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
+  return concatBytes(
+    segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) =>
+      x.bytes
+    ),
+  );
 }
 
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous

package/src/store.ts

@@ -306,8 +306,17 @@ export interface Store {
   nodeCount(): number;
 
   // ── soft resonance over node gists ─────────────────────────────────────
-  /** The k nodes whose gist resonates most with v. */
-  resonate(v: Vec, k: number): Promise<Hit[]>;
+  /** The k nodes whose gist resonates most with v.  `exhaustive` widens the
+   *  IVF probe to every cluster (see {@link AbstractStore.efFor}'s doc) —
+   *  for refusal-path-only callers where an approximate top-√C-clusters
+   *  search is not the same discriminator as the caller's own byte-exact
+   *  verification (the substitution bridge's proposal channel): a rarer
+   *  paraphrase can score lower than hundreds of unrelated hits by pure
+   *  fold-geometry structural distance (a middle-of-string mismatch
+   *  perturbs the tree hash far more than a tail mismatch of the same
+   *  byte length) and so never even reach a probed cluster, no matter how
+   *  large k is — k only reorders WITHIN the clusters already probed. */
+  resonate(v: Vec, k: number, exhaustive?: boolean): Promise<Hit[]>;
   /** Mark a node as a RESONANCE TARGET — promote its gist into the content
    *  index so {@link resonate} can find it.  A node's gist is captured at intern
    *  but indexed LAZILY (only targets are indexed; the ~99.5% intermediate DAG
@@ -1636,7 +1645,7 @@ export abstract class AbstractStore implements Store {
 
   // ── Soft resonance ─────────────────────────────────────────────────────
 
-  async resonate(v: Vec, k: number): Promise<Hit[]> {
+  async resonate(v: Vec, k: number, exhaustive = false): Promise<Hit[]> {
     await this._ensureReady();
     // Synchronous flush of any buffered index writes: the FIRST resonance
     // after a large ingest pays that flush here, so it shows up in respond
@@ -1649,17 +1658,22 @@ export abstract class AbstractStore implements Store {
     // same values still hits.  Lazy-init: null after any index write; the
     // first miss after a flush recreates it.  When voteRegions resonates
     // identical perceived sub-regions, only the first call descends the ANN.
-    const rk = vecKey(v) + ":" + k;
+    const rk = vecKey(v) + ":" + k + (exhaustive ? ":x" : "");
     const cache = this._resonateCache;
     if (cache) {
       const hit = cache.get(rk);
       if (hit !== undefined) return hit;
     }
 
+    const clusters = this._vecContentClusterCount();
     const results = this._vecContentQuery(
       normalize(copy(v)),
       k * this.overfetch,
-      this.efFor(this._vecContentClusterCount()),
+      // Exhaustive: probe every cluster (ef ≥ 4·clusters guarantees the
+      // IVF's own ef→nprobe=ceil(ef/4) mapping reaches all of them) — the
+      // natural ceiling for a search that is ALREADY refusal-path-only and
+      // must not miss a candidate hiding in an unprobed cluster.
+      exhaustive ? 4 * clusters : this.efFor(clusters),
     );
     const out: Hit[] = [];
     for (const r of results) {

package/test/36-already-answered-fusion.test.mjs

@@ -0,0 +1,128 @@
+// 36-already-answered-fusion.test.mjs — a point of attention whose own
+// learnt continuation is ALREADY present later in the query must not be
+// fused in again.
+//
+// This is Category A of a two-part defect found while investigating a
+// multi-turn dialogue's garbled fusion ("Thank you very much!" fused with
+// unrelated points of attention).  One of those points (root "Hello",
+// anchor of the greeting) traced back to substantial, genuine regions —
+// not noise — and its learnt continuation ("Hello! How can I assist you
+// today?") was found to be VERBATIM present earlier in the query, because
+// it was Sema's own prior reply, appended by addTurn the same way any turn
+// is appended.  Re-surfacing it is redundant: the query has already spoken
+// its own answer.
+//
+// Deliberately turn-agnostic and Mind-bookkeeping-free: Mind's multi-turn
+// API is strictly a computational optimization (incremental fold reuse) —
+// it must never be the thing inference depends on for correctness.  This
+// check uses only what already exists for ANY accumulated byte stream,
+// single-shot or multi-turn: `follow()` (the same content-addressed
+// continuation walk `reason()`'s own echo guard already uses,
+// `ctx.store.prevCount(qId) > 0`, just applied per-candidate-root instead
+// of to the whole query) and plain byte containment.  A query that embeds
+// its own prior exchange — via real respondTurn(), or a caller manually
+// pasting a transcript into one respond() call — is caught identically.
+//
+// (Category B — coincidental echo of a short, generic CURRENT-turn phrase
+// against unrelated corpus content, unrelated to staleness — is a SEPARATE
+// defect, not addressed here; see the investigation notes for why breadth
+// and regionSupport both fail to discriminate it from genuine fusion.)
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { Mind } from "../dist/src/index.js";
+import { SQliteStore } from "../dist/src/store-sqlite.js";
+import { fuseAttention } from "../dist/src/mind/reasoning.js";
+import { gistOf } from "../dist/src/mind/primitives.js";
+
+const enc = (s) => new TextEncoder().encode(s);
+const dec = (b) =>
+  new TextDecoder().decode(b.filter((x) => x !== 0)).replace(/\s+/g, " ")
+    .trim();
+
+// "greet" leads to "reply-greet" — a learnt exchange the query embeds BOTH
+// halves of, exactly the shape addTurn produces (ask, then the system's own
+// reply, concatenated raw).  "red circle" is a genuine, unrelated second
+// topic (test/34's own binding corpus) whose own continuation is nowhere
+// in the query — the ordinary multi-topic case, which must still fuse.
+const CORPUS = [
+  ["greet", "reply-greet"],
+  ["red", "is a color"],
+  ["blue", "is a color"],
+  ["circle", "is a shape"],
+  ["square", "is a shape"],
+  ["red circle", "answer alpha"],
+  ["red square", "answer beta"],
+  ["blue circle", "answer gamma"],
+  ["blue square", "answer delta"],
+];
+
+const mk = (seed) =>
+  new Mind({ seed, store: new SQliteStore({ path: ":memory:" }) });
+
+test("fuseAttention: a root whose continuation is already answered in the query is excluded", async () => {
+  const m = mk(1);
+  await m.ingest(CORPUS);
+  // "greet"'s own continuation, "reply-greet", is embedded right in the
+  // query — the ask and its answer both present, exactly like a
+  // conversation's own turn + reply.  "red circle" is a genuine further
+  // topic with no such embedded answer.
+  const q = enc("greet reply-greet then red then circle");
+  const roots = await m.climbAttention(q, 24);
+  const greet = roots.find((r) => r.start === 0);
+  const redCircle = roots.find((r) => r.start !== 0);
+  assert.ok(greet, "expected the 'greet' root at query offset 0");
+  assert.ok(redCircle, "expected the 'red circle' binding root");
+
+  const guide = gistOf(m, q);
+  const primary = enc("PRIMARYANCHORTEXT");
+  // forest[0] is never independently projected (see reasoning.ts: it is
+  // treated as already primary's own source) — a placeholder keeps BOTH
+  // real roots in `rest`, where this filter actually applies.
+  const placeholder = { ...greet, start: q.length, end: q.length };
+  const pre = {
+    attention: async () => ({
+      roots: [placeholder, greet, redCircle],
+      ranked: [placeholder, greet, redCircle],
+    }),
+    guide,
+  };
+  const out = dec(await fuseAttention(m, q, primary, pre));
+  assert.ok(
+    !out.includes("reply-greet"),
+    `a root whose continuation is already answered in the query must not fuse in, got "${out}"`,
+  );
+  assert.ok(
+    out.includes("answer alpha"),
+    `a genuine further topic with no embedded answer must still fuse in, got "${out}"`,
+  );
+  await m.store.close();
+});
+
+test("fuseAttention: ordinary multi-topic fusion (no embedded answers) is completely unaffected", async () => {
+  const m = mk(1);
+  await m.ingest(CORPUS);
+  const q = enc("red then circle");
+  const roots = await m.climbAttention(q, 24);
+  assert.equal(roots.length, 1, "expected the single joint binding root");
+  // Force a second, independent (unclimbed-style) inclusion path by
+  // reusing the SAME root twice at different synthetic positions, neither
+  // of which has an embedded answer — a pure regression check that the
+  // new filter doesn't fire when there is nothing to catch.
+  const guide = gistOf(m, q);
+  const primary = enc("PRIMARYANCHORTEXT");
+  const placeholder = { ...roots[0], start: q.length, end: q.length };
+  const pre = {
+    attention: async () => ({
+      roots: [placeholder, roots[0]],
+      ranked: [placeholder, roots[0]],
+    }),
+    guide,
+  };
+  const out = dec(await fuseAttention(m, q, primary, pre));
+  assert.ok(
+    out.includes("answer alpha"),
+    `an ordinary further topic must still fuse in, got "${out}"`,
+  );
+  await m.store.close();
+});