@hviana/sema 0.2.3 → 0.2.5

package/dist/src/mind/traverse.js

@@ -6,7 +6,6 @@
 // The PROJECTIONS built on these walks (follow, conceptHop, reverseContext,
 // project) live in match.ts — the elementary match-and-project operation.
 import { cosine } from "../vec.js";
-import { consensusFloor } from "../geometry.js";
 import { gistOf, read } from "./primitives.js";
 const structCaches = new WeakMap();
 function getStructCache(ctx) {
@@ -90,7 +89,22 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
     // is withdrawn.  On a small store the floor stays ≤ √N and the atom
     // climbs exactly as before, so single-letter facts keep working.
     if (id < 0 && atomIsHub(ctx, contextCount)) {
-        const reach = { roots: [], contextsReached: 0, saturated: true };
+        const bound0 = Math.ceil(Math.sqrt(Math.max(2, contextCount)));
+        const reach = {
+            roots: [],
+            contextsReached: 0,
+            saturated: true,
+            ...(ctx.trace
+                ? {
+                    saturation: {
+                        reason: "byte-atom-commonality",
+                        node: id,
+                        observed: atomReach(ctx, contextCount),
+                        limit: bound0,
+                    },
+                }
+                : {}),
+        };
         memo?.set(id, reach);
         return reach;
     }
@@ -99,6 +113,10 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
     const seen = new Set([id]);
     const ctxSeen = new Set();
     let saturated = false;
+    // Provenance of the FIRST decision that saturated this climb — allocated
+    // only when a trace is requested (see AncestorReach.saturation's doc); the
+    // climb itself never reads it back.
+    let satStop;
     // EXPAND-UNTIL-DECIDED: a reach is consumed either as a VOTE (which needs
     // contextsReached exactly, and only while ≤ √N — beyond that the region is
     // non-discriminative) or as an ABSTENTION (saturated — whose roots and
@@ -140,16 +158,46 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
             roots.push(x);
             if (hasNx)
                 ctxSeen.add(x);
-            if (pc > bound)
-                return false; // decided: ≥ pc > √N distinct contexts
+            if (pc > bound) {
+                // decided: ≥ pc > √N distinct contexts
+                if (ctx.trace) {
+                    satStop = {
+                        reason: "predecessor-fan-in",
+                        node: x,
+                        observed: pc,
+                        limit: bound,
+                    };
+                }
+                return false;
+            }
             for (const p of ctx.store.prevFirst(x, bound))
                 ctxSeen.add(p);
-            if (ctxSeen.size > bound)
-                return false; // decided
+            if (ctxSeen.size > bound) {
+                // decided
+                if (ctx.trace) {
+                    satStop = {
+                        reason: "distinct-context-limit",
+                        node: x,
+                        observed: ctxSeen.size,
+                        limit: bound,
+                    };
+                }
+                return false;
+            }
         }
         const parents = ctx.store.parentsFirst(x, bound + 1);
-        if (parents.length > bound)
-            return false; // decided: hub
+        if (parents.length > bound) {
+            // decided: hub
+            if (ctx.trace) {
+                satStop = {
+                    reason: "parent-fan-out",
+                    node: x,
+                    observed: parents.length,
+                    limit: bound,
+                };
+            }
+            return false;
+        }
         let fresh = 0;
         for (const p of parents) {
             if (!seen.has(p)) {
@@ -160,8 +208,18 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
         }
         if (fresh > 1) {
             lateral += fresh - 1;
-            if (lateral > bound)
-                return false; // decided: cone-wide hub
+            if (lateral > bound) {
+                // decided: cone-wide hub
+                if (ctx.trace) {
+                    satStop = {
+                        reason: "lateral-cone-limit",
+                        node: x,
+                        observed: lateral,
+                        limit: bound,
+                    };
+                }
+                return false;
+            }
         }
         return true;
     };
@@ -226,7 +284,12 @@ export function edgeAncestors(ctx, id, contextCount, memo) {
             }
         }
     }
-    const reach = { roots, contextsReached: ctxSeen.size, saturated };
+    const reach = {
+        roots,
+        contextsReached: ctxSeen.size,
+        saturated,
+        ...(saturated && satStop ? { saturation: satStop } : {}),
+    };
     memo?.set(id, reach);
     return reach;
 }
@@ -440,20 +503,22 @@ export function chooseNext(ctx, id, guide) {
             bestMass = mass;
         }
     }
-    // A pick among GENUINELY competing continuations still needs to clear the
-    // same genuine-corroboration floor {@link consensusFloor} draws for every
-    // other consumer that turns distinct-context support into a vote (the
-    // climb's recallByResonance/commitVotes) — below it, "most corroborated"
-    // is only the least-thin echo among several, not real evidence.  Gated to
-    // corpus scale large enough that this echo is even possible (the same
-    // scale {@link atomIsHub} already switches on for the identical reason:
-    // at small N every edge IS the evidence there is).  A hub id has no
-    // meaningful "distinct context" reading at all (id < 0 atoms are excluded
-    // from this path already, since chooseNext only ever sees real edges).
-    const N = corpusN(ctx);
-    if (capped.length > 1 && atomIsHub(ctx, N) && bestSupport < consensusFloor(N)) {
-        return undefined;
-    }
+    // NO consensusFloor gate here (tried and reverted — see
+    // test/40-choosenext-scale-guard.test.mjs): that floor is calibrated for
+    // POOLED, IDF-weighted CLIMB VOTES (recallByResonance, commitVotes), where
+    // each corroborating region contributes at most ln N and the floor grows
+    // with N exactly as that per-region ceiling does (HOW_IT_WORKS.md §8.6).
+    // `bestSupport` here is a different kind of quantity — a raw prevCount of
+    // how many training contexts predicted ONE destination, bounded by how
+    // often that specific fact was retold, never by corpus size N.  Gating an
+    // N-invariant count against an N-growing threshold guarantees failure
+    // once N is large enough, discarding genuinely, structurally dominant
+    // edges (observed: a fact corroborated 2-to-1-1-1 refused at N≈325K,
+    // falling back to a noisy concept-hop).  The loop above already IS the
+    // "genuinely competing" test: a tie leaves first-inserted as the pick
+    // (test/30's own pinned behaviour); a strict winner is real evidence
+    // regardless of corpus scale.  Matches HOW_IT_WORKS.md §25's own
+    // chooseNext pseudocode, which has no such floor.
     // Trace is built lazily — the filter + map below only execute when a
     // trace listener is attached, so the common (no-trace) path pays only
     // for the prevCount calls in the loop above, never for extra rItemShort

package/dist/src/mind/types.d.ts

@@ -74,6 +74,23 @@ export interface Attention {
      *  consensus; one that does not is a coincidental single-region echo —
      *  see test/35-attention-confidence.test.mjs. */
     breadth: number;
+    /** DISPERSION: the number of distinct clusters this point's contributing
+     *  regions form, merging any two whose gap is under one river-fold
+     *  quantum W.  Neither breadth NOR raw region count discriminates a
+     *  genuine further topic from a coincidental echo (both were tried and
+     *  falsified — breadth starves a genuine, evenly-split multi-topic query,
+     *  since no root in a real N-way split can exceed half the vote; raw
+     *  count doesn't separate them either, since a short, structurally simple
+     *  echo racks up as many corroborating regions as a real topic does).
+     *  Dispersion asks a different question: not how MUCH evidence, but how
+     *  many separate PLACES in the query corroborate it.  A coincidental
+     *  match — one local phrase resonating with an unrelated stored form —
+     *  is structurally confined to ONE cluster no matter how strong its vote;
+     *  a genuine further topic is named in its own distinctive wording
+     *  somewhere the query's scaffolding does not reach, always a SEPARATE
+     *  cluster from whatever else corroborates it.  See
+     *  test/37-cluster-dispersion-fusion.test.mjs. */
+    clusters: number;
 }
 /** Both read-outs of one consensus climb. */
 export interface AttentionRead {
@@ -116,11 +133,29 @@ export interface RegionVote {
      *  Defaults to 1 when absent. */
     absorbed?: number;
 }
+/** The structural gate that first decided an {@link edgeAncestors} climb was
+ *  saturated (an abstention, not a discriminative conclusion) — pure
+ *  instrumentation for {@link ClimbConsensusData}'s reach trace; it never
+ *  feeds back into the climb itself. */
+export type SaturationReason = "byte-atom-commonality" | "predecessor-fan-in" | "distinct-context-limit" | "parent-fan-out" | "lateral-cone-limit";
+/** One saturation stop's provenance: which reason fired, at which node, the
+ *  observed count against the bound that decided it. */
+export interface SaturationStop {
+    reason: SaturationReason;
+    node: number;
+    observed: number;
+    limit: number;
+}
 /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
 export interface AncestorReach {
     roots: number[];
     contextsReached: number;
     saturated: boolean;
+    /** The saturation gate that stopped this climb, when {@link saturated} is
+     *  true and a trace was requested — see {@link edgeAncestors}.  Absent for
+     *  a non-saturated reach, and absent (even when saturated) when no trace
+     *  was requested — instrumentation must not allocate when tracing is off. */
+    saturation?: SaturationStop;
 }
 /** Saturated-interval information for the noise-drop gate. */
 export interface SaturationInfo {
@@ -226,9 +261,29 @@ export interface MindContext extends GraphSearchHost {
 export declare const ALL = 2147483647;
 /** Splice every chosen span in order — the whole cover as one byte string. */
 export declare function spliceAll(segs: Seg[]): Uint8Array | null;
+/** Whether a chosen span RESTATES the query rather than answering it: its
+ *  SUBSTITUTED bytes (an edge followed from a recognised site, not the
+ *  site's own literal text read back) already occur elsewhere in the query
+ *  — the same principle recall.ts's tiers apply to a whole-query projection
+ *  ("a projection that is a proper byte-subspan of the query restates part
+ *  of the question").  A LITERAL span (the site's own bytes, unchanged) is
+ *  exempt: naming what's already there at its OWN position is not a
+ *  substitution.  A recognised site that is itself an entire PRIOR TURN of
+ *  a multi-turn query is exactly this shape: it carries a genuine learnt
+ *  continuation, but that continuation is something the asker already said
+ *  moments later in the SAME query, not a new answer.  Below one river
+ *  window, byte overlap is chance, not evidence — the same floor
+ *  identityBar and reachThreshold hold every other structural-overlap claim
+ *  to. */
+export declare function segRestatesQuery(s: Seg, query: Uint8Array, queryLen: number, W: number): boolean;
 /** Lift the answer out of the cover for think: the recognised region, free of
- *  the asker's surrounding (unrecognised) framing. */
-export declare function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null;
+ *  the asker's surrounding (unrecognised) framing — and free of any chosen
+ *  span that only RESTATES content the query already contains (see {@link
+ *  segRestatesQuery}).  A restating span is excluded from both the framing
+ *  (lo/hi) decision and the final concatenation: it is stale, not a second
+ *  answer, but the OTHER spans a derivation chose are independent evidence
+ *  and must not be discarded along with it. */
+export declare function liftAnswer(segs: Seg[], queryLen: number, query: Uint8Array, W: number): Uint8Array | null;
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */
 export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];

package/dist/src/mind/types.js

@@ -2,7 +2,7 @@
 //
 // GraphSearchHost is defined first (minimal imports) so GraphSearch can import
 // it without pulling in the full MindContext.
-import { concatBytes } from "../bytes.js";
+import { bytesEqual, concatBytes, indexOf } from "../bytes.js";
 import { dominates } from "../geometry.js";
 // ═══════════════════════════════════════════════════════════════════════════
 // FREE FUNCTIONS (pure, no state)
@@ -15,13 +15,44 @@ export function spliceAll(segs) {
         return null;
     return concatBytes(segs.map((s) => s.bytes));
 }
+/** Whether a chosen span RESTATES the query rather than answering it: its
+ *  SUBSTITUTED bytes (an edge followed from a recognised site, not the
+ *  site's own literal text read back) already occur elsewhere in the query
+ *  — the same principle recall.ts's tiers apply to a whole-query projection
+ *  ("a projection that is a proper byte-subspan of the query restates part
+ *  of the question").  A LITERAL span (the site's own bytes, unchanged) is
+ *  exempt: naming what's already there at its OWN position is not a
+ *  substitution.  A recognised site that is itself an entire PRIOR TURN of
+ *  a multi-turn query is exactly this shape: it carries a genuine learnt
+ *  continuation, but that continuation is something the asker already said
+ *  moments later in the SAME query, not a new answer.  Below one river
+ *  window, byte overlap is chance, not evidence — the same floor
+ *  identityBar and reachThreshold hold every other structural-overlap claim
+ *  to. */
+export function segRestatesQuery(s, query, queryLen, W) {
+    if (!s.rec)
+        return false;
+    const literal = s.j - s.i === s.bytes.length &&
+        bytesEqual(s.bytes, query.subarray(s.i, s.j));
+    if (literal)
+        return false;
+    return s.bytes.length >= W && s.bytes.length < queryLen &&
+        indexOf(query, s.bytes, 0) >= 0;
+}
 /** Lift the answer out of the cover for think: the recognised region, free of
- *  the asker's surrounding (unrecognised) framing. */
-export function liftAnswer(segs, queryLen) {
+ *  the asker's surrounding (unrecognised) framing — and free of any chosen
+ *  span that only RESTATES content the query already contains (see {@link
+ *  segRestatesQuery}).  A restating span is excluded from both the framing
+ *  (lo/hi) decision and the final concatenation: it is stale, not a second
+ *  answer, but the OTHER spans a derivation chose are independent evidence
+ *  and must not be discarded along with it. */
+export function liftAnswer(segs, queryLen, query, W) {
+    const restated = segs.map((s) => segRestatesQuery(s, query, queryLen, W));
     const recognised = [];
-    for (let k = 0; k < segs.length; k++)
-        if (segs[k].rec)
+    for (let k = 0; k < segs.length; k++) {
+        if (segs[k].rec && !restated[k])
             recognised.push(k);
+    }
     if (recognised.length === 0)
         return null;
     if (recognised.length === 1) {
@@ -42,13 +73,13 @@ export function liftAnswer(segs, queryLen) {
         if (s.computed && s.i > 0)
             return s.bytes;
         if (dominates(s.j - s.i, queryLen)) {
-            return concatBytes(segs.map((x) => x.bytes));
+            return concatBytes(segs.filter((_, k) => !restated[k]).map((x) => x.bytes));
         }
         return s.bytes;
     }
     const lo = recognised[0];
     const hi = recognised[recognised.length - 1];
-    return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
+    return concatBytes(segs.slice(lo, hi + 1).filter((_, k) => !restated[lo + k]).map((x) => x.bytes));
 }
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */

package/dist/src/store.d.ts

@@ -139,8 +139,17 @@ export interface Store {
      *  question is decided, with per-page work bounded by `limit`. */
     containersSlice(child: NodeId, offset: number, limit: number): NodeId[];
     nodeCount(): number;
-    /** The k nodes whose gist resonates most with v. */
-    resonate(v: Vec, k: number): Promise<Hit[]>;
+    /** The k nodes whose gist resonates most with v.  `exhaustive` widens the
+     *  IVF probe to every cluster (see {@link AbstractStore.efFor}'s doc) —
+     *  for refusal-path-only callers where an approximate top-√C-clusters
+     *  search is not the same discriminator as the caller's own byte-exact
+     *  verification (the substitution bridge's proposal channel): a rarer
+     *  paraphrase can score lower than hundreds of unrelated hits by pure
+     *  fold-geometry structural distance (a middle-of-string mismatch
+     *  perturbs the tree hash far more than a tail mismatch of the same
+     *  byte length) and so never even reach a probed cluster, no matter how
+     *  large k is — k only reorders WITHIN the clusters already probed. */
+    resonate(v: Vec, k: number, exhaustive?: boolean): Promise<Hit[]>;
     /** Mark a node as a RESONANCE TARGET — promote its gist into the content
      *  index so {@link resonate} can find it.  A node's gist is captured at intern
      *  but indexed LAZILY (only targets are indexed; the ~99.5% intermediate DAG
@@ -593,7 +602,7 @@ export declare abstract class AbstractStore implements Store {
      *
      *  Iterative explicit-queue walk: the call stack never sees tree depth. */
     protected indexSubtree(root: NodeId): void;
-    resonate(v: Vec, k: number): Promise<Hit[]>;
+    resonate(v: Vec, k: number, exhaustive?: boolean): Promise<Hit[]>;
     indexedVectorCount(): number;
     lastResonateReads(): number;
     /** How many physical compaction attempts have failed this session.  Zero in

package/dist/src/store.js

@@ -1135,7 +1135,7 @@ export class AbstractStore {
         }
     }
     // ── Soft resonance ─────────────────────────────────────────────────────
-    async resonate(v, k) {
+    async resonate(v, k, exhaustive = false) {
         await this._ensureReady();
         // Synchronous flush of any buffered index writes: the FIRST resonance
         // after a large ingest pays that flush here, so it shows up in respond
@@ -1148,14 +1148,20 @@ export class AbstractStore {
         // same values still hits.  Lazy-init: null after any index write; the
         // first miss after a flush recreates it.  When voteRegions resonates
         // identical perceived sub-regions, only the first call descends the ANN.
-        const rk = vecKey(v) + ":" + k;
+        const rk = vecKey(v) + ":" + k + (exhaustive ? ":x" : "");
         const cache = this._resonateCache;
         if (cache) {
             const hit = cache.get(rk);
             if (hit !== undefined)
                 return hit;
         }
-        const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, this.efFor(this._vecContentClusterCount()));
+        const clusters = this._vecContentClusterCount();
+        const results = this._vecContentQuery(normalize(copy(v)), k * this.overfetch, 
+        // Exhaustive: probe every cluster (ef ≥ 4·clusters guarantees the
+        // IVF's own ef→nprobe=ceil(ef/4) mapping reaches all of them) — the
+        // natural ceiling for a search that is ALREADY refusal-path-only and
+        // must not miss a candidate hiding in an unprobed cluster.
+        exhaustive ? 4 * clusters : this.efFor(clusters));
         const out = [];
         for (const r of results) {
             const id = r.id;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hviana/sema",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Sema: a non-parametric, instance-based reasoning system.",
   "type": "module",
   "main": "dist/src/index.js",

package/src/geometry.ts

@@ -7,7 +7,7 @@
 //      identically regardless of what follows — pure structural stability.
 //   3. The same rule recurses level after level until one root remains.
 
-import { normalize, Vec } from "./vec.js";
+import { addInto, copy, normalize, Vec, zeros } from "./vec.js";
 import { Sema, sema, Space } from "./sema.js";
 import { Alphabet } from "./alphabet.js";
 
@@ -461,8 +461,10 @@ function fold2(space: Space, a: Folded, b: Folded): Folded {
 /** Plain river fold WITHOUT the final root normalize — the segment-level
  *  building block of {@link stablePrefixFold} (interiors must keep their
  *  byte-proportional magnitude; only the whole perception's root is ever
- *  normalized). */
-function riverFoldRaw(space: Space, row: Folded[]): Folded {
+ *  normalized).  Exported so callers that COMPOSE already-existing structural
+ *  parts into a hypothetical synthetic root (see {@link composeStructuralGist})
+ *  can feed the same raw primitive instead of duplicating its mathematics. */
+export function riverFoldRaw(space: Space, row: Folded[]): Folded {
   if (row.length === 0) {
     const z = new Float32Array(space.D);
     return { tree: sema(z, new Uint8Array(0), null), len: 0 };
@@ -477,6 +479,53 @@ function riverFoldRaw(space: Space, row: Folded[]): Folded {
   return level[0];
 }
 
+// ---- structural composition (synthesize from EXISTING structural parts) ----
+
+/** One already-existing structural vector to compose, paired with the byte
+ *  span (query-slot) length it stands in for.  `len`, not the vector's own
+ *  magnitude, is what {@link composeStructuralGist} restores — the composed
+ *  slot's NATURAL span, exactly as the linear river fold would carry it. */
+export interface StructuralPart {
+  v: Vec;
+  len: number;
+}
+
+/** Synthesize a hypothetical internal structure from already-existing
+ *  structural vectors — NOT from bytes.  This is the raw positional
+ *  composition the linear river fold already uses (see the folding header
+ *  above): each part is positionally bound into its own seat, its natural
+ *  span magnitude is preserved, the parts are linearly superposed, and only
+ *  the final synthetic root is normalized.  It never calls {@link gistOf}
+ *  (there is no `gistOf` here — geometry.ts has no store), never perceives a
+ *  concatenated byte string, and never interns or stores a new node: the
+ *  result is an opaque, ungrounded Vec for an ANN probe only. */
+export function composeStructuralGist(
+  space: Space,
+  parts: readonly StructuralPart[],
+): Vec {
+  const foldedParts: Folded[] = [];
+
+  for (const part of parts) {
+    if (part.len <= 0) continue;
+
+    const direction = copy(part.v);
+    normalize(direction);
+
+    const scaled = zeros(space.D);
+    addInto(scaled, direction, Math.sqrt(part.len));
+
+    foldedParts.push({ tree: sema(scaled), len: part.len });
+  }
+
+  if (foldedParts.length === 0) return zeros(space.D);
+
+  const rawRoot = riverFoldRaw(space, foldedParts);
+
+  const result = copy(rawRoot.tree.v);
+  normalize(result);
+  return result;
+}
+
 // ---- pyramid fold (incremental plain perception) ----
 
 /** The PLAIN fold's full level pyramid — every level's item list, bottom