@hviana/sema 0.5.2 → 0.5.3

package/src/mind/pipeline-mechanism.ts

@@ -16,14 +16,21 @@
 import type { AncestorReach, MindContext, Recognition } from "./types.js";
 import type { AttentionRead } from "./types.js";
 import type { ComputedSpan } from "../extension.js";
+import type { Hit } from "../store.js";
 import type { Vec } from "../vec.js";
 import { indexOf } from "../bytes.js";
-import { dominates } from "../geometry.js";
+import { conceptThreshold, dominates } from "../geometry.js";
 import { windowIds } from "./canonical.js";
 import { read, resolve } from "./primitives.js";
-import { alignGraded, type GradedRun, skillExemplar } from "./match.js";
+import {
+  alignGraded,
+  type FrameInstance,
+  frameSlots,
+  type GradedRun,
+  skillExemplar,
+} from "./match.js";
 import { climbAttentionAll } from "./attention.js";
-import { sharedReachMemo } from "./traverse.js";
+import { hubBound, sharedReachMemo } from "./traverse.js";
 
 // ── Precomputed ──────────────────────────────────────────────────────────────
 //
@@ -125,6 +132,116 @@ export class Precomputed {
     return meter ? meter.time(phase, fn) : fn();
   }
 
+  private _resonance?: Promise<ReadonlyArray<Hit>>;
+  /** The response's ONE top-k content-index read: the k learnt forms nearest
+   *  the whole-query gist, ranked.  Recall's every gist tier is built on it,
+   *  and {@link frames} assembles the frame inventory from it.
+   *
+   *  An ANN query is the single most expensive read in the engine, and two
+   *  mechanisms asking the same question of the same gist is the one
+   *  duplication a profile shows as doubled `annVectorReads` with nothing to
+   *  account for it.  Cached BY PROMISE, so a second caller awaits the first. */
+  resonance(): Promise<ReadonlyArray<Hit>> {
+    return this._resonance ??= this.shared(
+      "resonance",
+      () => this.ctx.store.resonate(this.guide, this.k),
+    );
+  }
+
+  private _wide?: Promise<ReadonlyArray<number>>;
+  /** The response's WIDE candidate list — the top-k when the query's gist has
+   *  no concept-level match anywhere, and an exhaustive √N read when it does.
+   *
+   *  Every mechanism that has to look PAST the top-k reads this one list: the
+   *  substitution bridge, prefix completion and the frame filler all did, and
+   *  it was memoised inside recall for exactly that reason (measured: 490 ms
+   *  median re-issued against 13 ms non-exhaustive, 36x).  A memo inside one
+   *  mechanism only serves that mechanism's own tiers, so it lives here now —
+   *  the same move `resonance` made for the top-k.
+   *
+   *  THE CONDITION IS THE TOP HIT'S SCORE, NOT THE CORPUS SIZE.  When nothing
+   *  ranks at concept level, an exhaustive ANN only scores more vectors below
+   *  the bar (profiled at 38K–40K annVectorReads per refusing query on a 325K-
+   *  context store); the structural channels — junction walks, anchor climbs,
+   *  the write side's window index — are the correct proposal source there,
+   *  because the ANN cannot propose what the gist cannot rank.  This was once
+   *  spelled `corpusN(ctx) <= (k · W)³`, which asks a different question and
+   *  answers it wrongly at exactly the scale it was written from: at N =
+   *  325,608 with k = 24 and W = 4 the cube is 884,736, so that store took the
+   *  exhaustive branch — the very branch measured above.  Measured cost of the
+   *  mismatch: substitutionBridge 8,544 ms of a 19,548 ms think (44%), against
+   *  1,248 ms and 14,218 ms without it, every answer byte-identical. */
+  wideResonance(): Promise<ReadonlyArray<number>> {
+    return this._wide ??= this.shared("wideResonance", async () => {
+      const hits = await this.resonance();
+      if (
+        hits.length > 0 &&
+        hits[0].score >= conceptThreshold(this.ctx.store.D)
+      ) {
+        const exhaustive = await this.ctx.store.resonate(
+          this.guide,
+          hubBound(this.ctx),
+          true,
+        );
+        return exhaustive.map((h) => h.id);
+      }
+      return hits.map((h) => h.id);
+    });
+  }
+
+  private _frames?: Promise<ReadonlyArray<FrameInstance>>;
+  /** THE FRAME INVENTORY — every ranked candidate that reads as an instance of
+   *  the same frame as the query, each with the query spans it leaves VARIABLE
+   *  ({@link FrameInstance}).  The one place the engine represents "a position
+   *  whose occupant comes from the context rather than the corpus".
+   *
+   *  AN INVENTORY, NOT AN ELECTION.  It reports every pairing and elects no
+   *  frame, deliberately: a slot is a property of a PAIRING, not of the query,
+   *  and different candidates put slots in different places.  Committing to one
+   *  reading here would push whichever consumer asked first onto everyone else
+   *  — the market's decoupling (§2.6) broken from inside the shared container,
+   *  and the population error §2.7 names.  Each consumer groups and commits
+   *  for its own question; reference elects the modal slot signature, and a
+   *  consumer wanting a different reading is not fighting this one.
+   *
+   *  NO LICENCE EITHER.  Knowing a span is variable is safe for every consumer
+   *  — it can only improve an alignment.  Knowing one may be VOICED through is
+   *  a different and much stronger claim, gated separately by
+   *  {@link carriesFillers}, which needs projections this must not perform. */
+  frames(): Promise<ReadonlyArray<FrameInstance>> {
+    return this._frames ??= this.shared("frames", async () => {
+      const ctx = this.ctx;
+      const W = ctx.space.maxGroup;
+      // PHRASE SCALE, the same bound the bridge and the frame filler put on a
+      // candidate's bytes: a form an order of magnitude longer than the query
+      // is not a candidate for BEING it with a span replaced.
+      const capBytes = this.query.length * W;
+      const out: FrameInstance[] = [];
+      for (const h of await this.resonance()) {
+        // REJECT BY LENGTH BEFORE RECONSTRUCTING (§2.8): `contentLen` is an
+        // indexed read, `bytesPrefix` rebuilds a subtree.  ONLY the phrase-scale
+        // cap is applied — it is a bounded-read discipline, not a judgement.
+        //
+        // A LOWER bound was here too (`dominates(len, query.length)`, on the
+        // reasoning that a candidate shorter than half the query cannot supply
+        // a frame that dominates it).  That is reference's gate wearing a cost
+        // argument's clothes, and it hid the very pairings another consumer
+        // needs: `What is the capital of France?` (30 B) against `What is the
+        // capital of the country where the Eiffel Tower is?` (61 B) was
+        // rejected before it was ever read — a definite description standing
+        // where a noun stands, which is exactly the shape the frame filler
+        // exists for.
+        const len = ctx.store.contentLen(h.id, capBytes + 1);
+        if (len === 0 || len > capBytes) continue;
+        const cand = ctx.store.bytesPrefix(h.id, capBytes + 1);
+        if (cand.length === 0 || cand.length > capBytes) continue;
+        const inst = frameSlots(ctx, this.query, cand, h.id);
+        if (inst !== null) out.push(inst);
+      }
+      return out;
+    });
+  }
+
   private _attention?: Promise<AttentionRead>;
   /** The full consensus climb (roots + ranked anchors) — the query-level
    *  evidence CAST, confluence, extraction, recall's scaffolding tier, and

package/src/mind/pipeline.ts

@@ -23,6 +23,8 @@ import { coverMechanism } from "./mechanisms/cover.js";
 import { castMechanism } from "./mechanisms/cast.js";
 import { confluenceMechanism } from "./mechanisms/confluence.js";
 import { extractionMechanism } from "./mechanisms/extraction.js";
+import { referenceMechanism } from "./mechanisms/reference.js";
+import { prefixMechanism } from "./mechanisms/prefix-completion.js";
 import { recallMechanism } from "./mechanisms/recall.js";
 
 // Re-exports: cover's pre-resolution helpers and the ALU adapter kept
@@ -60,13 +62,23 @@ async function collectComputed(
 // floor pruning every mechanism is already subject to — not by asking
 // "is this an extension?". Grade TIES keep the earlier candidate, so this
 // order is also the tie-break priority: cover, cast, confluence, extraction,
-// recall.
+// reference, recall.
+//
+// REFERENCE sits after extraction and before recall because that is what its
+// claim is worth: extraction READS a span out of the query (no synthesis),
+// reference voices one through a learnt slot, and recall's tiers degrade
+// toward echo and silence.  It does not PRUNE recall — its floor is two
+// projections, so recall's one-STEP floor still clears `worthRunning` — and it
+// is not meant to: both run, share one resonance read
+// (Precomputed.resonance), and the ladder decides.
 export const defaultMechanisms: PipelineMechanism[] = [
   coverMechanism,
   castMechanism,
   confluenceMechanism,
   extractionMechanism,
+  referenceMechanism,
   recallMechanism,
+  prefixMechanism,
 ];
 
 // ── think — the main inference pipeline ─────────────────────────────────────
@@ -76,8 +88,10 @@ export type Provenance =
   | "join"
   | "cover"
   | "extract"
+  | "reference"
   | "recall"
-  | "recall-echo";
+  | "recall-echo"
+  | "prefix";
 
 export interface Thought {
   bytes: Uint8Array;

package/src/mind/traverse.ts

@@ -9,7 +9,7 @@
 import { cosine, Vec } from "../vec.js";
 import type { AncestorReach, MindContext, SaturationStop } from "./types.js";
 import { gistOf, read } from "./primitives.js";
-import { leafIdRun } from "./canonical.js";
+import { canonicalWindows, leafIdPrefix, leafIdRun } from "./canonical.js";
 
 // ── Session structural memo ─────────────────────────────────────────────
 //
@@ -836,3 +836,94 @@ export function allWindowsAreScaffolding(
   }
   return sawOne;
 }
+
+// ── THE PREFIX SUPPLY ───────────────────────────────────────────────────────
+//
+// A RETRIEVAL capability, not a grounding one: "which trained forms does this
+// byte run OPEN?"  It lived inside a recall tier, which is the wrong altitude
+// — it reads the write side's own leaf-id window index and answers a question
+// about the STORE, so any mechanism may ask it.
+
+/** Trained forms the query may OPEN, proposed from the write side's own
+ *  leaf-id window index — the supply of last resort for prefix completion.
+ *
+ *  WHY A SECOND SUPPLY EXISTS.  The ranked list prefix completion normally reads
+ *  is a resonance list, and resonance cannot rank a proper prefix: measured on
+ *  the trained store, cos(prefix, form) falls from 0.9629 at a one-byte
+ *  truncation to 0.6206 at three bytes, against a reachThreshold of 0.8750.
+ *  Three bytes of truncation put the answer out of reach on GEOMETRY, not on a
+ *  bug, so no k and no re-ranking recovers it.
+ *
+ *  WHY THIS ROUTE WORKS WHERE THE FOLD DOES NOT.  A query's own fold is
+ *  useless here: content addressing is not phrase-position-invariant, so a
+ *  standalone prefix folds to a DIFFERENT node than the same bytes sitting
+ *  inside a longer deposit, and neither the prefix's own node nor its
+ *  ancestors lead to the deposit (measured: the 22-byte prefix of the
+ *  photosynthesis form resolves, is shared by 6 contexts, and does not have
+ *  the form among its ancestors).  Leaf ids ARE position-invariant — they are
+ *  content-addressed on single bytes — and `indexSubSpans` already interns a
+ *  flat branch over every canonical WINDOW of a deposit's leaf-id stream, with
+ *  containment edges to the chunks that window spans.  A query that is a
+ *  prefix therefore shares those window nodes exactly, and reaches the deposit
+ *  by climbing containment then parents.  Nothing is added to the write side;
+ *  this reads an index training already built.
+ *
+ *  BOUNDED (§2.8), AND WITH NO NEW THRESHOLD.  The window whose containment is
+ *  SMALLEST carries the most evidence, and one saturated at `hubBound` carries
+ *  none — that is the same √N reading of "hub" the rest of the mind uses, not
+ *  a tuned knob.  The upward walk spends a budget of `hubBound` nodes and
+ *  fans out by W, so a hub query enumerates nothing and the caller stays
+ *  silent rather than guessing (§2.13).  Measured on the trained store: the
+ *  photosynthesis form at a one-byte truncation picks a window with 52
+ *  containers, visits 446 nodes, and yields exactly ONE candidate that
+ *  survives the caller's byte compare — the form itself.
+ *
+ *  These are PROPOSALS only.  Every candidate still faces the byte-exact
+ *  prefix compare and all three guards below, so a wrong proposal costs one
+ *  bounded read and can never be voiced (§2.3). */
+export function formsOpenedBy(
+  ctx: MindContext,
+  query: Uint8Array,
+): number[] {
+  const store = ctx.store;
+  const W = ctx.space.maxGroup;
+  const run = leafIdPrefix(ctx, query);
+  // The widest canonical window is the most discriminative one the write side
+  // ever interned; a query too short to spell one carries no window evidence.
+  const len = canonicalWindows(W)[1];
+  if (run.length < len) return [];
+  const bound = hubBound(ctx);
+
+  let best: number | null = null;
+  let bestN = 0;
+  for (let off = 0; off + len <= run.length; off++) {
+    const wid = store.findBranch(run.slice(off, off + len));
+    if (wid === null) continue;
+    const n = store.containersSlice(wid, 0, bound).length;
+    // Empty says the window spans no chunk; saturated says it is a hub, whose
+    // containment discriminates nothing.  Neither is evidence.
+    if (n === 0 || n >= bound) continue;
+    if (best === null || n < bestN) {
+      best = wid;
+      bestN = n;
+    }
+  }
+  if (best === null) return [];
+
+  let frontier = store.containersSlice(best, 0, bound);
+  const seen = new Set<number>(frontier);
+  let budget = bound;
+  while (frontier.length > 0 && budget > 0) {
+    const next: number[] = [];
+    for (const f of frontier) {
+      if (budget-- <= 0) break;
+      for (const p of store.parentsFirst(f, W)) {
+        if (seen.has(p)) continue;
+        seen.add(p);
+        next.push(p);
+      }
+    }
+    frontier = next;
+  }
+  return [...seen];
+}

package/src/store.ts

@@ -970,16 +970,25 @@ export abstract class AbstractStore implements Store {
   /** Content (gist) index write buffer. */
   protected _contentBuffer: Array<{ id: NodeId; vector: Float32Array }> = [];
   /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
-  protected _haloBuffer = new Map<NodeId, Float32Array>();
+  protected _haloBuffer: Map<NodeId, Float32Array> = new Map<
+    NodeId,
+    Float32Array
+  >();
   /** Containment write buffer: child → new parents, merged on flush cadence. */
-  protected _containBuf = new Map<NodeId, Set<NodeId>>();
+  protected _containBuf: Map<NodeId, Set<NodeId>> = new Map<
+    NodeId,
+    Set<NodeId>
+  >();
 
   /** Dedup-target candidates still in the write buffer (keyed by id).  Only
    *  roots that have gained an edge/halo are targets; a fresh intermediate
    *  branch is never folded onto. */
-  protected _nearDedupBuf = new Map<NodeId, Float32Array>();
+  protected _nearDedupBuf: Map<NodeId, Float32Array> = new Map<
+    NodeId,
+    Float32Array
+  >();
   /** Ids currently in `_contentBuffer` (not yet flushed) — O(1) membership. */
-  protected _bufferedIds = new Set<NodeId>();
+  protected _bufferedIds: Set<NodeId> = new Set<NodeId>();
 
   // ── Transparent-chain cache ────────────────────────────────────────────
 

package/test/33-multi-candidate.test.mjs

@@ -171,18 +171,28 @@ test("3 — a near-tie between the winner and runner-up emits narrowDecision", a
     ["water is frigid so water is freezing", "freezing"],
   ]);
   // The near-tie has to be REAL: two candidates whose accounted spans are the
-  // same, separated by a single move.  `steel is hard so steel is` is that —
-  // extraction and recall both explain the whole prefix (grades 21011 and
-  // 21010, margin 1).
+  // same, separated by a single move.  `water is hard so water is ???` is that
+  // — extraction and recall both explain the same span (grades 25011 and
+  // 25010, margin 1).
   //
-  // The old fixture, `steel is frigid so steel is ???`, was a near-tie only
-  // while CAST could not fire on it: the counterfactual subject and the seat
-  // it fills sit on opposite sides of one recurring structure, and CAST's
-  // subject gate used to reject any point whose alignment continued PAST the
-  // seat — which that very recurrence guarantees.  With CAST firing, it leaves
-  // only "???" unexplained and wins by 22008; a decisive win is the opposite
-  // of what this test pins, and test 3b already covers it.
-  const { steps } = await trace(m, "steel is hard so steel is");
+  // THIS FIXTURE HAS NOW BEEN REPLACED TWICE, BOTH TIMES FOR THE SAME REASON:
+  // a mechanism that genuinely explains the query started firing, and a
+  // decisive win is the opposite of what this test pins.
+  //
+  //   * `steel is frigid so steel is ???` was a near-tie only while CAST could
+  //     not fire on it — CAST's subject gate used to reject any point whose
+  //     alignment continued PAST the seat, which that fixture's recurrence
+  //     guarantees.  With CAST firing it wins by 22008.
+  //   * `steel is hard so steel is` was a near-tie only while prefix
+  //     completion was a buried recall TIER.  As a market mechanism it grounds
+  //     that query for what it is — the literal opening of exactly one trained
+  //     form — at grade 1 against 21010, and answers `steel is hard so steel
+  //     is strong`.  That is a better answer, not a regression.
+  //
+  // The lesson for whoever re-fixtures this next: pick a query no mechanism can
+  // explain OUTRIGHT.  A trailing `???` is what keeps this one honest — it is
+  // not the prefix of anything trained.
+  const { steps } = await trace(m, "water is hard so water is ???");
   const narrow = stepsNamed(steps, "narrowDecision");
   assert.equal(narrow.length, 1, "expected exactly one narrowDecision step");
   assert.ok(/margin \d+ grade-unit/.test(narrow[0].note), narrow[0].note);

package/test/70-prefix-completion.test.mjs

@@ -36,7 +36,7 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { Mind } from "../dist/src/index.js";
 import { SQliteStore } from "../dist/src/store-sqlite.js";
-import { prefixCompletion } from "../dist/src/mind/prefix-completion.js";
+import { prefixCompletion } from "../dist/src/mind/mechanisms/prefix-completion.js";
 
 const enc = (s) => new TextEncoder().encode(s);
 const dec = new TextDecoder();

package/test/72-prefix-candidate-supply.test.mjs

@@ -8,7 +8,7 @@
 // 0.9629 at a one-byte truncation to 0.6206 at three bytes, against a
 // reachThreshold of 0.8750.  The mechanism's guards were therefore never
 // reached — the trace read `candidates: 24, opened: 0` — and the query answered
-// nothing.  `prefixCandidates` is the second SUPPLY that closes it, reading the
+// nothing.  `formsOpenedBy` is the second SUPPLY that closes it, reading the
 // leaf-id WINDOW index `indexSubSpans` already writes at deposit time.  No
 // ingestion, storage or fold change is involved: this test would pass on a
 // store trained before the supply existed.
@@ -16,7 +16,7 @@
 // WHY THE ASSERTIONS ARE SHAPED THIS WAY.  On a small fixture resonance may
 // well return the form by luck, and then an end-to-end "does it answer?" test
 // would pass with the supply deleted — pinning nothing.  So the contract is
-// asserted on `prefixCandidates` DIRECTLY, and the resonance list is asserted
+// asserted on `formsOpenedBy` DIRECTLY, and the resonance list is asserted
 // to lack the form, which is what makes the supply load-bearing rather than
 // redundant.
 //
@@ -27,10 +27,8 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { Mind } from "../dist/src/index.js";
 import { SQliteStore } from "../dist/src/store-sqlite.js";
-import {
-  prefixCandidates,
-  prefixCompletion,
-} from "../dist/src/mind/prefix-completion.js";
+import { prefixCompletion } from "../dist/src/mind/mechanisms/prefix-completion.js";
+import { formsOpenedBy } from "../dist/src/mind/traverse.js";
 import { gistOf, resolve } from "../dist/src/mind/primitives.js";
 
 const enc = (s) => new TextEncoder().encode(s);
@@ -71,7 +69,7 @@ test("a proper prefix reaches its trained form through the window supply", async
   // prefix is a LARGE-CORPUS property — cos falls to 0.6206 at a three-byte
   // truncation against a 0.8750 bar, on a 15.7M-node store — and cannot be
   // reproduced at this scale.  That is why the contract below is asserted on
-  // `prefixCandidates` DIRECTLY: deleting or emptying the supply fails this
+  // `formsOpenedBy` DIRECTLY: deleting or emptying the supply fails this
   // test regardless of what resonance happens to return.
   const ranked = (await m.store.resonate(gistOf(m, query), 64)).map((h) =>
     h.id
@@ -84,7 +82,7 @@ test("a proper prefix reaches its trained form through the window supply", async
   );
 
   // THE CONTRACT — the write side's own window index proposes the form.
-  const proposed = prefixCandidates(m, query);
+  const proposed = formsOpenedBy(m, query);
   assert.ok(
     proposed.includes(formId),
     `the window supply must propose the trained form the query opens ` +
@@ -105,7 +103,7 @@ test("a proper prefix reaches its trained form through the window supply", async
   // supply that widened until it found something would be the real defect.
   const hub = enc("The ");
   assert.equal(
-    prefixCompletion(m, hub, prefixCandidates(m, hub)),
+    prefixCompletion(m, hub, formsOpenedBy(m, hub)),
     null,
     "a query carrying no discriminative window must stay silent",
   );

package/test/74-prefix-trap-not-sprung-early.test.mjs

@@ -31,7 +31,7 @@ import { test } from "node:test";
 import assert from "node:assert/strict";
 import { Mind } from "../dist/src/index.js";
 import { SQliteStore } from "../dist/src/store-sqlite.js";
-import { prefixCompletion } from "../dist/src/mind/prefix-completion.js";
+import { prefixCompletion } from "../dist/src/mind/mechanisms/prefix-completion.js";
 import { resolve } from "../dist/src/mind/primitives.js";
 
 const enc = (s) => new TextEncoder().encode(s);