@hviana/sema 0.6.0 → 0.7.2

package/src/mind/pipeline-mechanism.ts

@@ -19,7 +19,7 @@ import type { ComputedSpan } from "../extension.js";
 import type { Hit } from "../store.js";
 import type { Vec } from "../vec.js";
 import { indexOf } from "../bytes.js";
-import { conceptThreshold, dominates } from "../geometry.js";
+import { dominates } from "../geometry.js";
 import { windowIds } from "./canonical.js";
 import { read, resolve } from "./primitives.js";
 import {
@@ -30,7 +30,7 @@ import {
   skillExemplar,
 } from "./match.js";
 import { climbAttentionAll } from "./attention.js";
-import { hubBound, sharedReachMemo } from "./traverse.js";
+import { sharedReachMemo } from "./traverse.js";
 
 // ── Precomputed ──────────────────────────────────────────────────────────────
 //
@@ -148,46 +148,17 @@ export class Precomputed {
     );
   }
 
-  private _wide?: Promise<ReadonlyArray<number>>;
-  /** The response's WIDE candidate list — the top-k when the query's gist has
-   *  no concept-level match anywhere, and an exhaustive √N read when it does.
-   *
-   *  Every mechanism that has to look PAST the top-k reads this one list: the
-   *  substitution bridge, prefix completion and the frame filler all did, and
-   *  it was memoised inside recall for exactly that reason (measured: 490 ms
-   *  median re-issued against 13 ms non-exhaustive, 36x).  A memo inside one
-   *  mechanism only serves that mechanism's own tiers, so it lives here now —
-   *  the same move `resonance` made for the top-k.
-   *
-   *  THE CONDITION IS THE TOP HIT'S SCORE, NOT THE CORPUS SIZE.  When nothing
-   *  ranks at concept level, an exhaustive ANN only scores more vectors below
-   *  the bar (profiled at 38K–40K annVectorReads per refusing query on a 325K-
-   *  context store); the structural channels — junction walks, anchor climbs,
-   *  the write side's window index — are the correct proposal source there,
-   *  because the ANN cannot propose what the gist cannot rank.  This was once
-   *  spelled `corpusN(ctx) <= (k · W)³`, which asks a different question and
-   *  answers it wrongly at exactly the scale it was written from: at N =
-   *  325,608 with k = 24 and W = 4 the cube is 884,736, so that store took the
-   *  exhaustive branch — the very branch measured above.  Measured cost of the
-   *  mismatch: substitutionBridge 8,544 ms of a 19,548 ms think (44%), against
-   *  1,248 ms and 14,218 ms without it, every answer byte-identical. */
-  wideResonance(): Promise<ReadonlyArray<number>> {
-    return this._wide ??= this.shared("wideResonance", async () => {
-      const hits = await this.resonance();
-      if (
-        hits.length > 0 &&
-        hits[0].score >= conceptThreshold(this.ctx.store.D)
-      ) {
-        const exhaustive = await this.ctx.store.resonate(
-          this.guide,
-          hubBound(this.ctx),
-          true,
-        );
-        return exhaustive.map((h) => h.id);
-      }
-      return hits.map((h) => h.id);
-    });
-  }
+  // REMOVED — the WIDE exhaustive-√N resonance list (`wideResonance`).  It ran
+  // `resonate(guide, √N, exhaustive=true)` whenever the top hit cleared
+  // conceptThreshold, so consumers could look "past the top-k".  Every consumer
+  // only ever needed ≤ 2·recallQueryK proposals (the substitution bridge's own
+  // candidate cap) or a content-addressed answer (prefix completion's
+  // formsOpenedBy), and every proposal is byte-verified downstream (§2.3), so
+  // the exhaustive scan bought recall at O(index) cost for an O(k) need —
+  // measured: 244K annVectorReads per refusing query, ~1.5 s, every answer
+  // byte-identical to a top-k read.  The two consumers now read `resonance()`
+  // (the one top-k read) and the write side's window index respectively — see
+  // recall.ts and prefix-completion.ts.
 
   private _frames?: Promise<ReadonlyArray<FrameInstance>>;
   /** THE FRAME INVENTORY — every ranked candidate that reads as an instance of

package/src/mind/pipeline.ts

@@ -129,6 +129,31 @@ export interface NarrowDecisionData {
   margin: number;
 }
 
+/** Structured payload of the "regimePrediction" rationale step — the R8
+ *  observation exposed as data.  After the first mechanism (cover, which §2.6
+ *  runs first) grounds or abstains, the market's whole outcome is already
+ *  determined by the one cost ladder: the consensus climb runs exactly when
+ *  `worthRunning(2 * STEP)` is true — CAST (floor 2·STEP) is the cheapest
+ *  mechanism that first-touches it, and confluence (3·STEP) / extraction
+ *  (CONCEPT+STEP) are only reached after CAST is.  An incumbent at or below
+ *  that floor prunes CAST and, with it, the climb (retrieval); anything above
+ *  — or no incumbent — runs the full market and the climb (composition).
+ *  Purely observational; never read by inference. */
+export interface RegimePredictionData {
+  version: 1;
+  /** retrieval | composition — the two regimes R1 measured as a ~100× cost
+   *  step. */
+  regime: "retrieval" | "composition";
+  /** The incumbent's grade once the first mechanism's turn is over (it ran, or
+   *  it was skipped), or null when nothing has grounded — `best === null`,
+   *  which is composition with no incumbent. */
+  incumbentGrade: number | null;
+  /** The cheapest composition floor in grade units (`grade(2 * STEP)` = 2,
+   *  CAST's floor) — the bar the incumbent must sit at or below for the
+   *  consensus climb to be skipped. */
+  climbFloorGrade: number;
+}
+
 /** Think: a single lightest-derivation exploration of the Sema graph.
  *
  *  Every answer travels the same path:
@@ -161,10 +186,25 @@ export async function think(
 
   // ── Pre-computation ──────────────────────────────────────────────────
   const mechanisms = mechs ?? defaultMechanisms;
-  const rec = recognise(ctx, query);
+  const meter = ctx.meter;
+  // recognition is a shared analysis (§2.14 contract 5): it does the query's
+  // own store work (perceive → foldTree → resolve), which used to land in
+  // `think` and in nothing narrower — the meter's one accounting surface must
+  // charge it to itself, exactly as attention/weave/resonance are charged.
+  // SYNCHRONOUS phase: recognition is on the sync side of §2.10's seam, so it
+  // is timed with `timeSync` — wrapping it in a promise would make a profiled
+  // response await where an unprofiled one does not.
+  const rec = meter
+    ? meter.timeSync("recognise", () => recognise(ctx, query))
+    : recognise(ctx, query);
 
   // Phase 1: collect computed spans from mechanisms that implement parse()
-  const computed = await collectComputed(ctx, mechanisms, query);
+  const computed = meter
+    ? await meter.time(
+      "collectComputed",
+      () => collectComputed(ctx, mechanisms, query),
+    )
+    : await collectComputed(ctx, mechanisms, query);
 
   if (computed.length > 0) {
     ctx.trace?.step(
@@ -191,6 +231,9 @@ export async function think(
   // method on Precomputed, first-touched by whichever mechanism's floor
   // survives its cheap gates and the worthRunning check.  A query no
   // mechanism climbs for (e.g. one an extension decided) never climbs.
+  // NOT phased: the constructor itself is trivial (it only derives `k`), so a
+  // phase here would add a zero-work entry to every profiled report — the meter
+  // attributes WORK (§2.14); the trace already represents structure.
   const pre = new Precomputed(ctx, query, rec, computed, ctx._edgeGuide);
 
   // ── Grounding: ONE lightest-derivation choice among the mechanisms ────
@@ -255,12 +298,66 @@ export async function think(
   const worthRunning = (floor: number) =>
     best === null || grade(floor) < grade(best.weight);
 
+  // REGIME PREDICTION (R8) — observational only.  Once the FIRST mechanism has
+  // had its turn (cover, which §2.6 places first and floors at 0), the market's
+  // outcome is already determined by the one cost ladder: the consensus climb
+  // runs exactly when `worthRunning(2 * STEP)` is true — CAST (floor 2·STEP) is
+  // the cheapest mechanism that first-touches it, so an incumbent at or below
+  // grade 2 prunes CAST and, with it, confluence (3·STEP) and extraction
+  // (CONCEPT+STEP) (retrieval); anything above — or no incumbent — runs the
+  // full market and the climb (composition).  The predicate is `worthRunning`,
+  // the same function the loop itself uses — nothing is computed here that the
+  // engine had not already computed, and nothing is read back by inference.
+  //
+  // EMITTED BEFORE THE SECOND MECHANISM'S FLOOR, never after some mechanism's
+  // run: a "prediction" published after the fact could assert "the climb will
+  // not run" about a climb that already ran — which is what happens whenever
+  // the first mechanism is SKIPPED (null floor or pruned) and the block sits at
+  // the end of the first mechanism that actually ran.  Emitting on entry to
+  // iteration 1 makes the claim true by construction, whatever the first
+  // mechanism did, and keeps the payload identical on the ordinary path (the
+  // incumbent cannot change between the two positions).
+  let regimeReported = false;
+  const reportRegime = () => {
+    if (regimeReported) return;
+    regimeReported = true;
+    const climbFloorGrade = grade(2 * STEP);
+    // TS narrows `best` to null in the outer flow (it cannot see the closure
+    // assignments in `consider`) — cast back, the same read-back as `decided`
+    // below.
+    const incumbent = best as Candidate | null;
+    const incumbentGrade = incumbent === null ? null : grade(incumbent.weight);
+    const regime: "retrieval" | "composition" = worthRunning(2 * STEP)
+      ? "composition"
+      : "retrieval";
+    ctx.trace?.step(
+      "regimePrediction",
+      [rItem(query, "query")],
+      [],
+      regime === "retrieval"
+        ? `retrieval regime — incumbent grade ${incumbentGrade} ≤ climb floor ${climbFloorGrade}, ` +
+          `so no mechanism floored above that grade runs; the consensus climb will not run`
+        : `composition regime — ${
+          incumbentGrade === null
+            ? "no incumbent (nothing grounded)"
+            : `incumbent grade ${incumbentGrade}`
+        } above climb floor ${climbFloorGrade}, so the full market and climb run`,
+      undefined,
+      {
+        version: 1,
+        regime,
+        incumbentGrade,
+        climbFloorGrade,
+      } satisfies RegimePredictionData,
+    );
+  };
   // Phase 3: grounding loop
   // Per-mechanism accounting (src/meter.ts).  The market's whole premise is
   // that mechanisms compete on one cost scale — so the profiling read-out is
   // also per-mechanism, uniformly: the loop never asks which one it holds.
-  const meter = ctx.meter;
-  for (const mech of mechanisms) {
+  for (let mi = 0; mi < mechanisms.length; mi++) {
+    const mech = mechanisms[mi];
+    if (mi > 0) reportRegime();
     const floor = meter
       ? await meter.time(
         `${mech.name}.floor`,
@@ -309,6 +406,10 @@ export async function think(
       });
     }
   }
+  // A market of ONE mechanism never reaches iteration 1; the step is still
+  // emitted exactly once per think(), so a consumer never has to ask whether
+  // the list was long enough for the prediction to exist.
+  reportRegime();
 
   // (TS cannot see the closure assignments into `best` and narrows it to its
   // initial null, so the read-back needs the assertion.)
@@ -508,7 +609,7 @@ export async function think(
       reasoned,
       pre,
       unclimbed,
-      decided.accounted,
+      primarySpans,
     );
 
   done(

package/src/mind/recognition.ts

@@ -32,8 +32,25 @@ import type { Leaf, Site } from "./graph-search.js";
  *     the longest known leaf, chained into flat branches.  Names forms the
  *     query's own cut cannot, and records sub-leaf boundaries as `splits`.
  *
- *  Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus. */
-export function recognise(ctx: MindContext, bytes: Uint8Array): Recognition {
+ *  Both O(n · maxGroup) bounded O(1) probes — never a scan of the corpus.
+ *
+ *  ONE READING PER BYTE STREAM, deliberately: there is no "cheap mode" that
+ *  skips the edge-trim fallbacks.  A `trimmed` variant was tried and REFUTED
+ *  twice over.  Its premise — "the trims only recover misaligned FRAGMENTS, so
+ *  a consumer whose gate rejects fragments loses nothing" — is false: the
+ *  left/right trim loops below exist precisely to find WHOLE trained forms
+ *  embedded at an offset the query's own fold did not cut, and such a form has
+ *  no structural parents or containers, so it passes the pivot's fragment gate
+ *  and is exactly the candidate a multi-hop chain steps through.  Skipping them
+ *  narrows the pivot's evidence silently.  And a per-caller variant has to key
+ *  the memo by the variant, which breaks the "computed at most once" property
+ *  (§2.11): the pipeline recognises a grounded answer untrimmed for
+ *  `preConsumed`, and the pivot then recognises the same bytes again — the
+ *  saving inverts into a doubling on the path it was measured for. */
+export function recognise(
+  ctx: MindContext,
+  bytes: Uint8Array,
+): Recognition {
   // Content-keyed memo — works for both single-turn respond() and multi-turn
   // respondTurn() (where the map persists across calls).  ALWAYS consulted,
   // regardless of tracing — matching perceive()'s own memo, which carries no

package/src/mind/resonance.ts

@@ -12,7 +12,7 @@ import { concat2, concatBytes, indexOf } from "../bytes.js";
 import type { MindContext } from "./types.js";
 import { gistOf, read, resolve, walkTree } from "./primitives.js";
 import { perceive } from "./primitives.js";
-import { argmaxBy, argmaxCosine, candidateGist, hubBound } from "./traverse.js";
+import { argmaxCosine, candidateGist, hubBound } from "./traverse.js";
 import {
   cachedRead,
   type Junction,
@@ -365,6 +365,12 @@ export async function pivotInto(
     }
     for (const c of n.kids) queue.push(c); // breadth-first: larger regions first
   }
+  // THE FULL recognition, memo-shared with every other reader of these bytes.
+  // A "skip the edge trims here" variant was refuted (see recognise's own
+  // note): those trims are what find a WHOLE trained form embedded at an
+  // offset the answer's fold did not cut, and such a form is parentless,
+  // container-free and edge-bearing — i.e. exactly what the filter below
+  // ADMITS as a pivot, not what it rejects.
   const rec = recognise(ctx, answer);
   for (const s of rec.sites) {
     if (!consumed.has(s.payload) && ctx.store.hasNext(s.payload)) {
@@ -373,54 +379,83 @@ export async function pivotInto(
   }
   // Byte containment, longest wins — the answer literally contains the
   // pivot's bytes, and the biggest well-evidenced span is the real pivot.
-  const found = argmaxBy(
-    scored.keys(),
-    (id) => {
-      // A PIVOT MUST BE A THING THE CORPUS DEPOSITED, NOT A PIECE OF ONE.
-      // "Longest wins" ranks candidates but never asks whether the winner is
-      // an entity at all, and by the time a chain reaches here `consumeAll`
-      // has taken the answer's real contexts — so on a corpus of
-      // near-identical records the field is left to whatever interned
-      // fragments remain.  Measured on a 200-line templated log corpus, query
-      // "what happened to request_id=1042 and request_id=1077?": CAST
-      // produced the correct comparison and one `pivotStep` replaced it
-      // wholesale, pivoting through `s=70` — a four-byte tail of
-      // `latency_ms=70` — onto an unrelated record (`handled 1130`).
-      //
-      // The separator is NOT length.  Measured against the multi-hop tests'
-      // own pivots: `Paris` (5 bytes), `Jupiter` (7), `lithium` (7), `Mona
-      // Lisa` (9) against junk `s=70` (4) — a two-quantum floor, which
-      // confluence.ts applies to a meet for the same "one window is not an
-      // entity" reason, discards three of the four legitimate pivots.
-      // Entities are simply short.
-      //
-      // What separates them is STRUCTURAL, and the store already holds it:
-      //
-      //   s=70       parents 2  containers 1  prevCount 0  halo no
-      //   Paris      parents 0  containers 0  prevCount 1  halo yes
-      //   Jupiter    parents 0  containers 0  prevCount 1  halo yes
-      //   lithium    parents 0  containers 0  prevCount 1  halo yes
-      //   Mona Lisa  parents 0  containers 0  prevCount 1  halo yes
-      //
-      // A deposited whole — a context or an answer — is interned in its own
-      // right and has neither structural parents nor containment links.  A
-      // fragment is addressable ONLY because window interning made its span
-      // addressable inside something bigger, and that containment is exactly
-      // what `parents`/`containers` record.  Reasoning steps THROUGH a fact;
-      // a span that was never a fact on its own is not one to step through.
-      // No constant enters — it is a structural predicate, not a threshold.
-      if (ctx.store.hasParents(id) || ctx.store.hasContainers(id)) {
-        return -Infinity;
-      }
-      const bytes = read(ctx, id);
-      if (indexOf(answer, bytes, 0) < 0) return -Infinity;
-      for (const v of voiced) if (indexOf(v, bytes, 0) >= 0) return -Infinity;
-      return bytes.length;
-    },
-    0,
-    true,
-  );
-  return found?.item ?? null;
+  //
+  // REAL SATURATION, not a hard cap: the score IS the candidate's byte
+  // length, so the scan is DECIDED the moment the first candidate that passes
+  // every filter is found in DESCENDING length order — a shorter candidate can
+  // never outscore it.  `contentLen` (the prefix-capped length read, §2.8) is
+  // the cheap ordering key, and the first-inserted tie-break is made explicit
+  // (`a.index - b.index`) so equal lengths keep `scored`'s insertion order —
+  // exactly the tie argmaxBy(strict) used to keep.  The bytes of at most ONE
+  // winning candidate are read; every shorter candidate the probes proposed is
+  // skipped without reconstruction, where the old argmax read them all.
+  const ranked = [...scored.keys()]
+    .map((id, index) => ({
+      id,
+      index,
+      len: ctx.store.contentLen(id, answer.length + 1),
+    }))
+    .sort((a, b) => b.len - a.len || a.index - b.index);
+  let pivotId: number | null = null;
+  for (const c of ranked) {
+    const id = c.id;
+    // A ZERO-LENGTH candidate is not a pivot.  `argmaxBy(…, 0, strict)` used to
+    // carry this floor in its threshold argument, and dropping it here would
+    // admit an empty node: `indexOf(answer, <empty>)` returns 0, so every
+    // filter below passes and the chain would hop through nothing (§2.13 —
+    // empty bytes are truthy).
+    if (c.len === 0) continue;
+    // A PIVOT MUST BE A THING THE CORPUS DEPOSITED, NOT A PIECE OF ONE.
+    // "Longest wins" ranks candidates but never asks whether the winner is
+    // an entity at all, and by the time a chain reaches here `consumeAll`
+    // has taken the answer's real contexts — so on a corpus of
+    // near-identical records the field is left to whatever interned
+    // fragments remain.  Measured on a 200-line templated log corpus, query
+    // "what happened to request_id=1042 and request_id=1077?": CAST
+    // produced the correct comparison and one `pivotStep` replaced it
+    // wholesale, pivoting through `s=70` — a four-byte tail of
+    // `latency_ms=70` — onto an unrelated record (`handled 1130`).
+    //
+    // The separator is NOT length.  Measured against the multi-hop tests'
+    // own pivots: `Paris` (5 bytes), `Jupiter` (7), `lithium` (7), `Mona
+    // Lisa` (9) against junk `s=70` (4) — a two-quantum floor, which
+    // confluence.ts applies to a meet for the same "one window is not an
+    // entity" reason, discards three of the four legitimate pivots.
+    // Entities are simply short.
+    //
+    // What separates them is STRUCTURAL, and the store already holds it:
+    //
+    //   s=70       parents 2  containers 1  prevCount 0  halo no
+    //   Paris      parents 0  containers 0  prevCount 1  halo yes
+    //   Jupiter    parents 0  containers 0  prevCount 1  halo yes
+    //   lithium    parents 0  containers 0  prevCount 1  halo yes
+    //   Mona Lisa  parents 0  containers 0  prevCount 1  halo yes
+    //
+    // A deposited whole — a context or an answer — is interned in its own
+    // right and has neither structural parents nor containment links.  A
+    // fragment is addressable ONLY because window interning made its span
+    // addressable inside something bigger, and that containment is exactly
+    // what `parents`/`containers` record.  Reasoning steps THROUGH a fact;
+    // a span that was never a fact on its own is not one to step through.
+    // No constant enters — it is a structural predicate, not a threshold.
+    if (ctx.store.hasParents(id) || ctx.store.hasContainers(id)) continue;
+    // A candidate whose bytes are LONGER than the answer cannot be a
+    // substring of it — `indexOf` would return −1 regardless.  Prune by
+    // length BEFORE reconstructing the bytes: `read` is an UNCAPPED read
+    // (AGENTS §2.8), and a resonated context far longer than the answer is
+    // exactly the candidate that makes it cost a whole deposit's worth of
+    // reconstruction for a containment test that must fail.  `contentLen`
+    // with the `answer.length + 1` cap is the prefix-capped length read the
+    // same contract prescribes; the prune is byte-identical to the old
+    // `indexOf` miss (it returns −1 for a needle longer than the haystack).
+    if (c.len > answer.length) continue;
+    const bytes = read(ctx, id);
+    if (indexOf(answer, bytes, 0) < 0) continue;
+    if (voiced.some((v) => indexOf(v, bytes, 0) >= 0)) continue;
+    pivotId = id;
+    break;
+  }
+  return pivotId;
 }
 
 /** Which of the given labelled forms a span MEANS — generic resonance over

package/src/store.ts

@@ -1269,7 +1269,27 @@ export abstract class AbstractStore implements Store {
       parts.push(child);
       got += child.length;
     }
-    return concat(parts);
+    const out = concat(parts);
+    // Cache the BRANCH too, not just the leaf above.  Reconstruction is a pure
+    // function of the store, so this is a transparent cache in the strict sense
+    // — an eviction costs a re-walk and nothing else — which is exactly what
+    // `_bytesCache`'s "smallest"/"clock" configuration is for.
+    //
+    // Caching only leaves made every branch re-walk its whole subtree on every
+    // request, and the DAG is hash-consed, so the same children recur under many
+    // parents.  Measured on the 18.9M-node store, ONE 1,314-byte query:
+    // 20,021,474 `_prefix` calls over 469,083 distinct ids (42.7x reuse) to
+    // produce 87,789 results — 97.7% of the work re-derived bytes it had already
+    // built.  One single-byte leaf was reconstructed 2,599,984 times.  Measuring
+    // reuse at the TOP level only shows 1.1x and hides all of it.
+    //
+    // Only a COMPLETE reconstruction may be cached: `_prefix` is also called
+    // with a cap, and a truncated prefix stored under `id` would be served as
+    // if it were the node's whole content by the `_bytesCache` hit above.
+    // `got < maxLen` is that proof — the walk ran out of children before it ran
+    // out of budget, so nothing below was truncated either.
+    if (got < maxLen) this._bytesCache.set(id, out);
+    return out;
   }
 
   contentLen(id: NodeId, cap = Infinity): number {