@hviana/sema 0.9.0 → 0.9.2

package/src/mind/pipeline.ts

@@ -14,9 +14,10 @@ import { PASS, STEP } from "./graph-search.js";
 import type { ComputedSpan } from "../extension.js";
 import { gistOf, read, resolve } from "./primitives.js";
 import { recognise } from "./recognition.js";
-import { fuseAttention, reason } from "./reasoning.js";
+import { fusionLayer, walkLayer } from "./reasoning.js";
 import {
   closed,
+  closeOver,
   type DerivationState,
   remainderOf,
   type Span,
@@ -26,8 +27,14 @@ import {
 } from "./derivation.js";
 import { rItem } from "./trace.js";
 import { unexplainedLabel } from "./rationale.js";
-import { hubBound } from "./traverse.js";
-import { type PipelineMechanism, Precomputed } from "./pipeline-mechanism.js";
+import { hubBound, offsetCanon, scaffoldExtents } from "./traverse.js";
+import { windowIndex, witness } from "./evidence.js";
+import { indexOf } from "../bytes.js";
+import {
+  type MechanismResult,
+  type PipelineMechanism,
+  Precomputed,
+} from "./pipeline-mechanism.js";
 import { coverMechanism } from "./mechanisms/cover.js";
 import { castMechanism } from "./mechanisms/cast.js";
 import { confluenceMechanism } from "./mechanisms/confluence.js";
@@ -38,7 +45,7 @@ import { recallMechanism } from "./mechanisms/recall.js";
 
 // Re-exports: cover's pre-resolution helpers and the ALU adapter kept
 // importable from the pipeline module (their historical home).
-export { resolveConcepts, resolveConnectors } from "./mechanisms/cover.js";
+export { offerConcepts, offerConnectors } from "./mechanisms/cover.js";
 export { aluToMechanism } from "./mechanisms/alu.js";
 
 // ── Extension dispatch (pre-loop parse) ─────────────────────────────────────
@@ -161,6 +168,10 @@ export interface RegimePredictionData {
    *  CAST's floor) — the bar the incumbent must sit at or below for the
    *  consensus climb to be skipped. */
   climbFloorGrade: number;
+  /** The lowest grade reached by a mechanism run AHEAD of a dearer one (see
+   *  the grounding loop), when one ran and grounded: a bound below
+   *  `climbFloorGrade` makes the regime retrieval whatever the incumbent. */
+  boundGrade?: number;
 }
 
 /** Think: a single lightest-derivation exploration of the Sema graph.
@@ -180,6 +191,13 @@ export async function think(
   if (query.length === 0) return null;
 
   ctx._edgeGuide = gistOf(ctx, query);
+  {
+    const asked = offsetCanon(ctx, query);
+    ctx._edgeAsked = {
+      bytes: asked,
+      index: windowIndex(asked, ctx.space.maxGroup),
+    };
+  }
   ctx._edgeChoice.clear();
 
   const t = ctx.trace?.enter("think", [rItem(query, "query")]);
@@ -335,7 +353,7 @@ export async function think(
     // below.
     const incumbent = best as Candidate | null;
     const incumbentGrade = incumbent === null ? null : grade(incumbent.weight);
-    const regime: "retrieval" | "composition" = worthRunning(2 * STEP)
+    const regime: "retrieval" | "composition" = worthDeclared(2 * STEP)
       ? "composition"
       : "retrieval";
     ctx.trace?.step(
@@ -343,8 +361,10 @@ export async function think(
       [rItem(query, "query")],
       [],
       regime === "retrieval"
-        ? `retrieval regime — incumbent grade ${incumbentGrade} ≤ climb floor ${climbFloorGrade}, ` +
-          `so no mechanism floored above that grade runs; the consensus climb will not run`
+        ? (incumbentGrade !== null && incumbentGrade <= climbFloorGrade
+          ? `retrieval regime — incumbent grade ${incumbentGrade} ≤ climb floor ${climbFloorGrade}, `
+          : `retrieval regime — grade ${bound} already reached by a mechanism run ahead, below climb floor ${climbFloorGrade}, `) +
+          `so no mechanism floored above that grade runs; CAST will not climb`
         : `composition regime — ${
           incumbentGrade === null
             ? "no incumbent (nothing grounded)"
@@ -356,6 +376,7 @@ export async function think(
         regime,
         incumbentGrade,
         climbFloorGrade,
+        ...(bound !== Infinity ? { boundGrade: bound } : {}),
       } satisfies RegimePredictionData,
     );
   };
@@ -363,19 +384,139 @@ export async function think(
   // Per-mechanism accounting (src/meter.ts).  The market's whole premise is
   // that mechanisms compete on one cost scale — so the profiling read-out is
   // also per-mechanism, uniformly: the loop never asks which one it holds.
-  for (let mi = 0; mi < mechanisms.length; mi++) {
-    const mech = mechanisms[mi];
-    if (mi > 0) reportRegime();
+  //
+  // A CHEAPER BOUND IS LOOKED AT BEFORE A DEARER ONE IS PAID FOR.
+  //
+  // The declared order is the tie-break priority, and the pruning above is
+  // only as strong as the incumbent it has: a mechanism floored LOWER than the
+  // one about to invest, but declared after it, could not prune it.  Measured
+  // on the 31.7M-node store: a lowercased dialogue turn (#97 of the battery)
+  // was won by recall at grade 1 — after CAST (floor grade 2) had paid the
+  // consensus climb and the weave, confluence (3) a reach climb, and extraction
+  // and reference their reads: ~10 s of a 12.6 s response, for candidates that
+  // could not win.
+  //
+  // So before mechanism `m` first-touches anything, every LATER mechanism whose
+  // bound is strictly lower runs AHEAD of it, cheapest bound first, and the
+  // lowest grade any of them reaches becomes `bound`.  A mechanism whose floor
+  // grade exceeds `bound` is then skipped.  THE DECISION IS UNCHANGED — the
+  // same candidate wins as in the declared order:
+  //   • a mechanism `p` run ahead with best grade g bounds the final grade by
+  //     g: in the declared order p either runs (its candidate is weighed) or is
+  //     pruned by an incumbent already at or below p's floor ≤ g;
+  //   • so a mechanism floored above `bound` has only candidates the final
+  //     winner strictly outgrades — and with every candidate above `bound`
+  //     dropped, every mechanism floored at or below it meets the same
+  //     run-or-prune decision (`f < incumbent` iff `f < min(incumbent,
+  //     bound + 1)` for f ≤ bound) and yields the same candidates;
+  //   • and the winner is chosen from the candidates at or below `bound`, in
+  //     declared order — `consider` replays them where they are declared.
+  // Equal-grade floors are NOT skipped (≤, not <): a mechanism declared earlier
+  // keeps the tie it would have won.  Running `p` ahead is never extra work:
+  // what prunes p in the declared order is a candidate at or below p's floor,
+  // which only a mechanism floored at or below it can produce — and every such
+  // mechanism is either already run or run ahead of p.
+  //
+  // The bound is learnt by asking `floor` with a `worthRunning` that refuses:
+  // under the investment discipline (pipeline-mechanism.ts) a floor that cannot
+  // pay returns its bound UNINVESTED, so the question costs no analysis.
+  const refuse = () => false;
+  const probed: Array<number | null | undefined> = new Array(
+    mechanisms.length,
+  );
+  const probeGrade = async (i: number): Promise<number | null> => {
+    if (probed[i] === undefined) {
+      const f = await mechanisms[i].floor(ctx, query, pre, refuse);
+      probed[i] = f === null ? null : grade(f);
+    }
+    return probed[i]!;
+  };
+  /** Per mechanism: its floor once computed, and its results once run. */
+  const floors = new Map<number, number | null>();
+  const runs = new Map<number, MechanismResult[]>();
+  let bound = Infinity;
+  const worthAhead = (floor: number) =>
+    grade(floor) <
+      Math.min(best === null ? Infinity : grade(best.weight), bound);
+  const worthDeclared = (floor: number) =>
+    worthRunning(floor) && grade(floor) <= bound;
+  const floorOf = async (
+    i: number,
+    worth: (floor: number) => boolean,
+  ): Promise<number | null> => {
+    if (floors.has(i)) return floors.get(i)!;
+    const mech = mechanisms[i];
     const floor = meter
       ? await meter.time(
         `${mech.name}.floor`,
-        () => mech.floor(ctx, query, pre, worthRunning),
+        () => mech.floor(ctx, query, pre, worth),
       )
-      : await mech.floor(ctx, query, pre, worthRunning);
+      : await mech.floor(ctx, query, pre, worth);
     if (meter) {
       if (floor === null) meter.mechanismSkips++;
       else meter.mechanismFloors++;
     }
+    floors.set(i, floor);
+    return floor;
+  };
+  const runOf = async (i: number): Promise<MechanismResult[]> => {
+    let results = runs.get(i);
+    if (results === undefined) {
+      const mech = mechanisms[i];
+      if (meter) meter.mechanismRuns++;
+      results = meter
+        ? await meter.time(`${mech.name}.run`, () => mech.run(ctx, query, pre))
+        : await mech.run(ctx, query, pre);
+      runs.set(i, results);
+    }
+    return results;
+  };
+  const runAhead = async (mi: number) => {
+    const g = await probeGrade(mi);
+    if (g === null) return;
+    const ahead: Array<[number, number]> = [];
+    for (let j = mi + 1; j < mechanisms.length; j++) {
+      if (floors.has(j)) continue;
+      const gj = await probeGrade(j);
+      if (gj !== null && gj < g) ahead.push([gj, j]);
+    }
+    ahead.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+    for (const [gj, j] of ahead) {
+      // Only what the declared order would also run: a bound that cannot beat
+      // what is already held is not even asked for its real floor.
+      if (
+        !(gj < Math.min(best === null ? Infinity : grade(best.weight), bound))
+      ) {
+        continue;
+      }
+      const floor = await floorOf(j, worthAhead);
+      if (floor === null || !worthAhead(floor)) continue;
+      ctx.trace?.step(
+        "runAhead",
+        [],
+        [],
+        `${mechanisms[j].name} runs ahead of ${
+          mechanisms[mi].name
+        } — its floor (grade ${grade(floor)}) is below ${
+          mechanisms[mi].name
+        }'s (grade ${g}), so its result bounds what ${
+          mechanisms[mi].name
+        } could win`,
+      );
+      for (const r of await runOf(j)) {
+        // `consider` drops an empty answer, so it bounds nothing.
+        if (r.bytes.length === 0) continue;
+        bound = Math.min(bound, grade(weigh(r.accounted, r.moves)));
+      }
+    }
+  };
+  for (let mi = 0; mi < mechanisms.length; mi++) {
+    const mech = mechanisms[mi];
+    if (mi > 0) {
+      await runAhead(mi);
+      reportRegime();
+    }
+    const floor = await floorOf(mi, worthDeclared);
     if (floor === null) {
       ctx.trace?.step(
         "skipMechanism",
@@ -385,6 +526,16 @@ export async function think(
       );
       continue;
     }
+    if (grade(floor) > bound) {
+      if (meter) meter.mechanismsBounded++;
+      ctx.trace?.step(
+        "skipMechanism",
+        [],
+        [],
+        `${mech.name} skipped — floor ${floor} cannot beat grade ${bound}, already reached by a mechanism run ahead`,
+      );
+      continue;
+    }
     if (!worthRunning(floor)) {
       ctx.trace?.step(
         "skipMechanism",
@@ -396,10 +547,7 @@ export async function think(
       );
       continue;
     }
-    if (meter) meter.mechanismRuns++;
-    const results = meter
-      ? await meter.time(`${mech.name}.run`, () => mech.run(ctx, query, pre))
-      : await mech.run(ctx, query, pre);
+    const results = await runOf(mi);
     for (const r of results) {
       // ONE FORMULA, EVERY CANDIDATE: the chart's derivation reports how many
       // discrete moves it made and which bytes it could not recognise; the
@@ -544,25 +692,34 @@ export async function think(
   // without evidence stays owed, and a later transition pays it only by carrying
   // it (the law reads the window; see derivation.ts).  Same reading, one
   // definition — not a second spelling of it here.
-  const explained: Array<[number, number]> = [
+  const priced: Array<[number, number]> = [
     ...decided.accounted,
     ...pre.computed.map((u): [number, number] => [u.i, u.j]),
-  ].filter(([a, b]) =>
+  ];
+  const explained = priced.filter(([a, b]) =>
     windowOf([a, b], answer, query, ctx.space.maxGroup) !== null
   );
-  // WHAT THE CONSTRUCTION WITHHOLDS, at or above one quantum: the difference between
-  // the remainder paid in full and the remainder paid by carrying.  Both readings
-  // are the law's, so the floor is applied once and in one place.
-  const paidInFull = remainderOf(
+  // THE DERIVATION IS BORN OWING ITS DISCRIMINATIVE MATERIAL.  The bytes a
+  // corpus-global scaffolding window reaches (traverse.ts, `scaffoldExtents`)
+  // are nobody's debt — otherwise a later step could claim to pay them by
+  // restating ` is `, which every fact holds, and the law (which measures
+  // carrying by any window of what is owed) would admit it; the substitution
+  // bridge reads gaps the same way (`explainedSpan`).  PRICING is untouched:
+  // the ladder still charges every unexplained byte, because for a question
+  // made of nothing but scaffolding (`How are you today?`) covering those bytes
+  // IS the evidence.
+  const scaffold = scaffoldExtents(ctx, query);
+  const paid = remainderOf(
     query.length,
-    [
-      ...decided.accounted,
-      ...pre.computed.map((u): [number, number] => [u.i, u.j]),
-    ],
+    [...explained, ...scaffold],
     ctx.space.maxGroup,
   );
-  const paid = remainderOf(query.length, explained, ctx.space.maxGroup);
+  // WHAT THE CONSTRUCTION WITHHOLDS, at or above one quantum: the difference between
+  // the remainder paid in full and the remainder paid by carrying.  Both readings
+  // are the law's, so the floor is applied once and in one place — and the
+  // paid-in-full reading is computed only when a meter will read it.
   if (ctx.meter) {
+    const paidInFull = remainderOf(query.length, priced, ctx.space.maxGroup);
     ctx.meter.groundingWithheldBytes += unaccountedBytes(paid) -
       unaccountedBytes(paidInFull);
   }
@@ -577,8 +734,31 @@ export async function think(
   const uncovered = state.remainder;
 
   // ── Post-grounding, gated by the declaration and the remainder ────────
+  // WHAT THE GROUNDING SPOKE FOR, when it did not declare it: the forms inside
+  // the answer that the ASKER already holds.  An answer is the asker's material
+  // plus what the corpus added — the fact's own value — and only the former was
+  // spoken for: re-entering it walks back to the question.  The latter is what a
+  // chain continues THROUGH.  Consuming every form recognised in the answer
+  // consumed that too, so a chain could start only where recognition happened
+  // to miss the next entity — measured on 133 held-out 2Wiki compositional
+  // questions, 3 pivot steps in all once interior recognition named the entity
+  // inside every fact.  Witnessed exactly, as `chooseNext`'s exact tier reads the
+  // question (evidence.ts); a form below one window cannot be witnessed, and is
+  // consumed as before.
+  const asked = ctx._edgeAsked;
   const preConsumed = declaredUsed ??
-    new Set(recognise(ctx, answer).sites.map((s) => s.payload));
+    new Set(
+      recognise(ctx, answer).sites
+        .filter((s) =>
+          asked === null || s.end - s.start < ctx.space.maxGroup ||
+          witness(
+            offsetCanon(ctx, answer.subarray(s.start, s.end)),
+            [asked.index],
+            ctx.space.maxGroup,
+          ).complete
+        )
+        .map((s) => s.payload),
+    );
   // A grounding that DECLARED itself complete is not extended: the answer is
   // already a trained form's own continuation, reached through an identity
   // claim about the query, so a multi-hop pivot could only chain past the
@@ -607,8 +787,17 @@ export async function think(
   // `preConsumed` is derived by re-recognising the answer — "everything in
   // it", not "what it voiced" — and a containment rule over that would
   // suppress every pivot the answer legitimately contains.
+  //
+  // …and a continuation the ANSWER ITSELF holds was voiced, not withheld.  A
+  // substitution that answers with its anchor's own continuation (`The director
+  // of Beat Girl is Edmond T. Gréville.`) declared that anchor used, so reading
+  // its continuations as withheld refused every pivot inside the answer — the
+  // chain could never step past the entity the first hop introduced.
   const voiced = declaredUsed === undefined ? [] : [...declaredUsed].flatMap(
-    (id) => ctx.store.nextFirst(id, hubBound(ctx)).map((n) => read(ctx, n)),
+    (id) =>
+      ctx.store.nextFirst(id, hubBound(ctx))
+        .map((n) => read(ctx, n))
+        .filter((v) => indexOf(answer, v, 0) < 0),
   );
   // WHAT THIS BRANCH READ, published where it was read.  Post-grounding decides
   // by the DECLARATION (`decided.used`, which becomes `voiced`), by what the
@@ -664,89 +853,59 @@ export async function think(
     meter.postGroundingRemainderSpans += uncovered.length;
     meter.postGroundingRemainderBytes += unaccountedBytes(uncovered);
   }
-  // THE WALK THAT DOES NOT RUN, NAMED — the audit's point 1 could not attribute
-  // real questions that stopped with no note anywhere.  They never reached the
-  // offer: `decided.complete` says the grounding supplied a fixed point, so the
-  // walk is skipped BY DESIGN and the state is the grounding's own.  That state
-  // carries `fixed: true` (the trace already reports it), and the law's first
-  // clause refuses any continuation against it — so there is no `null` here to
-  // read as exhaustion, and the `Offer` contract is not in play at all.  The
-  // silent stop is therefore a NAMED state, not a gap.
+  // ── THE CLOSURE ENGINE ───────────────────────────────────────────────
   //
-  // THE WALK CONSUMES AND RETURNS A STATE.  It is handed the derivation's own —
-  // the grounding's product, accounting, remainder and cost — and hands back the
-  // state it advanced to, so what follows reads a state rather than bytes plus a
-  // tuple rebuilt here.  A supplied fixed point is the one case where the walk
-  // does not run at all, and then the state is the grounding's own.
-  const extension = decided.complete ? undefined : meter
-    ? await meter.time(
-      "reason",
-      () => reason(ctx, query, state, preConsumed, pre, voiced),
-    )
-    : await reason(ctx, query, state, preConsumed, pre, voiced);
-  const reasoned = extension ?? state;
-
-  // Fuse only when the query has a genuine REMAINDER no mechanism's
-  // structural evidence touched at all.  `decided.accounted` alone
-  // undercounts this: it is a COST-LADDER quantity (cover.ts prices its
-  // masked/computed spans at near-zero and deliberately leaves them out of
-  // `accounted` so PASS-bridged bytes are still charged), not a coverage
-  // one — a query fully explained by one computed span plus bridged
-  // connectors can report `accounted: []` while nothing is actually left
-  // unexplained.  The genuine remainder is what NEITHER the winning
-  // candidate's accounted spans NOR any recognised extension's computed
-  // span (`pre.computed` — every mechanism's parse() output, ALU included)
-  // ever touched.  A remainder under one river-fold quantum (W, the same
-  // floor cover.ts's restatedSpan and the honesty-density bar above both
-  // use) is bridging punctuation/whitespace, never a second topic —
-  // observed: a single space between two fully-computed arithmetic spans
-  // ("2+2 3+3") registered as "unaccounted" and pulled in an unrelated
-  // corpus fact, corrupting "4 6" into "4 63".
-  // THE GATE ASKS THE LAW, and that is an OPTIMISATION, not a tidy-up: the state
-  // above ALREADY carries the remainder (`remainderOf`, per-span, with the W
-  // floor applied), so asking it costs nothing, while the total this line used to
-  // compute (`unaccounted(explained)`) was one more sum over the spans on every
-  // response.  The two readings are the same condition, not two: the ACCOUNTING
-  // applies the same W floor the gate does, so a gap below one quantum never
-  // survives into `explained` and the total cannot reach W without some single
-  // gap reaching it.  Measured over twelve constructions at W = 4 (test/136.3,
-  // which pins the equivalence and both sides of it).
-  // Whether the winning candidate's entire recognised substance is
-  // COMPUTED — every accounted span exactly a pre.computed span, nothing
-  // from a genuinely recognised/climbed site.  fuseAttention's lone-root
-  // shortcut assumes a single point of attention already IS primary's own
-  // source; that assumption is exactly backwards for a pure computation
-  // (an ALU result has no anchor of its own) — see fuseAttention's
-  // `unclimbed` parameter, gated there by Attention.breadth so a
-  // coincidental echo (which this flag alone cannot distinguish) is still
-  // rejected.
+  // The grounding's state is CLOSED under the law by two layers, in order: the
+  // multi-hop WALK (`walkLayer`: a forward absorb or a pivot, offered one at a
+  // time) and the multi-topic FUSION (`fusionLayer`: one composed transition).
+  // Every step either layer offers goes through the same `closure` and the same
+  // law; this function no longer sequences them or gates them by hand.
+  //
+  // WHAT USED TO BE TWO HAND-WRITTEN GATES IS NOW THE LAW'S, OR THE LAYER'S:
+  //   • a declared-complete grounding (`fixed`) admits no transition — the
+  //     engine enters no layer for it (the law's first clause), which is the
+  //     walk-skip and the fusion-skip this branch used to spell separately;
+  //   • a CLOSED derivation has nothing left for a second topic to account for,
+  //     so the fusion layer does not ENGAGE — read off the state the walk
+  //     reached, the one fusion is actually offered against.
+  //
+  // What the fusion needs from the grounding — whether its substance is purely
+  // computed (`unclimbed`) and where it stands in the query (`primarySpans`) —
+  // is the grounding's own evidence, resolved here where both readings are in
+  // hand.
+  //
+  // Whether the winning candidate's entire recognised substance is COMPUTED —
+  // every accounted span exactly a pre.computed span, nothing from a genuinely
+  // recognised/climbed site.  fuseAttention's lone-root shortcut assumes a
+  // single point of attention already IS primary's own source; that assumption
+  // is exactly backwards for a pure computation (an ALU result has no anchor of
+  // its own) — gated there by Attention.breadth so a coincidental echo (which
+  // this flag alone cannot distinguish) is still rejected.
   const unclimbed = state.accounted.length > 0 &&
     state.accounted.every(([i, j]) =>
       pre.computed.some((u) => u.i === i && u.j === j)
     );
-  // Where the winning grounding stands in the query — fusion places primary
-  // by it (see fuseAttention's `primarySpans`).  `accounted` is the
-  // cost-ladder read and is authoritative when non-empty; when it is empty
-  // the grounding is a pure COMPUTATION, whose evidence is its computed span.
-  // Exactly the cost-ladder-vs-coverage distinction `explained` above draws,
-  // read here for POSITION instead of for coverage — and resolved here, where
-  // both readings are in hand, rather than inside fuseAttention.
+  // Where the winning grounding stands in the query — fusion places primary by
+  // it.  `accounted` is the cost-ladder read and is authoritative when
+  // non-empty; when it is empty the grounding is a pure COMPUTATION, whose
+  // evidence is its computed span — the cost-ladder-vs-coverage distinction
+  // `explained` above draws, read here for POSITION instead of coverage.
   const primarySpans: ReadonlyArray<Span> = state.accounted.length > 0
     ? state.accounted
     : pre.computed.map((u): [number, number] => [u.i, u.j]);
-  const fused = closed(state) ? reasoned : meter
-    ? await meter.time(
-      "fuse",
-      () => fuseAttention(ctx, query, reasoned, pre, unclimbed, primarySpans),
-    )
-    : await fuseAttention(
-      ctx,
-      query,
-      reasoned,
-      pre,
-      unclimbed,
-      primarySpans,
-    );
+  const fused = await closeOver(
+    state,
+    query,
+    ctx.space.maxGroup,
+    [
+      walkLayer(ctx, query, preConsumed, pre, voiced),
+      {
+        ...fusionLayer(ctx, query, pre, unclimbed, primarySpans),
+        engages: (d: DerivationState) => !closed(d),
+      },
+    ],
+    meter ? (name, walk) => meter.time(name, walk) : undefined,
+  );
 
   done(
     fused.product,