@hviana/sema 0.8.2 → 0.8.5

package/src/mind/attention.ts

@@ -64,6 +64,7 @@ import {
 } from "./junction.js";
 import type { Vec } from "../vec.js";
 import { indexOf } from "../bytes.js";
+import { restates } from "./derivation.js";
 import type { RationaleItem } from "./rationale.js";
 import { rDeriv, rItem, rNode, traceDerivation } from "./trace.js";
 
@@ -180,6 +181,10 @@ export interface ConsensusAnchorTrace {
 
   pooledVote: number;
   idfVote: number;
+  /** The LARGEST single-region contribution behind this anchor — the bar
+   *  recall's own gate reads (mechanisms/recall.ts).  Published so the one
+   *  decision-making quantity the climb computes is not invisible. */
+  peak: number;
 
   candidateBreadth: number;
   contributingVotes: number;
@@ -1243,15 +1248,18 @@ export async function voteRegions(
       }
       contrastiveMargin = margin;
       // Scaled by what this region does NOT address — see `cov` above.
-      const noiseFloor = estimatorNoise(ctx.store.D) * (1 - cov);
-      if (margin <= noiseFloor) {
+      // The bar THIS gate applies: the estimator's noise scaled by what the
+      // region does NOT address (`cov`).  ONE definition, used by the rejection
+      // path below and by the voted payload — the trace reports the applied bar.
+      const appliedFloor = estimatorNoise(ctx.store.D) * (1 - cov);
+      if (margin <= appliedFloor) {
         recordRegion("contrastive-margin-rejection", {
           selected,
           reachNode: voterId,
           idf,
           dfWeight: wf,
           contrastiveMargin: margin,
-          contrastiveNoiseFloor: noiseFloor,
+          contrastiveNoiseFloor: appliedFloor,
           ...(contrastiveRival ? { contrastiveRival } : {}),
         });
         continue;
@@ -1305,7 +1313,12 @@ export async function voteRegions(
       ...(contrastiveMargin !== undefined
         ? {
           contrastiveMargin,
-          contrastiveNoiseFloor: estimatorNoise(ctx.store.D),
+          // THE BAR THE GATE ACTUALLY APPLIED — the same expression
+          // the rejection path's `appliedFloor` defines, inline here because
+          // this payload is built in a scope that does not carry that local.
+          // Publishing the raw estimatorNoise(D) instead made a region that
+          // PASSED look closer to its limit than it was.
+          contrastiveNoiseFloor: estimatorNoise(ctx.store.D) * (1 - cov),
           ...(contrastiveRival ? { contrastiveRival } : {}),
         }
         : {}),
@@ -1438,7 +1451,15 @@ export function poolVotes(
     },
     pool,
   };
-  lightestDerivation(system);
+  // THE SEARCH WAS THE ONE LAYER WITH NO TIME.  The climb's phases are timed
+  // (voteRegions, structuralResonance, crossRegion) but the pooled derivation
+  // was not, so any cost or gain inside it stayed invisible.
+  // `timeSync`, not `time`: the search is SYNCHRONOUS, and wrapping it in a
+  // promise only to time it would make the profiled path wait where the
+  // unprofiled one does not (meter.ts's own contract).
+  if (ctx.meter) {
+    ctx.meter.timeSync("climb.derivation", () => lightestDerivation(system));
+  } else lightestDerivation(system);
 
   const votes = new Map<number, number>();
   const votesIdf = new Map<number, number>();
@@ -1476,12 +1497,24 @@ export function poolVotes(
   // The LARGEST single region's contribution to this anchor's pooled vote.
   // The pool is a SUM (deliberately — see the pooling note above), so it says
   // how much evidence there is in total, never whether any ONE place in the
-  // query carries evidence on its own.  Consumers that hold an anchor to
-  // consensusFloor(N) = ln(N) + 1/2 need the latter: that bar prices ONE
-  // region's maximally-discriminative evidence (ln N is the IDF of content
-  // reaching a single context), so comparing a six-region sum against it is a
-  // dimensional error.  Recorded here, beside the count, because this is the
-  // only place the per-region contributions are still separable.
+  // query carries evidence on its own.  Recorded here, beside the count,
+  // because this is the only place the per-region contributions are still
+  // separable.
+  //
+  // THE BAR IS THE POOLED FLOOR, AND IT WAS ONCE CLAIMED OTHERWISE HERE.
+  // This comment used to say that holding an anchor to consensusFloor(N)
+  // "prices ONE region's evidence", so comparing a six-region sum against it
+  // was "a dimensional error".  THAT WAS FALSE.  `thresholds.md` §2 derives
+  // `consensusFloor` as the POOLED-vote significance floor ("each region
+  // contributes at most ln(N/c) <= ln(N); ln(N) + 1/2 demands ..."), and the
+  // climb weights by IDF, so the sum and the floor are in ONE dimension —
+  // which is exactly why `recall.ts` gates `forest[0].idfVote` against it and
+  // why `commitVotes` does too.  The other two weighting modes DO leave that
+  // dimension (by at most ln 2, two-sided: `direct` deflates a region and
+  // `combined` inflates it), and the gates therefore read the IDF sum, which
+  // is mode-independent; `test/55` tests 19 and 20 pin both halves — the sum
+  // as the reading the bar is derived for, and the absence of any gate
+  // inversion across the three modes.
   const regionPeak = new Map<number, number>();
   const steps: DerivationStep[] = [];
   let order = 0;
@@ -1665,6 +1698,7 @@ export function commitVotes(
         anchor,
         vote,
         peak: regionPeak.get(anchor) ?? 0,
+        idfVote: votesIdf.get(anchor) ?? 0,
         start: s.start,
         end: s.end,
         breadth: (regionSupport.get(anchor) ?? 0) / totalRegions,
@@ -1674,6 +1708,16 @@ export function commitVotes(
         ),
       };
     })
+    // THE ORDER IS NOT A PREFERENCE: with equal evidence it decides ADMISSION,
+    // through the stable sort and the first-come overlap absorption below.
+    // Measured on test/34's corpus, query "red": the two candidates (`red
+    // circle` and `red square`) carry IDENTICAL `vote` and IDENTICAL `idfVote`
+    // (1.3863 each, three seeds), so this comparator leaves them tied and the
+    // stable sort keeps the ENUMERATION order — which is corpus-determined and
+    // admits `red square`, 60/60 seeds.  Adding an id tie-break (`|| a.anchor -
+    // b.anchor`) picks `red circle` instead and makes a single region reach the
+    // JOINT context, which is the premise `test/34` exists to protect.  The
+    // gates read IDF; this line only decides who gets looked at first.
     .sort((a, b) => b.vote - a.vote);
   const overlaps = (a: Attention, b: Attention) =>
     a.start < b.end && b.start < a.end;
@@ -1719,6 +1763,13 @@ export function commitVotes(
       rank,
       pooledVote: point.vote,
       idfVote: votesIdf.get(point.anchor) ?? 0,
+      // The LARGEST single-region contribution behind this anchor — the bar
+      // recall's own gate reads (mechanisms/recall.ts: forest[0].peak > LN2),
+      // and until now the only decision-making quantity the climb computed and
+      // did not publish.  `regionPeak` reached `ranked` (see its build below)
+      // and stopped there.  Published, not recomputed: the value is the one the
+      // climb already carries.
+      peak: point.peak,
       candidateBreadth: regions.length,
       contributingVotes: regionAxioms.get(point.anchor) ?? 0,
       contributingEvidence: regionSupport.get(point.anchor) ?? 0,
@@ -1748,6 +1799,18 @@ export function commitVotes(
     let passesConsensusFloor: boolean | undefined;
     let pastLeadingSaturation: boolean | undefined;
     let tiedWithDominant: boolean | undefined;
+    // ── ONE OF THREE ADMISSIONS, AND THEY ARE NOT THE SAME READING ────────
+    // This block admits by VOTES: per-region evidence pooled, gated on the
+    // natural break and on consensusFloor, with the dominant allowed to bypass
+    // both.  `structuralResonance` admits by a MARGIN over the estimator's own
+    // noise, and `crossRegionVotes` admits by STRUCTURE (which regions may pair
+    // at all, with at least one side individually discriminative).  Read
+    // together they look like one policy written three times; they are three
+    // different measurements of the same question ("is this evidence?"), and
+    // unifying them would average three readings into one — the mistake
+    // `extraction.ts` records as "do not unify the two into one machine".
+    // What they DO share, and must keep sharing, is the discipline of deriving
+    // every bar from D/W/N rather than choosing it (thresholds.md).
     const rejectionReasons: AnchorRejectionReason[] = [];
     if (absorbed) {
       status = "overlap";
@@ -1757,9 +1820,75 @@ export function commitVotes(
       pastLeadingSaturation = pastLeading;
       const vote = votesIdf.get(point.anchor) ?? 0;
       if (roots.length === 0) {
-        // The first non-overlapping root is DOMINANT and bypasses the two
-        // vote thresholds (it always grounds) — only the leading-saturation
-        // gate still applies to it.
+        // THE DOMINANCE PRIVILEGE, AND THE TENSION IT CARRIES (measured).
+        //
+        // The first non-overlapping candidate is DOMINANT: it bypasses both
+        // vote gates below and grounds on its own; only the leading-saturation
+        // gate still applies to it.  The privilege is load-bearing — analogies,
+        // substitutions and composed contexts are precisely candidates the
+        // query does NOT contain, and the engine loses them without it.
+        //
+        // WHICH candidate receives it, though, is decided by this loop's ORDER.
+        // That order comes from `ranked`, and when two candidates carry equal
+        // evidence the stable sort preserves the ENUMERATION order, so the
+        // privilege is allocated by an ordering rather than by a rule.
+        //
+        // Measured on test/34's corpus, query "red":
+        //
+        //   0:#77 vote=1.3863 idf=1.3863 [0,3) | 1:#49 vote=1.3863 idf=1.3863 [0,3)
+        //
+        // Both candidates (`red square` #77, `red circle` #49) have IDENTICAL
+        // `vote` AND IDENTICAL `idfVote` over the SAME support span, so the
+        // comparator leaves them tied, the second is absorbed as "overlap", and
+        // the first grounds.  With this build's enumeration order that first is
+        // `red square`, 60/60 seeds, and `red circle` — the JOINT context — is
+        // never reached by "red" alone.  That is the premise test/34 exists to
+        // protect: no single region reaches the joint context, which is what
+        // makes the binding query unreachable without direct region
+        // interaction.
+        //
+        // THE TENSION: the premise therefore holds BY ENUMERATION ORDER, not by
+        // a rule, so any change to this ordering can move the privilege onto the
+        // joint context and let one region reach it.  Measured: adding
+        // `|| a.anchor - b.anchor` — the lowest-id tie-break that AGENTS.md §2
+        // sanctions as an equivalent corpus-determined tie-break — does exactly
+        // that: "red" then attends to `red circle`, test/34 fails 6/1, and the
+        // canonical suite reports 1 failure.
+        //
+        // TWO ATTEMPTS TO MAKE IT A RULE, BOTH REFUTED BY MEASUREMENT:
+        //
+        //  1. EVIDENCE SEPARATION.  Grant the privilege only when the first
+        //     candidate's evidence is separated from the next distinct
+        //     candidate's by more than the co-dominant band (sqrt(k) *
+        //     estimatorNoise(D)).  Refuted: that band exists to ADMIT the
+        //     anchors the estimator cannot separate from the dominant — its own
+        //     documented purpose — so withholding the privilege on ties removes
+        //     the very case it was written for.  Suite: 4 failures (the two
+        //     co-dominant band laws, breadth/scale invariance, test/29 D2).
+        //
+        //  2. QUERY-OWNED CONTENT.  Grant the privilege only to a candidate
+        //     that IS a recognised region's identity (regions.some(r => r.id ===
+        //     point.anchor)).  Measured: for "circle" that identity IS the
+        //     ranked candidate, so the privilege stays and `circle` grounds; for
+        //     "red" the identity is the `red` node itself while the candidates
+        //     are the conjunctions, so neither is privileged; for "red then
+        //     circle" the composed context carries idf 3.958 and clears both
+        //     gates on its own evidence.  All four control queries came out
+        //     right — and the suite: 10 failures, six of them in the
+        //     analogy/counterfactual/CAST suites ("an analogy still transfers
+        //     from a structure the query never names"; "a substitute the query
+        //     NAMES may still be voiced").  Refuted: the privilege exists to
+        //     admit what the query does NOT contain, so identity is the wrong
+        //     axis.
+        //
+        // WHAT A FUTURE ATTEMPT MUST RESPECT: whatever allocates this privilege
+        // has to (a) keep it available to candidates the query does not contain
+        // — analogies, substitutions, compositions — and (b) not depend on the
+        // estimator's ordering among anchors of equal evidence, because that
+        // ordering is not a fact about the corpus.  No lever satisfying both has
+        // been found.  Until one is, this premise rests on the enumeration order
+        // recorded above, and test/34 is the only test that notices if it
+        // moves.
         dominant = true;
         if (pastLeading) {
           status = "root";
@@ -1768,8 +1897,15 @@ export function commitVotes(
           rejectionReasons.push("leading-saturation");
         }
       } else {
-        passesNaturalBreak = vote >= rootCut;
-        passesConsensusFloor = vote >= floor;
+        // THE FLOOR AND THE BREAK READ THE IDF WEIGHTING.  `floor` is derived
+        // for pooled IDF-weighted votes, and `rootCut` comes from the IDF
+        // distribution (`idfDesc`), so gating the mode-dependent `vote` against
+        // either let a weighting mode change an admission (measured: anchor 87,
+        // inverse 2.682 admitted vs direct 1.468 refused).  Reading the IDF sum
+        // makes the verdict mode-independent, and changes nothing in the
+        // engine's own mode, where the two readings coincide.
+        passesNaturalBreak = point.idfVote >= rootCut;
+        passesConsensusFloor = point.idfVote >= floor;
         // CO-DOMINANT — an anchor the estimator cannot separate from the
         // dominant inherits the dominant's exemption, because that exemption's
         // only warrant is being TOP, and "top" is not a fact about the corpus
@@ -2467,6 +2603,13 @@ export async function structuralResonance(
     });
   };
 
+  // ── ADMISSION BY MARGIN, not by votes (see voteRegions' note) ─────────
+  // What this site measures: how far the best ANN proposal's effective score
+  // (score × semanticConfidence) stands above the runner-up's, against
+  // `estimatorNoise(D)`.  What it does NOT measure: how many regions voted,
+  // or whether the query's regions agree — that is voteRegions' question, and
+  // here a synthetic gist has already replaced them.  The two bars are both
+  // derived (thresholds.md), and neither is a tuning of the other.
   let selected: StructuralResonanceProposal | null = null;
   let selectedReach: AncestorReach | null = null;
   let selectedIdf = 0;
@@ -2582,6 +2725,15 @@ async function crossRegionVotes(
   // successfully reconstructed while probing one pair must not be read and
   // perceived again while probing another pair in the same climb.
   const siblingGistMemo = new Map<number, CachedSiblingGist>();
+  // ── ADMISSION BY STRUCTURE, not by a bar (see voteRegions' note) ──────
+  // What this site decides: WHICH regions may pair at all — a region that
+  // already voted (individually discriminative), or a KNOWN non-voting one as
+  // the weak side of a pair whose other side voted; never two non-voting
+  // regions, and never a span contained in a maximal one whose reading is
+  // exact.  The bar (the container's idf) comes later, on the candidate.  So
+  // its "rejection reasons" name structural disqualifications — a different
+  // vocabulary because it answers a different question, and the three
+  // taxonomies stay separate for the same reason the readings do.
   const votedSpans = new Set<string>();
   for (const rv of rvs.votes) votedSpans.add(`${rv.start},${rv.end}`);
   const seen = new Set<string>();
@@ -2978,7 +3130,7 @@ async function crossRegionVotes(
               Math.min(li, ri),
               Math.max(li + left.length, ri + right.length),
             );
-            if (indexOf(query, joined, 0) >= 0) {
+            if (restates(query, joined, 0)) {
               if (structuralTrace) structuralTrace.selfEvidenceRejected++;
               continue; // query says it itself
             }

package/src/mind/canonical.ts

@@ -88,6 +88,49 @@ export function leafIdPrefix(
   return ids;
 }
 
+/** Which prefixes of `prefix ‖ tail` are STORED NODES — as lengths in the
+ *  tail's own coordinates, ascending, excluding the empty one.  This is the
+ *  candidate set a rule needs to join an already-stored prefix to a suffix it
+ *  has not stored: a key names a relation exactly when `prefix ‖ tail[0..p]` IS
+ *  a node, and a key can end strictly inside the tail without sitting on any
+ *  fold boundary (a stored member's end is the end of ITS OWN stream, and the
+ *  fold never emits a cut at a stream's end).  Measured: "stockholm mayor"
+ *  exists, leads on, and its boundary 6 is in neither the tail's cuts nor the
+ *  concatenation's.
+ *
+ *  ONE cheap content-addressed probe per offset — `leafIdPrefix` walks the bytes
+ *  once (a point probe each), `findBranch` hashes the growing kid run — and NO
+ *  `resolve`, which is what keeps this off the O(suffix) vector folds the
+ *  recognition path pays.  It stops at the first byte that was never interned,
+ *  which costs nothing real: a stored key's bytes are interned by construction. */
+export function keyEnds(
+  ctx: MindContext,
+  prefix: Uint8Array,
+  tail: Uint8Array,
+): number[] {
+  if (prefix.length === 0 || tail.length === 0) return [];
+  const joined = new Uint8Array(prefix.length + tail.length);
+  joined.set(prefix, 0);
+  joined.set(tail, prefix.length);
+  const ids = leafIdPrefix(ctx, joined);
+  if (ids.length < prefix.length) return [];
+  const ends: number[] = [];
+  // The kid run GROWS by one id per offset; `findBranch` wants an array, so the
+  // run is built once and pushed into, never re-sliced.  Re-slicing
+  // `ids.slice(0, prefix.length + p)` per offset made this O(|tail| ·
+  // (|prefix| + |tail|)) — quadratic in the tail, where the learning path this
+  // follows slices a run that SHRINKS.  Same ends, linear copying.
+  const run = ids.slice(0, prefix.length);
+  // The loop ENDS at the first byte that was never interned (`ids.length`):
+  // every later prefix contains it, so none of them can be a node either — this
+  // is where the scan stops, not a silent truncation of the answer.
+  for (let p = 1; prefix.length + p <= ids.length; p++) {
+    run.push(ids[prefix.length + p - 1]);
+    if (ctx.store.findBranch(run) !== null) ends.push(p);
+  }
+  return ends;
+}
+
 /** The canonical W-window node ids of a byte stream, offset → id — the
  *  CONTENT-ADDRESSED IDENTITY of every W-sized slice, under which any content
  *  two deposits share IS the same node (hash-consing paid the comparison at