@hviana/sema 0.8.1 → 0.8.3

package/src/mind/graph-search.ts

@@ -238,6 +238,14 @@ export interface DerivationItem {
  *  {@link GraphSearch}'s rules fired, recovered from the rule's premise/
  *  conclusion shape (the rules carry no label, so this classifies by structure,
  *  the single place that maps rule geometry to a name). */
+// CLOSED ON PURPOSE — AND ONLY THIS ONE IS.  A derivation move is BRANCHED ON
+// (`classifyMove`, the rationale's readers, MOVE_NOTE's fallback), so it is a
+// closed union: adding one without teaching every reader is a compile error,
+// which is what a closed vocabulary buys.  The MECHANISM names in
+// `rationale.ts` are the opposite case — written and displayed, never
+// branched on — and they stay free strings that COMPOSE with the nesting
+// (`["respond", "think", "recognise"]`).  That asymmetry is deliberate; do not
+// "fix" it by uniting the two (see test/126 for the pipeline half of it).
 export type DerivationMove =
   | "axiom" // a seed: a perceived leaf, a recognised form, or a computed result
   | "follow-edge" // form→form via a continuation edge (STEP) — the core "what follows what"
@@ -375,6 +383,14 @@ export class GraphSearch {
     private readonly host: GraphSearchHost,
   ) {}
 
+  /** The nodes the QUERY canonically names — the same identity the store's keys
+   *  were written through.  A byte-exact test is not enough: the query writes
+   *  `Eiffel Tower country` and the deposited node is `eiffel tower country`, so
+   *  a join that filters the query's own subject by RAW bytes re-admits it —
+   *  measured: that is the trap's wrong answer (`The capital of Eiffel Tower
+   *  country is Berlin.`).  Cached by query identity, because the search is
+   *  reused across responses. */
+
   /* * The hub bound √N (bounded-reads.md) — the ONE
    *  fan-out cap, stated here rather than imported from `traverse.ts` because
    *  this module is deliberately host-based (it holds a bare Store, never a
@@ -703,7 +719,13 @@ export class GraphSearch {
           return this.coverRules(it, coversDone, coverableByStart);
         }
         if (it.kind === "form") {
-          return this.formRules(it, conceptTarget, substitutions, nodeBytes);
+          return this.formRules(
+            it,
+            conceptTarget,
+            substitutions,
+            nodeBytes,
+            queryLen,
+          );
         }
         return this.outRules(it, {
           W,
@@ -808,6 +830,7 @@ export class GraphSearch {
     conceptTarget: ReadonlyMap<number, number>,
     substitutions: ReadonlyMap<number, Uint8Array> | undefined,
     nodeBytes: (n: number) => Uint8Array,
+    queryLen: number,
   ): Iterable<Rule<GItem>> {
     // Articulation: emit voice bytes at the recognised span; the hop/concept/
     // emit chain is suppressed — the form contributes only its substitute.
@@ -843,7 +866,45 @@ export class GraphSearch {
       // guard then dead-ends it) with no way to reach the forward edge.
       // Forking offers every continuation as its own rule so the one that
       // genuinely advances (not a duplicate) is still reachable.
+      // A CHAIN HOP OFFERS ONLY WHAT THE QUESTION CAN PAY FOR.
+      //
+      // `hubBound` = √N is the READ cap — every read here stays inside it — but
+      // it is not an exploration bound: measured, a hub of degree 1083 sits
+      // BELOW √N = 1559, so a hop offered all 1083 continuations, the chart grew
+      // to 3113 outs for a two-word question, and since every out with an
+      // uncovered tail probes its tail's prefixes (measured: 16 885 canonical
+      // probes = 87% of that query's work, and its 270 MB peak / 256 MB OOM),
+      // the cost came from OFFERING rather than from reading.
+      //
+      // The bound is derived, not tuned: a derivation of L hops consumes ~L
+      // units of the question, so a hop cannot be paid for by offering more
+      // continuations than the question has units —
+      // `ceil(queryLen / W)`, floored at 2 for plurality. It is QUERY-sized
+      // (invariant 5: no per-query read grows with N) and it leaves `hubBound`
+      // and every read untouched.
+      // THE OFFER IS THE CORPUS'S OWN STRUCTURE, and the search pays for
+      // exploring it.  There is no offer cap here any more: the traversal cap I
+      // had put on this hop was a short-circuit — it bounded what a hop could
+      // OFFER instead of charging for it — and it was not needed.
+      //
+      // MEASURED in the regime where it used to bite (`hubBound = ceil(√N)`
+      // GREATER than the hub's degree — reached in a fixture by choosing the
+      // degree below √N, so the trained store is not needed): with the cap the
+      // offer was 8/9/9 continuations at degrees 35/70/120; without it, 52/84/120
+      // — and the WORK is LINEAR in the degree, not quadratic: pushes 262/296/332,
+      // perceptions 530/592/757, while the PEAK is identical with and without the
+      // cap (218/415/689 MB against 215/410/662) because it is set by the store,
+      // not by the fan-out.  What made this hop expensive was never the fan-out
+      // breadth: it was the per-offer work, two duplicate/oversized computations
+      // since removed (the per-offset canonical scan, and the tail scan now
+      // restricted to the fold's boundaries).
+      //
+      // The residual, stated: the trained store's hub (degree 1 083) is an
+      // EXTRAPOLATION from this linear shape, not a measurement.
       const nx = this.store.nextFirst(it.node, this.hubBound());
+      // Count what is OFFERED, not what was read: the evidence-preferred
+      // continuation is yielded too, even when it lies outside the cap.
+      if (this.host.meter) this.host.meter.chainOffers += nx.length + 1;
       if (nx.length) {
         // The SAME evidence-weighted disambiguation the first hop uses
         // (below) identifies the most-corroborated continuation.  Yielding
@@ -852,10 +913,14 @@ export class GraphSearch {
         // arrivals at an EQUAL cost (`cost < current`, strictly) — so
         // among same-depth sibling forks that tie in cost, the
         // evidence-backed edge wins deterministically, never by
-        // exploration-order luck.  `preferred`, when set, is necessarily
-        // an element of `nx` (chooseNext reads the identical hub-bounded
-        // set — see traverse.ts), so a plain skip-in-place suffices; no
-        // second array need be allocated to reorder it to the front.
+        // exploration-order luck.  `preferred` is NOT necessarily an element
+        // of `nx` any more: `nx` is the capped read above, while `chooseNext`
+        // reads its own hub-bounded set (see traverse.ts) — so the evidence
+        // pick may lie outside the cap, and it is still yielded FIRST on
+        // purpose.  The cap bounds what the hop EXPLORES; it must never make
+        // the evidence-ranked continuation unreachable, or the derivation the
+        // corpus corroborates would lose to exploration order.  Nothing is
+        // reallocated: `nx` is walked with a skip-in-place for the duplicate.
         const preferred = nx.length > 1
           ? this.host.chooseNext?.(it.node)
           : undefined;
@@ -1078,12 +1143,51 @@ export class GraphSearch {
       // concepts/connectors either (those need the caller's async
       // pre-resolution) — the recursion follows edges and fusion, which is what
       // a deeper rewrite chain is made of.
+      if (this.host.meter) this.host.meter.recompletes++;
       const rec = this.host.recogniseSpan(bytes);
       const kids = new Set(nrec.kids);
+      // THE NODE'S OWN KIDS ARE SITES BY STRUCTURE — recognition cannot be the
+      // only source.  A produced composite is decomposed by ITS OWN SHAPE, and
+      // at hub scale the recognition of a produced span returns the WHOLE while
+      // deliberately suppressing its atoms (the off-boundary suppression), so
+      // the kid filter below would admit nothing at all and the chain would end
+      // at the intermediate composite.  Measured on the chain
+      // `seed → "p q" → (p→r, q→s) → "r s" → "m n"`: below the flip it reaches
+      // "m n" with fuse+recompose, above it stops at "p q" — the trace shows
+      // `recognise("p q") ⇒ form "p q"` alone, no parts.
+      //
+      // Laying the node's kids out over its own bytes restores exactly the
+      // decomposition the node's tree already states; a kid recognition ALREADY
+      // offers is skipped, so below the flip the seed set is byte-identical to
+      // what it was.  The filter's guarantee is untouched: nothing beyond the
+      // node's own kids may enter.
+      const recognised = rec.sites.filter((s) => kids.has(s.payload));
+      // DEDUPED BY SPAN, not by payload.  A node whose kids repeat (`"abab"`
+      // folds as ["ab","ab"]) has TWO occurrences of the same node at different
+      // offsets; a payload-keyed set suppressed the structural site for BOTH, so
+      // the second occurrence had no site at all.  The set now holds the spans
+      // recognition already covers, and a structural site is added exactly when
+      // nothing covers that PLACE — O(1) lookups, no extra reads.
+      const seenSpans = new Set(recognised.map((s) => `${s.start}:${s.end}`));
+      const structural: Site[] = [];
+      {
+        let off = 0;
+        for (const k of nrec.kids) {
+          const len = this.store.bytesPrefix(k, ALL).length;
+          if (!seenSpans.has(`${off}:${off + len}`) && len > 0) {
+            structural.push({
+              start: off,
+              end: Math.min(off + len, bytes.length),
+              payload: k,
+            });
+          }
+          off += len;
+        }
+      }
       const solved = this.solve(
         bytes.length,
         {
-          sites: rec.sites.filter((s) => kids.has(s.payload)),
+          sites: [...recognised, ...structural],
           leaves: rec.leaves,
           splits: rec.splits,
           starts: rec.starts,
@@ -1163,6 +1267,14 @@ export class GraphSearch {
     if (!this.host.recogniseSpan) return;
     const tail = queryBytes.subarray(fact.j, queryLen);
     if (tail.length === 0) return;
+    // Report ONLY the invocations that could have joined: the search asks this
+    // rule for every finalized out with a node, which includes the one-byte
+    // outs the cover bridges with — measured, 68 refusals for a single
+    // 3-relation query, all of them letters.  A form shorter than one window is
+    // not a fact a join could travel through, so it is not a refusal worth
+    // reporting; W is the same line the rest of the mind draws between a chance
+    // overlap and a form.
+    const reportable = fact.bytes.length >= this.maxGroup;
     // The entity candidates are the forms the fact's own bytes CONTAIN — the
     // same recogniser the query went through, so the evidence standard is the
     // query's.  A byte atom is never a subject; the fact's own node is the span
@@ -1171,12 +1283,66 @@ export class GraphSearch {
     // LENDS it when it can (Mind does, with the response-scoped struct cache),
     // and a bare host falls back to the raw-store probe, so the search stays
     // host-based.
-    const leading = this.host.recogniseSpan(fact.bytes).sites.filter((s) =>
-      s.payload >= 0 && s.payload !== fact.node &&
-      (this.host.leadsSomewhere !== undefined
-        ? this.host.leadsSomewhere(s.payload)
-        : this.store.hasNext(s.payload) || this.store.hasHalo(s.payload))
+    const factRec = this.host.recogniseSpan(fact.bytes);
+    const leads = (id: number): boolean =>
+      this.host.leadsSomewhere !== undefined
+        ? this.host.leadsSomewhere(id)
+        : this.store.hasNext(id) || this.store.hasHalo(id);
+    // THE QUERY'S OWN SUBJECT, CANONICALLY.  The filter used raw bytes and the
+    // store's nodes are canonical, so `Eiffel Tower country` in the query did
+    // not match the deposited `eiffel tower country` — measured, that is the
+    // trap's wrong answer.
+    //
+    // TAKEN FROM THE RECOGNITION THE RESPONSE ALREADY COMPUTED — the host's
+    // `recogniseSpan`, the same surface the rest of this search uses — not from
+    // a second offset scan.  `canonicalQueryNodes` re-derived, per byte offset,
+    // what `recognise` had already resolved once per query (its memo is keyed by
+    // content), and that scan was the largest single cost the DIANOT join added:
+    // measured against the pre-change tree, the same fixture and the same test
+    // were 21 s slower with the scan than without it.  A recognised site IS a
+    // canonical node of the query that can lead somewhere, which is exactly the
+    // set this filter wants, and it costs nothing to read.
+    const queryNodes = new Set<number>(
+      (this.host.recogniseSpan?.(queryBytes)?.sites ?? []).map((s) =>
+        s.payload
+      ),
     );
+    // TWO SOURCES, ONE ADMISSION.  The recognition of a STORED WHOLE returns the
+    // whole and stops — measured: for `The director of Eva is Gustaf Molander.`
+    // it yields exactly ONE site, the fact's own node — so the entity a join
+    // exists for is never proposed.  The canonical fold is the second source,
+    // and the scan runs only for a FORM (≥ W: a one-byte out is not something to
+    // join through, and running it per letter measured 20-26 s in test/99).
+    const W = this.maxGroup;
+    const proposed = new Map<number, Uint8Array>();
+    // The SOURCE of each proposal travels with it: a refusal that names only the
+    // bytes leaves the next reader guessing which path proposed them — three
+    // attempts at the chained join were spent fixing paths that never produced
+    // the offending candidate.
+    const source = new Map<number, string>();
+    for (const s of factRec.sites) {
+      if (s.payload >= 0 && leads(s.payload)) {
+        proposed.set(s.payload, this.store.bytesPrefix(s.payload, ALL));
+        source.set(s.payload, "recognised site");
+      }
+    }
+    if (this.host.canonResolve !== undefined && fact.bytes.length >= W) {
+      const canon = this.host.canonResolve.bind(this.host);
+      for (let start = 0; start < fact.bytes.length; start++) {
+        for (let end = fact.bytes.length; end - start >= W; end--) {
+          const id = canon(fact.bytes.subarray(start, end));
+          if (id === null) continue;
+          if (leads(id)) {
+            proposed.set(id, this.store.bytesPrefix(id, ALL));
+            source.set(id, "canonical fold");
+          }
+          break; // the longest form at this offset wins
+        }
+      }
+    }
+    const leading = [...proposed]
+      .filter(([payload]) => payload !== fact.node && !queryNodes.has(payload))
+      .map(([payload, bytes]) => ({ payload, bytes }));
     // …then prefer the entity the query did NOT name, and the MAXIMAL one.  The
     // join exists to reach the subject the query never wrote, so:
     //   • a candidate the query already contains is the query's OWN subject, and
@@ -1187,32 +1353,120 @@ export class GraphSearch {
     //     introduces ("Timur" must not win over "Timur Bekmambetov").
     // Byte work over bytes already read, and the pruning REMOVES the
     // resolve()/nextFirst() probes these candidates would have paid.
+    if (leading.length === 0) {
+      if (this.host.meter) this.host.meter.joinNoEntity++;
+      if (reportable) {
+        // Report WHAT the recognition returned, not just that nothing led: the
+        // count and the first few site texts are the difference between "the
+        // fact was not recognised" and "it was recognised but nothing led".
+        const seen = factRec.sites.slice(0, 3).map((s) =>
+          this.store.bytesPrefix(s.payload, ALL)
+        );
+        this.host.reportSearch?.(
+          "deriveThroughMiss",
+          [fact.bytes, tail, ...seen],
+          `no entity inside the fact leads anywhere — ${factRec.sites.length} site(s) recognised inside it`,
+        );
+      }
+    }
     const candidates = leading
-      .map((s) => ({
-        payload: s.payload,
-        bytes: this.store.bytesPrefix(s.payload, ALL),
-      }))
-      .filter((c) => indexOf(queryBytes, c.bytes, 0) < 0)
       .filter((c, _i, all) =>
         !all.some((o) =>
           o.bytes.length > c.bytes.length && indexOf(o.bytes, c.bytes, 0) >= 0
         )
       );
     for (const c of candidates) {
-      const key = this.host.resolve(concat2(c.bytes, tail));
-      if (key === null) continue;
-      const nx = this.store.nextFirst(key, 1);
-      if (nx.length === 0) continue;
+      // THE SHORTEST TAIL PREFIX WHOSE KEY ALSO LEADS SOMEWHERE.
+      //
+      // The conclusion covers only that prefix, so the rest of the tail stays
+      // for the step after — which is what CHAINING is (a whole-tail key
+      // consumed the whole remainder and made every join terminal: measured,
+      // `joinFired=0` on a three-relation query).
+      //
+      // A KEY THAT RESOLVES IS NOT ENOUGH.  Measured with a dry run of these
+      // very primitives: for the candidate `Sweden` the first tail prefix that
+      // resolves is `" "` — the key `Sweden ` (a trailing space) — and it leads
+      // NOWHERE (`nextFirst` = 0), so accepting it refused the join while the
+      // key that names the fact (`sweden capital`) sat one prefix further.  The
+      // loop therefore asks BOTH questions before accepting, and keeps looking
+      // otherwise.
+      //
+      // EXACT FIRST, THEN CANONICAL: the corpus holds both identities.
+      let key: number | null = null;
+      let used = 0;
+      // The continuation the accepted key leads to, carried out of the loop:
+      // the loop already HAD to read it to accept the key (a key that leads
+      // nowhere is not the relation), so re-reading it after the loop was a
+      // duplicate read and a branch that could never be taken.
+      let next: number | null = null;
+      let keyBytes = c.bytes;
+      // THE CANDIDATE ENDS ARE THE PREFIXES THAT ARE STORED NODES, ASCENDING.
+      // The key is `entity + prefix`, and it names a relation exactly when that
+      // concatenation IS a node — so the ends come from a content-addressed
+      // probe per offset (the host's `contentKeyEnds`, the learning path's own
+      // mechanism: one leaf walk plus one `findBranch` per offset, no `resolve`),
+      // never from the fold's boundaries.  A boundary is not a proxy: a stored
+      // member's end is the end of ITS OWN stream, and the fold never cuts at a
+      // stream's end — measured, "stockholm mayor" exists, leads on to the mayor
+      // fact, and its boundary 6 sits in neither the tail's cuts ([4,7]) nor the
+      // concatenation's.  Filtering the scan by "is this a node?" cannot change
+      // the winner: a position that is not a node cannot resolve, so skipping it
+      // is invisible; and the order stays SHORTEST FIRST, which is a semantic
+      // law, not an optimisation (test/106, test/108 pin it).
+      //
+      // A host that cannot answer falls back to every prefix: exact and
+      // complete, at a `resolve` per offset.  A host that CAN answer is
+      // authoritative even when it answers "none" — if no prefix is a node then
+      // no key exists to resolve, so enumerating would only pay nulls.  (A key
+      // reachable through the CANONICAL equivalence alone and ending off every
+      // node end is therefore not tried here; that dimension is unreachable on
+      // this path by construction and is not part of the exact-key law.)
+      const ends = this.host.contentKeyEnds?.(c.bytes, tail);
+      const candidateEnds = function* (): Generator<number> {
+        if (ends !== undefined) {
+          yield* ends;
+          return;
+        }
+        for (let p = 1; p <= tail.length; p++) yield p;
+      };
+      for (const len of candidateEnds()) {
+        keyBytes = concat2(c.bytes, tail.subarray(0, len));
+        const k = this.host.resolve(keyBytes) ??
+          this.host.canonResolve?.(keyBytes) ??
+          null;
+        if (k === null) continue;
+        const nx = this.store.nextFirst(k, 1);
+        if (nx.length === 0) continue;
+        key = k;
+        used = len;
+        next = nx[0];
+        break;
+      }
+      if (key === null) {
+        if (this.host.meter) this.host.meter.joinNoKey++;
+        if (reportable) {
+          this.host.reportSearch?.(
+            "deriveThroughMiss",
+            [c.bytes, tail, keyBytes],
+            `no learnt key names this entity and tail together ` +
+              `(candidate #${c.payload}, from the ${
+                source.get(c.payload) ?? "unknown"
+              } source)`,
+          );
+        }
+        continue;
+      }
+      if (this.host.meter) this.host.meter.joinFired++;
       yield {
         premises: [fact],
         conclusion: {
           kind: "out",
           i: fact.i,
-          j: queryLen,
-          bytes: this.store.bytesPrefix(nx[0], ALL),
+          j: fact.j + used,
+          bytes: this.store.bytesPrefix(next!, ALL),
           cover: true,
           rec: true,
-          node: nx[0],
+          node: next!,
           throughFact: true,
         },
         cost: STEP,

package/src/mind/index.ts

@@ -4,7 +4,12 @@
 // exported from mind/mind.ts directly.
 
 export { Mind } from "./mind.js";
-export type { Input, Response } from "./mind.js";
+export type {
+  CorpusTextPair,
+  CorpusTextResult,
+  Input,
+  Response,
+} from "./mind.js";
 export type { ComputedSpan, ExtensionHost } from "./mind.js";
 export type {
   MechanismResult,
@@ -38,3 +43,5 @@ export type {
   NarrowDecisionData,
   Provenance,
 } from "./pipeline.js";
+export { sampleCorpus, searchCorpus } from "./corpus.js";
+export type { CorpusMiss, CorpusPair, CorpusResult } from "./corpus.js";