@hviana/sema 0.8.1 → 0.8.2

package/src/mind/corpus.ts

@@ -0,0 +1,202 @@
+// corpus.ts — read the trained memory back out of the DAG, AS DATA.
+//
+// A trained experience pair IS one continuation edge: `src` is the context that
+// was deposited, `dst` is what the mind learnt follows it.  Reading them back
+// uses the store's own structure and its own indexes — no auxiliary index is
+// built, nothing is written, and NOTHING HERE KNOWS ABOUT TEXT: this layer takes
+// bytes and returns bytes.  The text case is one helper on the Mind
+// (`searchCorpusText`), which encodes, calls this, and decodes.
+//
+// WHERE EACH STAGE COMES FROM (ported from the demo's `explore.ts`, which
+// hand-rolled its own resolution):
+//
+//   1. PERCEIVE, CONTENT-ADDRESS and ADMIT the query — `recognise()`, the SAME
+//      machinery an answer goes through.  It returns the sites: the query spans
+//      that content-addressed to a stored node that can lead somewhere.  The
+//      demo's own recursive `findLeaf`/`findBranch` walk was a second
+//      implementation of exactly this, and it is NOT ported.
+//   2. CLIMB the structural `kid` table from each resolved site to the
+//      edge-bearing contexts above it (`edgeAncestors`, traverse.ts), weighting
+//      each context by how much query content reached it.
+//   3. READ the continuation off the edge table (`nextFirst`).
+//
+// COST is set by how much of the QUERY resolves, never by the size of the
+// store: the sites are what recognition already found, the climb is bounded by
+// the declared `corpusClimbs`/`corpusContextsPerClimb`, and every store call is
+// a point probe or a capped read.  All work is accounted by the store's own
+// meter hooks when a response's meter is open — there is no second instrument.
+//
+// WHAT THIS IS NOT.  Exact content addressing, not fuzzy keyword search: a query
+// shares results with a stored note when it shares actual chunk-aligned content
+// with it.  An arbitrary mid-word fragment resolves to nothing, and the honest
+// answer there is "nothing matched" — which is why `sampleCorpus` exists, and
+// why the miss is reported as a STATE rather than as prose (the text helper
+// turns it into words).
+
+import { recognise } from "./recognition.js";
+import { edgeAncestors } from "./traverse.js";
+import type { MindContext } from "./types.js";
+
+/** One stored experience pair, as bytes. */
+export interface CorpusPair {
+  context: Uint8Array;
+  continuation: Uint8Array;
+  contextId: number;
+  continuationId: number;
+  /** Bytes of the query this pair was matched on — 0 when browsing. */
+  matchedBytes: number;
+  /** True when the stored bytes ran past the declared preview capacity. */
+  contextTruncated: boolean;
+  continuationTruncated: boolean;
+}
+
+/** Why a search produced no pairs.  A STATE, so a caller's own layer can say it
+ *  in its own words — the byte layer does not speak. */
+export type CorpusMiss = "matched" | "nothing-resolved" | "no-continuations";
+
+export interface CorpusResult {
+  pairs: CorpusPair[];
+  /** Query subtrees that content-addressed to a real stored node. */
+  resolved: number;
+  /** Distinct edge-bearing contexts the climb reached. */
+  reached: number;
+  /** Distinct contexts that carry a learnt continuation, store-wide. */
+  totalContexts: number;
+  /** True when these are browse samples rather than search results. */
+  browsed: boolean;
+  miss: CorpusMiss;
+}
+
+/** A context node as a pair, or null when it carries no continuation. */
+function pairOf(
+  ctx: MindContext,
+  id: number,
+  matchedBytes: number,
+): CorpusPair | null {
+  const outs = ctx.store.nextFirst(id, 1);
+  if (outs.length === 0) return null;
+  const cap = ctx.cfg.corpusPreviewBytes;
+  const context = ctx.store.bytesPrefix(id, cap + 1);
+  const continuation = ctx.store.bytesPrefix(outs[0], cap + 1);
+  const contextTruncated = context.length > cap;
+  const continuationTruncated = continuation.length > cap;
+  return {
+    context: contextTruncated ? context.subarray(0, cap) : context,
+    continuation: continuationTruncated
+      ? continuation.subarray(0, cap)
+      : continuation,
+    contextId: id,
+    continuationId: outs[0],
+    matchedBytes,
+    contextTruncated,
+    continuationTruncated,
+  };
+}
+
+/** Which stored notes does this query reach?  BYTES in, BYTES out.
+ *
+ *  Exact content addressing through the machinery that already exists: the
+ *  query's recognised sites are the resolved subtrees, the climb goes up from
+ *  the biggest first, and a pair is a context that carries a continuation. */
+export function searchCorpus(
+  ctx: MindContext,
+  queryBytes: Uint8Array,
+  limit?: number,
+): CorpusResult {
+  const store = ctx.store;
+  const want = Math.max(
+    1,
+    Math.min(limit ?? ctx.cfg.corpusContextsPerClimb, ctx.cfg.corpusLimitMax),
+  );
+  // A resolved subtree must account for at least one window (W): a single
+  // character resolves against almost any store and means nothing.  W is the
+  // mind's own line between chance and evidence — derived, never declared.
+  const floor = ctx.space.maxGroup;
+  const resolved = recognise(ctx, queryBytes).sites
+    .map((s) => ({ id: s.payload, len: store.contentLen(s.payload, 512) }))
+    .filter((r) => r.len >= floor);
+  // Biggest first, lowest id breaking ties: a clause is evidence, a character is
+  // noise, and equal evidence must not be decided by iteration order.
+  const byLength = resolved
+    .sort((a, b) => b.len - a.len || a.id - b.id)
+    .slice(0, ctx.cfg.corpusClimbs);
+
+  // Weight each context by how much query content reached it.
+  const weight = new Map<number, number>();
+  for (const { id, len } of byLength) {
+    for (
+      const root of edgeAncestors(ctx, id, ctx.cfg.corpusContextsPerClimb).roots
+    ) {
+      weight.set(root, (weight.get(root) ?? 0) + len);
+    }
+  }
+
+  const pairs: CorpusPair[] = [];
+  const ranked = [...weight.entries()].sort(
+    (a, b) => b[1] - a[1] || a[0] - b[0],
+  );
+  for (const [id, w] of ranked) {
+    if (pairs.length >= want) break;
+    const pair = pairOf(ctx, id, w);
+    if (pair) pairs.push(pair);
+  }
+  return {
+    pairs,
+    resolved: byLength.length,
+    reached: weight.size,
+    totalContexts: store.edgeSourceCount(),
+    browsed: false,
+    miss: pairs.length > 0
+      ? "matched"
+      : weight.size === 0
+      ? "nothing-resolved"
+      : "no-continuations",
+  };
+}
+
+/** Browse real pairs, striding the id space so the sample is spread rather than
+ *  one local cluster.  DETERMINISTIC: `from` is the caller's own offset, so
+ *  browsing twice with different offsets shows different notes without a random
+ *  draw (the demo drew `Math.random()`, which the engine cannot do — same seed,
+ *  same order, same query must mean the same answer). */
+export function sampleCorpus(
+  ctx: MindContext,
+  limit?: number,
+  from = 0,
+): CorpusResult {
+  const store = ctx.store;
+  const want = Math.max(
+    1,
+    Math.min(limit ?? ctx.cfg.corpusContextsPerClimb, ctx.cfg.corpusLimitMax),
+  );
+  const total = store.nodeCount();
+  const probes = ctx.cfg.corpusSampleProbes;
+  const floorBytes = ctx.cfg.corpusSampleFloorBytes;
+  const pairs: CorpusPair[] = [];
+  // EACH CONTEXT AT MOST ONCE.  Striding the id space revisits ids when the
+  // store is small relative to the probe budget (measured: a 160-node store
+  // returned the SAME pair six times for `limit: 6`), and a browse that repeats
+  // itself is not a browse.  The demo had the same hole; it is invisible only on
+  // a store far larger than the probe budget.
+  const seen = new Set<number>();
+  for (let i = 0; i < probes && pairs.length < want && total > 0; i++) {
+    const slot = (i / probes + from) % 1;
+    const id = Math.floor(slot * total);
+    if (seen.has(id)) continue;
+    if (!store.has(id) || !store.hasNext(id)) continue;
+    if (store.contentLen(id, floorBytes) < floorBytes) continue;
+    const pair = pairOf(ctx, id, 0);
+    if (pair) {
+      seen.add(id);
+      pairs.push(pair);
+    }
+  }
+  return {
+    pairs,
+    resolved: 0,
+    reached: pairs.length,
+    totalContexts: store.edgeSourceCount(),
+    browsed: true,
+    miss: pairs.length > 0 ? "matched" : "no-continuations",
+  };
+}

package/src/mind/graph-search.ts

@@ -375,6 +375,14 @@ export class GraphSearch {
     private readonly host: GraphSearchHost,
   ) {}
 
+  /** The nodes the QUERY canonically names — the same identity the store's keys
+   *  were written through.  A byte-exact test is not enough: the query writes
+   *  `Eiffel Tower country` and the deposited node is `eiffel tower country`, so
+   *  a join that filters the query's own subject by RAW bytes re-admits it —
+   *  measured: that is the trap's wrong answer (`The capital of Eiffel Tower
+   *  country is Berlin.`).  Cached by query identity, because the search is
+   *  reused across responses. */
+
   /* * The hub bound √N (bounded-reads.md) — the ONE
    *  fan-out cap, stated here rather than imported from `traverse.ts` because
    *  this module is deliberately host-based (it holds a bare Store, never a
@@ -703,7 +711,13 @@ export class GraphSearch {
           return this.coverRules(it, coversDone, coverableByStart);
         }
         if (it.kind === "form") {
-          return this.formRules(it, conceptTarget, substitutions, nodeBytes);
+          return this.formRules(
+            it,
+            conceptTarget,
+            substitutions,
+            nodeBytes,
+            queryLen,
+          );
         }
         return this.outRules(it, {
           W,
@@ -808,6 +822,7 @@ export class GraphSearch {
     conceptTarget: ReadonlyMap<number, number>,
     substitutions: ReadonlyMap<number, Uint8Array> | undefined,
     nodeBytes: (n: number) => Uint8Array,
+    queryLen: number,
   ): Iterable<Rule<GItem>> {
     // Articulation: emit voice bytes at the recognised span; the hop/concept/
     // emit chain is suppressed — the form contributes only its substitute.
@@ -843,7 +858,45 @@ export class GraphSearch {
       // guard then dead-ends it) with no way to reach the forward edge.
       // Forking offers every continuation as its own rule so the one that
       // genuinely advances (not a duplicate) is still reachable.
+      // A CHAIN HOP OFFERS ONLY WHAT THE QUESTION CAN PAY FOR.
+      //
+      // `hubBound` = √N is the READ cap — every read here stays inside it — but
+      // it is not an exploration bound: measured, a hub of degree 1083 sits
+      // BELOW √N = 1559, so a hop offered all 1083 continuations, the chart grew
+      // to 3113 outs for a two-word question, and since every out with an
+      // uncovered tail probes its tail's prefixes (measured: 16 885 canonical
+      // probes = 87% of that query's work, and its 270 MB peak / 256 MB OOM),
+      // the cost came from OFFERING rather than from reading.
+      //
+      // The bound is derived, not tuned: a derivation of L hops consumes ~L
+      // units of the question, so a hop cannot be paid for by offering more
+      // continuations than the question has units —
+      // `ceil(queryLen / W)`, floored at 2 for plurality. It is QUERY-sized
+      // (invariant 5: no per-query read grows with N) and it leaves `hubBound`
+      // and every read untouched.
+      // THE OFFER IS THE CORPUS'S OWN STRUCTURE, and the search pays for
+      // exploring it.  There is no offer cap here any more: the traversal cap I
+      // had put on this hop was a short-circuit — it bounded what a hop could
+      // OFFER instead of charging for it — and it was not needed.
+      //
+      // MEASURED in the regime where it used to bite (`hubBound = ceil(√N)`
+      // GREATER than the hub's degree — reached in a fixture by choosing the
+      // degree below √N, so the trained store is not needed): with the cap the
+      // offer was 8/9/9 continuations at degrees 35/70/120; without it, 52/84/120
+      // — and the WORK is LINEAR in the degree, not quadratic: pushes 262/296/332,
+      // perceptions 530/592/757, while the PEAK is identical with and without the
+      // cap (218/415/689 MB against 215/410/662) because it is set by the store,
+      // not by the fan-out.  What made this hop expensive was never the fan-out
+      // breadth: it was the per-offer work, two duplicate/oversized computations
+      // since removed (the per-offset canonical scan, and the tail scan now
+      // restricted to the fold's boundaries).
+      //
+      // The residual, stated: the trained store's hub (degree 1 083) is an
+      // EXTRAPOLATION from this linear shape, not a measurement.
       const nx = this.store.nextFirst(it.node, this.hubBound());
+      // Count what is OFFERED, not what was read: the evidence-preferred
+      // continuation is yielded too, even when it lies outside the cap.
+      if (this.host.meter) this.host.meter.chainOffers += nx.length + 1;
       if (nx.length) {
         // The SAME evidence-weighted disambiguation the first hop uses
         // (below) identifies the most-corroborated continuation.  Yielding
@@ -852,10 +905,14 @@ export class GraphSearch {
         // arrivals at an EQUAL cost (`cost < current`, strictly) — so
         // among same-depth sibling forks that tie in cost, the
         // evidence-backed edge wins deterministically, never by
-        // exploration-order luck.  `preferred`, when set, is necessarily
-        // an element of `nx` (chooseNext reads the identical hub-bounded
-        // set — see traverse.ts), so a plain skip-in-place suffices; no
-        // second array need be allocated to reorder it to the front.
+        // exploration-order luck.  `preferred` is NOT necessarily an element
+        // of `nx` any more: `nx` is the capped read above, while `chooseNext`
+        // reads its own hub-bounded set (see traverse.ts) — so the evidence
+        // pick may lie outside the cap, and it is still yielded FIRST on
+        // purpose.  The cap bounds what the hop EXPLORES; it must never make
+        // the evidence-ranked continuation unreachable, or the derivation the
+        // corpus corroborates would lose to exploration order.  Nothing is
+        // reallocated: `nx` is walked with a skip-in-place for the duplicate.
         const preferred = nx.length > 1
           ? this.host.chooseNext?.(it.node)
           : undefined;
@@ -1080,10 +1137,48 @@ export class GraphSearch {
       // a deeper rewrite chain is made of.
       const rec = this.host.recogniseSpan(bytes);
       const kids = new Set(nrec.kids);
+      // THE NODE'S OWN KIDS ARE SITES BY STRUCTURE — recognition cannot be the
+      // only source.  A produced composite is decomposed by ITS OWN SHAPE, and
+      // at hub scale the recognition of a produced span returns the WHOLE while
+      // deliberately suppressing its atoms (the off-boundary suppression), so
+      // the kid filter below would admit nothing at all and the chain would end
+      // at the intermediate composite.  Measured on the chain
+      // `seed → "p q" → (p→r, q→s) → "r s" → "m n"`: below the flip it reaches
+      // "m n" with fuse+recompose, above it stops at "p q" — the trace shows
+      // `recognise("p q") ⇒ form "p q"` alone, no parts.
+      //
+      // Laying the node's kids out over its own bytes restores exactly the
+      // decomposition the node's tree already states; a kid recognition ALREADY
+      // offers is skipped, so below the flip the seed set is byte-identical to
+      // what it was.  The filter's guarantee is untouched: nothing beyond the
+      // node's own kids may enter.
+      const recognised = rec.sites.filter((s) => kids.has(s.payload));
+      // DEDUPED BY SPAN, not by payload.  A node whose kids repeat (`"abab"`
+      // folds as ["ab","ab"]) has TWO occurrences of the same node at different
+      // offsets; a payload-keyed set suppressed the structural site for BOTH, so
+      // the second occurrence had no site at all.  The set now holds the spans
+      // recognition already covers, and a structural site is added exactly when
+      // nothing covers that PLACE — O(1) lookups, no extra reads.
+      const seenSpans = new Set(recognised.map((s) => `${s.start}:${s.end}`));
+      const structural: Site[] = [];
+      {
+        let off = 0;
+        for (const k of nrec.kids) {
+          const len = this.store.bytesPrefix(k, ALL).length;
+          if (!seenSpans.has(`${off}:${off + len}`) && len > 0) {
+            structural.push({
+              start: off,
+              end: Math.min(off + len, bytes.length),
+              payload: k,
+            });
+          }
+          off += len;
+        }
+      }
       const solved = this.solve(
         bytes.length,
         {
-          sites: rec.sites.filter((s) => kids.has(s.payload)),
+          sites: [...recognised, ...structural],
           leaves: rec.leaves,
           splits: rec.splits,
           starts: rec.starts,
@@ -1163,6 +1258,14 @@ export class GraphSearch {
     if (!this.host.recogniseSpan) return;
     const tail = queryBytes.subarray(fact.j, queryLen);
     if (tail.length === 0) return;
+    // Report ONLY the invocations that could have joined: the search asks this
+    // rule for every finalized out with a node, which includes the one-byte
+    // outs the cover bridges with — measured, 68 refusals for a single
+    // 3-relation query, all of them letters.  A form shorter than one window is
+    // not a fact a join could travel through, so it is not a refusal worth
+    // reporting; W is the same line the rest of the mind draws between a chance
+    // overlap and a form.
+    const reportable = fact.bytes.length >= this.maxGroup;
     // The entity candidates are the forms the fact's own bytes CONTAIN — the
     // same recogniser the query went through, so the evidence standard is the
     // query's.  A byte atom is never a subject; the fact's own node is the span
@@ -1171,12 +1274,66 @@ export class GraphSearch {
     // LENDS it when it can (Mind does, with the response-scoped struct cache),
     // and a bare host falls back to the raw-store probe, so the search stays
     // host-based.
-    const leading = this.host.recogniseSpan(fact.bytes).sites.filter((s) =>
-      s.payload >= 0 && s.payload !== fact.node &&
-      (this.host.leadsSomewhere !== undefined
-        ? this.host.leadsSomewhere(s.payload)
-        : this.store.hasNext(s.payload) || this.store.hasHalo(s.payload))
+    const factRec = this.host.recogniseSpan(fact.bytes);
+    const leads = (id: number): boolean =>
+      this.host.leadsSomewhere !== undefined
+        ? this.host.leadsSomewhere(id)
+        : this.store.hasNext(id) || this.store.hasHalo(id);
+    // THE QUERY'S OWN SUBJECT, CANONICALLY.  The filter used raw bytes and the
+    // store's nodes are canonical, so `Eiffel Tower country` in the query did
+    // not match the deposited `eiffel tower country` — measured, that is the
+    // trap's wrong answer.
+    //
+    // TAKEN FROM THE RECOGNITION THE RESPONSE ALREADY COMPUTED — the host's
+    // `recogniseSpan`, the same surface the rest of this search uses — not from
+    // a second offset scan.  `canonicalQueryNodes` re-derived, per byte offset,
+    // what `recognise` had already resolved once per query (its memo is keyed by
+    // content), and that scan was the largest single cost the DIANOT join added:
+    // measured against the pre-change tree, the same fixture and the same test
+    // were 21 s slower with the scan than without it.  A recognised site IS a
+    // canonical node of the query that can lead somewhere, which is exactly the
+    // set this filter wants, and it costs nothing to read.
+    const queryNodes = new Set<number>(
+      (this.host.recogniseSpan?.(queryBytes)?.sites ?? []).map((s) =>
+        s.payload
+      ),
     );
+    // TWO SOURCES, ONE ADMISSION.  The recognition of a STORED WHOLE returns the
+    // whole and stops — measured: for `The director of Eva is Gustaf Molander.`
+    // it yields exactly ONE site, the fact's own node — so the entity a join
+    // exists for is never proposed.  The canonical fold is the second source,
+    // and the scan runs only for a FORM (≥ W: a one-byte out is not something to
+    // join through, and running it per letter measured 20-26 s in test/99).
+    const W = this.maxGroup;
+    const proposed = new Map<number, Uint8Array>();
+    // The SOURCE of each proposal travels with it: a refusal that names only the
+    // bytes leaves the next reader guessing which path proposed them — three
+    // attempts at the chained join were spent fixing paths that never produced
+    // the offending candidate.
+    const source = new Map<number, string>();
+    for (const s of factRec.sites) {
+      if (s.payload >= 0 && leads(s.payload)) {
+        proposed.set(s.payload, this.store.bytesPrefix(s.payload, ALL));
+        source.set(s.payload, "recognised site");
+      }
+    }
+    if (this.host.canonResolve !== undefined && fact.bytes.length >= W) {
+      const canon = this.host.canonResolve.bind(this.host);
+      for (let start = 0; start < fact.bytes.length; start++) {
+        for (let end = fact.bytes.length; end - start >= W; end--) {
+          const id = canon(fact.bytes.subarray(start, end));
+          if (id === null) continue;
+          if (leads(id)) {
+            proposed.set(id, this.store.bytesPrefix(id, ALL));
+            source.set(id, "canonical fold");
+          }
+          break; // the longest form at this offset wins
+        }
+      }
+    }
+    const leading = [...proposed]
+      .filter(([payload]) => payload !== fact.node && !queryNodes.has(payload))
+      .map(([payload, bytes]) => ({ payload, bytes }));
     // …then prefer the entity the query did NOT name, and the MAXIMAL one.  The
     // join exists to reach the subject the query never wrote, so:
     //   • a candidate the query already contains is the query's OWN subject, and
@@ -1187,32 +1344,104 @@ export class GraphSearch {
     //     introduces ("Timur" must not win over "Timur Bekmambetov").
     // Byte work over bytes already read, and the pruning REMOVES the
     // resolve()/nextFirst() probes these candidates would have paid.
+    if (leading.length === 0) {
+      if (this.host.meter) this.host.meter.joinNoEntity++;
+      if (reportable) {
+        // Report WHAT the recognition returned, not just that nothing led: the
+        // count and the first few site texts are the difference between "the
+        // fact was not recognised" and "it was recognised but nothing led".
+        const seen = factRec.sites.slice(0, 3).map((s) =>
+          this.store.bytesPrefix(s.payload, ALL)
+        );
+        this.host.reportSearch?.(
+          "deriveThroughMiss",
+          [fact.bytes, tail, ...seen],
+          `no entity inside the fact leads anywhere — ${factRec.sites.length} site(s) recognised inside it`,
+        );
+      }
+    }
     const candidates = leading
-      .map((s) => ({
-        payload: s.payload,
-        bytes: this.store.bytesPrefix(s.payload, ALL),
-      }))
-      .filter((c) => indexOf(queryBytes, c.bytes, 0) < 0)
       .filter((c, _i, all) =>
         !all.some((o) =>
           o.bytes.length > c.bytes.length && indexOf(o.bytes, c.bytes, 0) >= 0
         )
       );
     for (const c of candidates) {
-      const key = this.host.resolve(concat2(c.bytes, tail));
-      if (key === null) continue;
-      const nx = this.store.nextFirst(key, 1);
-      if (nx.length === 0) continue;
+      // THE SHORTEST TAIL PREFIX WHOSE KEY ALSO LEADS SOMEWHERE.
+      //
+      // The conclusion covers only that prefix, so the rest of the tail stays
+      // for the step after — which is what CHAINING is (a whole-tail key
+      // consumed the whole remainder and made every join terminal: measured,
+      // `joinFired=0` on a three-relation query).
+      //
+      // A KEY THAT RESOLVES IS NOT ENOUGH.  Measured with a dry run of these
+      // very primitives: for the candidate `Sweden` the first tail prefix that
+      // resolves is `" "` — the key `Sweden ` (a trailing space) — and it leads
+      // NOWHERE (`nextFirst` = 0), so accepting it refused the join while the
+      // key that names the fact (`sweden capital`) sat one prefix further.  The
+      // loop therefore asks BOTH questions before accepting, and keeps looking
+      // otherwise.
+      //
+      // EXACT FIRST, THEN CANONICAL: the corpus holds both identities.
+      let key: number | null = null;
+      let used = 0;
+      // The continuation the accepted key leads to, carried out of the loop:
+      // the loop already HAD to read it to accept the key (a key that leads
+      // nowhere is not the relation), so re-reading it after the loop was a
+      // duplicate read and a branch that could never be taken.
+      let next: number | null = null;
+      let keyBytes = c.bytes;
+      // THE PREFIX ENDS ARE THE TAIL'S OWN FOLD BOUNDARIES, not every byte
+      // length.  The key is `entity + prefix`, and the prefix that names a
+      // stored relation ends where the fold cuts: measured over four join-firing
+      // queries, 5 of 5 accepted keys ended on a boundary (or the tail's end)
+      // while the byte-by-byte scan spent 153 probes where 14 boundaries would
+      // do.  Same criterion — resolves AND leads — same shortest-first order, so
+      // the answer is the same one the enumeration found; only the candidates
+      // come from the structure instead of from the byte count.  A host with no
+      // boundary rule falls back to the enumeration.
+      const cuts = this.host.contentCuts?.(tail);
+      const ends = cuts && cuts.length > 0
+        ? [...cuts.filter((c) => c > 0 && c < tail.length), tail.length]
+        : Array.from({ length: tail.length }, (_, i) => i + 1);
+      for (const len of ends) {
+        keyBytes = concat2(c.bytes, tail.subarray(0, len));
+        const k = this.host.resolve(keyBytes) ??
+          this.host.canonResolve?.(keyBytes) ??
+          null;
+        if (k === null) continue;
+        const nx = this.store.nextFirst(k, 1);
+        if (nx.length === 0) continue;
+        key = k;
+        used = len;
+        next = nx[0];
+        break;
+      }
+      if (key === null) {
+        if (this.host.meter) this.host.meter.joinNoKey++;
+        if (reportable) {
+          this.host.reportSearch?.(
+            "deriveThroughMiss",
+            [c.bytes, tail, keyBytes],
+            `no learnt key names this entity and tail together ` +
+              `(candidate #${c.payload}, from the ${
+                source.get(c.payload) ?? "unknown"
+              } source)`,
+          );
+        }
+        continue;
+      }
+      if (this.host.meter) this.host.meter.joinFired++;
       yield {
         premises: [fact],
         conclusion: {
           kind: "out",
           i: fact.i,
-          j: queryLen,
-          bytes: this.store.bytesPrefix(nx[0], ALL),
+          j: fact.j + used,
+          bytes: this.store.bytesPrefix(next!, ALL),
           cover: true,
           rec: true,
-          node: nx[0],
+          node: next!,
           throughFact: true,
         },
         cost: STEP,

package/src/mind/index.ts

@@ -4,7 +4,12 @@
 // exported from mind/mind.ts directly.
 
 export { Mind } from "./mind.js";
-export type { Input, Response } from "./mind.js";
+export type {
+  CorpusTextPair,
+  CorpusTextResult,
+  Input,
+  Response,
+} from "./mind.js";
 export type { ComputedSpan, ExtensionHost } from "./mind.js";
 export type {
   MechanismResult,
@@ -38,3 +43,5 @@ export type {
   NarrowDecisionData,
   Provenance,
 } from "./pipeline.js";
+export { sampleCorpus, searchCorpus } from "./corpus.js";
+export type { CorpusMiss, CorpusPair, CorpusResult } from "./corpus.js";