@hviana/sema 0.8.1 → 0.8.3

package/dist/src/mind/graph-search.js

@@ -194,6 +194,13 @@ export class GraphSearch {
         this.maxGroup = maxGroup;
         this.host = host;
     }
+    /** The nodes the QUERY canonically names — the same identity the store's keys
+     *  were written through.  A byte-exact test is not enough: the query writes
+     *  `Eiffel Tower country` and the deposited node is `eiffel tower country`, so
+     *  a join that filters the query's own subject by RAW bytes re-admits it —
+     *  measured: that is the trap's wrong answer (`The capital of Eiffel Tower
+     *  country is Berlin.`).  Cached by query identity, because the search is
+     *  reused across responses. */
     /* * The hub bound √N (bounded-reads.md) — the ONE
      *  fan-out cap, stated here rather than imported from `traverse.ts` because
      *  this module is deliberately host-based (it holds a bare Store, never a
@@ -461,7 +468,7 @@ export class GraphSearch {
                     return this.coverRules(it, coversDone, coverableByStart);
                 }
                 if (it.kind === "form") {
-                    return this.formRules(it, conceptTarget, substitutions, nodeBytes);
+                    return this.formRules(it, conceptTarget, substitutions, nodeBytes, queryLen);
                 }
                 return this.outRules(it, {
                     W,
@@ -547,7 +554,7 @@ export class GraphSearch {
     }
     /** form(i,j,node,via): follow the graph out of `node`, or (in articulation)
      *  emit its substitute voice directly. */
-    *formRules(it, conceptTarget, substitutions, nodeBytes) {
+    *formRules(it, conceptTarget, substitutions, nodeBytes, queryLen) {
         // Articulation: emit voice bytes at the recognised span; the hop/concept/
         // emit chain is suppressed — the form contributes only its substitute.
         if (substitutions) {
@@ -581,7 +588,46 @@ export class GraphSearch {
             // guard then dead-ends it) with no way to reach the forward edge.
             // Forking offers every continuation as its own rule so the one that
             // genuinely advances (not a duplicate) is still reachable.
+            // A CHAIN HOP OFFERS ONLY WHAT THE QUESTION CAN PAY FOR.
+            //
+            // `hubBound` = √N is the READ cap — every read here stays inside it — but
+            // it is not an exploration bound: measured, a hub of degree 1083 sits
+            // BELOW √N = 1559, so a hop offered all 1083 continuations, the chart grew
+            // to 3113 outs for a two-word question, and since every out with an
+            // uncovered tail probes its tail's prefixes (measured: 16 885 canonical
+            // probes = 87% of that query's work, and its 270 MB peak / 256 MB OOM),
+            // the cost came from OFFERING rather than from reading.
+            //
+            // The bound is derived, not tuned: a derivation of L hops consumes ~L
+            // units of the question, so a hop cannot be paid for by offering more
+            // continuations than the question has units —
+            // `ceil(queryLen / W)`, floored at 2 for plurality. It is QUERY-sized
+            // (invariant 5: no per-query read grows with N) and it leaves `hubBound`
+            // and every read untouched.
+            // THE OFFER IS THE CORPUS'S OWN STRUCTURE, and the search pays for
+            // exploring it.  There is no offer cap here any more: the traversal cap I
+            // had put on this hop was a short-circuit — it bounded what a hop could
+            // OFFER instead of charging for it — and it was not needed.
+            //
+            // MEASURED in the regime where it used to bite (`hubBound = ceil(√N)`
+            // GREATER than the hub's degree — reached in a fixture by choosing the
+            // degree below √N, so the trained store is not needed): with the cap the
+            // offer was 8/9/9 continuations at degrees 35/70/120; without it, 52/84/120
+            // — and the WORK is LINEAR in the degree, not quadratic: pushes 262/296/332,
+            // perceptions 530/592/757, while the PEAK is identical with and without the
+            // cap (218/415/689 MB against 215/410/662) because it is set by the store,
+            // not by the fan-out.  What made this hop expensive was never the fan-out
+            // breadth: it was the per-offer work, two duplicate/oversized computations
+            // since removed (the per-offset canonical scan, and the tail scan now
+            // restricted to the fold's boundaries).
+            //
+            // The residual, stated: the trained store's hub (degree 1 083) is an
+            // EXTRAPOLATION from this linear shape, not a measurement.
             const nx = this.store.nextFirst(it.node, this.hubBound());
+            // Count what is OFFERED, not what was read: the evidence-preferred
+            // continuation is yielded too, even when it lies outside the cap.
+            if (this.host.meter)
+                this.host.meter.chainOffers += nx.length + 1;
             if (nx.length) {
                 // The SAME evidence-weighted disambiguation the first hop uses
                 // (below) identifies the most-corroborated continuation.  Yielding
@@ -590,10 +636,14 @@ export class GraphSearch {
                 // arrivals at an EQUAL cost (`cost < current`, strictly) — so
                 // among same-depth sibling forks that tie in cost, the
                 // evidence-backed edge wins deterministically, never by
-                // exploration-order luck.  `preferred`, when set, is necessarily
-                // an element of `nx` (chooseNext reads the identical hub-bounded
-                // set — see traverse.ts), so a plain skip-in-place suffices; no
-                // second array need be allocated to reorder it to the front.
+                // exploration-order luck.  `preferred` is NOT necessarily an element
+                // of `nx` any more: `nx` is the capped read above, while `chooseNext`
+                // reads its own hub-bounded set (see traverse.ts) — so the evidence
+                // pick may lie outside the cap, and it is still yielded FIRST on
+                // purpose.  The cap bounds what the hop EXPLORES; it must never make
+                // the evidence-ranked continuation unreachable, or the derivation the
+                // corpus corroborates would lose to exploration order.  Nothing is
+                // reallocated: `nx` is walked with a skip-in-place for the duplicate.
                 const preferred = nx.length > 1
                     ? this.host.chooseNext?.(it.node)
                     : undefined;
@@ -818,10 +868,50 @@ export class GraphSearch {
             // concepts/connectors either (those need the caller's async
             // pre-resolution) — the recursion follows edges and fusion, which is what
             // a deeper rewrite chain is made of.
+            if (this.host.meter)
+                this.host.meter.recompletes++;
             const rec = this.host.recogniseSpan(bytes);
             const kids = new Set(nrec.kids);
+            // THE NODE'S OWN KIDS ARE SITES BY STRUCTURE — recognition cannot be the
+            // only source.  A produced composite is decomposed by ITS OWN SHAPE, and
+            // at hub scale the recognition of a produced span returns the WHOLE while
+            // deliberately suppressing its atoms (the off-boundary suppression), so
+            // the kid filter below would admit nothing at all and the chain would end
+            // at the intermediate composite.  Measured on the chain
+            // `seed → "p q" → (p→r, q→s) → "r s" → "m n"`: below the flip it reaches
+            // "m n" with fuse+recompose, above it stops at "p q" — the trace shows
+            // `recognise("p q") ⇒ form "p q"` alone, no parts.
+            //
+            // Laying the node's kids out over its own bytes restores exactly the
+            // decomposition the node's tree already states; a kid recognition ALREADY
+            // offers is skipped, so below the flip the seed set is byte-identical to
+            // what it was.  The filter's guarantee is untouched: nothing beyond the
+            // node's own kids may enter.
+            const recognised = rec.sites.filter((s) => kids.has(s.payload));
+            // DEDUPED BY SPAN, not by payload.  A node whose kids repeat (`"abab"`
+            // folds as ["ab","ab"]) has TWO occurrences of the same node at different
+            // offsets; a payload-keyed set suppressed the structural site for BOTH, so
+            // the second occurrence had no site at all.  The set now holds the spans
+            // recognition already covers, and a structural site is added exactly when
+            // nothing covers that PLACE — O(1) lookups, no extra reads.
+            const seenSpans = new Set(recognised.map((s) => `${s.start}:${s.end}`));
+            const structural = [];
+            {
+                let off = 0;
+                for (const k of nrec.kids) {
+                    const len = this.store.bytesPrefix(k, ALL).length;
+                    if (!seenSpans.has(`${off}:${off + len}`) && len > 0) {
+                        structural.push({
+                            start: off,
+                            end: Math.min(off + len, bytes.length),
+                            payload: k,
+                        });
+                    }
+                    off += len;
+                }
+            }
             const solved = this.solve(bytes.length, {
-                sites: rec.sites.filter((s) => kids.has(s.payload)),
+                sites: [...recognised, ...structural],
                 leaves: rec.leaves,
                 splits: rec.splits,
                 starts: rec.starts,
@@ -892,6 +982,14 @@ export class GraphSearch {
         const tail = queryBytes.subarray(fact.j, queryLen);
         if (tail.length === 0)
             return;
+        // Report ONLY the invocations that could have joined: the search asks this
+        // rule for every finalized out with a node, which includes the one-byte
+        // outs the cover bridges with — measured, 68 refusals for a single
+        // 3-relation query, all of them letters.  A form shorter than one window is
+        // not a fact a join could travel through, so it is not a refusal worth
+        // reporting; W is the same line the rest of the mind draws between a chance
+        // overlap and a form.
+        const reportable = fact.bytes.length >= this.maxGroup;
         // The entity candidates are the forms the fact's own bytes CONTAIN — the
         // same recogniser the query went through, so the evidence standard is the
         // query's.  A byte atom is never a subject; the fact's own node is the span
@@ -900,10 +998,62 @@ export class GraphSearch {
         // LENDS it when it can (Mind does, with the response-scoped struct cache),
         // and a bare host falls back to the raw-store probe, so the search stays
         // host-based.
-        const leading = this.host.recogniseSpan(fact.bytes).sites.filter((s) => s.payload >= 0 && s.payload !== fact.node &&
-            (this.host.leadsSomewhere !== undefined
-                ? this.host.leadsSomewhere(s.payload)
-                : this.store.hasNext(s.payload) || this.store.hasHalo(s.payload)));
+        const factRec = this.host.recogniseSpan(fact.bytes);
+        const leads = (id) => this.host.leadsSomewhere !== undefined
+            ? this.host.leadsSomewhere(id)
+            : this.store.hasNext(id) || this.store.hasHalo(id);
+        // THE QUERY'S OWN SUBJECT, CANONICALLY.  The filter used raw bytes and the
+        // store's nodes are canonical, so `Eiffel Tower country` in the query did
+        // not match the deposited `eiffel tower country` — measured, that is the
+        // trap's wrong answer.
+        //
+        // TAKEN FROM THE RECOGNITION THE RESPONSE ALREADY COMPUTED — the host's
+        // `recogniseSpan`, the same surface the rest of this search uses — not from
+        // a second offset scan.  `canonicalQueryNodes` re-derived, per byte offset,
+        // what `recognise` had already resolved once per query (its memo is keyed by
+        // content), and that scan was the largest single cost the DIANOT join added:
+        // measured against the pre-change tree, the same fixture and the same test
+        // were 21 s slower with the scan than without it.  A recognised site IS a
+        // canonical node of the query that can lead somewhere, which is exactly the
+        // set this filter wants, and it costs nothing to read.
+        const queryNodes = new Set((this.host.recogniseSpan?.(queryBytes)?.sites ?? []).map((s) => s.payload));
+        // TWO SOURCES, ONE ADMISSION.  The recognition of a STORED WHOLE returns the
+        // whole and stops — measured: for `The director of Eva is Gustaf Molander.`
+        // it yields exactly ONE site, the fact's own node — so the entity a join
+        // exists for is never proposed.  The canonical fold is the second source,
+        // and the scan runs only for a FORM (≥ W: a one-byte out is not something to
+        // join through, and running it per letter measured 20-26 s in test/99).
+        const W = this.maxGroup;
+        const proposed = new Map();
+        // The SOURCE of each proposal travels with it: a refusal that names only the
+        // bytes leaves the next reader guessing which path proposed them — three
+        // attempts at the chained join were spent fixing paths that never produced
+        // the offending candidate.
+        const source = new Map();
+        for (const s of factRec.sites) {
+            if (s.payload >= 0 && leads(s.payload)) {
+                proposed.set(s.payload, this.store.bytesPrefix(s.payload, ALL));
+                source.set(s.payload, "recognised site");
+            }
+        }
+        if (this.host.canonResolve !== undefined && fact.bytes.length >= W) {
+            const canon = this.host.canonResolve.bind(this.host);
+            for (let start = 0; start < fact.bytes.length; start++) {
+                for (let end = fact.bytes.length; end - start >= W; end--) {
+                    const id = canon(fact.bytes.subarray(start, end));
+                    if (id === null)
+                        continue;
+                    if (leads(id)) {
+                        proposed.set(id, this.store.bytesPrefix(id, ALL));
+                        source.set(id, "canonical fold");
+                    }
+                    break; // the longest form at this offset wins
+                }
+            }
+        }
+        const leading = [...proposed]
+            .filter(([payload]) => payload !== fact.node && !queryNodes.has(payload))
+            .map(([payload, bytes]) => ({ payload, bytes }));
         // …then prefer the entity the query did NOT name, and the MAXIMAL one.  The
         // join exists to reach the subject the query never wrote, so:
         //   • a candidate the query already contains is the query's OWN subject, and
@@ -914,30 +1064,110 @@ export class GraphSearch {
         //     introduces ("Timur" must not win over "Timur Bekmambetov").
         // Byte work over bytes already read, and the pruning REMOVES the
         // resolve()/nextFirst() probes these candidates would have paid.
+        if (leading.length === 0) {
+            if (this.host.meter)
+                this.host.meter.joinNoEntity++;
+            if (reportable) {
+                // Report WHAT the recognition returned, not just that nothing led: the
+                // count and the first few site texts are the difference between "the
+                // fact was not recognised" and "it was recognised but nothing led".
+                const seen = factRec.sites.slice(0, 3).map((s) => this.store.bytesPrefix(s.payload, ALL));
+                this.host.reportSearch?.("deriveThroughMiss", [fact.bytes, tail, ...seen], `no entity inside the fact leads anywhere — ${factRec.sites.length} site(s) recognised inside it`);
+            }
+        }
         const candidates = leading
-            .map((s) => ({
-            payload: s.payload,
-            bytes: this.store.bytesPrefix(s.payload, ALL),
-        }))
-            .filter((c) => indexOf(queryBytes, c.bytes, 0) < 0)
             .filter((c, _i, all) => !all.some((o) => o.bytes.length > c.bytes.length && indexOf(o.bytes, c.bytes, 0) >= 0));
         for (const c of candidates) {
-            const key = this.host.resolve(concat2(c.bytes, tail));
-            if (key === null)
-                continue;
-            const nx = this.store.nextFirst(key, 1);
-            if (nx.length === 0)
+            // THE SHORTEST TAIL PREFIX WHOSE KEY ALSO LEADS SOMEWHERE.
+            //
+            // The conclusion covers only that prefix, so the rest of the tail stays
+            // for the step after — which is what CHAINING is (a whole-tail key
+            // consumed the whole remainder and made every join terminal: measured,
+            // `joinFired=0` on a three-relation query).
+            //
+            // A KEY THAT RESOLVES IS NOT ENOUGH.  Measured with a dry run of these
+            // very primitives: for the candidate `Sweden` the first tail prefix that
+            // resolves is `" "` — the key `Sweden ` (a trailing space) — and it leads
+            // NOWHERE (`nextFirst` = 0), so accepting it refused the join while the
+            // key that names the fact (`sweden capital`) sat one prefix further.  The
+            // loop therefore asks BOTH questions before accepting, and keeps looking
+            // otherwise.
+            //
+            // EXACT FIRST, THEN CANONICAL: the corpus holds both identities.
+            let key = null;
+            let used = 0;
+            // The continuation the accepted key leads to, carried out of the loop:
+            // the loop already HAD to read it to accept the key (a key that leads
+            // nowhere is not the relation), so re-reading it after the loop was a
+            // duplicate read and a branch that could never be taken.
+            let next = null;
+            let keyBytes = c.bytes;
+            // THE CANDIDATE ENDS ARE THE PREFIXES THAT ARE STORED NODES, ASCENDING.
+            // The key is `entity + prefix`, and it names a relation exactly when that
+            // concatenation IS a node — so the ends come from a content-addressed
+            // probe per offset (the host's `contentKeyEnds`, the learning path's own
+            // mechanism: one leaf walk plus one `findBranch` per offset, no `resolve`),
+            // never from the fold's boundaries.  A boundary is not a proxy: a stored
+            // member's end is the end of ITS OWN stream, and the fold never cuts at a
+            // stream's end — measured, "stockholm mayor" exists, leads on to the mayor
+            // fact, and its boundary 6 sits in neither the tail's cuts ([4,7]) nor the
+            // concatenation's.  Filtering the scan by "is this a node?" cannot change
+            // the winner: a position that is not a node cannot resolve, so skipping it
+            // is invisible; and the order stays SHORTEST FIRST, which is a semantic
+            // law, not an optimisation (test/106, test/108 pin it).
+            //
+            // A host that cannot answer falls back to every prefix: exact and
+            // complete, at a `resolve` per offset.  A host that CAN answer is
+            // authoritative even when it answers "none" — if no prefix is a node then
+            // no key exists to resolve, so enumerating would only pay nulls.  (A key
+            // reachable through the CANONICAL equivalence alone and ending off every
+            // node end is therefore not tried here; that dimension is unreachable on
+            // this path by construction and is not part of the exact-key law.)
+            const ends = this.host.contentKeyEnds?.(c.bytes, tail);
+            const candidateEnds = function* () {
+                if (ends !== undefined) {
+                    yield* ends;
+                    return;
+                }
+                for (let p = 1; p <= tail.length; p++)
+                    yield p;
+            };
+            for (const len of candidateEnds()) {
+                keyBytes = concat2(c.bytes, tail.subarray(0, len));
+                const k = this.host.resolve(keyBytes) ??
+                    this.host.canonResolve?.(keyBytes) ??
+                    null;
+                if (k === null)
+                    continue;
+                const nx = this.store.nextFirst(k, 1);
+                if (nx.length === 0)
+                    continue;
+                key = k;
+                used = len;
+                next = nx[0];
+                break;
+            }
+            if (key === null) {
+                if (this.host.meter)
+                    this.host.meter.joinNoKey++;
+                if (reportable) {
+                    this.host.reportSearch?.("deriveThroughMiss", [c.bytes, tail, keyBytes], `no learnt key names this entity and tail together ` +
+                        `(candidate #${c.payload}, from the ${source.get(c.payload) ?? "unknown"} source)`);
+                }
                 continue;
+            }
+            if (this.host.meter)
+                this.host.meter.joinFired++;
             yield {
                 premises: [fact],
                 conclusion: {
                     kind: "out",
                     i: fact.i,
-                    j: queryLen,
-                    bytes: this.store.bytesPrefix(nx[0], ALL),
+                    j: fact.j + used,
+                    bytes: this.store.bytesPrefix(next, ALL),
                     cover: true,
                     rec: true,
-                    node: nx[0],
+                    node: next,
                     throughFact: true,
                 },
                 cost: STEP,

package/dist/src/mind/index.d.ts

@@ -1,5 +1,5 @@
 export { Mind } from "./mind.js";
-export type { Input, Response } from "./mind.js";
+export type { CorpusTextPair, CorpusTextResult, Input, Response, } from "./mind.js";
 export type { ComputedSpan, ExtensionHost } from "./mind.js";
 export type { MechanismResult, PipelineMechanism, Precomputed, } from "./pipeline-mechanism.js";
 export type { InspectRationale, RationaleItem, RationaleStep, } from "./rationale.js";
@@ -7,3 +7,5 @@ export type { AnchorRejectionReason, ClimbConsensusData, ConsensusAnchorTrace, C
 export type { AncestorReach, AttentionRead, SaturationReason, SaturationStop, } from "./types.js";
 export type { DepositReport } from "./learning.js";
 export type { DecideGroundingData, NarrowDecisionData, Provenance, } from "./pipeline.js";
+export { sampleCorpus, searchCorpus } from "./corpus.js";
+export type { CorpusMiss, CorpusPair, CorpusResult } from "./corpus.js";

package/dist/src/mind/index.js

@@ -3,3 +3,4 @@
 // Re-exports the Mind class and all public types that were previously
 // exported from mind/mind.ts directly.
 export { Mind } from "./mind.js";
+export { sampleCorpus, searchCorpus } from "./corpus.js";

package/dist/src/mind/match.d.ts

@@ -78,9 +78,14 @@ export interface AlignGap {
 }
 /** Extend a seed match (query offset qo ↔ candidate offset co) to its maximal
  *  common run, then walk outward in both directions collecting further common
- *  runs of at least W bytes across bounded mismatch gaps (each side ≤
- *  chainReach).  Returns the matched query spans and the mismatch pairs
- *  between consecutive runs.
+ *  runs of at least W bytes across mismatch gaps.  Each gap's LENGTH is the
+ *  pair's own extent (a gap cannot be longer than the bytes it spans) and the
+ *  sweep's WORK is proportional to the bytes a run spans (the context's windows
+ *  are indexed once, then the query's are walked) — the arity bound
+ *  (`chainReach`) used to cap BOTH, and truncated every learned frame whose
+ *  slot was longer.  Each sweep owns its own budget, so an exhausted right
+ *  sweep never starves the left one.  Returns the matched query spans and the
+ *  mismatch pairs between consecutive runs.
  *
  *  This is the SEEDED aligner, distinct from {@link alignRuns}: that one finds
  *  every run two structures share anywhere (a weave), this one reads two
@@ -195,7 +200,7 @@ export declare function frameSlots(ctx: MindContext, query: Uint8Array, cand: Ui
  *  ANCHOR that the query displaced.  Neither implies the other, and the
  *  observed failures pass the restatement guard cleanly.
  *
- *  Three conditions, all byte-exact and all necessary:
+ *  Four conditions, all byte-exact and all necessary:
  *
  *  1. the query and the anchor must be ONE STRUCTURE — what they share has to
  *     dominate the query, or the query is not a variant of the anchor at all