@hviana/sema 0.9.1 → 0.9.2

package/src/mind/evidence.ts

@@ -0,0 +1,107 @@
+// evidence.ts — what of a stored form the material at hand WITNESSES.
+//
+// A stored form is WITNESSED by some material when every one of its bytes lies
+// inside a W-window that occurs literally in that material — in any order, at
+// any place.  It is the order-free reading of correspondence (`alignRuns` is
+// its run-producing sibling, `junctionContainersFrom(…, unordered)` its
+// container-finding one): a learnt form is evidenced by bytes that hold all of
+// its windows, whichever way the asker happened to arrange them.
+//
+// The material is a LIST of sources because what a derivation has at hand is
+// more than the question: the node it stands on is material too.  A trained
+// question `Edmond T. Gréville place of death` is in neither the asker's
+// `Where was the place of death of the director of film Beat Girl?` nor the
+// fact `The director of Beat Girl is Edmond T. Gréville.` the first hop reached
+// — and both together hold every byte of it.  The witness records which source
+// held each window, so a consumer can tell the question's share (what the asker
+// SAID about the form) from what the derivation itself brought.
+//
+// Exact, deterministic and linear: one window index per source, one probe per
+// window of the form.  Below one window a byte agreement is chance (the floor
+// identityBar, attestedQ and the site test all draw), so a form shorter than W
+// is never witnessed.
+//
+// Layering: bytes only — importable from traverse.ts and everything above it.
+
+import { latin1 } from "../bytes.js";
+
+/** Where each W-window of one source first occurs.  Built once per source and
+ *  reused across every form asked against it. */
+export type WindowIndex = Map<string, number>;
+
+export function windowIndex(bytes: Uint8Array, W: number): WindowIndex {
+  const index: WindowIndex = new Map();
+  for (let o = 0; o + W <= bytes.length; o++) {
+    const key = latin1(bytes.subarray(o, o + W));
+    if (!index.has(key)) index.set(key, o);
+  }
+  return index;
+}
+
+/** The windows of `index` (over `bytes`) that `spoken` does not hold — what
+ *  of the question no product of the derivation has restated yet.  The keys
+ *  keep their positions in `bytes`, so a witnessing span still names the
+ *  asker's bytes. */
+export function unspoken(
+  index: WindowIndex,
+  spoken: ReadonlyArray<WindowIndex>,
+): WindowIndex {
+  const out: WindowIndex = new Map();
+  for (const [key, at] of index) {
+    if (!spoken.some((s) => s.has(key))) out.set(key, at);
+  }
+  return out;
+}
+
+/** How a form is witnessed by a list of sources. */
+export interface Witnessing {
+  /** Every byte of the form lies in a window some source holds. */
+  complete: boolean;
+  /** The windows SOURCE 0 alone supplied — windows no later source holds — as
+   *  merged spans of source 0.  With the question as source 0, this is what the
+   *  asker said about the form that the derivation did not already have. */
+  spans: Array<[number, number]>;
+  /** Bytes of source 0 inside `spans`. */
+  bytes: number;
+}
+
+/** Witness `form` against `sources` (their window indexes, same order).  A
+ *  window is credited to the LAST source holding it — so source 0 is credited
+ *  only with what nothing else at hand supplies. */
+export function witness(
+  form: Uint8Array,
+  indexes: ReadonlyArray<WindowIndex>,
+  W: number,
+): Witnessing {
+  const none: Witnessing = { complete: false, spans: [], bytes: 0 };
+  if (form.length < W || indexes.length === 0) return none;
+  const covered = new Uint8Array(form.length);
+  const own: Array<[number, number]> = [];
+  for (let o = 0; o + W <= form.length; o++) {
+    const key = latin1(form.subarray(o, o + W));
+    let at = -1;
+    let from = -1;
+    for (let s = indexes.length - 1; s >= 0; s--) {
+      const p = indexes[s].get(key);
+      if (p !== undefined) {
+        at = p;
+        from = s;
+        break;
+      }
+    }
+    if (from < 0) continue;
+    covered.fill(1, o, o + W);
+    if (from === 0) own.push([at, at + W]);
+  }
+  for (let i = 0; i < form.length; i++) if (!covered[i]) return none;
+  own.sort((a, b) => a[0] - b[0]);
+  const spans: Array<[number, number]> = [];
+  for (const [s, e] of own) {
+    const last = spans[spans.length - 1];
+    if (last !== undefined && s <= last[1]) last[1] = Math.max(last[1], e);
+    else spans.push([s, e]);
+  }
+  let bytes = 0;
+  for (const [s, e] of spans) bytes += e - s;
+  return { complete: true, spans, bytes };
+}

package/src/mind/mechanisms/cover.ts

@@ -18,7 +18,12 @@ import type {
   Site,
 } from "../graph-search.js";
 import { read, resolve } from "../primitives.js";
-import { guidedFirst, hubBound } from "../traverse.js";
+import {
+  guidedFirst,
+  hubBound,
+  namedContinuations,
+  scaffoldSpans,
+} from "../traverse.js";
 import { conceptHop } from "../match.js";
 import { bridge } from "../resonance.js";
 import { liftAnswer, liftedScaffolding, segRestatesQuery } from "../types.js";
@@ -229,7 +234,7 @@ export const coverMechanism: PipelineMechanism = {
 
     // Masking: computed spans are authoritative.  Remove recognised sites
     // that overlap any computed span before building the cover search.
-    const sites = computed.length === 0
+    let sites = computed.length === 0
       ? rec.sites
       : rec.sites.filter((s) =>
         !computed.some((u) => s.start < u.j && u.i < s.end)
@@ -254,6 +259,45 @@ export const coverMechanism: PipelineMechanism = {
       );
     }
 
+    // A FRAGMENT ANSWERS OTHER QUESTIONS.  A recognised form that sits inside
+    // other forms (it has structural parents or containers) holds continuations
+    // because the forms it is a piece of were asked — suffix inheritance gives
+    // `f death` the answer of every `… place of death` question.  Choosing one of
+    // several such continuations by popularity voices some other question's
+    // answer, and the cover used to account the fragment's bytes as explained by
+    // it.  A fragment with several continuations leads somewhere FOR THIS
+    // QUESTION only when the question names one (traverse.ts, the exact tier).
+    // A form that is (nearly) the whole question is not a piece of it: the
+    // question says nothing beyond it, so its continuations answer THIS question.
+    const asked = ctx._edgeAsked;
+    const W0 = ctx.space.maxGroup;
+    const leading = asked === null
+      ? sites
+      : sites.filter((s) =>
+        query.length - (s.end - s.start) < W0 ||
+        !(ctx.store.hasParents(s.payload) ||
+          ctx.store.hasContainers(s.payload)) ||
+        ctx.store.nextFirst(s.payload, 2).length < 2 ||
+        namedContinuations(ctx, s.payload, asked) !== null
+      );
+    if (leading.length < sites.length) {
+      if (ctx.meter) {
+        ctx.meter.unaskedFragments += sites.length - leading.length;
+      }
+      ctx.trace?.step(
+        "unaskedFragments",
+        sites.filter((s) => !leading.includes(s)).map((s) =>
+          rItem(query.subarray(s.start, s.end), "fragment", s.payload, [
+            s.start,
+            s.end,
+          ])
+        ),
+        [],
+        "a piece of other forms whose continuations the question names none of — it answers other questions",
+      );
+      sites = leading;
+    }
+
     if (sites.length === 0 && computed.length === 0) return [];
 
     const connectors = ctx.meter
@@ -382,8 +426,19 @@ export const coverMechanism: PipelineMechanism = {
     // restated span contributes nothing to the composed answer, so it must
     // not be priced as if it did (PASS-carried bytes are priced already;
     // the diagnostic label reflects the same distinction).
+    //
+    // …and a span made of nothing but corpus-global scaffolding names nothing:
+    // every window of it is a piece of more than √N stored forms, so the form
+    // recognised there — the song `What`, in the trained store — is one of
+    // thousands it could be.  Voicing that form's fact explains no byte the
+    // asker owes: measured, `What country is Jerry Bock a citizen of?` was won
+    // by `The performer of What is Melinda Marx.` glued onto the right fact.
+    const scaffold = scaffoldSpans(ctx, query);
     const accounted: Array<[number, number]> = segs
-      .filter((s) => s.rec && !segRestatesQuery(s, query, query.length, W))
+      .filter((s) =>
+        s.rec && !segRestatesQuery(s, query, query.length, W) &&
+        !scaffold.some(([a, b]) => a <= s.i && s.j <= b)
+      )
       .map((s) => [s.i, s.j]);
 
     return [{

package/src/mind/mechanisms/recall.ts

@@ -15,7 +15,13 @@ import {
 import type { MindContext } from "../types.js";
 import { gistOf, read, resolve } from "../primitives.js";
 import { indexOf } from "../../bytes.js";
-import { allWindowsAreScaffolding, corpusN, hubBound } from "../traverse.js";
+import {
+  allWindowsAreScaffolding,
+  askedEvidence,
+  corpusN,
+  guidedFirst,
+  hubBound,
+} from "../traverse.js";
 import {
   follow,
   project,
@@ -154,10 +160,18 @@ export async function recallByResonance(
         g !== null && g.length > 0 &&
         !lawRestates(query, g, 0, { proper: true })
       ) {
+        // EVIDENCE TRAVELS: when the question NAMED the continuation followed
+        // (traverse.ts, the exact tier — `Who is the father of Frederick II?`
+        // witnesses the father fact's own question `Frederick II father`), the
+        // spans that named it are explained too, not only the argument.
+        const hop = guidedFirst(ctx, arg.payload);
+        const named = hop === undefined
+          ? []
+          : askedEvidence(ctx, arg.payload, hop);
         return ground(
           g,
           "argument binding — the query's sole edge-source constituent, continuation followed",
-          [[arg.start, arg.end]],
+          [[arg.start, arg.end], ...named],
           STEP,
         );
       }

package/src/mind/mind.ts

@@ -141,6 +141,7 @@ import {
   leadsSomewhere,
 } from "./traverse.js";
 import { invalidateJunctionCache } from "./junction.js";
+import type { WindowIndex } from "./evidence.js";
 import { follow } from "./match.js";
 import { recognise, segment } from "./recognition.js";
 import { meaningOf } from "./resonance.js";
@@ -333,6 +334,8 @@ export class Mind implements MindContext {
    *  first-edge behaviour) and the reverse projection uses it for
    *  reverse-recall disambiguation via `chooseAmong`. */
   _edgeGuide: Vec | null = null;
+  /** See {@link MindContext._edgeAsked}. */
+  _edgeAsked: { bytes: Uint8Array; index: WindowIndex } | null = null;
   /** Per-response memo of {@link chooseNext} picks — ensures every mechanism
    *  of a single response follows the SAME continuation for each ambiguous
    *  context node. */
@@ -647,6 +650,7 @@ export class Mind implements MindContext {
     this.canon = null;
     this.canonMemo = null;
     this._edgeGuide = null;
+    this._edgeAsked = null;
     this._edgeChoice.clear();
   }
 

package/src/mind/pipeline.ts

@@ -27,7 +27,9 @@ import {
 } from "./derivation.js";
 import { rItem } from "./trace.js";
 import { unexplainedLabel } from "./rationale.js";
-import { hubBound } from "./traverse.js";
+import { hubBound, offsetCanon, scaffoldExtents } from "./traverse.js";
+import { windowIndex, witness } from "./evidence.js";
+import { indexOf } from "../bytes.js";
 import {
   type MechanismResult,
   type PipelineMechanism,
@@ -189,6 +191,13 @@ export async function think(
   if (query.length === 0) return null;
 
   ctx._edgeGuide = gistOf(ctx, query);
+  {
+    const asked = offsetCanon(ctx, query);
+    ctx._edgeAsked = {
+      bytes: asked,
+      index: windowIndex(asked, ctx.space.maxGroup),
+    };
+  }
   ctx._edgeChoice.clear();
 
   const t = ctx.trace?.enter("think", [rItem(query, "query")]);
@@ -690,7 +699,21 @@ export async function think(
   const explained = priced.filter(([a, b]) =>
     windowOf([a, b], answer, query, ctx.space.maxGroup) !== null
   );
-  const paid = remainderOf(query.length, explained, ctx.space.maxGroup);
+  // THE DERIVATION IS BORN OWING ITS DISCRIMINATIVE MATERIAL.  The bytes a
+  // corpus-global scaffolding window reaches (traverse.ts, `scaffoldExtents`)
+  // are nobody's debt — otherwise a later step could claim to pay them by
+  // restating ` is `, which every fact holds, and the law (which measures
+  // carrying by any window of what is owed) would admit it; the substitution
+  // bridge reads gaps the same way (`explainedSpan`).  PRICING is untouched:
+  // the ladder still charges every unexplained byte, because for a question
+  // made of nothing but scaffolding (`How are you today?`) covering those bytes
+  // IS the evidence.
+  const scaffold = scaffoldExtents(ctx, query);
+  const paid = remainderOf(
+    query.length,
+    [...explained, ...scaffold],
+    ctx.space.maxGroup,
+  );
   // WHAT THE CONSTRUCTION WITHHOLDS, at or above one quantum: the difference between
   // the remainder paid in full and the remainder paid by carrying.  Both readings
   // are the law's, so the floor is applied once and in one place — and the
@@ -711,8 +734,31 @@ export async function think(
   const uncovered = state.remainder;
 
   // ── Post-grounding, gated by the declaration and the remainder ────────
+  // WHAT THE GROUNDING SPOKE FOR, when it did not declare it: the forms inside
+  // the answer that the ASKER already holds.  An answer is the asker's material
+  // plus what the corpus added — the fact's own value — and only the former was
+  // spoken for: re-entering it walks back to the question.  The latter is what a
+  // chain continues THROUGH.  Consuming every form recognised in the answer
+  // consumed that too, so a chain could start only where recognition happened
+  // to miss the next entity — measured on 133 held-out 2Wiki compositional
+  // questions, 3 pivot steps in all once interior recognition named the entity
+  // inside every fact.  Witnessed exactly, as `chooseNext`'s exact tier reads the
+  // question (evidence.ts); a form below one window cannot be witnessed, and is
+  // consumed as before.
+  const asked = ctx._edgeAsked;
   const preConsumed = declaredUsed ??
-    new Set(recognise(ctx, answer).sites.map((s) => s.payload));
+    new Set(
+      recognise(ctx, answer).sites
+        .filter((s) =>
+          asked === null || s.end - s.start < ctx.space.maxGroup ||
+          witness(
+            offsetCanon(ctx, answer.subarray(s.start, s.end)),
+            [asked.index],
+            ctx.space.maxGroup,
+          ).complete
+        )
+        .map((s) => s.payload),
+    );
   // A grounding that DECLARED itself complete is not extended: the answer is
   // already a trained form's own continuation, reached through an identity
   // claim about the query, so a multi-hop pivot could only chain past the
@@ -741,8 +787,17 @@ export async function think(
   // `preConsumed` is derived by re-recognising the answer — "everything in
   // it", not "what it voiced" — and a containment rule over that would
   // suppress every pivot the answer legitimately contains.
+  //
+  // …and a continuation the ANSWER ITSELF holds was voiced, not withheld.  A
+  // substitution that answers with its anchor's own continuation (`The director
+  // of Beat Girl is Edmond T. Gréville.`) declared that anchor used, so reading
+  // its continuations as withheld refused every pivot inside the answer — the
+  // chain could never step past the entity the first hop introduced.
   const voiced = declaredUsed === undefined ? [] : [...declaredUsed].flatMap(
-    (id) => ctx.store.nextFirst(id, hubBound(ctx)).map((n) => read(ctx, n)),
+    (id) =>
+      ctx.store.nextFirst(id, hubBound(ctx))
+        .map((n) => read(ctx, n))
+        .filter((v) => indexOf(answer, v, 0) < 0),
   );
   // WHAT THIS BRANCH READ, published where it was read.  Post-grounding decides
   // by the DECLARATION (`decided.used`, which becomes `voiced`), by what the

package/src/mind/reasoning.ts

@@ -8,10 +8,17 @@ import { bytesEqual, indexOf, latin1 } from "../bytes.js";
 import type { Attention, MindContext } from "./types.js";
 import { read, resolve } from "./primitives.js";
 import { countClusters } from "./attention.js";
-import { hubBound } from "./traverse.js";
+import {
+  guidedFirst,
+  hubBound,
+  namedContinuations,
+  offsetCanon,
+} from "./traverse.js";
+import { unspoken, type WindowIndex, windowIndex } from "./evidence.js";
 import { containsSpan, follow, haloSiblings, project } from "./match.js";
 import { joinWithBridge, pivotInto } from "./resonance.js";
 import {
+  closed,
   closeOver,
   type ClosureLayer,
   type Continuation,
@@ -102,6 +109,12 @@ export function walkLayer(
   // fire — the same visibility trade chooseNext documents.
   const bound = hubBound(ctx);
   const consumed = new Set<number>();
+  /** The windows each product of this derivation holds — what of the question
+   *  the derivation has already restated, so a later step is NAMED only by
+   *  question material none of them said. */
+  const spoken: WindowIndex[] = [];
+  const speak = (product: Uint8Array) =>
+    spoken.push(windowIndex(offsetCanon(ctx, product), ctx.space.maxGroup));
   /** `prev` lets a caller hand in an already-read reverse-edge list — hop 0
    *  reuses the guard's, below, instead of re-reading it. */
   const consumeNode = (
@@ -122,6 +135,7 @@ export function walkLayer(
   const begin = async (d0: DerivationState): Promise<boolean> => {
     const answer = d0.product;
     startedFrom = answer;
+    speak(answer);
     // Echo guard: a query that is ITSELF a learnt continuation (some context's
     // answer) is being asked back at the system — hopping forward from it would
     // chain through the very fact that produced it and echo the conversation
@@ -402,16 +416,48 @@ export function walkLayer(
       );
       return null;
     }
+    // THE STEP THE QUESTION ASKS FOR.  The hop `follow` took is NAMED when one
+    // of its establishing contexts is witnessed by what of the question no
+    // product has restated yet, plus the pivot itself (traverse.ts,
+    // `namedContinuations`): `Where was the place of death of the director of
+    // film Beat Girl?` names `Edmond T. Gréville place of death` once the first
+    // hop has restated `director` and `Beat Girl` — and `Who is the father of
+    // Frederick II?` names nothing past `Peter III of Aragon`, because `father`
+    // was restated by the hop that reached him.  A named step MOVES to structure
+    // the question asked for, so it is admitted on that ground; an unnamed one
+    // must carry the question's remainder as before, and from a CLOSED
+    // derivation — nothing left owed — it is not offered at all: the question
+    // asks for no further step, and the law would otherwise admit any.
+    const asked = ctx._edgeAsked;
+    let named = false;
+    if (asked !== null) {
+      const unsaid = {
+        bytes: asked.bytes,
+        index: unspoken(asked.index, spoken),
+      };
+      const names = namedContinuations(ctx, pivot, unsaid);
+      const hop = guidedFirst(ctx, pivot);
+      named = names !== null && hop !== undefined && names.includes(hop);
+    }
+    if (!named && !producerOwnsShape && closed(d)) {
+      ctx.trace?.step(
+        "pivotUnasked",
+        [rItem(cur, "answer"), rNode(ctx, pivot, "pivot")],
+        [rItem(fc, "withheld")],
+        "the derivation owes nothing and the question names no further step — not offered",
+      );
+      return null;
+    }
     pending = { kind: "pivot", cur, pivot, fc };
-    // Offered with the identity species ONLY when the grounding declared what it
-    // speaks for; otherwise the law requires this step to carry question
-    // material the grounding left unaccounted, which is the drift the extension
-    // tests pin — refused by the law's own measure, not by a private window
-    // test.
+    // Offered with the identity species when the grounding declared what it
+    // speaks for, or when the question names the step; otherwise the law
+    // requires this step to carry question material the grounding left
+    // unaccounted, which is the drift the extension tests pin — refused by the
+    // law's own measure, not by a private window test.
     return {
       product: fc,
       contains: true,
-      reaches: producerOwnsShape,
+      reaches: producerOwnsShape || named,
       cost: STEP,
     };
   };
@@ -422,6 +468,7 @@ export function walkLayer(
     witnesses: ReadonlyArray<Witness>,
   ): void => {
     meterTransition(ctx, before, after, witnesses);
+    speak(after.product);
     const p = pending;
     pending = null;
     if (p === null) return;