@hviana/sema 0.9.0 → 0.9.2

package/src/store-sqlite.ts

@@ -146,6 +146,17 @@ CREATE TABLE IF NOT EXISTS canon (
   id INTEGER NOT NULL,
   PRIMARY KEY (h, id)
 ) WITHOUT ROWID;
+-- The canon index's negative filter, persisted so an open does not rescan the
+-- h column (seconds on a trained store).  Written in the SAME transaction as
+-- the canon rows it covers, stamped with the meta 'canon.upto' of that commit;
+-- a stamp that disagrees with the meta (rows a writer added without it) makes
+-- the filter stale, and it is rebuilt from the table instead.
+CREATE TABLE IF NOT EXISTS canon_bloom (
+  id   INTEGER PRIMARY KEY CHECK (id = 1),
+  bits BLOB NOT NULL,
+  n    INTEGER NOT NULL,
+  upto TEXT NOT NULL
+);
 -- CONSTITUENT SKETCH (Store.sketchGet/sketchPut): the bottom-k minimal
 -- constituents of a node's subtree, k = √D, chosen by identity hash.  The blob is
 -- a packed int32 little-endian run, already in hash order; an EMPTY blob is a
@@ -251,6 +262,20 @@ export class SQliteStore extends AbstractStore implements Store {
   private _bloom: NodeBloom | null = null;
   /** Dedup probes answered by the filter alone this session (observability). */
   bloomSkips = 0;
+  /** The same negative filter over the CANON index's key hashes.  Recognition's
+   *  canonical admission and the join's canonical entity scan ask `canonFind`
+   *  once per probed span, and almost every answer is "no such key" — on an
+   *  index that is often EMPTY (it is built only by `buildCanonIndex`).
+   *  Loaded on first use from `canon_bloom` when its stamp matches the meta,
+   *  else built by one sequential scan of the h column; kept exact on
+   *  `canonAdd` (rebuilt bigger from the table, uncommitted rows included,
+   *  when growth saturates it) and persisted with the commit that wrote the
+   *  rows.  The canon table is never deleted from, so the filter can never
+   *  hold a false negative: a miss it reports is a miss the query would have
+   *  returned. */
+  private _canonBloom: NodeBloom | null = null;
+  /** The in-memory canon filter differs from the persisted one. */
+  private _canonBloomDirty = false;
 
   private _insertNode: any = null;
   private _insertKid: any = null;
@@ -509,7 +534,10 @@ export class SQliteStore extends AbstractStore implements Store {
   }
 
   protected _dbClose(): void {
-    if (this.sqlite) this._bloomPersist();
+    if (this.sqlite) {
+      this._bloomPersist();
+      this._canonBloomPersist();
+    }
     if (this.content) {
       this.content.close();
       this.content = null;
@@ -534,6 +562,8 @@ export class SQliteStore extends AbstractStore implements Store {
 
   protected _dbCommitTx(): void {
     if (!this._inTx || !this.sqlite) return;
+    // The canon filter commits WITH the rows it covers — see canon_bloom.
+    this._canonBloomPersist();
     this._inTx = false; // clear first so a throw can't wedge us mid-commit
     this.sqlite.exec("COMMIT");
   }
@@ -615,6 +645,15 @@ export class SQliteStore extends AbstractStore implements Store {
     return row ? row.id : null;
   }
 
+  /** The node filter alone: never a false negative (every inserted hash is
+   *  added before any probe can see it), so `false` is exact. */
+  protected override _dbFlatMayExist(h: number, bytes: Uint8Array): boolean {
+    if (this._bloom === null) return super._dbFlatMayExist(h, bytes);
+    if (this._bloom.mightContain(h)) return true;
+    this.bloomSkips++;
+    return false;
+  }
+
   protected _dbFindBranchByKids(
     h: number,
     packed: Uint8Array,
@@ -1053,9 +1092,61 @@ export class SQliteStore extends AbstractStore implements Store {
     // bulk index build coalesces instead of paying autocommit per row.
     this._dbBeginTx();
     this._insCanon.run(h, id);
+    const b = this._canonFilter();
+    b.add(h);
+    // Rebuilt bigger at once, from the table as THIS connection sees it — the
+    // persisted filter cannot stand in, it lacks this transaction's rows.
+    if (b.saturated) this._canonBloom = this._canonScan();
+    this._canonBloomDirty = true;
+  }
+
+  /** The canon filter: the persisted one when its stamp matches the meta,
+   *  else built from the index as it stands. */
+  private _canonFilter(): NodeBloom {
+    if (this._canonBloom === null) {
+      const row = this.sqlite!.prepare(
+        "SELECT bits, n, upto FROM canon_bloom WHERE id = 1",
+      ).get() as { bits: Uint8Array; n: number; upto: string } | undefined;
+      if (row !== undefined && row.upto === this._canonStamp()) {
+        const b = new NodeBloom(31 - Math.clz32(row.bits.length * 8));
+        b.bits.set(row.bits);
+        b.n = row.n;
+        this._canonBloom = b;
+      } else {
+        this._canonBloom = this._canonScan();
+        this._canonBloomDirty = true;
+      }
+    }
+    return this._canonBloom;
+  }
+
+  private _canonScan(): NodeBloom {
+    const b = new NodeBloom(bloomLog2For(this.canonCount()));
+    const scan = this.sqlite!.prepare("SELECT h FROM canon");
+    scan.setReturnArrays(true);
+    for (const r of scan.iterate() as IterableIterator<[number]>) b.add(r[0]);
+    return b;
+  }
+
+  /** What a persisted canon filter is stamped with: the incremental build's
+   *  cursor, which every canon writer advances in the transaction it writes. */
+  private _canonStamp(): string {
+    return this._dbGetMeta("canon.upto") ?? "";
+  }
+
+  private _canonBloomPersist(): void {
+    const b = this._canonBloom;
+    if (b === null || !this._canonBloomDirty || !this.sqlite) return;
+    this.sqlite.prepare(
+      "INSERT INTO canon_bloom (id, bits, n, upto) VALUES (1, ?, ?, ?) " +
+        "ON CONFLICT(id) DO UPDATE SET bits = excluded.bits, " +
+        "n = excluded.n, upto = excluded.upto",
+    ).run(b.bits, b.n, this._canonStamp());
+    this._canonBloomDirty = false;
   }
 
   canonFind(h: number): number[] {
+    if (!this._canonFilter().mightContain(h)) return [];
     if (!this._selCanon) {
       this._selCanon = this.sqlite!.prepare(
         "SELECT id FROM canon WHERE h = ?",

package/src/store.ts

@@ -29,9 +29,10 @@
 // essentials required for SQLite + VectorDatabase communication.
 
 import { addInto, copy, dot, normalize, Vec } from "./vec.js";
-import { DEFAULT_CONFIG, type StoreConfig } from "./config.js";
+import type { StoreConfig } from "./config.js";
 import { identityBar } from "./geometry.js";
 import type { Meter } from "./meter.js";
+import { latin1 } from "./bytes.js";
 
 /** A node id: a dense, non-negative integer assigned in creation order. */
 export type NodeId = number;
@@ -340,6 +341,20 @@ export interface Store {
    *  bytes — the allocation-free probe span scanners use.  Optional: a store
    *  without it is simply probed through `findBranch`. */
   findFlatBranch?(bytes: Uint8Array): NodeId | null;
+  /** Whether a flat branch with these bytes MAY exist.  `false` is EXACT — no
+   *  such node exists, and a {@link findFlatBranch} would return null; `true`
+   *  means only that a lookup is needed.  The existence half of the probe,
+   *  without its verification: a caller that will resolve the bytes anyway
+   *  (and so verify them) asks this to refuse a miss without paying for the
+   *  lookup a hit would repeat.  Optional, like the probe it answers for. */
+  flatBranchMayExist?(bytes: Uint8Array): boolean;
+  /** A {@link findFlatBranch} over spans of ONE buffer: `probe(start, end)`
+   *  answers exactly `findFlatBranch(bytes.subarray(start, end))`, but a span's
+   *  content hash extends the one its start was last probed at, so a scanner
+   *  that sweeps ends upward per start pays O(1) per probe instead of
+   *  O(span).  The buffer must not change while the prober is used.
+   *  Optional, like the probe it accelerates. */
+  flatSpans?(bytes: Uint8Array): (start: number, end: number) => NodeId | null;
   /** The branch nodes that list `id` among their children — the reverse of
    *  `get(id).kids`. Lets the structural DAG be climbed upward, from a
    *  recognised fragment to the larger learned forms that contain it. */
@@ -499,6 +514,12 @@ export interface Store {
    *  the search's fuse guard) must ask this instead: one row read, a mass
    *  compare, no decode. */
   hasHalo(id: NodeId): boolean;
+  /** THE ADMISSION PREDICATE — whether `id` LEADS SOMEWHERE: it bears a
+   *  continuation edge ({@link hasNext}) or a halo ({@link hasHalo}).  A form
+   *  that leads nowhere contributes nothing to any derivation.  This is the ONE
+   *  raw definition; `traverse.ts`'s `leadsSomewhere` is the same predicate with
+   *  its edge tier memoised for the response.  Two point probes at most. */
+  leadsSomewhere(id: NodeId): boolean;
   /** How many episode signatures were poured into `id`'s halo — the DIRECT
    *  measure of distributional evidence (each training pair pours once, so
    *  repetition counts, unlike {@link prevCount}, which counts DISTINCT
@@ -662,11 +683,14 @@ export function unpackKids(blob: Uint8Array): NodeId[] {
 
 /** 32-bit FNV-1a of a byte blob — the integer content hash `idx_node_h` keys
  *  on. Collisions are resolved by verifying the stored blob, never trusted. */
+const FNV_OFFSET = 0x811c9dc5 >>> 0;
+const FNV_PRIME = 0x01000193;
+
 function hashOf(bytes: Uint8Array): number {
-  let h = 0x811c9dc5 >>> 0;
+  let h = FNV_OFFSET;
   for (let i = 0; i < bytes.length; i++) {
     h ^= bytes[i];
-    h = Math.imul(h, 0x01000193) >>> 0;
+    h = Math.imul(h, FNV_PRIME) >>> 0;
   }
   return h >>> 0;
 }
@@ -957,6 +981,9 @@ export abstract class AbstractStore implements Store {
   /** Exact-content dedup: content-key → node id. Intrinsic compression. */
   protected readonly _leafKey: BoundedMap<string, NodeId>;
   protected readonly _branchKey: BoundedMap<string, NodeId>;
+  /** {@link findFlatBranch}'s hits, keyed by the bytes themselves (latin1 —
+   *  exact, unlike the hash keys above). */
+  protected readonly _flatKey: BoundedMap<string, NodeId>;
   /** Reconstructed-bytes read cache (regenerable), keyed by node id. */
   protected readonly _bytesCache: BoundedMap<NodeId, Uint8Array>;
   /** contentLen memo — content is immutable, so entries never invalidate. */
@@ -1063,6 +1090,12 @@ export abstract class AbstractStore implements Store {
     this.compactEveryNWrites = config.compactEveryNWrites;
     this._leafKey = new BoundedMap(config.dedupCacheMax);
     this._branchKey = new BoundedMap(config.dedupCacheMax);
+    this._flatKey = new BoundedMap(
+      config.dedupCacheMax,
+      undefined,
+      "lru",
+      "clock",
+    );
     this._bytesCache = new BoundedMap(
       config.bytesCacheMax,
       (v) => v.byteLength,
@@ -1400,12 +1433,80 @@ export abstract class AbstractStore implements Store {
    *  answers most of those with no I/O at all, so the allocations dominated.
    *
    *  Pass a subarray: it is a view, so a caller scanning spans of a query
-   *  allocates nothing per probe.  Deliberately NOT memoized — its callers
-   *  probe many spans that miss, and a key string per probe is the cost this
-   *  exists to remove. */
+   *  allocates nothing per probe that misses.  HITS ARE MEMOIZED, MISSES ARE
+   *  NOT: the negative filter answers first, so a span that is not stored
+   *  builds no key; one that may be pays a key and then usually skips the
+   *  lookup — the spans the identity fold names are segments, a few bytes
+   *  each, asked over and over by every span that contains them.  A node, once
+   *  minted, keeps its bytes and its id, so a cached hit never goes stale. */
   findFlatBranch(bytes: Uint8Array): NodeId | null {
+    return this._findFlatHashed(hashOf(bytes), bytes);
+  }
+
+  /** {@link Store.flatSpans}.  Each start keeps the hash of the span it was
+   *  last probed to, plus the hash one byte short of it; a probe ending at or
+   *  past that end extends it, one ending one byte short of it reuses the
+   *  second, and any other probe hashes from the start.  `hashOf` is FNV-1a,
+   *  a left fold over the bytes, so an extended hash IS the span's hash —
+   *  same key, same filter answer, same lookup. */
+  flatSpans(bytes: Uint8Array): (start: number, end: number) => NodeId | null {
+    const n = bytes.length;
+    const runEnd = new Int32Array(n + 1).fill(-1);
+    const runH = new Uint32Array(n + 1);
+    const shortH = new Uint32Array(n + 1);
+    return (start, end) => {
+      let h: number;
+      let from: number;
+      const e0 = runEnd[start];
+      if (e0 >= 0 && e0 <= end) {
+        h = runH[start];
+        from = e0;
+      } else if (e0 >= 0 && e0 - 1 === end && end > start) {
+        h = shortH[start];
+        from = end;
+      } else {
+        h = FNV_OFFSET;
+        from = start;
+      }
+      let prev = h;
+      for (let i = from; i < end; i++) {
+        prev = h;
+        h = Math.imul(h ^ bytes[i], FNV_PRIME) >>> 0;
+      }
+      if (from < end) {
+        runEnd[start] = end;
+        runH[start] = h;
+        shortH[start] = prev;
+      }
+      return this._findFlatHashed(h, bytes.subarray(start, end));
+    };
+  }
+
+  /** {@link findFlatBranch} past the hash: `h` must be `hashOf(bytes)`. */
+  private _findFlatHashed(h: number, bytes: Uint8Array): NodeId | null {
     if (this.meter) this.meter.branchLookups++;
-    return this._dbFindBranchByLeaf(hashOf(bytes), bytes);
+    if (!this._dbFlatMayExist(h, bytes)) return null;
+    const key = bytes.length <= DEDUP_KEY_MAX ? latin1(bytes) : null;
+    if (key !== null) {
+      const hit = this._flatKey.get(key);
+      if (hit !== undefined) return hit;
+    }
+    const id = this._dbFindBranchByLeaf(h, bytes);
+    if (id !== null && key !== null) this._flatKey.set(key, id);
+    return id;
+  }
+
+  /** {@link Store.flatBranchMayExist} — the backend's negative filter when it
+   *  keeps one ({@link _dbFlatMayExist}), else the lookup itself. */
+  flatBranchMayExist(bytes: Uint8Array): boolean {
+    return this._dbFlatMayExist(hashOf(bytes), bytes);
+  }
+
+  /** Default: no filter, so anything MAY exist and the lookup decides.  A
+   *  backend with a negative filter over node hashes overrides this to answer
+   *  from the filter alone. */
+  protected _dbFlatMayExist(_h: number, _bytes: Uint8Array): boolean {
+    return true;
   }
 
   findBranch(kids: NodeId[]): NodeId | null {
@@ -2310,6 +2411,11 @@ export abstract class AbstractStore implements Store {
     return r !== null && r.mass >= this.minHaloMass;
   }
 
+  /** {@link Store.leadsSomewhere} — edge first (the cheaper, commoner probe). */
+  leadsSomewhere(id: NodeId): boolean {
+    return this.hasNext(id) || this.hasHalo(id);
+  }
+
   async pourHalo(id: NodeId, add: Vec): Promise<void> {
     await this._ensureReady();
     // A node with a halo is a genuine resonance target — the consensus climb

package/test/105-derive-through-reports-its-refusal.test.mjs

@@ -52,10 +52,13 @@ const misses = (steps) =>
 
 test("the join's refusal is reported, naming the candidate and tail it tried", async () => {
   const mind = await chain();
-  // The THREE-relation query: the second join is still refused (the rule
-  // concludes terminal — the study's other half), so this is where the refusal
-  // is observable.  The two-relation one now JOINS (measured, and pinned by
-  // test/99's spec), which is why this test moved here.
+  // The THREE-relation query: the chain joins through to Stockholm, and on the
+  // way the fact it stands on ("…Gustaf Molander is Sweden.") offers its other
+  // entity, "gustaf molander", with the remaining tail " capital" — a key no
+  // deposit learnt, so that join is refused, and that refusal is observable.
+  // A join is priced only for a fact a derivation STANDS ON (graph-search.ts,
+  // solve), so the refusal reported is that fact's — not one from a fact the
+  // exploration merely reached.
   const steps = [];
   await mind.respond("eva director country capital", (s) => steps.push(s));
   const got = misses(steps);
@@ -63,7 +66,7 @@ test("the join's refusal is reported, naming the candidate and tail it tried", a
   const named = got.some((s) => {
     const parts = (s.inputs ?? []).map((i) => String(i.text));
     return parts.some((t) => t.includes("gustaf molander")) &&
-      parts.some((t) => t.includes("country"));
+      parts.some((t) => t.includes("capital"));
   });
   assert.ok(named, "the report must name the candidate and the tail it tried");
   // …and WHERE the candidate came from: a refusal that names only bytes leaves

package/test/106-the-join-fires.test.mjs

@@ -92,3 +92,24 @@ test("a query with no tail is answered directly, not by a join", async () => {
   );
   await mind.store.close();
 });
+
+test("the asker's wording that already holds the answer's form is not spliced into it", async () => {
+  // Articulation revoices an answer form in the asker's words when the two
+  // keep the same company (halo).  `eva` and `eva director` keep the SAME
+  // company here — both lead to the same fact — but the asker's wording holds
+  // the form plus a further word: splicing it in is not a re-voicing, it adds
+  // the asker's other words to the answer.  The DAG cannot see the containment
+  // (the fold cuts `eva director` as `eva d|irector`), so it is read off the
+  // bytes.  Observed before: "The director of eva director is Gustaf Molander."
+  const lower = "The director of eva is Gustaf Molander.";
+  const store = new SQliteStore({ path: ":memory:" });
+  const mind = new Mind({ seed: 7, store });
+  await mind.ingest([
+    ["eva", lower],
+    ["eva director", lower],
+    ["gustaf molander", F2],
+    ["gustaf molander country", F2],
+  ]);
+  assert.equal(text(await mind.respond("eva director")).trim(), lower);
+  await mind.store.close();
+});

package/test/111-the-cover-assembly-is-counted.test.mjs

@@ -46,15 +46,18 @@ test("the pairwise assembly is counted, with no rationale attached", async () =>
 });
 
 test("the n-ary interior's allowance is counted", async () => {
-  // Three sites in one query: the n-ary path passes an allowance that grows
-  // with the intermediate answers' bytes — the quantity that would grow with a
-  // hub query's answers.
+  // Three touching sites in one query, each answered: the n-ary path passes an
+  // allowance that grows with the intermediate answers' bytes — the quantity
+  // that would grow with a hub query's answers.  The connectors are bridged
+  // only for the pairs the search REACHES (ConnectorLicence), so the lightest
+  // cover must stand on all three answers: a query recognised whole (an
+  // `ab cd` learnt as its own context) never reaches its parts' pair.
   const mind = await fixture([
     ["ab", "x"],
     ["cd", "y"],
-    ["ab cd", "z"],
+    ["ef", "z"],
   ]);
-  await mind.respondText("ab cd");
+  await mind.respondText("abcdef");
   const c = mind.lastCost?.counters ?? {};
   assert.ok(
     (c.coverAllowanceBytes ?? 0) > 0,

package/test/128-the-leads-somewhere-pair-agrees.test.mjs

@@ -1,17 +1,18 @@
 // 128-the-leads-somewhere-pair-agrees.test.mjs — THE PAIR, PINNED
 //
-// "Does this node lead somewhere?" is answered in TWO homes, and they cannot
-// share a name: `traverse.ts`'s `leadsSomewhere` is `cachedHasNext || hasHalo`,
-// and `primitives.ts` spells out `hasNext || hasHalo` because `traverse.ts`
-// imports that file (a cycle) — the comment there says so.  Two spellings of
-// one predicate is a drift risk, so this pins the AGREEMENT, node by node,
-// through the observable surface only: `mind.leadsSomewhere(id)` against
-// `store.hasNext(id) || store.hasHalo(id)`.
+// "Does this node lead somewhere?" has ONE raw definition — the store's
+// `leadsSomewhere` (`hasNext || hasHalo`) — and ONE memoised reading of it,
+// `traverse.ts`'s `leadsSomewhere` (`cachedHasNext || hasHalo`), which the Mind
+// lends to the search.  The callers that used to spell the pair out by hand
+// (`primitives.ts`, the search's bare-host fallback) now ask the store.  This
+// pins the AGREEMENT, node by node, through the observable surface only:
+// `mind.leadsSomewhere(id)` against `store.leadsSomewhere(id)`, and both against
+// the two probes the definition is made of.
 //
 // It is NOT a claim that the two are the same function: one reads a memo, the
 // other reads the store.  The claim is that the memoised reading and the store
-// reading agree — what makes the spelled-out copy correct, and what would break
-// first if `cachedHasNext` ever outlived a store write.
+// reading agree — what would break first if `cachedHasNext` ever outlived a
+// store write.
 //
 // THE REACH OF THIS PIN, said as it is: halos are poured on the WRITE side, so
 // a synthetic corpus of this shape carries NONE (measured: 0 nodes with a halo
@@ -49,16 +50,21 @@ test("the two homes of 'leads somewhere' agree, node by node", async () => {
   let withHalo = 0;
   for (let id = 0; id < total; id++) {
     const named = m.leadsSomewhere(id);
-    const spelled = m.store.hasNext(id) || m.store.hasHalo(id);
+    const stored = m.store.leadsSomewhere(id);
     const hasEdge = m.store.hasNext(id);
     const hasHalo = m.store.hasHalo(id);
     if (hasEdge) withEdge++;
     if (hasHalo) withHalo++;
+    assert.equal(
+      stored,
+      hasEdge || hasHalo,
+      `node ${id}: the store's definition must be its two probes`,
+    );
     assert.equal(
       named,
-      spelled,
+      stored,
       `node ${id}: traverse.leadsSomewhere says ${named}, ` +
-        `primitives' spelling says ${spelled} (hasNext=${hasEdge}, hasHalo=${hasHalo})`,
+        `the store says ${stored} (hasNext=${hasEdge}, hasHalo=${hasHalo})`,
     );
   }
 

package/test/136-the-two-named-limits.test.mjs

@@ -77,8 +77,9 @@ test("136.1 the fuse gate asks the LAW — one condition, one home, one less sum
   );
   assert.match(
     src,
-    /const fused = closed\(state\)/,
-    "the fuse gate must ask the law's `closed(state)` — the condition the state " +
+    /engages: \(d: DerivationState\) => !closed\(d\)/,
+    "the fuse gate must ask the law's `closed` of the state fusion is offered " +
+      "against (the fusion layer engages only while it is open) — the condition the state " +
       "already carries.  If it went back to summing `unaccounted(explained)`, that " +
       "is a second reading of one condition AND an extra sum per response: 136.3 " +
       "measures the equivalence, so re-read it before restoring the total",

package/test/137-the-law-lives-once-and-below.test.mjs

@@ -398,3 +398,24 @@ test("137.6 no export is genuinely dead — the triage, as a guard", async () =>
     "genuinely dead exports found — remove them, or state where they are consumed",
   );
 });
+
+test("137.7 no transition of the derivation is built by hand outside the law", async () => {
+  // `advance` is the ONLY transition (derivation.ts).  A state spread with a new
+  // product — `{ ...state, product: … }` — is a transition the law never saw:
+  // the fusion used to hand-build one when the law refused it, which is exactly
+  // the bypass a fixed point exists to forbid.  Comments are stripped, so prose
+  // that DESCRIBES the shape cannot satisfy or break this.
+  const { text } = await graph();
+  const offenders = [...text.entries()]
+    .filter(([f]) => f !== "src/mind/derivation.ts")
+    .filter(([, s]) =>
+      /\.\.\.\s*(?:state|d0|d|reasoned|closed_)\s*,\s*product\s*:/.test(s)
+    )
+    .map(([f]) => f);
+  assert.deepEqual(
+    offenders,
+    [],
+    "a derivation state is advanced by hand outside derivation.ts — offer the " +
+      "step to the law (`admissible` + `advance`) instead",
+  );
+});