@hviana/sema 0.9.0 → 0.9.2

package/src/mind/primitives.ts

@@ -8,34 +8,20 @@ import { Sema } from "../sema.js";
 import {
   bytesToTree,
   contentFoldIncremental,
+  contentIdentity,
   Grid,
   gridToTree,
   hilbertBytes,
   stackGrids,
 } from "../geometry.js";
 import { canonHash } from "../canon.js";
-import { bytesEqual } from "../bytes.js";
+import { bytesEqual, concatBytes, latin1 } from "../bytes.js";
 import { ALL } from "./types.js";
 import type { Input, MindContext } from "./types.js";
 import type { ContentFold } from "../geometry.js";
 
 // ── Address: bytes → node ──────────────────────────────────────────────
 
-/** The content key of a byte span — one latin1 char per byte, an exact,
- *  collision-free encoding.  Spans on the perception path are query-scale
- *  (windows, regions, candidate spans), so key construction is far cheaper
- *  than the river fold it deduplicates. */
-export function latin1Key(bytes: Uint8Array): string {
-  // Batched String.fromCharCode — avoids the O(n²) cost of repeated += on
-  // potentially-large query spans, and stays well under the ~65536 arg limit.
-  const n = bytes.length;
-  let s = "";
-  for (let i = 0; i < n; i += 4096) {
-    s += String.fromCharCode(...bytes.subarray(i, Math.min(i + 4096, n)));
-  }
-  return s;
-}
-
 /** The {@link perceive} memo key: the span's content PLUS the boundary set it
  *  was folded under.  The tree is a function of BOTH — the same bytes fold
  *  plainly with no boundaries and into a left-nested stable-prefix shape with
@@ -50,7 +36,7 @@ export function perceiveKey(
   bytes: Uint8Array,
   boundaries?: readonly number[],
 ): string {
-  const k = latin1Key(bytes);
+  const k = latin1(bytes);
   return boundaries === undefined || boundaries.length === 0
     ? k
     : k + "\u0000" + boundaries.join(",");
@@ -154,7 +140,7 @@ export function perceiveDeposit(
     .filter((L) => L >= 2 && L < bytes.length)
     .sort((a, b) => b - a);
   for (const L of lens) {
-    const hit = ctx._depositTrees.get(latin1Key(bytes.subarray(0, L)));
+    const hit = ctx._depositTrees.get(latin1(bytes.subarray(0, L)));
     if (hit !== undefined) {
       prev = hit.content;
       break;
@@ -173,7 +159,7 @@ export function perceiveDeposit(
       ctx._depositLens.clear();
       ctx._depositTrees.clear();
     }
-    ctx._depositTrees.set(latin1Key(bytes), { content: folded.fold });
+    ctx._depositTrees.set(latin1(bytes), { content: folded.fold });
     ctx._depositLens.add(bytes.length);
   }
   return folded.tree;
@@ -252,12 +238,10 @@ export function foldTree(
     return { end, node };
   }
   let pos = start;
-  let known = true;
-  const kids: number[] = [];
+  const kids: Array<number | null> = [];
   for (const k of n.kids) {
     const r = foldTree(ctx, k, pos, visit);
-    if (r.node === null) known = false;
-    else if (known) kids.push(r.node);
+    kids.push(r.node);
     pos = r.end;
   }
   // Same store-probe elision as the leaf case: a cached entry already names
@@ -265,11 +249,10 @@ export function foldTree(
   // id need not be re-derived.  Using it also keeps a warm walk's ids
   // bit-identical to a cold walk's rather than re-deriving them from children
   // that may themselves have come from cache.
-  const node = cached !== undefined
-    ? cached.id
-    : known
-    ? ctx.store.findBranch(kids)
-    : null;
+  const named = cached !== undefined
+    ? { id: cached.id, byBytes: false }
+    : branchNaming(ctx, kids, treeBytes(n));
+  const node = named.id;
   visit?.(n, start, pos, node);
   if (node !== null && ctx._resolvedSubtrees) {
     ctx._resolvedSubtrees.set(n, { id: node, len: pos - start });
@@ -277,15 +260,113 @@ export function foldTree(
   return { end: pos, node };
 }
 
+/** A perceived subtree's bytes, its leaves in order. */
+function treeBytes(n: Sema): Uint8Array {
+  const parts: Uint8Array[] = [];
+  const walk = (x: Sema): void => {
+    if (x.kids === null) parts.push(x.leaf ?? new Uint8Array(0));
+    else for (const k of x.kids) walk(k);
+  };
+  walk(n);
+  return concatBytes(parts);
+}
+
+/** The EXACT content-addressed node of a byte stream — `foldTree(perceive)`,
+ *  read for identity alone.
+ *
+ *  A fold names a branch only when every child is named, so identity needs the
+ *  fold's SHAPE and the store's answer per node, never its vectors:
+ *  {@link contentIdentity} walks the same shape (geometry.ts — one grouping
+ *  rule, two algebras) and asks the store bottom-up, building no D-dimensional
+ *  gist and leaving nothing in the perception memo.  Each node is named the
+ *  way the store's write side names it ({@link branchNaming}).  `test/148` pins
+ *  the agreement with the full fold over random and corpus spans. */
+export function exactNode(ctx: MindContext, bytes: Uint8Array): number | null {
+  return exactNaming(ctx, bytes).id;
+}
+
+/** {@link exactNode}, with whether the span's own name was found only through
+ *  its BYTES — its children named no branch, and the flat node over the same
+ *  bytes did ({@link branchNaming}).  That is where the exact lookup used to
+ *  MISS, so it is where {@link resolve} still asks the canonical class: the
+ *  class may hold the learnt member that leads somewhere, which a flat index
+ *  entry need not (measured: `tonight` named an edge-less window and
+ *  pre-empted the case-folded `Tonight` whose edge a composition stood on).
+ *  Recognition's probes reach it through `resolve`; asking it again for the
+ *  perceived tree's own byte-named groups changed none of 116 real queries and
+ *  no test, so it is not asked there. */
+export function exactNaming(
+  ctx: MindContext,
+  bytes: Uint8Array,
+): { id: number | null; byBytes: boolean } {
+  if (bytes.length === 0) {
+    return { id: foldTree(ctx, perceive(ctx, bytes), 0).node, byBytes: false };
+  }
+  if (ctx.meter) ctx.meter.identityBytes += bytes.length;
+  let byBytes = false;
+  const id = contentIdentity(
+    ctx.space,
+    ctx.alphabet,
+    bytes,
+    (from, to) =>
+      to - from === 1
+        ? ctx.store.findLeaf(bytes.subarray(from, to))
+        : flatNode(ctx, bytes.subarray(from, to)),
+    (kids, from, to) => {
+      const named = branchNaming(ctx, kids, bytes.subarray(from, to));
+      if (from === 0 && to === bytes.length) byBytes = named.byBytes;
+      return named.id;
+    },
+  );
+  return { id, byBytes };
+}
+
+/** The flat node over a span's single-byte atoms — the node every deposit
+ *  interns for its whole input and for each canonical window (learning.ts
+ *  `deposit`, `indexSubSpans`).  The store's negative filter refuses most
+ *  misses without a lookup. */
+function flatNode(ctx: MindContext, span: Uint8Array): number | null {
+  const store = ctx.store;
+  return store.findFlatBranch
+    ? store.findFlatBranch(span)
+    : store.findBranch(Array.from(span, (b) => -(b + 1)));
+}
+
+/** THE READ SIDE NAMES A BRANCH EXACTLY AS THE WRITE SIDE DID.  `intern`
+ *  (store.ts) names a branch by its children; when they name none, it looks up
+ *  the flat node over the same bytes and REUSES it (step 1b, "same bytes, same
+ *  node") — so a deposit whose fold grouped `ver` + `!` was stored with the
+ *  window `ver!` as that child.  Reading by the children alone could never
+ *  name such a deposit again: measured on the 31.7M-node store, 8 of 80 stored
+ *  dialogue turns asked verbatim resolved to nothing (a 25-byte turn, a final
+ *  `?` or `!`, …) and fell to the composition path.  Same order as the write
+ *  side: the children first, the bytes when they name nothing — and an unnamed
+ *  child does not settle it, since the write side minted that child and still
+ *  reached step 1b. */
+function branchNaming(
+  ctx: MindContext,
+  kids: ReadonlyArray<number | null>,
+  span: Uint8Array,
+): { id: number | null; byBytes: boolean } {
+  if (kids.every((k) => k !== null)) {
+    const id = ctx.store.findBranch(kids as number[]);
+    if (id !== null) return { id, byBytes: false };
+  }
+  if (kids.length < 2) return { id: null, byBytes: false };
+  const id = flatNode(ctx, span);
+  if (id !== null && ctx.meter) ctx.meter.flatBranchNames++;
+  return { id, byBytes: id !== null };
+}
+
 /** The canonical node id of a byte span: perceive it in isolation — the way
  *  training did — and recover its root bottom-up.  Returns null if any part is
  *  unknown. */
 export function resolve(ctx: MindContext, bytes: Uint8Array): number | null {
   if (bytes.length === 0) return null;
   if (ctx.meter) ctx.meter.resolves++;
-  const exact = foldTree(ctx, perceive(ctx, bytes), 0).node;
-  if (exact !== null) return exact;
-  return canonResolve(ctx, bytes);
+  const { id: exact, byBytes } = exactNaming(ctx, bytes);
+  if (exact !== null && !byBytes) return exact;
+  return canonResolve(ctx, bytes) ?? exact;
 }
 
 /** Equivalence-class resolution: when the exact content-addressed lookup
@@ -305,7 +386,7 @@ export function canonResolve(
   if (canon === null || !store.canonFind) return null;
   if (bytes.length < 2) return null;
   const memo = ctx.canonMemo;
-  const memoKey = memo ? latin1Key(bytes) : "";
+  const memoKey = memo ? latin1(bytes) : "";
   if (memo) {
     const hit = memo.get(memoKey);
     if (hit !== undefined) return hit;
@@ -320,7 +401,7 @@ export function canonResolve(
   // skips identity rows) — the exact content-addressed lookup of the
   // canonical bytes finds it directly.
   if (key.length !== bytes.length || !bytesEqual(key, bytes)) {
-    const direct = foldTree(ctx, perceive(ctx, key), 0).node;
+    const direct = exactNode(ctx, key);
     if (direct !== null) return set(direct);
   }
   if (ctx.meter) ctx.meter.canonLookups++;
@@ -336,17 +417,12 @@ export function canonResolve(
     // resolved for these bytes is their FOLD — the deposit-shaped node that
     // carries the edges and halos.  Re-folding the candidate's bytes lands
     // on exactly the node the canonical-case query would have found.
-    const folded = foldTree(ctx, perceive(ctx, bytesOf), 0).node;
+    const folded = exactNode(ctx, bytesOf);
     const use = folded ?? id;
-    // THE ADMISSION PREDICATE, by its own pair of probes: `traverse.ts`'s
-    // `leadsSomewhere` is edge-or-halo, and `hasHalo` is the one that carries
-    // the mass bar (`mass >= minHaloMass`).  Asking `haloMass(use) > 0` instead
-    // is the same answer only while `minHaloMass <= 1` (its default): raise the
-    // bar and this site would rank a node as leading on evidence the law
-    // refuses.  Calling `leadsSomewhere` here is not possible — `traverse.ts`
-    // imports THIS file, so it would be a cycle — which is why the pair is
-    // spelled out rather than named.
-    const leads = store.hasNext(use) || store.hasHalo(use);
+    // THE ADMISSION PREDICATE, asked of the store that owns it (edge or halo,
+    // the halo tier carrying the mass bar).  Asking `haloMass(use) > 0` instead
+    // would agree only while `minHaloMass <= 1`.
+    const leads = store.leadsSomewhere(use);
     if (
       best === null || (leads && !bestLeads) ||
       (leads === bestLeads && use < best)