@hviana/sema 0.4.2 → 0.4.4

package/dist/src/derive/test/derive.test.js

@@ -0,0 +1,122 @@
+// Self-contained tests for the `derive` library: the lightest-derivation
+// engine (including a multi-premise *bridge* rule), the on-demand trie matcher,
+// and the optimal cover. Uses node:test; no dependency on sema.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { coverSequence, lightestDerivation, Trie, } from "../src/index.js";
+// ── trie: on-demand matching, no length bound ──────────────────────────────
+test("trie matchesAt reports forms beginning exactly at a position", () => {
+    const t = new Trie();
+    t.insert([1, 2], "ab");
+    t.insert([1, 2, 3], "abc");
+    t.insert([2, 3], "bc");
+    const at0 = t.matchesAt([1, 2, 3, 4], 0).map((m) => m.payload);
+    assert.deepEqual(at0, ["ab", "abc"]); // both forms starting at 0, shortest first
+    const at1 = t.matchesAt([1, 2, 3, 4], 1).map((m) => m.payload);
+    assert.deepEqual(at1, ["bc"]);
+    const at3 = t.matchesAt([1, 2, 3, 4], 3).map((m) => m.payload);
+    assert.deepEqual(at3, []); // nothing learned here → dead-ends immediately
+});
+test("trie scan finds every occurrence; duplicate inserts share an id", () => {
+    const t = new Trie();
+    const id1 = t.insert([7, 7], 1);
+    const id2 = t.insert([7, 7], 2); // same content
+    assert.equal(id1, id2);
+    assert.equal(t.size, 1);
+    const hits = t.scan([7, 7, 7]);
+    assert.equal(hits.length, 2); // [0,2) and [1,3)
+});
+// ── engine: Dijkstra/Knuth core with a bridge (multi-premise) rule ─────────
+test("lightestDerivation solves a hypergraph with a bridge premise", () => {
+    // Items: "A", "B", "AB". Axioms A (3) and B (4). A binary bridge A ∧ B → AB
+    // at cost 1. The only derivation of AB costs 3 + 4 + 1 = 8.
+    const A = "A", B = "B", AB = "AB";
+    const system = {
+        key: (s) => s,
+        axioms: () => [{ item: A, cost: 3 }, { item: B, cost: 4 }],
+        isGoal: (s) => s === AB,
+        *rules(item) {
+            // Bridge fires from either premise; the engine waits until both are known.
+            if (item === A)
+                yield { premises: [A, B], conclusion: AB, cost: 1 };
+            if (item === B)
+                yield { premises: [A, B], conclusion: AB, cost: 1 };
+        },
+    };
+    const d = lightestDerivation(system);
+    assert.ok(d);
+    assert.equal(d.item, AB);
+    assert.equal(d.cost, 8);
+    assert.equal(d.premises.length, 2); // it really used the bridge
+});
+test("lightestDerivation picks the cheaper of competing derivations", () => {
+    // Two ways to reach the goal G from axiom S: S→G cost 10, or S→M→G cost 3+3.
+    const system = {
+        key: (s) => s,
+        axioms: () => [{ item: "S", cost: 0 }],
+        isGoal: (s) => s === "G",
+        *rules(item) {
+            if (item === "S") {
+                yield { premises: ["S"], conclusion: "G", cost: 10 };
+                yield { premises: ["S"], conclusion: "M", cost: 3 };
+            }
+            if (item === "M")
+                yield { premises: ["M"], conclusion: "G", cost: 3 };
+        },
+    };
+    const d = lightestDerivation(system);
+    assert.ok(d);
+    assert.equal(d.cost, 6); // via M, not the direct 10
+    assert.equal(d.rule?.premises[0], "M");
+});
+test("lightestDerivation returns null when the goal is unreachable", () => {
+    const system = {
+        key: (n) => "" + n,
+        axioms: () => [{ item: 0, cost: 0 }],
+        isGoal: (n) => n === 99,
+        *rules(n) {
+            if (n < 3)
+                yield { premises: [n], conclusion: n + 1, cost: 1 };
+        },
+    };
+    assert.equal(lightestDerivation(system), null);
+});
+// ── cover: optimal, not greedy ─────────────────────────────────────────────
+test("coverSequence maximises coverage, beating greedy longest-match", () => {
+    // Over [0,4): one long span [1,4) (len 3) vs two short [0,2),[2,4) (cover 4).
+    // Greedy-longest takes [1,4) and covers 3; the optimal cover takes the pair.
+    const cover = coverSequence(4, [
+        { start: 1, end: 4, payload: "long" },
+        { start: 0, end: 2, payload: "left" },
+        { start: 2, end: 4, payload: "right" },
+    ]);
+    assert.equal(cover.covered, 4);
+    assert.equal(cover.uncovered, 0);
+    assert.deepEqual(cover.spans.map((s) => s.payload), ["left", "right"]);
+});
+test("coverSequence prefers fewer, longer spans on ties (unit weight)", () => {
+    // [0,3) covers the same as [0,1)+[1,3) but in one span → preferred.
+    const cover = coverSequence(3, [
+        { start: 0, end: 3, payload: "whole" },
+        { start: 0, end: 1, payload: "a" },
+        { start: 1, end: 3, payload: "b" },
+    ]);
+    assert.deepEqual(cover.spans.map((s) => s.payload), ["whole"]);
+});
+test("coverSequence leaves genuinely uncovered gaps uncovered", () => {
+    const cover = coverSequence(10, [
+        { start: 0, end: 3, payload: "x" },
+        { start: 6, end: 9, payload: "y" },
+    ]);
+    assert.equal(cover.covered, 6);
+    assert.equal(cover.uncovered, 4);
+    assert.deepEqual(cover.spans.map((s) => s.payload), ["x", "y"]);
+});
+test("coverSequence honours weights when coverage ties", () => {
+    // Both single-span covers cover all 2 symbols; the lighter weight wins.
+    const cover = coverSequence(2, [
+        { start: 0, end: 2, weight: 5, payload: "heavy" },
+        { start: 0, end: 2, weight: 1, payload: "light" },
+    ]);
+    assert.deepEqual(cover.spans.map((s) => s.payload), ["light"]);
+});

package/dist/src/extension.d.ts

@@ -0,0 +1,37 @@
+/** A half-open byte span of a query an extension computed, and the canonical
+ *  result bytes it is authoritative for. */
+export interface ComputedSpan {
+    i: number;
+    j: number;
+    bytes: Uint8Array;
+}
+/** The generic capabilities the mind lends every extension — nothing here
+ *  names any particular extension, and every member is machinery the mind
+ *  already has:
+ *
+ *  MEANING (resonance and grounding — the same mechanisms recall uses):
+ *   • meaningOf — which of some labelled forms does a span mean?  Pure gist
+ *     nearness; the extension supplies its own vocabulary and gives the answer
+ *     its own reading.
+ *   • continuation — the grounded form the corpus continues a form to (the
+ *     continuation fixpoint recall grounds answers with), or null when the
+ *     form leads nowhere.  The mind answers only "where does this form lead?";
+ *     what that continuation MEANS is the extension's business.
+ *
+ *  GEOMETRY (the perception tree's own structure):
+ *   • segment — coherent runs by the alphabet-space merge the perception tree
+ *     uses, so an extension's notion of "separator" is the learnt geometry's.
+ *   • reach — the river's grouping capacity (maxGroup), bounding how far apart
+ *     two spans may sit and still be read as one construction. */
+export interface ExtensionHost {
+    meaningOf(bytes: Uint8Array, anchors: ReadonlyArray<{
+        name: string;
+        form: Uint8Array;
+    }>): Promise<string | null>;
+    continuation(bytes: Uint8Array): Promise<Uint8Array | null>;
+    segment(bytes: Uint8Array): Array<{
+        i: number;
+        j: number;
+    }>;
+    reach: number;
+}

package/dist/src/extension.js

@@ -0,0 +1,7 @@
+// extension.ts — shared types for computational extensions.
+//
+// Extensions (ALU, user computation packs) implement the PipelineMechanism
+// interface (pipeline-mechanism.ts) — this file holds only the types that
+// both the extension author and the pipeline need, without coupling either
+// to a specific mechanism.
+export {};

package/dist/src/geometry.d.ts

@@ -0,0 +1,175 @@
+import { Vec } from "./vec.js";
+import { Sema, Space } from "./sema.js";
+import { Alphabet } from "./alphabet.js";
+/** The store's geometric identity bar: cosine ≥ 1 − 1/√D is the similarity at
+ *  which `intern` already treats two gists as the SAME node.  Recall reuses it
+ *  to accept a near-identical query, and the climb to accept a containing form —
+ *  one derived constant, never a tuned threshold.  NOTE: this fixed bar is
+ *  the ESTIMATOR floor of an identity claim; a whole-span claim over a span
+ *  longer than the perception quantum must use the scale-aware
+ *  {@link identityBar}, which converts the tolerated fraction into bytes. */
+export declare function mergeThreshold(D: number): number;
+/** The scale-aware IDENTITY bar for a whole-span resonance claim over a span
+ *  of `len` bytes.  Under the linear fold a cosine reads "fraction of aligned
+ *  shared bytes", so a FIXED cosine bar admits a byte budget that grows with
+ *  the span: 1 − 1/√D over a 4·√D-byte span tolerates four whole river
+ *  windows of foreign content while still claiming "near-identical".  An
+ *  identity claim may tolerate at most ONE river window W — the perception
+ *  quantum, the same single-window budget near-dedup's differsByOneWindow
+ *  grants — so the bar is 1 − W/len, floored at mergeThreshold(D), below
+ *  which the RaBitQ estimator cannot certify identity anyway.  This is the
+ *  angle+magnitude form of the identity test: the ANGLE carries the shared
+ *  fraction, the span's MAGNITUDE (√len, the linear fold's own norm) converts
+ *  the tolerated fraction into tolerated bytes.  Derived from W, D and the
+ *  span; never tuned. */
+export declare function identityBar(D: number, maxGroup: number, len: number): number;
+/** The reach bar: half a river quantum, derived from the fold's own geometry.
+ *  A branch folds up to `maxGroup` children, so two forms that differ in ONE
+ *  whole child — the smallest distinction perception can mean — sit at cosine
+ *  ≈ 1 − 1/maxGroup.  Half that quantum, 1 − 1/(2·maxGroup), is closer than any
+ *  single-child difference can be: a positional echo of the same content.
+ *
+ *  Recall uses this as its confidence floor: a query whose nearest resonant
+ *  form sits below this bar is structurally unrelated to everything in the store
+ *  — further than any single-child variant — and the system returns null rather
+ *  than fabricate an answer from an unrelated form.  Derived, never tuned. */
+export declare function reachThreshold(maxGroup: number): number;
+/** The estimator's own noise floor: 1/√D — ONE standard deviation of the
+ *  cosine between two independent random vectors in D dimensions (the same σ
+ *  {@link significanceBar} takes three of).  It is the smallest difference in
+ *  cosine that is distinguishable from the rotation-uniformised RaBitQ
+ *  estimation error (see the MEASUREMENT CAVEAT above): a contrastive margin
+ *  below it is quantisation noise, not evidence.  The consensus climb gates a
+ *  region's vote on its discriminative margin clearing this floor — the
+ *  minimal "above noise" bar, one σ, not the stricter 3σ relatedness bar.
+ *  Derived, never tuned. */
+export declare function estimatorNoise(D: number): number;
+/** The statistical-significance bar for whole-query resonance: 3/√D.
+ *  In D dimensions the expected cosine of two independent random vectors is 0
+ *  with standard deviation 1/√D.  A cosine ≥ 3/√D is three standard deviations
+ *  above chance — the query is statistically related to the store, not merely
+ *  sharing random byte noise.  Below this bar the consensus climb (which trusts
+ *  sub-region resonance) is skipped: there is no evidence the query belongs to
+ *  the same distribution as the stored content.  Derived, never tuned. */
+export declare function significanceBar(D: number): number;
+/** The concept (halo) threshold: the cosine above which two nodes share a
+ *  distributional concept.  A halo is a superposition of episode signatures in
+ *  D-dimensional space, so the expected cosine between two unrelated halos is 0
+ *  with standard deviation 1/√D.  The structural midpoint 0.5 separates "more
+ *  similar than not" from noise; the +0.5/√D term adds one half-sigma margin
+ *  that vanishes as D → ∞, accounting for the wider noise band at lower D
+ *  without inventing a tuned constant.  At D=1024 this gives 0.516, within
+ *  3% of 0.5 — existing behavior is preserved while threshold and D move
+ *  together.  Derived, never tuned. */
+export declare function conceptThreshold(D: number): number;
+/** The HALF-DOMINANCE predicate: whether a part covering `partLen` of a
+ *  whole of `wholeLen` covers STRICTLY more than half of it.  A span that
+ *  dominates its whole can no longer discriminate the whole's own content —
+ *  the one test behind liftAnswer's keep-the-frame rule, collectRegions'
+ *  wrapper exclusion, and CAST's frame-depth majority (each cites this).
+ *  CAST's frame-FRACTION gate is the deliberately CLOSED variant (≥ ½ is
+ *  already unusable there) and stays inline where it is documented.
+ *  Derived from the structural midpoint, never tuned. */
+export declare function dominates(partLen: number, wholeLen: number): boolean;
+/** The consensus-vote significance floor: ln(N) + 1/2, where N is the number
+ *  of learnt contexts (edge sources).  A single region's IDF-weighted vote for
+ *  an anchor reached through c contexts is at most ln(N/c) ≤ ln(N); the +1/2
+ *  demands the pooled vote exceed what ONE maximally-specific region could
+ *  contribute by half a unit — i.e. genuine corroboration beyond a lone
+ *  region's echo at this corpus scale.  The ONE floor both consumers gate on:
+ *  recallByResonance trusting a climb anchor, and commitVotes admitting a
+ *  further point of attention.  Defined once here so the two can never
+ *  drift apart.  Derived from N, never tuned. */
+export declare function consensusFloor(N: number): number;
+/** The coverage bar for the reach (interior) index, when vector-similarity
+ *  gating is used.  Returns the concept threshold — the structural midpoint
+ *  (~0.5 at D=1024) where two forms are "more similar than not."
+ *
+ *  Currently UNUSED in the hot training path: interior nodes are indexed
+ *  unconditionally (hash-cons dedup bounds the index naturally).
+ *  Post-hoc structural compaction ({@link Store.compactContentIndex})
+ *  replaces runtime coverage gating with a batch pass that removes
+ *  structurally-isolated entries.  Derived, never tuned. */
+export declare function coverageBar(_maxGroup: number, D: number): number;
+export interface Folded {
+    tree: Sema;
+    /** Byte length of the subtree — carried incrementally so the stable-prefix
+     *  boundary scan never re-walks subtrees (the old per-level walk was
+     *  O(n log n) over the whole input). */
+    len: number;
+}
+export interface Grid {
+    width: number;
+    height: number;
+    channels: number;
+    data: Uint8Array;
+    dims?: number[];
+}
+export declare function contentBoundaries(space: Space, bytes: Uint8Array): number[];
+/** Find the longest prefix of `bytes` whose leaf-id signature matches a
+ *  known branch via `lookup`.  Returns the byte-length of that prefix, or 0. */
+export declare function knownPrefixLength(bytes: Uint8Array, leafAt: (i: number) => number | null, lookup: (leafIds: number[]) => number | null): number;
+/** Bytes → Sema tree.  `leafAt` and `lookup` are store capabilities for
+ *  detecting previously-stored prefixes so the river can split at the
+ *  correct boundary.  Pass them through from `perceive`; the geometry
+ *  computes the stable prefix internally.
+ *
+ *  `boundaries` is the CALLER-computed stable-prefix boundary set (§10.3):
+ *  strictly-increasing proper byte offsets, each the length of a prefix that
+ *  is already a stored whole-stream form.  When given, the fold splits into
+ *  the segments between consecutive boundaries — each folded independently,
+ *  exactly as it folded when it was learned — and the segment roots join
+ *  LEFT-NESTED (((s₀·s₁)·s₂)…), so every learnt cumulative-context root
+ *  reappears as an identical subtree (and, by hash-consing, the very same
+ *  node) inside the grown stream.  This is what lets a conversation's next
+ *  turn extend perception instead of refolding it: identical prefixes
+ *  produce identical subtrees regardless of what follows them. */
+export declare function bytesToTree(space: Space, alphabet: Alphabet, bytes: Uint8Array, leafAt?: (i: number) => number | null, lookup?: (leafIds: number[]) => number | null, boundaries?: readonly number[]): Sema;
+/** A stable-prefix fold's reusable state: the segment edge offsets and each
+ *  segment's independently-folded root ({@link riverFoldRaw} output).  A
+ *  grown stream whose boundary set EXTENDS a previous fold's reuses every
+ *  matching segment's Folded unchanged (segments fold independently by
+ *  construction, so reuse is bit-identical to refolding) and folds only the
+ *  new right-edge segment — O(turn) per extension.  Purely a cache: the
+ *  produced tree never depends on cache state. */
+export interface StableFold {
+    edges: number[];
+    segs: Folded[];
+}
+/** {@link stablePrefixFold} with incremental segment reuse — same cuts, same
+ *  segment folds, same left-nested join, same single root normalize; `prev`
+ *  only elides recomputing segments whose [start,end) offsets it already
+ *  folded over a byte-identical prefix (the caller keys the cache by
+ *  content).  Requires a non-empty effective boundary set. */
+export declare function stablePrefixFoldIncremental(space: Space, alphabet: Alphabet, bytes: Uint8Array, boundaries: readonly number[], prev?: StableFold): {
+    tree: Sema;
+    fold: StableFold;
+};
+/** Plain river fold WITHOUT the final root normalize — the segment-level
+ *  building block of {@link stablePrefixFold} (interiors must keep their
+ *  byte-proportional magnitude; only the whole perception's root is ever
+ *  normalized).  Exported so callers that COMPOSE already-existing structural
+ *  parts into a hypothetical synthetic root (see {@link composeStructuralGist})
+ *  can feed the same raw primitive instead of duplicating its mathematics. */
+export declare function riverFoldRaw(space: Space, row: Folded[]): Folded;
+/** One already-existing structural vector to compose, paired with the byte
+ *  span (query-slot) length it stands in for.  `len`, not the vector's own
+ *  magnitude, is what {@link composeStructuralGist} restores — the composed
+ *  slot's NATURAL span, exactly as the linear river fold would carry it. */
+export interface StructuralPart {
+    v: Vec;
+    len: number;
+}
+/** Synthesize a hypothetical internal structure from already-existing
+ *  structural vectors — NOT from bytes.  This is the raw positional
+ *  composition the linear river fold already uses (see the folding header
+ *  above): each part is positionally bound into its own seat, its natural
+ *  span magnitude is preserved, the parts are linearly superposed, and only
+ *  the final synthetic root is normalized.  It never calls {@link gistOf}
+ *  (there is no `gistOf` here — geometry.ts has no store), never perceives a
+ *  concatenated byte string, and never interns or stores a new node: the
+ *  result is an opaque, ungrounded Vec for an ANN probe only. */
+export declare function composeStructuralGist(space: Space, parts: readonly StructuralPart[]): Vec;
+export declare function hilbertBytes(grid: Grid): Uint8Array;
+export declare function gridToTree(space: Space, alphabet: Alphabet, grid: Grid): Sema;
+export declare function stackGrids(frames: Grid[]): Grid;