@hviana/sema 0.4.2 → 0.4.4

package/dist/src/mind/learning.js

@@ -0,0 +1,270 @@
+// learning.ts — ingest and deposition (Section 7 of the mind).
+//
+//   Learning is DEPOSITION: perceive a stream into a tree and intern every
+//   node.  A fact is an EDGE between node ids; recall traverses edges.
+import { bindSeat, companySignature, isChunk } from "../sema.js";
+import { changedNodes } from "./types.js";
+import { inputBytes, latin1Key, perceiveDeposit, resolve, } from "./primitives.js";
+import { canonicalWindows, leafIdPrefix } from "./canonical.js";
+import { fold as foldVecs } from "../sema.js";
+/** Intern a perceived tree into node ids, bottom-up, sharing equal subtrees.
+ *  Returns the root node id and a map from tree nodes to their ids.
+ *
+ *  Memoized by NODE IDENTITY (ctx._internIds): the pyramid fold shares a
+ *  prefix's subtree OBJECTS across an accumulated context's deposits, and a
+ *  node already interned needs nothing again — its id is permanent
+ *  (content-addressed) and its intern-time side effects (gist capture, kid
+ *  rows) fired at first mint; re-interning was pure lookups.  A memo hit
+ *  therefore skips the WHOLE shared subtree, making the intern walk
+ *  O(new nodes) per deposit instead of O(context).  Only the hit node
+ *  itself enters `ids`; descendants stay reachable via the memo (see
+ *  idOf in indexSubSpans and the changedNodes prune). */
+export async function internTreeIds(ctx, node, ids) {
+    const known = ctx._internIds.get(node);
+    if (known !== undefined) {
+        ids.set(node, known);
+        return known;
+    }
+    let id;
+    if (node.kids === null) {
+        id = await ctx.store.putLeaf(node.leaf ?? new Uint8Array(0), node.v);
+    }
+    else {
+        const kds = [];
+        for (const k of node.kids)
+            kds.push(await internTreeIds(ctx, k, ids));
+        id = await ctx.store.putBranch(kds, node.v);
+    }
+    ids.set(node, id);
+    ctx._internIds.set(node, id);
+    return id;
+}
+/** Index flat branches for sub-spans of a deposit's byte stream, linked to
+ *  their structural chunks via durable CONTAINMENT edges. */
+export async function indexSubSpans(ctx, tree, ids) {
+    const chunkOf = [];
+    const streamIds = [];
+    const streamVecs = [];
+    const collect = (n) => {
+        if (isChunk(n)) {
+            // A chunk inside a memo-skipped shared subtree is absent from `ids`;
+            // the intern memo still knows it (same object).  A miss on both (the
+            // WeakMap entry was collected) only forfeits the seenBefore skip.
+            const chunkId = ids.get(n) ?? ctx._internIds.get(n);
+            for (const k of n.kids) {
+                const lid = k.leaf ? ctx.store.findLeaf(k.leaf) : null;
+                if (lid === null)
+                    return false;
+                streamIds.push(lid);
+                streamVecs.push(k.v);
+                chunkOf.push(chunkId);
+            }
+            return true;
+        }
+        if (n.kids) {
+            for (const k of n.kids)
+                if (!collect(k))
+                    return false;
+        }
+        return true;
+    };
+    if (!collect(tree))
+        return false;
+    const W = ctx.space.maxGroup; // write side of the canonical contract
+    const prev = ctx._prevSeen;
+    const seenBefore = (off, len) => {
+        if (!prev)
+            return false;
+        for (let i = off; i < off + len; i++) {
+            const c = chunkOf[i];
+            if (c === undefined || !prev.has(c))
+                return false;
+        }
+        return true;
+    };
+    const lens = streamIds.length >= W ? canonicalWindows(W) : [streamIds.length];
+    for (const len of lens) {
+        if (len < 1)
+            continue;
+        for (let off = 0; off + len <= streamIds.length; off++) {
+            if (seenBefore(off, len))
+                continue;
+            const winIds = streamIds.slice(off, off + len);
+            const flatId = ctx.store.findBranch(winIds) ??
+                await ctx.store.putBranch(winIds, foldVecs(ctx.space, streamVecs.slice(off, off + len)));
+            for (let i = off; i < off + len; i++) {
+                const c = chunkOf[i];
+                if (c !== undefined)
+                    ctx.store.addContainer(flatId, c);
+            }
+        }
+    }
+    return true;
+}
+/** Perceive, intern, and index a single input.  Returns the perceived tree,
+ *  root id, id map, and the changed (new) subtrees for halo reinforcement. */
+export async function deposit(ctx, input, track, conversational = false) {
+    const bytes = inputBytes(ctx, input);
+    // Deposit-shaped perception: stable-prefix tree SEEDING (see
+    // perceiveDeposit) — an accumulated context re-folds only its new suffix,
+    // O(turn) instead of O(context) per conversation turn.  Cache-only here
+    // (no store-probe fallback): a knownPrefixLength scan on every novel fact
+    // would cost O(n²) hashing, while conversation replays are always warm —
+    // re-deposition replays from the first turn, rebuilding the cache as it
+    // goes.  `conversational` scopes the STABLE-PREFIX variant (turn-boundary
+    // folding, matching query-time perception) to ingestPair's own growing
+    // context argument — a bare ingestOne deposit whose bytes merely happen
+    // to extend an earlier UNRELATED deposit (no conversational relationship)
+    // must keep the plain fold, or two coincidentally-prefix-sharing facts
+    // would stop sharing structure with each other.
+    const tree = perceiveDeposit(ctx, bytes, conversational);
+    const ids = new Map();
+    const rootId = await internTreeIds(ctx, tree, ids);
+    const indexed = await indexSubSpans(ctx, tree, ids);
+    const leafIds = leafIdPrefix(ctx, bytes);
+    if (leafIds.length === bytes.length && leafIds.length >= 2) {
+        await ctx.store.putBranch(leafIds, tree.v);
+    }
+    const changed = (track && ctx._prevSeen)
+        ? changedNodes(tree, ids, ctx._prevSeen)
+        : [tree];
+    if (track)
+        ctx._prevSeen = indexed ? new Set(ids.values()) : null;
+    return { tree, rootId, ids, changed };
+}
+/** Ingest a single input (a bare experience, no continuation). */
+export async function ingestOne(ctx, input) {
+    const { tree, rootId, ids } = await deposit(ctx, input, true);
+    ctx.store.indexTarget(rootId);
+    const parts = tree.kids
+        ? tree.kids.map((k) => ids.get(k))
+        : [rootId];
+    const stride = ctx.space.maxGroup;
+    if (parts.length > stride) {
+        for (let i = 0; i + stride < parts.length; i += stride) {
+            await ctx.store.link(parts[i], parts[i + stride]);
+        }
+        if ((parts.length - 1) % stride !== 0) {
+            const lastStart = Math.floor((parts.length - 1) / stride) * stride;
+            if (lastStart < parts.length - 1) {
+                await ctx.store.link(parts[lastStart], parts[parts.length - 1]);
+            }
+        }
+    }
+    else {
+        for (const id of parts)
+            ctx.store.indexTarget(id);
+    }
+    return Object.assign(tree, { id: rootId });
+}
+/** For each right-edge suffix of the context bytes, resolve it against the
+ *  store.  A suffix whose resolved node is already a known form inherits the
+ *  continuation edge.  Gate: ≥ 2 structural parents (reused across deposits),
+ *  or (halo > 0 ∧ already an edge source).  Pure answers do not qualify. */
+async function propagateSuffixes(ctx, src, dst) {
+    const W = ctx.space.maxGroup;
+    const bytes = ctx.store.bytes(src);
+    const n = bytes.length;
+    if (n < 2 * W)
+        return;
+    // Existence prefilter — the write side of the canonical contract: every
+    // deposit interns its WHOLE byte stream as a flat branch of per-byte leaf
+    // ids (deposit(), canonical.ts).  A suffix is a stored form exactly when
+    // that flat twin exists, so one content-hash probe per offset decides;
+    // only a hit pays for resolve()'s deposit-shaped perception.  This keeps
+    // the scan free of river folds — O(1) probes over cheap byte hashes
+    // instead of O(suffix) vector folds per offset.
+    const leafIds = leafIdPrefix(ctx, bytes);
+    for (let i = 1; i <= n - W; i++) {
+        if (ctx.store.findBranch(leafIds.slice(i)) === null)
+            continue;
+        const id = resolve(ctx, bytes.subarray(i));
+        if (id === null || id === src)
+            continue;
+        const known = ctx.store.parentsFirst(id, 2).length >= 2 ||
+            (ctx.store.haloMass(id) > 0 && ctx.store.hasNext(id));
+        if (!known)
+            continue;
+        await ctx.store.link(id, dst);
+    }
+}
+/** Ingest a pair (context, continuation) — learn an edge and pour halos.
+ *  Returns the deposited root ids (context, continuation) — a pure
+ *  read-out; callers that ignore it behave exactly as before. */
+export async function ingestPair(ctx, ctxInput, cont) {
+    const c = await deposit(ctx, ctxInput, true, true);
+    const cont_ = await deposit(ctx, cont, false);
+    const ctxId = c.rootId, contId = cont_.rootId;
+    // Stamp this turn's continuation onto its own cache entry — the proof a
+    // FUTURE, longer ctxInput needs (see perceiveDeposit) to recognise itself
+    // as this conversation's genuine next turn rather than an unrelated fact
+    // that happens to share this ctxInput's byte prefix.
+    {
+        const ctxBytes = inputBytes(ctx, ctxInput);
+        const entry = ctx._depositTrees.get(latin1Key(ctxBytes));
+        if (entry !== undefined)
+            entry.nextBytes = inputBytes(ctx, cont);
+    }
+    await ctx.store.link(ctxId, contId);
+    await propagateSuffixes(ctx, ctxId, contId);
+    // Halos pour company SIGNATURES (identity), not gists (content) — see
+    // companySignature in sema.ts.
+    const contSeat = bindSeat(ctx.space, companySignature(ctx.space, contId), 1);
+    for (const part of c.changed) {
+        const partId = c.ids.get(part);
+        await ctx.store.pourHalo(partId, contSeat);
+        await ctx.store.pourHalo(contId, bindSeat(ctx.space, companySignature(ctx.space, partId), 0));
+    }
+    return { ctxId, contId };
+}
+/** Dispatch the public ingest input shapes onto one-input / pair handlers —
+ *  THE one reading of ingest's polymorphic surface (scalar, (context,
+ *  continuation) pair, or a list mixing bare inputs and pairs).  Both ingest
+ *  paths — the direct one below and {@link CachedIngest} — route through
+ *  this, so the shape-detection can never drift between them again (the
+ *  ingest cache once re-implemented it and drifted). */
+export async function dispatchIngest(input, second, onOne, onPair) {
+    if (Array.isArray(input) && !(input instanceof Uint8Array) &&
+        input.width === undefined) {
+        const arr = input;
+        if (arr.length === 2 && !Array.isArray(arr[0]) && !Array.isArray(arr[1])) {
+            await onPair(arr[0], arr[1]);
+            return undefined;
+        }
+        for (const item of arr) {
+            if (Array.isArray(item) && item.length === 2) {
+                await onPair(item[0], item[1]);
+            }
+            else
+                await onOne(item);
+        }
+        return undefined;
+    }
+    if (second === undefined)
+        return onOne(input);
+    await onPair(input, second);
+    return undefined;
+}
+/** Ingest an input or array of inputs/pairs.  The public ingest entry point.
+ *
+ *  `onDeposit`, when given, is invoked once per ingested item with the
+ *  deposited root node ids ({@link DepositReport}) — item-level provenance
+ *  for tooling that needs to know which stored node an ingested item became.
+ *  Purely observational: the callback runs after the item's deposit
+ *  completed and nothing reads its result. */
+export async function ingest(ctx, input, second, onDeposit) {
+    let index = 0;
+    return dispatchIngest(input, second, async (i) => {
+        const r = await ingestOne(ctx, i);
+        onDeposit?.({ index: index++, kind: "one", contextId: r.id });
+        return r;
+    }, async (a, b) => {
+        const { ctxId, contId } = await ingestPair(ctx, a, b);
+        onDeposit?.({
+            index: index++,
+            kind: "pair",
+            contextId: ctxId,
+            continuationId: contId,
+        });
+    });
+}

package/dist/src/mind/match.d.ts

@@ -0,0 +1,181 @@
+import { Vec } from "../vec.js";
+import type { Hit } from "../store.js";
+import type { MindContext } from "./types.js";
+import type { Site } from "./graph-search.js";
+/** The graded LOCATE ladder: find `needle` in `haystack` starting at
+ *  `fromPos`, strictest matcher first, relaxing only when the stricter one
+ *  fails.  This is the read-out matcher skill extraction locates exemplar
+ *  frames with.
+ *
+ *  1. exact    — literal byte match (the fast path).
+ *  2. halo     — the needle's distributional role matches a recognised query
+ *                form (gate: conceptThreshold).
+ *  3. gist     — the needle's perceived gist matches a query segment
+ *                (gate: identityBar — scale-aware).
+ *
+ *  Returns the absolute byte position, or −1. */
+export declare function locate(ctx: MindContext, haystack: Uint8Array, needle: Uint8Array, fromPos: number, sites?: ReadonlyArray<Site>): number;
+/** The ALIGNED matcher: maximal literal matching runs between `query` and
+ *  `ct` (a learned context's bytes), by seed-and-extend over
+ *  `space.maxGroup`-sized n-gram seeds.  Where locate() finds ONE position of
+ *  a short frame, this finds EVERY run two whole structures share — the
+ *  matcher CAST detects a woven query with.  Returns non-overlapping runs
+ *  sorted by query position. */
+export declare function alignRuns(ctx: MindContext, query: Uint8Array, ct: Uint8Array): Array<{
+    qs: number;
+    qe: number;
+    cs: number;
+}>;
+/** A run from {@link alignGraded} — the ALIGNED matcher extended with the
+ *  same graded-evidence ladder as {@link locate}.  Literal runs carry
+ *  `weight = 1` (exact match is full evidence); halo-matched site runs carry
+ *  `weight = cosine` (measured evidence — the halo similarity itself).
+ *  `cs` is the structural byte position in the context regardless of run
+ *  kind, so the substitution/redirection schemas work unchanged on conceptual
+ *  alignment. */
+export interface GradedRun {
+    /** True for a run the CLIMB proposed rather than the byte matcher found — see
+     *  {@link computeWeave}'s phase 2.  Provenance only: a proposed run reaches
+     *  CAST already gated (dominant literal agreement, not frame, and over bytes
+     *  no literal run claimed), so every consumer reads it exactly like any other
+     *  run.  Filtering seat-shaped reads by this flag was tried and is NOT
+     *  needed — the gates, not the flag, are what make it safe. */
+    proposed?: boolean;
+    qs: number;
+    qe: number;
+    cs: number;
+    weight: number;
+}
+/** The GRADED alignment matcher: extends literal W-gram alignment
+ *  ({@link alignRuns}) with halo-matched recognised sites in query regions
+ *  that have no literal coverage.  Same ladder as {@link locate}: literal
+ *  first, then distributional role (halo-matched sites, gate:
+ *  conceptThreshold, enforced by {@link bestHaloMate}).  Returns weighted
+ *  runs sorted by query position.
+ *
+ *  `querySites` are the pre-computed recognition sites for the query
+ *  (optional — when absent, only literal alignment fires and graded degrades
+ *  to the original behaviour).  Context sites are recognised internally. */
+export declare function alignGraded(ctx: MindContext, query: Uint8Array, contextBytes: Uint8Array, querySites?: ReadonlyArray<Site>): GradedRun[];
+/** The IN-LIST halo matcher: the best halo-mate for `halo` among EXPLICIT
+ *  candidates, above the concept threshold — the list counterpart of
+ *  {@link haloSiblings}, which asks the halo INDEX for candidates instead.
+ *  Behind locate()'s halo step and articulation's voice matching; a third
+ *  "best halo among these" decision must come here, not inline. */
+export declare function bestHaloMate<T>(ctx: MindContext, halo: Vec, items: Iterable<T>, haloOf: (item: T) => Vec | null | undefined): {
+    item: T;
+    score: number;
+} | null;
+export declare function haloSiblings(ctx: MindContext, id: number, halo?: Vec | null, bar?: number): Promise<Hit[]>;
+/** Bundle the distributional company of every addressable W-window in a
+ *  byte span.  This is the query-time counterpart of the write-side halo
+ *  pours: no lexical unit or storage row is invented; the span is represented
+ *  by VSA superposition of the window concepts the store already knows.
+ *
+ *  Components are normalized before bundling so repetition mass remains
+ *  evidence about each stored node, not an accidental weight on one window
+ *  inside the composed phrase.  Returns null when the corpus provides no
+ *  distributional evidence for the span. */
+export declare function spanHalo(ctx: MindContext, bytes: Uint8Array, from?: number, to?: number): Vec | null;
+/** Distributional synonym evidence between arbitrary byte spans. Whole words
+ *  need not be independently interned: their stored W-window occurrences are
+ *  lifted to episode halos, bundled, and compared. The caller chooses the
+ *  derived gate appropriate to its claim (concept identity or analogy). */
+export declare function spanSynonymStrength(ctx: MindContext, a: Uint8Array, b: Uint8Array): number;
+/** The DISTRIBUTIONAL matcher between two nodes: mutual-nearest-neighbour
+ *  strength, not a pick.  Returns the direct halo cosine, or failing that the
+ *  highest mutual-halo-sibling min-score (second-order analogy), or failing
+ *  that the SHARED-FRAME strength (below) — the gate CAST's comparison
+ *  schema validates genuine analogs with (bar: significanceBar).
+ *
+ *  The result names its TIER alongside the score: `halo: true` means the
+ *  score cleared a significanceBar-gated HALO tier (direct company cosine
+ *  or mutual-sibling) — genuine distributional evidence; `halo: false`
+ *  means only the structural shared-frame fallback matched, a coverage
+ *  fraction with no bar of its own.  CAST's comparison gate treats the two
+ *  differently (see cast.ts): halo evidence stands alone, frame evidence
+ *  needs the query to have named the analog or the climb root to be
+ *  trusted. */
+export interface AnalogyEvidence {
+    score: number;
+    halo: boolean;
+}
+export declare function analogyStrength(ctx: MindContext, a: number, b: number): Promise<AnalogyEvidence>;
+/** The STRUCTURAL analogy tier: two nodes are analogs when their byte
+ *  streams share a LEARNT frame — a content-addressed flat form of at least
+ *  one full river window (W bytes, the perception quantum) that occurs in
+ *  BOTH.  This is what "playing the same role" means structurally: "Ice is
+ *  cold" and "Steel is hard" share the learnt " is " frame even though they
+ *  keep disjoint distributional company.  Halos measure company by IDENTITY
+ *  (company signatures — see sema.ts), so unrelated-company analogs must be
+ *  validated by the frame itself, not by content leaking through halo
+ *  vectors.  Strength is the shared learnt coverage of the SHORTER side —
+ *  a fraction, comparable to the cosine tiers above.  Derived: the window
+ *  is maxGroup, the same quantum differsByOneWindow and canonicalChunkId
+ *  measure by; no tuned constants. */
+export declare function sharedFrameStrength(ctx: MindContext, a: number, b: number): number;
+/** FORWARD through a synonym: the continuation an edge-less node borrows from
+ *  a concept (halo) sibling — resonate the node's halo, take the first
+ *  sibling above the concept threshold that itself has a direct edge. */
+export declare function conceptHop(ctx: MindContext, id: number): Promise<number | null>;
+/** FORWARD projection: follow continuation edges from a node to its fixpoint.
+ *  The first hop may cross a concept (halo) link — a synonym.  The rest
+ *  follow direct edges.  Convergence is intrinsic: the seen set guards
+ *  against cycles.  `guide` disambiguates multi-continuation nodes by
+ *  resonance. */
+export declare function follow(ctx: MindContext, id: number, guide?: Vec | null): Promise<Uint8Array | null>;
+/** REVERSE projection: the context a learnt continuation follows, voiced as
+ *  bytes.  A common continuation ("Yes.") follows MANY contexts; with a
+ *  `guide` the context whose gist resonates with the query wins (seat
+ *  symmetry) — without one, the most-corroborated context wins (poured halo
+ *  MASS, the direct measure of how many episodes established it), falling
+ *  back to first-learnt on equal mass.  Among many predecessors RECIPROCAL
+ *  ones (mutual edges) are preferred when any exist (RC5).  Callers that
+ *  HAVE a query gist must pass it, or they silently change disambiguation
+ *  regime.
+ *
+ *  `rev`, when the caller has already materialised prevOf (one read per
+ *  relation — a hub's reverse fan-in is corpus-sized), is reused instead of
+ *  refetched.  Returns null when there is no predecessor or the picked
+ *  context reads empty (a zero-length context is no grounding: an empty
+ *  Uint8Array is truthy, and returning it would flow a hollow "answer"
+ *  onward). */
+export declare function reverseContext(ctx: MindContext, id: number, guide?: Vec | null, rev?: readonly number[]): Uint8Array | null;
+/** THE projection: ground a matched node to answer bytes — FORWARD to its
+ *  continuation fixpoint (which may cross a concept hop), else REVERSE to
+ *  the context it follows.  This is the direction ladder every mechanism's
+ *  final grounding step reduces to. */
+export declare function project(ctx: MindContext, id: number, guide?: Vec | null): Promise<Uint8Array | null>;
+/** Check whether an anchor is a span-shaped skill exemplar: it represents a
+ *  fact whose context and answer together form a span-in-context pattern.
+ *  If the anchor has a nextOf continuation, that is the answer and the anchor
+ *  itself is the context.  Otherwise the anchor's prevOf parents provide
+ *  candidate contexts, and the longest one whose span is span-shaped wins. */
+export declare function skillExemplar(ctx: MindContext, anchor: number, guide?: Vec | null): Promise<{
+    contextBytes: Uint8Array;
+    answerBytes: Uint8Array;
+} | null>;
+/** Whether the answer is a SPARSE subsequence of the context (bytes in
+ *  order, arbitrary gaps) — the OPEN span-shape reading (see the section
+ *  note above).  This is what lets extraction validate a MULTI-PIECE
+ *  exemplar whose answer is stitched from several context runs — but it is
+ *  deliberately permissive, so it must never be used as evidence that one
+ *  span was "drawn from" another (see {@link containsSpan} for that).
+ *
+ *  There is deliberately NO containsSpan pre-check here: strict containment
+ *  IMPLIES the subsequence embedding (a contiguous run, or a resolved node —
+ *  whose content-addressed identity means its bytes occur contiguously — is
+ *  an in-order embedding with zero gaps), so the scan below decides alone,
+ *  with the same truth value.  The old pre-check re-perceived the context
+ *  (a full river fold) per CANDIDATE in skillExemplar's √N-capped loop —
+ *  pure cost, no discrimination. */
+export declare function isSpanShaped(_ctx: MindContext, context: Uint8Array, answer: Uint8Array): boolean;
+/** STRICT containment: the answer's resolved node appears in the context's
+ *  folded tree, or the answer occurs as one CONTIGUOUS byte run of the
+ *  context.  This is real evidence the answer was drawn from the context.
+ *  Fusion gates on this — the sparse-subsequence reading of
+ *  {@link isSpanShaped} is trivially satisfied by short answers over long
+ *  queries ("cold" is a gap-tolerant subsequence of most sentences holding
+ *  c…o…l…d in order), and gating fusion on it silently starved multi-topic
+ *  queries of their further points of attention. */
+export declare function containsSpan(ctx: MindContext, context: Uint8Array, answer: Uint8Array): boolean;