@hviana/sema 0.1.0 → 0.1.2

package/dist/src/mind/types.d.ts

@@ -0,0 +1,174 @@
+import type { Vec } from "../vec.js";
+import type { Sema } from "../sema.js";
+import type { BoundedMap, Store } from "../store.js";
+import type { Space } from "../sema.js";
+import type { Alphabet } from "../alphabet.js";
+import type { MindConfig } from "../config.js";
+import type { GraphSearch, Leaf, Seg, Site } from "./graph-search.js";
+import type { Rationale } from "./rationale.js";
+import type { FoldPyramid, Grid } from "../geometry.js";
+export type Input = string | Uint8Array | Grid | Grid[];
+/** The host capabilities GraphSearch consults during a cover.  MindContext
+ *  extends this so the Mind can pass itself as the host. */
+export interface GraphSearchHost {
+    resolve(bytes: Uint8Array): number | null;
+    recogniseSpan?(bytes: Uint8Array): {
+        sites: ReadonlyArray<Site>;
+        leaves: ReadonlyArray<Leaf>;
+        splits: ReadonlySet<number>;
+    };
+    chooseNext?(node: number): number | undefined;
+}
+export interface Recognition {
+    /** Forms that can lead somewhere — they have an edge or a halo. */
+    sites: Site[];
+    /** The query's perceived leaves (the search's covering axioms). */
+    leaves: Leaf[];
+    /** Sub-leaf positions where a form boundary falls between leaf edges. */
+    splits: Set<number>;
+}
+/** How the consensus climb weights a region's Document-Frequency reach. */
+export type DFMode = "inverse" | "direct" | "combined";
+/** One POINT OF ATTENTION the consensus climb resolved. */
+export interface Attention {
+    /** The learnt context this point resolves to. */
+    anchor: number;
+    /** IDF-weighted consensus vote — the strength that orders points. */
+    vote: number;
+    /** The union of the query byte-spans whose evidence supports this point. */
+    start: number;
+    end: number;
+}
+/** Both read-outs of one consensus climb. */
+export interface AttentionRead {
+    roots: Attention[];
+    ranked: Attention[];
+}
+/** A positioned region of a byte stream paired with its gist. */
+export interface Segment {
+    start: number;
+    end: number;
+    v: Vec;
+}
+/** A region of the query's perceived tree for the consensus climb. */
+export interface Region {
+    v: Vec;
+    start: number;
+    end: number;
+    chunk: boolean;
+    /** Whether the region's bytes resolve to a KNOWN node (content-addressed,
+     *  exact).  Exact regions vote with full weight; approximate ones pay the
+     *  contrastive margin (see voteRegions) — under the linear fold a raw
+     *  resonance score is byte-overlap, evidence only in excess of its best
+     *  rival conclusion. */
+    known: boolean;
+}
+/** Per-region vote data from the consensus climb's resonance pass. */
+export interface RegionVote {
+    start: number;
+    end: number;
+    canonicalFailed: boolean;
+    roots: readonly number[];
+    w: number;
+    wFocus: number;
+}
+/** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
+export interface AncestorReach {
+    roots: number[];
+    contextsReached: number;
+    saturated: boolean;
+}
+/** Saturated-interval information for the noise-drop gate. */
+export interface SaturationInfo {
+    leadingEnd: number;
+    hasLeading: boolean;
+    intervals: Array<{
+        start: number;
+        end: number;
+    }>;
+}
+/** The items of poolVotes' deduction system. */
+export type AItem = {
+    kind: "region";
+    ri: number;
+} | {
+    kind: "anchor";
+    id: number;
+} | {
+    kind: "anchorFocus";
+    id: number;
+};
+export interface MindContext extends GraphSearchHost {
+    store: Store;
+    space: Space;
+    alphabet: Alphabet;
+    cfg: MindConfig;
+    search: GraphSearch;
+    trace: Rationale | null;
+    climbMemo: WeakMap<Uint8Array, Map<string, AttentionRead>> | null;
+    /** Per-response memo of {@link recognise} keyed by the byte-array OBJECT
+     *  (think, articulate, and the post-grounding pre-consume all recognise the
+     *  same query/answer objects).  Valid because the store is read-only while
+     *  a response is in flight; bypassed when a trace is attached so every
+     *  recognise still emits its rationale step.  Null outside respond(). */
+    recogniseMemo: WeakMap<Uint8Array, Recognition> | null;
+    /** Per-response memo of {@link perceive} keyed by the byte-array OBJECT —
+     *  the GENERAL memo the result-level ones (recogniseMemo, climbMemo,
+     *  _gistCache) each partially compensate for: resolve(), gistOf(), and
+     *  every mechanism's re-perception of the same query/answer object hit it
+     *  (a reason hop used to fold the same answer three times).  Valid because
+     *  the store is read-only while a response is in flight and perception is
+     *  a pure function of bytes; only inference-shaped calls (plain Uint8Array,
+     *  no leafAt/lookup capabilities) are memoised, so the deposit path never
+     *  sees it.  Keyed by CONTENT (latin1 of the bytes), not object identity —
+     *  mechanisms materialise the same span in fresh subarrays constantly
+     *  (measured on a trained store: 46% of one response's perceptions were
+     *  byte-identical repeats an identity key missed).  NOT bypassed under
+     *  trace — perception emits no rationale steps, so there is nothing a memo
+     *  hit could swallow.  Null outside respond(). */
+    perceiveMemo: Map<string, Sema> | null;
+    _edgeGuide: Vec | null;
+    _edgeChoice: Map<number, number>;
+    _prevSeen: Set<number> | null;
+    /** Session cache of node-id → perceived gist, for candidate scoring
+     *  ({@link chooseAmong} in the reverse projection's recall path re-gists up to
+     *  √N contexts per pick — the measured bottleneck there).  `chooseNext` does
+     *  NOT use this cache; forward-edge disambiguation uses prevOf counts
+     *  (distributional evidence) instead of gist comparison, because for short
+     *  answer candidates the gist is dominated by accidental byte-pattern
+     *  correlations.  A node's bytes are immutable and perception is a pure
+     *  function of bytes, so an entry stays valid for the store's lifetime —
+     *  never invalidated.  Bounded LRU (byte-sized); a miss only re-perceives,
+     *  never a correctness risk. */
+    _gistCache: BoundedMap<number, Vec>;
+    /** DEPOSIT-path stable-prefix tree cache: content key (latin1) of a
+     *  deposited input → its plain-fold level PYRAMID, so the NEXT deposit
+     *  whose prefix it is (a conversation's accumulated context) reuses every
+     *  full aligned block and refolds only the right edge of each level —
+     *  O(turn) per deposit instead of O(context), with the tree BIT-IDENTICAL
+     *  to a from-scratch fold (see FoldPyramid).  Purely a performance cache.
+     *  Entry-count bounded (a pyramid is ~KB/byte of content; only the few
+     *  live conversation chains matter). */
+    _depositTrees: BoundedMap<string, FoldPyramid>;
+    /** The byte lengths present in {@link _depositTrees} — the candidate
+     *  prefix lengths probed (longest first).  Drifts on eviction (a stale
+     *  length only costs a miss); cleared with the map when it outgrows the
+     *  probe budget. */
+    _depositLens: Set<number>;
+    /** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
+     *  content-addressed id.  Valid forever (ids are permanent, Sema nodes
+     *  immutable); WeakMap, so entries live exactly as long as the pyramid
+     *  cache keeps the shared subtrees alive.  Lets internTreeIds skip whole
+     *  shared subtrees and indexSubSpans keep its seenBefore window skip. */
+    _internIds: WeakMap<Sema, number>;
+}
+/** Read a whole node's bytes. */
+export declare const ALL = 2147483647;
+/** Splice every chosen span in order — the whole cover as one byte string. */
+export declare function spliceAll(segs: Seg[]): Uint8Array | null;
+/** Lift the answer out of the cover for think: the recognised region, free of
+ *  the asker's surrounding (unrecognised) framing. */
+export declare function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null;
+/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
+ *  tracked deposit interned (`prevSeen`). */
+export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];

package/dist/src/mind/types.js

@@ -0,0 +1,84 @@
+// types.ts — all interfaces, types, and free functions for the mind.
+//
+// GraphSearchHost is defined first (minimal imports) so GraphSearch can import
+// it without pulling in the full MindContext.
+import { concatBytes } from "../bytes.js";
+import { dominates } from "../geometry.js";
+// ═══════════════════════════════════════════════════════════════════════════
+// FREE FUNCTIONS (pure, no state)
+// ═══════════════════════════════════════════════════════════════════════════
+/** Read a whole node's bytes. */
+export const ALL = 0x7fffffff;
+/** Splice every chosen span in order — the whole cover as one byte string. */
+export function spliceAll(segs) {
+    if (!segs.some((s) => s.rec))
+        return null;
+    return concatBytes(segs.map((s) => s.bytes));
+}
+/** Lift the answer out of the cover for think: the recognised region, free of
+ *  the asker's surrounding (unrecognised) framing. */
+export function liftAnswer(segs, queryLen) {
+    const recognised = [];
+    for (let k = 0; k < segs.length; k++)
+        if (segs[k].rec)
+            recognised.push(k);
+    if (recognised.length === 0)
+        return null;
+    if (recognised.length === 1) {
+        const s = segs[recognised[0]];
+        if (dominates(s.j - s.i, queryLen)) {
+            return concatBytes(segs.map((x) => x.bytes));
+        }
+        return s.bytes;
+    }
+    const lo = recognised[0];
+    const hi = recognised[recognised.length - 1];
+    return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
+}
+/** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
+ *  tracked deposit interned (`prevSeen`). */
+export function changedNodes(tree, ids, prevSeen) {
+    const newCount = new Map();
+    const count = (n) => {
+        const memo = newCount.get(n);
+        if (memo !== undefined)
+            return memo;
+        const id = ids.get(n);
+        // PRUNE: a node whose id the previous deposit already interned is old,
+        // and content addressing makes that transitive — the same id names the
+        // same content, so every descendant was interned then too.  The whole
+        // subtree counts 0 without walking it; with the pyramid fold sharing a
+        // conversation's prefix subtree, this is what keeps the changed-nodes
+        // read O(new nodes) instead of O(context).  (A node internTreeIds
+        // memo-skipped has an id here exactly when it is such a shared root.)
+        if (id !== undefined && prevSeen.has(id)) {
+            newCount.set(n, 0);
+            return 0;
+        }
+        let c = 1; // reachable only when NOT pruned above ⇒ this node is new
+        if (n.kids) {
+            for (const k of n.kids)
+                c += count(k);
+        }
+        newCount.set(n, c);
+        return c;
+    };
+    const total = count(tree);
+    if (total === 0)
+        return [tree];
+    let n = tree;
+    for (;;) {
+        if (n.kids === null)
+            return [n];
+        let holder = null;
+        for (const k of n.kids) {
+            if (newCount.get(k) === total) {
+                holder = k;
+                break;
+            }
+        }
+        if (holder === null)
+            return [n];
+        n = holder;
+    }
+}

package/dist/src/rabitq-hnsw/src/database.d.ts

@@ -0,0 +1,200 @@
+/** External ids are integers only. */
+export type ExternalId = number;
+export interface DatabaseOptions {
+    /** Path to the SQLite database file (use ":memory:" for a transient store). */
+    dbPath: string;
+    /** Vector dimensionality. Required for a new database; ignored when reopening. */
+    dim?: number;
+    /** Max neighbours per node on layers >= 1 (layer 0 uses 2*M). Default 16. */
+    M?: number;
+    /** Candidate list size during construction. Default 200. */
+    efConstruction?: number;
+    /** Candidate list size during search. Default 100. Tunable at runtime. */
+    efSearch?: number;
+    /** Bits used to scalar-quantise the query for the RaBitQ estimator.
+     *  Query-side only (stored codes and the graph are independent of it), so it
+     *  is tunable at reopen, like efSearch.  Default 8: 4 bits cannot resolve a
+     *  tight cluster's residual and costs ~18% self-recall there (test 2a); 8
+     *  bits restores it for the price of a Uint16 per-query LUT. */
+    queryBits?: number;
+    /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
+    rotationRounds?: number;
+    /** Seed for the rotation and graph levels. Default fixed. */
+    seed?: number;
+    /** Optional centroid the vectors are centered by before quantisation. */
+    centroid?: ArrayLike<number>;
+    /**
+     * RAM cache size in MiB -- the single memory knob. It sizes SQLite's page
+     * cache AND the immutable-code LRU (whose entry-capacity is derived from this
+     * budget and the code size — there is no second knob). Both are purely speed
+     * enhancements: correctness and the per-operation storage-read count are
+     * identical with it off. Pass 0 to run with essentially no cache. Default 64.
+     * Not persisted; set per open.
+     */
+    cacheSizeMb?: number;
+}
+export interface QueryResult {
+    id: ExternalId;
+    /** Estimated cosine distance (1 - cosine). */
+    distance: number;
+}
+export interface StorageStats {
+    /** Bytes a single Float32 copy of the vector would take. */
+    float32BytesPerVector: number;
+    /** Bytes of 1-bit code kept per vector (on disk). */
+    codeBytesPerVector: number;
+    /** Bytes kept per vector (just the code; excludes the graph adjacency). */
+    bytesPerVector: number;
+    /** float32BytesPerVector / bytesPerVector. */
+    compressionRatio: number;
+}
+/**
+ * A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
+ * stored entirely in SQLite at `dbPath`. The graph IS the database -- there is no
+ * load/save step and no in-RAM copy of the data, so resident memory stays flat as
+ * the collection grows and the store survives process restarts. Reopening the
+ * same path restores the exact configuration (the rotation is regenerated from
+ * the persisted seed), so the codes already on disk remain valid.
+ *
+ * External ids are integers. The original float vectors are never retained --
+ * only the sign codes -- so a 256-d vector costs 32 bytes instead of 1024. `get`
+ * returns the stored code, which can be fed straight back into
+ * `insert`/`update`/`query`.
+ */
+export declare class VectorDatabase {
+    readonly dim: number;
+    private readonly store;
+    private readonly quantizer;
+    private readonly index;
+    private readonly codeWords;
+    constructor(options: DatabaseOptions);
+    /** Number of live (non-deleted) vectors. */
+    get size(): number;
+    /**
+     * Physical node count including tombstones from deletes/updates. When it grows
+     * well beyond `size`, call `compact()` to reclaim the space on disk.
+     */
+    get physicalSize(): number;
+    get efSearch(): number;
+    set efSearch(value: number);
+    /** Distance computations performed during the most recent query. */
+    get lastQueryDistanceComputations(): number;
+    /**
+     * Storage row reads issued by the most recent query. This is the honest,
+     * cache-independent scalability metric: it counts every node/neighbour fetch
+     * that hit the database, so disabling the cache cannot hide a bad access pattern.
+     */
+    get lastQueryStorageReads(): number;
+    /** Per-vector storage cost of the index versus a Float32 baseline. */
+    get storage(): StorageStats;
+    has(id: ExternalId): boolean;
+    /** Stream every live external id (bounded memory). */
+    keys(): IterableIterator<ExternalId>;
+    /**
+     * Stream live entries whose INTERNAL id is > `after`, as
+     * {ext, internal} pairs in internal-id order.  Internal ids are assigned
+     * monotonically at insert and preserved by {@link compact}, so the largest
+     * internal id a caller has seen is a durable incremental watermark: a later
+     * call with it yields exactly the entries added since.
+     */
+    keysSince(after: number): IterableIterator<{
+        ext: ExternalId;
+        internal: number;
+    }>;
+    private checkId;
+    /**
+     * Convert a value to code bytes, selecting by length:
+     *  - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
+     *  - otherwise            -> a raw `dim`-vector, encoded first
+     * `codeWords < dim` for any dim >= 2, so the two never collide.
+     */
+    private toCodeBytes;
+    /**
+     * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
+     * detected by length. Throws if the id already exists (use `update`/`upsert`).
+     */
+    insert(id: ExternalId, value: ArrayLike<number>): void;
+    /** Insert with the caller owning the transaction (used by {@link upsertMany}).
+     *  `ef` optionally narrows the construction beam for this one vector (a
+     *  caller-declared cheap entry, e.g. a reach-only interior); omitted means
+     *  the index's configured efConstruction. */
+    private insertCore;
+    upsert(id: ExternalId, value: ArrayLike<number>): void;
+    /**
+     * Upsert many vectors under ONE transaction. The HNSW build touches the store
+     * on every wired edge, so a transaction per vector is one WAL commit per vector
+     * — which dominates a bulk load on disk. Wrapping the whole batch in a single
+     * transaction coalesces those commits into one while leaving the graph and its
+     * result identical (reads on the connection still see the uncommitted rows).
+     *
+     * This is purely about commit batching; it changes nothing about the storage
+     * model. Codes still live only in SQLite and are read on demand — the
+     * cache-independent per-operation read count is unchanged — so a batched load
+     * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
+     * rolls the whole batch back, so the caller treats it as the per-item path on
+     * failure.
+     */
+    upsertMany(entries: Array<{
+        id: ExternalId;
+        vector: ArrayLike<number>;
+        ef?: number;
+    }>): void;
+    /**
+     * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
+     * original vector is not retained; the code round-trips into insert/update/query.
+     */
+    get(id: ExternalId): Uint32Array | null;
+    /**
+     * Update the vector bound to an id (raw vector or code, detected by length).
+     * The previous node is tombstoned and a fresh one inserted. Throws if absent.
+     */
+    update(id: ExternalId, value: ArrayLike<number>): void;
+    /** Update with the caller owning the transaction (used by {@link upsertMany}). */
+    private updateCore;
+    /** Update when the live internal node id is already known.
+     *
+     *  Skips the write entirely when the new code equals the stored one — the
+     *  update would tombstone the node and replay a full graph insert for a
+     *  byte-identical result. This is not a corner case: content-addressed
+     *  callers re-upsert unchanged vectors wholesale after a restart (the same
+     *  content always encodes to the same code), and each such no-op otherwise
+     *  costs a tombstone (permanent routing/disk overhead until compaction)
+     *  plus an O(ef·log N) reinsert. */
+    private updateAt;
+    /**
+     * Delete many ids under ONE transaction — same commit-coalescing rationale
+     * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
+     * commit per id, which dominates a bulk prune. Absent ids are skipped.
+     * Returns the number of vectors actually removed.
+     */
+    deleteMany(ids: ExternalId[]): number;
+    /** Delete the vector bound to an id. Returns false if absent. */
+    delete(id: ExternalId): boolean;
+    /**
+     * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
+     * scans, up to their budget-derived caps.  Optional and purely a latency
+     * optimisation: a cold session otherwise pays the same warming through
+     * random point reads over its first minutes.  Call once after open on a
+     * session that will do sustained inserts/queries.  Returns rows warmed;
+     * a 0-budget database returns 0 immediately.
+     */
+    warmCache(): number;
+    /**
+     * Rebuild the graph from the live codes only, dropping tombstones and returning
+     * the freed pages to the filesystem. Lossless -- equivalent to the original
+     * build -- and needs no original vectors.
+     */
+    compact(): void;
+    /**
+     * k-NN search. The argument's length selects the mode:
+     *  - `dim` elements       -> a raw vector (accurate 4-bit-query estimator)
+     *  - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
+     * Detection is by length, so a code is recognised whether it is a Uint32Array
+     * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
+     */
+    query(query: ArrayLike<number>, k?: number, opts?: {
+        ef?: number;
+    }): QueryResult[];
+    /** Close the underlying database. The instance must not be used afterwards. */
+    close(): void;
+}