@hviana/sema 0.1.5 → 0.1.7

package/dist/src/mind/types.d.ts

@@ -11,156 +11,158 @@ export type Input = string | Uint8Array | Grid | Grid[];
 /** The host capabilities GraphSearch consults during a cover.  MindContext
  *  extends this so the Mind can pass itself as the host. */
 export interface GraphSearchHost {
-  resolve(bytes: Uint8Array): number | null;
-  recogniseSpan?(bytes: Uint8Array): {
-    sites: ReadonlyArray<Site>;
-    leaves: ReadonlyArray<Leaf>;
-    splits: ReadonlySet<number>;
-  };
-  chooseNext?(node: number): number | undefined;
+    resolve(bytes: Uint8Array): number | null;
+    recogniseSpan?(bytes: Uint8Array): {
+        sites: ReadonlyArray<Site>;
+        leaves: ReadonlyArray<Leaf>;
+        splits: ReadonlySet<number>;
+    };
+    chooseNext?(node: number): number | undefined;
 }
 export interface Recognition {
-  /** Forms that can lead somewhere — they have an edge or a halo. */
-  sites: Site[];
-  /** The query's perceived leaves (the search's covering axioms). */
-  leaves: Leaf[];
-  /** Sub-leaf positions where a form boundary falls between leaf edges. */
-  splits: Set<number>;
+    /** Forms that can lead somewhere — they have an edge or a halo. */
+    sites: Site[];
+    /** The query's perceived leaves (the search's covering axioms). */
+    leaves: Leaf[];
+    /** Sub-leaf positions where a form boundary falls between leaf edges. */
+    splits: Set<number>;
 }
 /** How the consensus climb weights a region's Document-Frequency reach. */
 export type DFMode = "inverse" | "direct" | "combined";
 /** One POINT OF ATTENTION the consensus climb resolved. */
 export interface Attention {
-  /** The learnt context this point resolves to. */
-  anchor: number;
-  /** IDF-weighted consensus vote — the strength that orders points. */
-  vote: number;
-  /** The union of the query byte-spans whose evidence supports this point. */
-  start: number;
-  end: number;
+    /** The learnt context this point resolves to. */
+    anchor: number;
+    /** IDF-weighted consensus vote — the strength that orders points. */
+    vote: number;
+    /** The union of the query byte-spans whose evidence supports this point. */
+    start: number;
+    end: number;
 }
 /** Both read-outs of one consensus climb. */
 export interface AttentionRead {
-  roots: Attention[];
-  ranked: Attention[];
+    roots: Attention[];
+    ranked: Attention[];
 }
 /** A positioned region of a byte stream paired with its gist. */
 export interface Segment {
-  start: number;
-  end: number;
-  v: Vec;
+    start: number;
+    end: number;
+    v: Vec;
 }
 /** A region of the query's perceived tree for the consensus climb. */
 export interface Region {
-  v: Vec;
-  start: number;
-  end: number;
-  chunk: boolean;
-  /** Whether the region's bytes resolve to a KNOWN node (content-addressed,
-   *  exact).  Exact regions vote with full weight; approximate ones pay the
-   *  contrastive margin (see voteRegions) — under the linear fold a raw
-   *  resonance score is byte-overlap, evidence only in excess of its best
-   *  rival conclusion. */
-  known: boolean;
+    v: Vec;
+    start: number;
+    end: number;
+    chunk: boolean;
+    /** Whether the region's bytes resolve to a KNOWN node (content-addressed,
+     *  exact).  Exact regions vote with full weight; approximate ones pay the
+     *  contrastive margin (see voteRegions) — under the linear fold a raw
+     *  resonance score is byte-overlap, evidence only in excess of its best
+     *  rival conclusion. */
+    known: boolean;
 }
 /** Per-region vote data from the consensus climb's resonance pass. */
 export interface RegionVote {
-  start: number;
-  end: number;
-  canonicalFailed: boolean;
-  roots: readonly number[];
-  w: number;
-  wFocus: number;
+    start: number;
+    end: number;
+    canonicalFailed: boolean;
+    roots: readonly number[];
+    w: number;
+    wFocus: number;
 }
 /** The edge-bearing contexts reached by climbing from a node, plus saturation info. */
 export interface AncestorReach {
-  roots: number[];
-  contextsReached: number;
-  saturated: boolean;
+    roots: number[];
+    contextsReached: number;
+    saturated: boolean;
 }
 /** Saturated-interval information for the noise-drop gate. */
 export interface SaturationInfo {
-  leadingEnd: number;
-  hasLeading: boolean;
-  intervals: Array<{
-    start: number;
-    end: number;
-  }>;
+    leadingEnd: number;
+    hasLeading: boolean;
+    intervals: Array<{
+        start: number;
+        end: number;
+    }>;
 }
 /** The items of poolVotes' deduction system. */
 export type AItem = {
-  kind: "region";
-  ri: number;
+    kind: "region";
+    ri: number;
 } | {
-  kind: "anchor";
-  id: number;
+    kind: "anchor";
+    id: number;
 } | {
-  kind: "anchorFocus";
-  id: number;
+    kind: "anchorFocus";
+    id: number;
 };
 export interface MindContext extends GraphSearchHost {
-  store: Store;
-  space: Space;
-  alphabet: Alphabet;
-  cfg: MindConfig;
-  search: GraphSearch;
-  trace: Rationale | null;
-  climbMemo: WeakMap<Uint8Array, Map<string, AttentionRead>> | null;
-  /** Per-response memo of {@link recognise} keyed by the byte-array OBJECT
-   *  (think, articulate, and the post-grounding pre-consume all recognise the
-   *  same query/answer objects).  Valid because the store is read-only while
-   *  a response is in flight; bypassed when a trace is attached so every
-   *  recognise still emits its rationale step.  Null outside respond(). */
-  recogniseMemo: WeakMap<Uint8Array, Recognition> | null;
-  /** Per-response memo of {@link perceive} keyed by the byte-array OBJECT —
-   *  the GENERAL memo the result-level ones (recogniseMemo, climbMemo,
-   *  _gistCache) each partially compensate for: resolve(), gistOf(), and
-   *  every mechanism's re-perception of the same query/answer object hit it
-   *  (a reason hop used to fold the same answer three times).  Valid because
-   *  the store is read-only while a response is in flight and perception is
-   *  a pure function of bytes; only inference-shaped calls (plain Uint8Array,
-   *  no leafAt/lookup capabilities) are memoised, so the deposit path never
-   *  sees it.  Keyed by CONTENT (latin1 of the bytes), not object identity —
-   *  mechanisms materialise the same span in fresh subarrays constantly
-   *  (measured on a trained store: 46% of one response's perceptions were
-   *  byte-identical repeats an identity key missed).  NOT bypassed under
-   *  trace — perception emits no rationale steps, so there is nothing a memo
-   *  hit could swallow.  Null outside respond(). */
-  perceiveMemo: Map<string, Sema> | null;
-  _edgeGuide: Vec | null;
-  _edgeChoice: Map<number, number>;
-  _prevSeen: Set<number> | null;
-  /** Session cache of node-id → perceived gist, for candidate scoring
-   *  ({@link chooseAmong} in the reverse projection's recall path re-gists up to
-   *  √N contexts per pick — the measured bottleneck there).  `chooseNext` does
-   *  NOT use this cache; forward-edge disambiguation uses prevOf counts
-   *  (distributional evidence) instead of gist comparison, because for short
-   *  answer candidates the gist is dominated by accidental byte-pattern
-   *  correlations.  A node's bytes are immutable and perception is a pure
-   *  function of bytes, so an entry stays valid for the store's lifetime —
-   *  never invalidated.  Bounded LRU (byte-sized); a miss only re-perceives,
-   *  never a correctness risk. */
-  _gistCache: BoundedMap<number, Vec>;
-  /** DEPOSIT-path stable-prefix tree cache: content key (latin1) of a
-   *  deposited input → its plain-fold level PYRAMID, so the NEXT deposit
-   *  whose prefix it is (a conversation's accumulated context) reuses every
-   *  full aligned block and refolds only the right edge of each level —
-   *  O(turn) per deposit instead of O(context), with the tree BIT-IDENTICAL
-   *  to a from-scratch fold (see FoldPyramid).  Purely a performance cache.
-   *  Entry-count bounded (a pyramid is ~KB/byte of content; only the few
-   *  live conversation chains matter). */
-  _depositTrees: BoundedMap<string, FoldPyramid>;
-  /** The byte lengths present in {@link _depositTrees} — the candidate
-   *  prefix lengths probed (longest first).  Drifts on eviction (a stale
-   *  length only costs a miss); cleared with the map when it outgrows the
-   *  probe budget. */
-  _depositLens: Set<number>;
-  /** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
-   *  content-addressed id.  Valid forever (ids are permanent, Sema nodes
-   *  immutable); WeakMap, so entries live exactly as long as the pyramid
-   *  cache keeps the shared subtrees alive.  Lets internTreeIds skip whole
-   *  shared subtrees and indexSubSpans keep its seenBefore window skip. */
-  _internIds: WeakMap<Sema, number>;
+    store: Store;
+    space: Space;
+    alphabet: Alphabet;
+    cfg: MindConfig;
+    search: GraphSearch;
+    trace: Rationale | null;
+    /** Memo of the consensus climb — content-keyed (latin1) so results
+     *  persist across conversation turns where the same byte spans recur.
+     *  Null outside respond(); during respondTurn() the conversation's
+     *  persistent map is swapped in. */
+    climbMemo: Map<string, Map<string, AttentionRead>> | null;
+    /** Memo of {@link recognise} — content-keyed (latin1) so recognised
+     *  forms carry forward across conversation turns.  Bypassed while a
+     *  trace is attached.  Null outside respond(). */
+    recogniseMemo: Map<string, Recognition> | null;
+    /** Memo of {@link perceive} — content-keyed (latin1).  The general
+     *  cache the result-level memos each partially compensate for.  NOT
+     *  bypassed under trace — perception emits no rationale steps.
+     *  Null outside respond(). */
+    perceiveMemo: Map<string, Sema> | null;
+    /** Subtree-resolution cache: Sema node → its store id and byte length.
+     *  Populated by {@link foldTree} during inference; checked before
+     *  walking children.  When a conversation's pyramid reuses prefix
+     *  subtrees, this cache lets {@link recognise} skip them entirely —
+     *  O(suffix) instead of O(context).  Mind-lifetime (WeakMap keys are
+     *  the Sema objects the pyramid keeps alive). */
+    _resolvedSubtrees: WeakMap<Sema, {
+        id: number;
+        len: number;
+    }> | null;
+    _edgeGuide: Vec | null;
+    _edgeChoice: Map<number, number>;
+    _prevSeen: Set<number> | null;
+    /** Session cache of node-id → perceived gist, for candidate scoring
+     *  ({@link chooseAmong} in the reverse projection's recall path re-gists up to
+     *  √N contexts per pick — the measured bottleneck there).  `chooseNext` does
+     *  NOT use this cache; forward-edge disambiguation uses prevOf counts
+     *  (distributional evidence) instead of gist comparison, because for short
+     *  answer candidates the gist is dominated by accidental byte-pattern
+     *  correlations.  A node's bytes are immutable and perception is a pure
+     *  function of bytes, so an entry stays valid for the store's lifetime —
+     *  never invalidated.  Bounded LRU (byte-sized); a miss only re-perceives,
+     *  never a correctness risk. */
+    _gistCache: BoundedMap<number, Vec>;
+    /** DEPOSIT-path stable-prefix tree cache: content key (latin1) of a
+     *  deposited input → its plain-fold level PYRAMID, so the NEXT deposit
+     *  whose prefix it is (a conversation's accumulated context) reuses every
+     *  full aligned block and refolds only the right edge of each level —
+     *  O(turn) per deposit instead of O(context), with the tree BIT-IDENTICAL
+     *  to a from-scratch fold (see FoldPyramid).  Purely a performance cache.
+     *  Entry-count bounded (a pyramid is ~KB/byte of content; only the few
+     *  live conversation chains matter). */
+    _depositTrees: BoundedMap<string, FoldPyramid>;
+    /** The byte lengths present in {@link _depositTrees} — the candidate
+     *  prefix lengths probed (longest first).  Drifts on eviction (a stale
+     *  length only costs a miss); cleared with the map when it outgrows the
+     *  probe budget. */
+    _depositLens: Set<number>;
+    /** Mind-lifetime intern memo by NODE IDENTITY: perceived-tree node → its
+     *  content-addressed id.  Valid forever (ids are permanent, Sema nodes
+     *  immutable); WeakMap, so entries live exactly as long as the pyramid
+     *  cache keeps the shared subtrees alive.  Lets internTreeIds skip whole
+     *  shared subtrees and indexSubSpans keep its seenBefore window skip. */
+    _internIds: WeakMap<Sema, number>;
 }
 /** Read a whole node's bytes. */
 export declare const ALL = 2147483647;
@@ -168,14 +170,7 @@ export declare const ALL = 2147483647;
 export declare function spliceAll(segs: Seg[]): Uint8Array | null;
 /** Lift the answer out of the cover for think: the recognised region, free of
  *  the asker's surrounding (unrecognised) framing. */
-export declare function liftAnswer(
-  segs: Seg[],
-  queryLen: number,
-): Uint8Array | null;
+export declare function liftAnswer(segs: Seg[], queryLen: number): Uint8Array | null;
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */
-export declare function changedNodes(
-  tree: Sema,
-  ids: Map<Sema, number>,
-  prevSeen: Set<number>,
-): Sema[];
+export declare function changedNodes(tree: Sema, ids: Map<Sema, number>, prevSeen: Set<number>): Sema[];

package/dist/src/mind/types.js

@@ -11,83 +11,74 @@ import { dominates } from "../geometry.js";
 export const ALL = 0x7fffffff;
 /** Splice every chosen span in order — the whole cover as one byte string. */
 export function spliceAll(segs) {
-  if (!segs.some((s) => s.rec)) {
-    return null;
-  }
-  return concatBytes(segs.map((s) => s.bytes));
+    if (!segs.some((s) => s.rec))
+        return null;
+    return concatBytes(segs.map((s) => s.bytes));
 }
 /** Lift the answer out of the cover for think: the recognised region, free of
  *  the asker's surrounding (unrecognised) framing. */
 export function liftAnswer(segs, queryLen) {
-  const recognised = [];
-  for (let k = 0; k < segs.length; k++) {
-    if (segs[k].rec) {
-      recognised.push(k);
+    const recognised = [];
+    for (let k = 0; k < segs.length; k++)
+        if (segs[k].rec)
+            recognised.push(k);
+    if (recognised.length === 0)
+        return null;
+    if (recognised.length === 1) {
+        const s = segs[recognised[0]];
+        if (dominates(s.j - s.i, queryLen)) {
+            return concatBytes(segs.map((x) => x.bytes));
+        }
+        return s.bytes;
     }
-  }
-  if (recognised.length === 0) {
-    return null;
-  }
-  if (recognised.length === 1) {
-    const s = segs[recognised[0]];
-    if (dominates(s.j - s.i, queryLen)) {
-      return concatBytes(segs.map((x) => x.bytes));
-    }
-    return s.bytes;
-  }
-  const lo = recognised[0];
-  const hi = recognised[recognised.length - 1];
-  return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
+    const lo = recognised[0];
+    const hi = recognised[recognised.length - 1];
+    return concatBytes(segs.slice(lo, hi + 1).map((x) => x.bytes));
 }
 /** The CHANGED NODES of a freshly-perceived `tree` against the node ids a previous
  *  tracked deposit interned (`prevSeen`). */
 export function changedNodes(tree, ids, prevSeen) {
-  const newCount = new Map();
-  const count = (n) => {
-    const memo = newCount.get(n);
-    if (memo !== undefined) {
-      return memo;
-    }
-    const id = ids.get(n);
-    // PRUNE: a node whose id the previous deposit already interned is old,
-    // and content addressing makes that transitive — the same id names the
-    // same content, so every descendant was interned then too.  The whole
-    // subtree counts 0 without walking it; with the pyramid fold sharing a
-    // conversation's prefix subtree, this is what keeps the changed-nodes
-    // read O(new nodes) instead of O(context).  (A node internTreeIds
-    // memo-skipped has an id here exactly when it is such a shared root.)
-    if (id !== undefined && prevSeen.has(id)) {
-      newCount.set(n, 0);
-      return 0;
-    }
-    let c = 1; // reachable only when NOT pruned above ⇒ this node is new
-    if (n.kids) {
-      for (const k of n.kids) {
-        c += count(k);
-      }
-    }
-    newCount.set(n, c);
-    return c;
-  };
-  const total = count(tree);
-  if (total === 0) {
-    return [tree];
-  }
-  let n = tree;
-  for (;;) {
-    if (n.kids === null) {
-      return [n];
-    }
-    let holder = null;
-    for (const k of n.kids) {
-      if (newCount.get(k) === total) {
-        holder = k;
-        break;
-      }
-    }
-    if (holder === null) {
-      return [n];
+    const newCount = new Map();
+    const count = (n) => {
+        const memo = newCount.get(n);
+        if (memo !== undefined)
+            return memo;
+        const id = ids.get(n);
+        // PRUNE: a node whose id the previous deposit already interned is old,
+        // and content addressing makes that transitive — the same id names the
+        // same content, so every descendant was interned then too.  The whole
+        // subtree counts 0 without walking it; with the pyramid fold sharing a
+        // conversation's prefix subtree, this is what keeps the changed-nodes
+        // read O(new nodes) instead of O(context).  (A node internTreeIds
+        // memo-skipped has an id here exactly when it is such a shared root.)
+        if (id !== undefined && prevSeen.has(id)) {
+            newCount.set(n, 0);
+            return 0;
+        }
+        let c = 1; // reachable only when NOT pruned above ⇒ this node is new
+        if (n.kids) {
+            for (const k of n.kids)
+                c += count(k);
+        }
+        newCount.set(n, c);
+        return c;
+    };
+    const total = count(tree);
+    if (total === 0)
+        return [tree];
+    let n = tree;
+    for (;;) {
+        if (n.kids === null)
+            return [n];
+        let holder = null;
+        for (const k of n.kids) {
+            if (newCount.get(k) === total) {
+                holder = k;
+                break;
+            }
+        }
+        if (holder === null)
+            return [n];
+        n = holder;
     }
-    n = holder;
-  }
 }

package/dist/src/rabitq-ivf/src/database.d.ts

@@ -0,0 +1,113 @@
+/** External ids are integers only. */
+export type ExternalId = number;
+export interface DatabaseOptions {
+    /** Path to the SQLite database file (use ":memory:" for a transient store). */
+    dbPath: string;
+    /** Vector dimensionality. Required for a new database; ignored when reopening. */
+    dim?: number;
+    /** Query breadth: clusters probed per query = ceil(efSearch / 4).  Default
+     *  64 (16 probes). Tunable at runtime and at reopen. */
+    efSearch?: number;
+    /** Bits used to scalar-quantise the query for the RaBitQ estimator.
+     *  Query-side only (stored codes are independent of it), so it is tunable
+     *  at reopen, like efSearch.  Default 8. */
+    queryBits?: number;
+    /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
+    rotationRounds?: number;
+    /** Seed for the rotation. Default fixed. */
+    seed?: number;
+    /** Optional centroid the vectors are centered by before quantisation. */
+    centroid?: ArrayLike<number>;
+    /** SQLite page-cache budget in MiB.  Purely a latency knob: correctness and
+     *  the per-operation storage-read count are identical with it at 0. */
+    cacheSizeMb?: number;
+}
+export interface QueryResult {
+    id: ExternalId;
+    /** Estimated cosine distance (1 - cosine). */
+    distance: number;
+}
+export interface StorageStats {
+    float32BytesPerVector: number;
+    codeBytesPerVector: number;
+    bytesPerVector: number;
+    compressionRatio: number;
+}
+/**
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`.  See
+ * ivf.ts for the index design.  The original float vectors are never
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
+ *
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
+ * walk).  Query cost is bounded by nprobe × cluster size — constant in N
+ * once the collection is past its first splits.
+ */
+export declare class VectorDatabase {
+    private readonly index;
+    constructor(options: DatabaseOptions);
+    get dim(): number;
+    /** Number of live (non-deleted) vectors. */
+    get size(): number;
+    /** Physical slot count including tombstones from deletes/updates. */
+    get physicalSize(): number;
+    /** Number of clusters (partitions) currently in the index. */
+    get clusterCount(): number;
+    get efSearch(): number;
+    set efSearch(value: number);
+    /** Distance computations performed during the most recent query. */
+    get lastQueryDistanceComputations(): number;
+    /** Storage row reads issued by the most recent query — the honest,
+     *  cache-independent scalability metric. */
+    get lastQueryStorageReads(): number;
+    get storage(): StorageStats;
+    private checkId;
+    /** Convert a value to code bytes, selecting by length: `codeWords`
+     *  elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
+    private toCodeBytes;
+    has(id: ExternalId): boolean;
+    /** Stream every live external id (bounded memory). */
+    keys(): IterableIterator<ExternalId>;
+    /** Stream live entries whose INTERNAL id is > `after` — a durable
+     *  incremental watermark (internal ids are monotone at insert and preserved
+     *  by update and compact). */
+    keysSince(after: number): IterableIterator<{
+        ext: ExternalId;
+        internal: number;
+    }>;
+    /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
+    get(id: ExternalId): Uint32Array | null;
+    insert(id: ExternalId, value: ArrayLike<number>): void;
+    update(id: ExternalId, value: ArrayLike<number>): void;
+    upsert(id: ExternalId, value: ArrayLike<number>): void;
+    /**
+     * Upsert many vectors under ONE transaction — one WAL commit for the whole
+     * batch instead of one per vector, with chunk appends to the same cluster
+     * coalesced in the index's write-back buffer.
+     */
+    upsertMany(entries: Array<{
+        id: ExternalId;
+        vector: ArrayLike<number>;
+    }>): void;
+    /** Delete many ids under ONE transaction. Absent ids are skipped.
+     *  Returns the number of vectors actually removed. */
+    deleteMany(ids: ExternalId[]): number;
+    /** Delete the vector bound to an id. Returns false if absent. */
+    delete(id: ExternalId): boolean;
+    /** Heat the SQLite page cache with sequential scans (latency only). */
+    warmCache(): number;
+    /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
+     *  VACUUM.  Internal ids and cluster assignment are preserved. */
+    compact(): void;
+    /**
+     * k-NN search. The argument's length selects the mode:
+     *  - `dim` elements       → a raw vector (accurate estimator)
+     *  - `codeWords` elements → a 1-bit code, by sign-bit Hamming
+     */
+    query(query: ArrayLike<number>, k?: number, opts?: {
+        ef?: number;
+    }): QueryResult[];
+    /** Close the underlying database. The instance must not be used afterwards. */
+    close(): void;
+}