@hviana/sema 0.1.5 → 0.1.7

package/src/mind/traverse.ts

@@ -115,6 +115,30 @@ export function edgeAncestors(
   const hit = memo?.get(id);
   if (hit !== undefined) return hit;
 
+  // BYTE-ATOM COMMONALITY.  A single-byte leaf (implicit negative id) has no
+  // structural parents BY CONSTRUCTION — atoms are never linked into the kid
+  // or contain tables — so this climb cannot observe its containment at all.
+  // The walk below would see only the atom's own direct edges and report
+  // contextsReached ≈ 1, turning the MOST common content in the store into
+  // the MOST discriminative voter (observed on a 325K-context store: every
+  // recognised single-letter site voted full ln N for the one fact whose
+  // continuation is that letter, and their pooled sum out-voted every
+  // genuine anchor).  An unmeasurable containment must not default to
+  // "maximally rare": it is bounded below by the uniform expectation over
+  // the byte alphabet — N contexts, each at least one chunk of up to W of
+  // the 256 possible atoms, reach ≥ N·W/256 contexts per atom on average
+  // (see {@link atomReach}).  When that floor itself exceeds the hub bound
+  // √N the atom is a hub at this corpus scale and the climb abstains
+  // (saturated) — the atom's own edges remain fully traversable (tier-0
+  // exact recall, chooseNext, project); only its say as a consensus voter
+  // is withdrawn.  On a small store the floor stays ≤ √N and the atom
+  // climbs exactly as before, so single-letter facts keep working.
+  if (id < 0 && atomIsHub(ctx, contextCount)) {
+    const reach = { roots: [], contextsReached: 0, saturated: true };
+    memo?.set(id, reach);
+    return reach;
+  }
+
   const bound = Math.ceil(Math.sqrt(contextCount));
   const roots: number[] = [];
   const seen = new Set<number>([id]);
@@ -257,6 +281,29 @@ export function prevOf(ctx: MindContext, id: number): number[] {
   return ctx.store.prev(id);
 }
 
+/** The uniform-expectation floor on a byte atom's corpus commonality: N
+ *  learnt contexts, each at least one perception chunk of up to W of the 256
+ *  possible byte values, contain a given atom in ≥ N·W/256 contexts on
+ *  average.  An atom's TRUE containment is unmeasurable (atoms carry no
+ *  kid/contain links by construction), so this floor is the honest stand-in:
+ *  derived entirely from the corpus scale N, the perception window W, and
+ *  the alphabet size — never tuned. */
+export function atomReach(ctx: MindContext, contextCount: number): number {
+  return Math.max(
+    1,
+    Math.ceil((contextCount * ctx.space.maxGroup) / 256),
+  );
+}
+
+/** Whether a byte atom is a hub at this corpus scale — its commonality floor
+ *  {@link atomReach} exceeds the hub bound √N.  Below it (small stores) an
+ *  atom votes and is recognised exactly as any stored form; above it the
+ *  alphabet is scaffolding everywhere and abstains. */
+export function atomIsHub(ctx: MindContext, contextCount: number): boolean {
+  return atomReach(ctx, contextCount) >
+    Math.ceil(Math.sqrt(Math.max(2, contextCount)));
+}
+
 /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
  *  The admission predicate recognition filters sites with (HOW_IT_WORKS
  *  §15.3): a form that leads nowhere contributes nothing to any derivation.

package/src/mind/types.ts

@@ -144,28 +144,27 @@ export interface MindContext extends GraphSearchHost {
   cfg: MindConfig;
   search: GraphSearch;
   trace: Rationale | null;
-  climbMemo: WeakMap<Uint8Array, Map<string, AttentionRead>> | null;
-  /** Per-response memo of {@link recognise} keyed by the byte-array OBJECT
-   *  (think, articulate, and the post-grounding pre-consume all recognise the
-   *  same query/answer objects).  Valid because the store is read-only while
-   *  a response is in flight; bypassed when a trace is attached so every
-   *  recognise still emits its rationale step.  Null outside respond(). */
-  recogniseMemo: WeakMap<Uint8Array, Recognition> | null;
-  /** Per-response memo of {@link perceive} keyed by the byte-array OBJECT —
-   *  the GENERAL memo the result-level ones (recogniseMemo, climbMemo,
-   *  _gistCache) each partially compensate for: resolve(), gistOf(), and
-   *  every mechanism's re-perception of the same query/answer object hit it
-   *  (a reason hop used to fold the same answer three times).  Valid because
-   *  the store is read-only while a response is in flight and perception is
-   *  a pure function of bytes; only inference-shaped calls (plain Uint8Array,
-   *  no leafAt/lookup capabilities) are memoised, so the deposit path never
-   *  sees it.  Keyed by CONTENT (latin1 of the bytes), not object identity —
-   *  mechanisms materialise the same span in fresh subarrays constantly
-   *  (measured on a trained store: 46% of one response's perceptions were
-   *  byte-identical repeats an identity key missed).  NOT bypassed under
-   *  trace — perception emits no rationale steps, so there is nothing a memo
-   *  hit could swallow.  Null outside respond(). */
+  /** Memo of the consensus climb — content-keyed (latin1) so results
+   *  persist across conversation turns where the same byte spans recur.
+   *  Null outside respond(); during respondTurn() the conversation's
+   *  persistent map is swapped in. */
+  climbMemo: Map<string, Map<string, AttentionRead>> | null;
+  /** Memo of {@link recognise} — content-keyed (latin1) so recognised
+   *  forms carry forward across conversation turns.  Bypassed while a
+   *  trace is attached.  Null outside respond(). */
+  recogniseMemo: Map<string, Recognition> | null;
+  /** Memo of {@link perceive} — content-keyed (latin1).  The general
+   *  cache the result-level memos each partially compensate for.  NOT
+   *  bypassed under trace — perception emits no rationale steps.
+   *  Null outside respond(). */
   perceiveMemo: Map<string, Sema> | null;
+  /** Subtree-resolution cache: Sema node → its store id and byte length.
+   *  Populated by {@link foldTree} during inference; checked before
+   *  walking children.  When a conversation's pyramid reuses prefix
+   *  subtrees, this cache lets {@link recognise} skip them entirely —
+   *  O(suffix) instead of O(context).  Mind-lifetime (WeakMap keys are
+   *  the Sema objects the pyramid keeps alive). */
+  _resolvedSubtrees: WeakMap<Sema, { id: number; len: number }> | null;
   _edgeGuide: Vec | null;
   _edgeChoice: Map<number, number>;
   _prevSeen: Set<number> | null;

package/src/rabitq-ivf/README.md

@@ -0,0 +1,56 @@
+# rabitq-ivf
+
+A small TypeScript library for **approximate nearest-neighbour search** using an
+**adaptive partitioned (IVF) index** over **1-bit RaBitQ codes**, with
+**cosine** distance. The entire index lives in **SQLite** — the durable copy of
+every code is on disk, resident memory is bounded (the pivot table plus a capped
+page cache, never the whole collection), and the index survives process
+restarts.
+
+The vectors are stored **only** as 1-bit RaBitQ codes — the original vectors are
+never kept — so the index is both fast _and_ tiny on disk:
+
+```
+a 1024-d vector:
+  Float32             : 1024 × 4 bytes = 4096 bytes
+  RaBitQ 1-bit (code) : 1024 × 1 bit   = 128 bytes    (32× smaller)
+```
+
+The code _is_ the entire per-vector representation. `get` returns the bare code,
+and `insert`, `update`, and `query` all accept either a raw vector or a code —
+detected by length.
+
+## The index
+
+- **Clusters, not a graph.** The collection is partitioned into clusters, each
+  with a binary pivot code and its member codes packed in fixed-size chunk
+  blobs.
+- **Insert = route + append.** Find the nearest pivot (one linear Hamming scan
+  of the RAM-resident pivot table) and append to that cluster's tail chunk. No
+  beam search, no neighbour rewiring — per-insert cost is essentially flat in
+  collection size.
+- **Query = probe + scan.** Rank all pivots with the accurate RaBitQ estimator,
+  scan the `ceil(efSearch/4)` nearest clusters with the same estimator, keep the
+  top k. Per-query storage reads are bounded by nprobe × chunks-per-cluster —
+  constant once the collection has split.
+- **Adaptive, deterministic splits.** A cluster reaching 4096 entries is
+  median-split on the margin between two farthest-point seeds (two exact halves,
+  cascade-proof), and both halves get fresh majority-bit pivots. There is no
+  RNG: the index is a pure function of the insertion sequence.
+- **Same durability discipline as the rest of Sema**: WAL, batched caller-owned
+  transactions (`upsertMany`), 1 KiB pages, 64 MiB WAL autocheckpoint.
+
+## Usage
+
+```ts
+import { VectorDatabase } from "./src/index.js";
+
+const db = new VectorDatabase({ dbPath: "vectors.db", dim: 1024 });
+db.upsertMany([{ id: 1, vector: v1 }, { id: 2, vector: v2 }]);
+const hits = db.query(q, 10); // [{ id, distance }]
+db.close();
+```
+
+`lastQueryStorageReads` reports the cache-independent storage-read count of the
+most recent query — the honest scalability witness the test suite asserts on
+(`test/35-ivf.test.mjs` at the repo root).

package/src/rabitq-ivf/src/database.ts

@@ -0,0 +1,276 @@
+import { IvfHit, IvfIndex } from "./ivf.js";
+
+/** External ids are integers only. */
+export type ExternalId = number;
+
+export interface DatabaseOptions {
+  /** Path to the SQLite database file (use ":memory:" for a transient store). */
+  dbPath: string;
+  /** Vector dimensionality. Required for a new database; ignored when reopening. */
+  dim?: number;
+  /** Query breadth: clusters probed per query = ceil(efSearch / 4).  Default
+   *  64 (16 probes). Tunable at runtime and at reopen. */
+  efSearch?: number;
+  /** Bits used to scalar-quantise the query for the RaBitQ estimator.
+   *  Query-side only (stored codes are independent of it), so it is tunable
+   *  at reopen, like efSearch.  Default 8. */
+  queryBits?: number;
+  /** Number of sign-flip + Hadamard rounds in the random rotation. Default 3. */
+  rotationRounds?: number;
+  /** Seed for the rotation. Default fixed. */
+  seed?: number;
+  /** Optional centroid the vectors are centered by before quantisation. */
+  centroid?: ArrayLike<number>;
+  /** SQLite page-cache budget in MiB.  Purely a latency knob: correctness and
+   *  the per-operation storage-read count are identical with it at 0. */
+  cacheSizeMb?: number;
+}
+
+export interface QueryResult {
+  id: ExternalId;
+  /** Estimated cosine distance (1 - cosine). */
+  distance: number;
+}
+
+export interface StorageStats {
+  float32BytesPerVector: number;
+  codeBytesPerVector: number;
+  bytesPerVector: number;
+  compressionRatio: number;
+}
+
+/**
+ * A persistent vector database: an adaptive PARTITIONED (IVF) index over
+ * 1-bit RaBitQ codes (cosine), stored entirely in SQLite at `dbPath`.  See
+ * ivf.ts for the index design.  The original float vectors are never
+ * retained — only the sign codes — so a 1024-d vector costs 128 bytes.
+ *
+ * Inserting is route-and-append: cost is essentially FLAT in collection size
+ * (one RAM scan of the pivot table + one chunk append — no per-insert graph
+ * walk).  Query cost is bounded by nprobe × cluster size — constant in N
+ * once the collection is past its first splits.
+ */
+export class VectorDatabase {
+  private readonly index: IvfIndex;
+
+  constructor(options: DatabaseOptions) {
+    if (
+      !options || typeof options.dbPath !== "string" ||
+      options.dbPath.length === 0
+    ) {
+      throw new Error("DatabaseOptions.dbPath (string) is required");
+    }
+    this.index = new IvfIndex(options.dbPath, options);
+  }
+
+  get dim(): number {
+    return this.index.dim;
+  }
+
+  /** Number of live (non-deleted) vectors. */
+  get size(): number {
+    return this.index.size;
+  }
+
+  /** Physical slot count including tombstones from deletes/updates. */
+  get physicalSize(): number {
+    return this.index.physicalSize;
+  }
+
+  /** Number of clusters (partitions) currently in the index. */
+  get clusterCount(): number {
+    return this.index.clusterCount;
+  }
+
+  get efSearch(): number {
+    return this.index.efSearch;
+  }
+  set efSearch(value: number) {
+    this.index.efSearch = value;
+  }
+
+  /** Distance computations performed during the most recent query. */
+  get lastQueryDistanceComputations(): number {
+    return this.index.lastQueryDistComps;
+  }
+
+  /** Storage row reads issued by the most recent query — the honest,
+   *  cache-independent scalability metric. */
+  get lastQueryStorageReads(): number {
+    return this.index.lastQueryStorageReads;
+  }
+
+  get storage(): StorageStats {
+    const f32 = this.dim * 4;
+    const bpv = this.index.bytesPerVector;
+    return {
+      float32BytesPerVector: f32,
+      codeBytesPerVector: bpv,
+      bytesPerVector: bpv,
+      compressionRatio: f32 / bpv,
+    };
+  }
+
+  private checkId(id: ExternalId): number {
+    // 32-bit signed range: chunk blobs store exts as int32 — a wider id
+    // would silently truncate, so it is rejected at the door instead.
+    if (!Number.isInteger(id) || id > 0x7fffffff || id < -0x80000000) {
+      throw new Error(`External id must be a 32-bit integer, got ${id}`);
+    }
+    return id;
+  }
+
+  /** Convert a value to code bytes, selecting by length: `codeWords`
+   *  elements → an existing 1-bit code; otherwise a raw `dim`-vector. */
+  private toCodeBytes(value: ArrayLike<number>): Uint8Array {
+    if (value.length === this.index.codeWords) {
+      return this.index.codeToBytes(value);
+    }
+    if (value.length !== this.dim) {
+      throw new Error(
+        `Vector dimension mismatch: expected ${this.dim}, got ${value.length}`,
+      );
+    }
+    return this.index.encodeToBytes(value);
+  }
+
+  has(id: ExternalId): boolean {
+    return this.index.has(this.checkId(id));
+  }
+
+  /** Stream every live external id (bounded memory). */
+  *keys(): IterableIterator<ExternalId> {
+    yield* this.index.keys();
+  }
+
+  /** Stream live entries whose INTERNAL id is > `after` — a durable
+   *  incremental watermark (internal ids are monotone at insert and preserved
+   *  by update and compact). */
+  *keysSince(
+    after: number,
+  ): IterableIterator<{ ext: ExternalId; internal: number }> {
+    yield* this.index.keysSince(after);
+  }
+
+  /** Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. */
+  get(id: ExternalId): Uint32Array | null {
+    const bytes = this.index.codeOf(this.checkId(id));
+    return bytes ? this.index.bytesToCode(bytes) : null;
+  }
+
+  insert(id: ExternalId, value: ArrayLike<number>): void {
+    this.index.begin();
+    try {
+      this.index.insert(this.checkId(id), this.toCodeBytes(value));
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  update(id: ExternalId, value: ArrayLike<number>): void {
+    this.index.begin();
+    try {
+      this.index.update(this.checkId(id), this.toCodeBytes(value));
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  upsert(id: ExternalId, value: ArrayLike<number>): void {
+    this.upsertMany([{ id, vector: value }]);
+  }
+
+  /**
+   * Upsert many vectors under ONE transaction — one WAL commit for the whole
+   * batch instead of one per vector, with chunk appends to the same cluster
+   * coalesced in the index's write-back buffer.
+   */
+  upsertMany(
+    entries: Array<{ id: ExternalId; vector: ArrayLike<number> }>,
+  ): void {
+    if (entries.length === 0) return;
+    this.index.begin();
+    try {
+      for (const e of entries) {
+        // One shared vmap probe decides insert vs update — the per-row point
+        // query is the dominant bulk-load cost, so it is never duplicated.
+        this.index.upsert(this.checkId(e.id), this.toCodeBytes(e.vector));
+      }
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+  }
+
+  /** Delete many ids under ONE transaction. Absent ids are skipped.
+   *  Returns the number of vectors actually removed. */
+  deleteMany(ids: ExternalId[]): number {
+    if (ids.length === 0) return 0;
+    let removed = 0;
+    this.index.begin();
+    try {
+      for (const id of ids) {
+        if (this.index.remove(this.checkId(id))) removed++;
+      }
+      this.index.commit();
+    } catch (e) {
+      this.index.rollback();
+      throw e;
+    }
+    return removed;
+  }
+
+  /** Delete the vector bound to an id. Returns false if absent. */
+  delete(id: ExternalId): boolean {
+    return this.deleteMany([id]) === 1;
+  }
+
+  /** Heat the SQLite page cache with sequential scans (latency only). */
+  warmCache(): number {
+    return this.index.warmCache();
+  }
+
+  /** Reclaim tombstoned slots by rewriting the clusters that carry any, then
+   *  VACUUM.  Internal ids and cluster assignment are preserved. */
+  compact(): void {
+    this.index.compact();
+  }
+
+  /**
+   * k-NN search. The argument's length selects the mode:
+   *  - `dim` elements       → a raw vector (accurate estimator)
+   *  - `codeWords` elements → a 1-bit code, by sign-bit Hamming
+   */
+  query(
+    query: ArrayLike<number>,
+    k = 10,
+    opts?: { ef?: number },
+  ): QueryResult[] {
+    let hits: IvfHit[];
+    if (query.length === this.index.codeWords) {
+      hits = this.index.searchKnnByCode(
+        this.index.codeToBytes(query),
+        k,
+        opts?.ef,
+      );
+    } else {
+      if (query.length !== this.dim) {
+        throw new Error(
+          `Vector dimension mismatch: expected ${this.dim}, got ${query.length}`,
+        );
+      }
+      hits = this.index.searchKnn(query, k, opts?.ef);
+    }
+    return hits;
+  }
+
+  /** Close the underlying database. The instance must not be used afterwards. */
+  close(): void {
+    this.index.close();
+  }
+}

package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts

@@ -5,11 +5,8 @@ export type {
   QueryResult,
   StorageStats,
 } from "./database.js";
-export { HnswIndex } from "./hnsw.js";
-export type { HnswParams, KnnHit } from "./hnsw.js";
+export { IvfIndex } from "./ivf.js";
+export type { IvfConfig, IvfHit } from "./ivf.js";
 export { RaBitQuantizer } from "./rabitq.js";
 export type { QueryContext, RaBitQOptions } from "./rabitq.js";
-export { Store } from "./store.js";
-export type { GlobalState, NodeRec, StoreConfig } from "./store.js";
-export { Heap } from "./heap.js";
 export { Prng } from "./prng.js";