@hviana/sema 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1687 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +17 -42
  51. package/dist/src/geometry.js +235 -266
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +749 -871
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +126 -185
  82. package/dist/src/mind/mind.d.ts +129 -154
  83. package/dist/src/mind/mind.js +258 -288
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +8 -32
  89. package/dist/src/mind/primitives.js +99 -117
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +183 -208
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +16 -58
  101. package/dist/src/mind/traverse.js +304 -357
  102. package/dist/src/mind/types.d.ts +120 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +169 -166
  117. package/dist/src/store-sqlite.js +762 -662
  118. package/dist/src/store.d.ts +586 -630
  119. package/dist/src/store.js +1423 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +13 -10
  123. package/package.json +1 -1
  124. package/src/alu/README.md +1 -1
  125. package/src/alu/src/index.ts +1 -1
  126. package/src/config.ts +19 -27
  127. package/src/index.ts +6 -11
  128. package/src/rabitq-ivf/README.md +56 -0
  129. package/src/rabitq-ivf/src/database.ts +276 -0
  130. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  131. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  132. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  133. package/src/store-sqlite.ts +196 -9
  134. package/src/store.ts +8 -32
  135. package/test/08-storage.test.mjs +3 -3
  136. package/test/14-scaling.test.mjs +2 -2
  137. package/test/35-ivf.test.mjs +263 -0
  138. package/test/36-bloom.test.mjs +123 -0
  139. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  140. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  141. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  142. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  143. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  144. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  145. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  146. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  147. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  148. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  149. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  150. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  151. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  152. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  153. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  154. package/src/rabitq-hnsw/README.md +0 -303
  155. package/src/rabitq-hnsw/src/database.ts +0 -492
  156. package/src/rabitq-hnsw/src/heap.ts +0 -90
  157. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  158. package/src/rabitq-hnsw/src/store.ts +0 -994
  159. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  160. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -0,0 +1,1165 @@
1
+ // ivf.ts — an adaptive PARTITIONED (IVF-style) index over 1-bit RaBitQ codes,
2
+ // persisted in SQLite.
3
+ //
4
+ // The collection is a flat set of CLUSTERS. Each cluster has a binary PIVOT
5
+ // code and holds its member codes in fixed-size CHUNK blobs. Inserting is
6
+ // "route + append": Hamming-scan the pivot table (RAM-resident, tiny) for the
7
+ // nearest cluster, then append the code to that cluster's tail chunk — no
8
+ // graph walk, no neighbour rewiring, so insert cost is essentially FLAT in
9
+ // collection size (the pivot scan is K = N/⌀cluster comparisons of a few
10
+ // dozen machine words; at 10M vectors that is ~0.1 ms of pure arithmetic).
11
+ // A graph-based index pays a beam search of dozens of random storage reads
12
+ // per insert here — route + append pays none.
13
+ //
14
+ // Querying ranks every pivot with the accurate RaBitQ estimator (one LUT pass
15
+ // per pivot), then linearly scans the NPROBE nearest clusters' chunks with
16
+ // the same estimator, keeping the top k in a bounded heap. Per-query storage
17
+ // reads are bounded by nprobe × (chunks per cluster) — a CONSTANT once the
18
+ // collection is large enough to split, independent of N.
19
+ //
20
+ // A cluster that reaches SPLIT_MAX entries is SPLIT in place: two seed codes
21
+ // are chosen deterministically (farthest from the old pivot, then farthest
22
+ // from the first), members are MEDIAN-split on the two-seed margin (two exact
23
+ // halves — nearer-seed assignment degenerates on tie-heavy distributions and
24
+ // cascades cluster count), and both halves get fresh majority-bit pivots.
25
+ // No RNG anywhere — the index is a pure function of the insertion sequence.
26
+ //
27
+ // Durability follows the store discipline used across Sema: WAL journal,
28
+ // batched caller-owned transactions (upsertMany), 1 KiB pages, and a 64 MiB
29
+ // autocheckpoint so hot pages coalesce in the WAL instead of being copied out
30
+ // per commit.
31
+ //
32
+ // Layout (all in one SQLite file):
33
+ // meta one row: quantizer geometry + efSearch + counters.
34
+ // cent one row per cluster: pivot code + chunk/entry counts.
35
+ // chunk one row per fixed-capacity chunk of a cluster, keyed by
36
+ // cid·2^16 + seq: a packed blob of [n | dead bitmap | ids | exts |
37
+ // codes]. Fixed-size blobs keep B-tree pages stable under the
38
+ // read-modify-write tail append.
39
+ // vmap one row per live vector: ext → (monotone internal id, cid, seq,
40
+ // slot). The internal id survives compaction, so the largest id a
41
+ // scan has seen is a durable incremental watermark (keysSince).
42
+ import { DatabaseSync } from "node:sqlite";
43
+ import { RaBitQuantizer } from "./rabitq.js";
44
+ /** Entries per chunk blob. 64 keeps the tail read-modify-write ≤ ~9 KiB at
45
+ * D=1024 while a full cluster is only SPLIT_MAX/64 blob reads per probe. */
46
+ const CHUNK_CAP = 64;
47
+ /** Entries (live + dead) at which a cluster splits in two. */
48
+ const SPLIT_MAX = 4096;
49
+ /** chunk rowid key = cid·SEQ_SPAN + seq. Chunks per cluster ≤ SPLIT_MAX /
50
+ * CHUNK_CAP = 64 ≪ 2^16, and cid·2^16 stays exact in a double for every
51
+ * reachable cid. */
52
+ const SEQ_SPAN = 1 << 16;
53
+ // ── chunk blob layout (fixed size) ─────────────────────────────────────────
54
+ // [0] u32 n entries used (live + dead)
55
+ // [4..12) u64 dead bitmap bit s set = slot s tombstoned
56
+ // [12..16) pad
57
+ // [16..) CHUNK_CAP × f64 internal ids
58
+ // [+..) CHUNK_CAP × i32 external ids
59
+ // [+..) CHUNK_CAP × codeBytes codes
60
+ const IDS_OFF = 16;
61
+ const EXTS_OFF = IDS_OFF + CHUNK_CAP * 8;
62
+ const CODES_OFF = EXTS_OFF + CHUNK_CAP * 4;
63
+ function chunkSize(codeBytes) {
64
+ return CODES_OFF + CHUNK_CAP * codeBytes;
65
+ }
66
+ /** Typed-array views over one chunk blob. The blob is allocated aligned
67
+ * (fresh Uint8Array), so the f64/i32 views are always valid. */
68
+ class Chunk {
69
+ buf;
70
+ n32; // [0] = n
71
+ dead; // 8 bytes at offset 4
72
+ ids;
73
+ exts;
74
+ constructor(buf) {
75
+ // A blob straight from SQLite may sit at an arbitrary byteOffset; copy to
76
+ // an aligned buffer only when needed.
77
+ if (buf.byteOffset % 8 !== 0)
78
+ buf = new Uint8Array(buf);
79
+ this.buf = buf;
80
+ this.n32 = new Uint32Array(buf.buffer, buf.byteOffset, 4);
81
+ this.dead = buf.subarray(4, 12);
82
+ this.ids = new Float64Array(buf.buffer, buf.byteOffset + IDS_OFF, CHUNK_CAP);
83
+ this.exts = new Int32Array(buf.buffer, buf.byteOffset + EXTS_OFF, CHUNK_CAP);
84
+ }
85
+ get n() {
86
+ return this.n32[0];
87
+ }
88
+ set n(v) {
89
+ this.n32[0] = v;
90
+ }
91
+ isDead(slot) {
92
+ return (this.dead[slot >> 3] & (1 << (slot & 7))) !== 0;
93
+ }
94
+ markDead(slot) {
95
+ this.dead[slot >> 3] |= 1 << (slot & 7);
96
+ }
97
+ codeAt(slot, codeBytes) {
98
+ return CODES_OFF + slot * codeBytes; // byte offset into buf
99
+ }
100
+ }
101
+ /** Bounded max-heap of (distance, ext) — keeps the k SMALLEST distances. */
102
+ class TopK {
103
+ k;
104
+ ds;
105
+ ids;
106
+ size = 0;
107
+ constructor(k) {
108
+ this.k = k;
109
+ this.ds = new Float64Array(k);
110
+ this.ids = new Float64Array(k);
111
+ }
112
+ get worst() {
113
+ return this.size < this.k ? Infinity : this.ds[0];
114
+ }
115
+ push(d, id) {
116
+ const { ds, ids } = this;
117
+ if (this.size < this.k) {
118
+ let i = this.size++;
119
+ ds[i] = d;
120
+ ids[i] = id;
121
+ for (;;) { // sift up
122
+ if (i === 0)
123
+ break;
124
+ const p = (i - 1) >> 1;
125
+ if (ds[p] >= ds[i])
126
+ break;
127
+ const td = ds[p], ti = ids[p];
128
+ ds[p] = ds[i];
129
+ ids[p] = ids[i];
130
+ ds[i] = td;
131
+ ids[i] = ti;
132
+ i = p;
133
+ }
134
+ return;
135
+ }
136
+ if (d >= ds[0])
137
+ return;
138
+ ds[0] = d;
139
+ ids[0] = id;
140
+ let i = 0;
141
+ for (;;) { // sift down
142
+ const l = 2 * i + 1, r = l + 1;
143
+ let m = i;
144
+ if (l < this.size && ds[l] > ds[m])
145
+ m = l;
146
+ if (r < this.size && ds[r] > ds[m])
147
+ m = r;
148
+ if (m === i)
149
+ break;
150
+ const td = ds[m], ti = ids[m];
151
+ ds[m] = ds[i];
152
+ ids[m] = ids[i];
153
+ ds[i] = td;
154
+ ids[i] = ti;
155
+ i = m;
156
+ }
157
+ }
158
+ drain() {
159
+ const out = new Array(this.size);
160
+ for (let i = 0; i < this.size; i++) {
161
+ out[i] = { id: this.ids[i], distance: this.ds[i] < 0 ? 0 : this.ds[i] };
162
+ }
163
+ out.sort((a, b) => a.distance - b.distance);
164
+ return out;
165
+ }
166
+ }
167
+ export class IvfIndex {
168
+ db;
169
+ quantizer;
170
+ codeBytes;
171
+ chunkBytes;
172
+ _efSearch;
173
+ // ── RAM-resident routing state (O(K); K = clusters ≈ N / 2048) ──────────
174
+ /** Pivot codes, K × codeBytes, contiguous — the insert/query routing scan
175
+ * is one linear pass over this array. */
176
+ centCodes = new Uint8Array(0);
177
+ /** Aligned 32-bit word view of {@link centCodes} — the routing scan's
178
+ * operand (4 bytes per XOR+popcount instead of byte re-composition). */
179
+ centWords = new Uint32Array(0);
180
+ /** Per-cluster entry count (live + dead) — the split trigger. */
181
+ centEntries = new Int32Array(0);
182
+ /** Per-cluster chunk count (tail seq = count − 1). */
183
+ centChunks = new Int32Array(0);
184
+ K = 0;
185
+ // global counters (persisted in meta at commit)
186
+ live = 0;
187
+ totalSlots = 0;
188
+ nextId = 1;
189
+ /** Storage row reads (chunk blob fetches) — the cache-independent
190
+ * scalability witness, same discipline as the graph index it replaces. */
191
+ reads = 0;
192
+ lastQueryDistComps = 0;
193
+ lastQueryStorageReads = 0;
194
+ // Per-transaction write-back buffer of dirty chunks, doubling as a CLEAN
195
+ // read cache when a memory budget was granted. The `reads` counter counts
196
+ // SQLite fall-throughs only, so with `cacheSizeMb: 0` clean retention is
197
+ // DISABLED (dirty chunks must still buffer within a transaction — that is
198
+ // write coalescing, not read caching) and every re-fetch is an honest
199
+ // storage read. Budget-derived cap; flushed entries stay readable from
200
+ // SQLite, so eviction is safe anywhere outside the dirty set.
201
+ chunkCache = new Map();
202
+ dirtyChunks = new Set();
203
+ dirtyCents = new Set();
204
+ metaDirty = false;
205
+ cacheEnabled;
206
+ chunkCacheMax;
207
+ sSelChunk;
208
+ sUpsChunk;
209
+ sDelChunk;
210
+ sSelVmap;
211
+ sInsVmap;
212
+ sUpdVmap;
213
+ sDelVmap;
214
+ sUpsCent;
215
+ sMeta;
216
+ constructor(dbPath, options) {
217
+ this.db = new DatabaseSync(dbPath);
218
+ const kb = Math.max(0, Math.floor((options.cacheSizeMb ?? 64) * 1024));
219
+ this.cacheEnabled = kb > 0;
220
+ // 1 KiB pages (chunk blobs span ~9 pages; cent/vmap rows are small), WAL
221
+ // with a 64 MiB autocheckpoint (see store-sqlite.ts for the rationale —
222
+ // the 1000-page default forced a synchronous checkpoint per batch).
223
+ this.db.exec("PRAGMA page_size = 1024;");
224
+ this.db.exec(`
225
+ PRAGMA journal_mode = WAL;
226
+ PRAGMA synchronous = NORMAL;
227
+ PRAGMA temp_store = MEMORY;
228
+ PRAGMA cache_size = ${kb > 0 ? `-${kb}` : "0"};
229
+ PRAGMA wal_autocheckpoint = 65536;
230
+ PRAGMA foreign_keys = OFF;
231
+ `);
232
+ this.db.exec(`
233
+ CREATE TABLE IF NOT EXISTS meta (
234
+ id INTEGER PRIMARY KEY CHECK (id = 0),
235
+ format TEXT NOT NULL,
236
+ dim INTEGER NOT NULL,
237
+ query_bits INTEGER NOT NULL,
238
+ rotation_rounds INTEGER NOT NULL,
239
+ seed INTEGER NOT NULL,
240
+ centroid BLOB,
241
+ code_words INTEGER NOT NULL,
242
+ padded_dim INTEGER NOT NULL,
243
+ ef_search INTEGER NOT NULL,
244
+ live INTEGER NOT NULL DEFAULT 0,
245
+ total INTEGER NOT NULL DEFAULT 0,
246
+ next_id INTEGER NOT NULL DEFAULT 1
247
+ );
248
+ CREATE TABLE IF NOT EXISTS cent (
249
+ cid INTEGER PRIMARY KEY,
250
+ code BLOB NOT NULL,
251
+ chunks INTEGER NOT NULL,
252
+ entries INTEGER NOT NULL
253
+ );
254
+ CREATE TABLE IF NOT EXISTS chunk (
255
+ key INTEGER PRIMARY KEY,
256
+ blob BLOB NOT NULL
257
+ );
258
+ CREATE TABLE IF NOT EXISTS vmap (
259
+ ext INTEGER PRIMARY KEY,
260
+ id INTEGER NOT NULL,
261
+ cid INTEGER NOT NULL,
262
+ seq INTEGER NOT NULL,
263
+ slot INTEGER NOT NULL
264
+ );
265
+ CREATE INDEX IF NOT EXISTS idx_vmap_id ON vmap(id);
266
+ `);
267
+ // ── config: load stored, or initialise from options ──
268
+ const row = this.db.prepare("SELECT * FROM meta WHERE id = 0").get();
269
+ let cfg;
270
+ if (row) {
271
+ if (row.format !== "ivf1") {
272
+ throw new Error(`vector index at ${dbPath} has format '${String(row.format)}', ` +
273
+ `expected 'ivf1' — delete the .vec files and rebuild ` +
274
+ `(Store.repairContentIndex regenerates the content index)`);
275
+ }
276
+ let centroid = null;
277
+ const cb = row.centroid;
278
+ if (cb) {
279
+ centroid = new Float64Array(row.dim);
280
+ new Uint8Array(centroid.buffer).set(cb);
281
+ }
282
+ cfg = {
283
+ dim: row.dim,
284
+ efSearch: options.efSearch ?? row.ef_search,
285
+ queryBits: options.queryBits ?? row.query_bits,
286
+ rotationRounds: row.rotation_rounds,
287
+ seed: row.seed,
288
+ centroid,
289
+ codeWords: row.code_words,
290
+ paddedDim: row.padded_dim,
291
+ };
292
+ if (options.efSearch !== undefined &&
293
+ options.efSearch !== row.ef_search) {
294
+ this.db.prepare("UPDATE meta SET ef_search = ? WHERE id = 0")
295
+ .run(options.efSearch);
296
+ }
297
+ if (options.queryBits !== undefined &&
298
+ options.queryBits !== row.query_bits) {
299
+ this.db.prepare("UPDATE meta SET query_bits = ? WHERE id = 0")
300
+ .run(options.queryBits);
301
+ }
302
+ this.live = row.live;
303
+ this.totalSlots = row.total;
304
+ this.nextId = row.next_id;
305
+ }
306
+ else {
307
+ if (!Number.isInteger(options.dim) || options.dim <= 0) {
308
+ throw new Error("dim must be a positive integer for a new database");
309
+ }
310
+ const dim = options.dim;
311
+ const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
312
+ cfg = {
313
+ dim,
314
+ efSearch: options.efSearch ?? 64,
315
+ queryBits: options.queryBits ?? 8,
316
+ rotationRounds: options.rotationRounds ?? 3,
317
+ seed: (options.seed ?? 0x1234abcd) >>> 0,
318
+ centroid: options.centroid ? Float64Array.from(options.centroid) : null,
319
+ codeWords: probe.codeWords,
320
+ paddedDim: probe.paddedDim,
321
+ };
322
+ const centroidBlob = cfg.centroid
323
+ ? new Uint8Array(cfg.centroid.buffer, 0, cfg.centroid.byteLength)
324
+ : null;
325
+ this.db.prepare("INSERT INTO meta (id, format, dim, query_bits, rotation_rounds, " +
326
+ "seed, centroid, code_words, padded_dim, ef_search) " +
327
+ "VALUES (0, 'ivf1', ?, ?, ?, ?, ?, ?, ?, ?)").run(cfg.dim, cfg.queryBits, cfg.rotationRounds, cfg.seed, centroidBlob, cfg.codeWords, cfg.paddedDim, cfg.efSearch);
328
+ }
329
+ this.quantizer = new RaBitQuantizer(cfg.dim, {
330
+ queryBits: cfg.queryBits,
331
+ rounds: cfg.rotationRounds,
332
+ seed: cfg.seed,
333
+ centroid: cfg.centroid ?? undefined,
334
+ });
335
+ if (this.quantizer.codeWords !== cfg.codeWords) {
336
+ throw new Error("Stored code geometry does not match the quantizer (corrupt database?)");
337
+ }
338
+ this.codeBytes = cfg.codeWords * 4;
339
+ this.chunkBytes = chunkSize(this.codeBytes);
340
+ // Clean-chunk retention cap, derived from the memory budget (~9 KiB per
341
+ // chunk at D=1024). With the budget at 0 only dirty chunks are held.
342
+ this.chunkCacheMax = this.cacheEnabled
343
+ ? Math.max(64, Math.min(Math.floor((kb * 1024) / this.chunkBytes), 65536))
344
+ : 64;
345
+ this._efSearch = cfg.efSearch;
346
+ this.dim = cfg.dim;
347
+ this.prepareAll();
348
+ this.loadCents();
349
+ }
350
+ dim;
351
+ prepareAll() {
352
+ this.sSelChunk = this.db.prepare("SELECT blob FROM chunk WHERE key = ?");
353
+ this.sSelChunk.setReturnArrays(true);
354
+ this.sUpsChunk = this.db.prepare("INSERT INTO chunk (key, blob) VALUES (?, ?) " +
355
+ "ON CONFLICT(key) DO UPDATE SET blob = excluded.blob");
356
+ this.sDelChunk = this.db.prepare("DELETE FROM chunk WHERE key = ?");
357
+ this.sSelVmap = this.db.prepare("SELECT id, cid, seq, slot FROM vmap WHERE ext = ?");
358
+ this.sSelVmap.setReturnArrays(true);
359
+ this.sInsVmap = this.db.prepare("INSERT INTO vmap (ext, id, cid, seq, slot) VALUES (?, ?, ?, ?, ?)");
360
+ this.sUpdVmap = this.db.prepare("UPDATE vmap SET cid = ?, seq = ?, slot = ? WHERE ext = ?");
361
+ this.sDelVmap = this.db.prepare("DELETE FROM vmap WHERE ext = ?");
362
+ this.sUpsCent = this.db.prepare("INSERT INTO cent (cid, code, chunks, entries) VALUES (?, ?, ?, ?) " +
363
+ "ON CONFLICT(cid) DO UPDATE SET code = excluded.code, " +
364
+ "chunks = excluded.chunks, entries = excluded.entries");
365
+ this.sMeta = this.db.prepare("UPDATE meta SET live = ?, total = ?, next_id = ? WHERE id = 0");
366
+ }
367
+ /** Load the pivot table into RAM — K rows, ~(codeBytes + 8) bytes each. */
368
+ loadCents() {
369
+ const rows = this.db.prepare("SELECT cid, code, chunks, entries FROM cent ORDER BY cid").all();
370
+ this.K = rows.length === 0 ? 0 : rows[rows.length - 1].cid + 1;
371
+ const cap = Math.max(16, this.K);
372
+ this.centCodes = new Uint8Array(cap * this.codeBytes);
373
+ this.centWords = new Uint32Array(this.centCodes.buffer);
374
+ this.centEntries = new Int32Array(cap);
375
+ this.centChunks = new Int32Array(cap);
376
+ for (const r of rows) {
377
+ this.centCodes.set(r.code, r.cid * this.codeBytes);
378
+ this.centEntries[r.cid] = r.entries;
379
+ this.centChunks[r.cid] = r.chunks;
380
+ }
381
+ // Super-pivots are derived state — rebuild lazily from scratch.
382
+ this.superWords = new Uint32Array(0);
383
+ this.superStale.clear();
384
+ const G = IvfIndex.SUPER_G;
385
+ for (let g = 0; g * G < this.K; g++)
386
+ this.superStale.add(g);
387
+ }
388
+ growCents() {
389
+ const cap = this.centEntries.length;
390
+ if (this.K < cap)
391
+ return;
392
+ const ncap = cap * 2;
393
+ const codes = new Uint8Array(ncap * this.codeBytes);
394
+ codes.set(this.centCodes);
395
+ this.centCodes = codes;
396
+ this.centWords = new Uint32Array(codes.buffer);
397
+ const e = new Int32Array(ncap);
398
+ e.set(this.centEntries);
399
+ this.centEntries = e;
400
+ const c = new Int32Array(ncap);
401
+ c.set(this.centChunks);
402
+ this.centChunks = c;
403
+ }
404
+ // ── counters ────────────────────────────────────────────────────────────
405
+ get size() {
406
+ return this.live;
407
+ }
408
+ get physicalSize() {
409
+ return this.totalSlots;
410
+ }
411
+ get clusterCount() {
412
+ return this.K;
413
+ }
414
+ get bytesPerVector() {
415
+ return this.codeBytes;
416
+ }
417
+ get efSearch() {
418
+ return this._efSearch;
419
+ }
420
+ set efSearch(v) {
421
+ this._efSearch = Math.max(1, v | 0);
422
+ this.db.prepare("UPDATE meta SET ef_search = ? WHERE id = 0")
423
+ .run(this._efSearch);
424
+ }
425
+ resetReads() {
426
+ this.reads = 0;
427
+ }
428
+ // ── chunk IO (write-back cached within a transaction) ───────────────────
429
+ /** Fetch a chunk, serving dirty/cached copies first. `retain` controls
430
+ * whether a clean fetch enters the cache — read-only consumers pass the
431
+ * cacheEnabled flag so the `cacheSizeMb: 0` mode stays honestly uncached;
432
+ * writers always retain (the copy is about to become dirty). */
433
+ chunkAt(cid, seq, retain = true) {
434
+ const key = cid * SEQ_SPAN + seq;
435
+ const hit = this.chunkCache.get(key);
436
+ if (hit !== undefined)
437
+ return hit;
438
+ this.reads++;
439
+ const row = this.sSelChunk.get(key);
440
+ const c = new Chunk(row ? row[0] : new Uint8Array(this.chunkBytes));
441
+ if (retain)
442
+ this.cacheChunk(key, c);
443
+ return c;
444
+ }
445
+ cacheChunk(key, c) {
446
+ if (this.chunkCache.size >= this.chunkCacheMax)
447
+ this.flushChunks();
448
+ this.chunkCache.set(key, c);
449
+ }
450
+ markDirty(cid, seq, c) {
451
+ const key = cid * SEQ_SPAN + seq;
452
+ if (!this.chunkCache.has(key))
453
+ this.chunkCache.set(key, c);
454
+ this.dirtyChunks.add(key);
455
+ }
456
+ /** Write every dirty chunk row; drop the clean cache only when retention
457
+ * is off (no budget) — under a budget the warm set survives the flush.
458
+ * Runs inside the caller's transaction. */
459
+ flushChunks() {
460
+ for (const key of this.dirtyChunks) {
461
+ const c = this.chunkCache.get(key);
462
+ this.sUpsChunk.run(key, c.buf);
463
+ }
464
+ this.dirtyChunks.clear();
465
+ if (!this.cacheEnabled || this.chunkCache.size >= this.chunkCacheMax) {
466
+ this.chunkCache.clear();
467
+ }
468
+ }
469
+ flushCents() {
470
+ for (const cid of this.dirtyCents) {
471
+ this.sUpsCent.run(cid, this.centCodes.subarray(cid * this.codeBytes, (cid + 1) * this.codeBytes), this.centChunks[cid], this.centEntries[cid]);
472
+ }
473
+ this.dirtyCents.clear();
474
+ if (this.metaDirty) {
475
+ this.sMeta.run(this.live, this.totalSlots, this.nextId);
476
+ this.metaDirty = false;
477
+ }
478
+ }
479
+ /** Persist all buffered state. MUST be called before the enclosing
480
+ * transaction commits (upsertMany/deleteMany do; single-op paths too). */
481
+ flushAll() {
482
+ this.flushChunks();
483
+ this.flushCents();
484
+ }
485
+ // ── routing ─────────────────────────────────────────────────────────────
486
+ //
487
+ // TWO-LEVEL: a flat pivot scan is O(K) per insert with K growing as N/2048,
488
+ // which quietly becomes the dominant insert cost at millions of vectors
489
+ // (profiled: 33% of a 400k bulk load, and rising). Pivots are therefore
490
+ // grouped into SUPER-GROUPS of SUPER_G consecutive cids; each group carries
491
+ // a majority-bit super-pivot (pure RAM, derived — rebuilt from centCodes on
492
+ // open, marked stale when any member pivot changes). Routing scans the
493
+ // K/SUPER_G super-pivots, takes the SUPER_TOP nearest groups, and scans
494
+ // only their members exactly: O(K/64 + 256) word-Hamming rows per insert.
495
+ // Deterministic (majority + fixed tie order). Routing is a placement
496
+ // heuristic — a near-best cluster is as good as the best for recall, since
497
+ // queries probe many clusters — so the approximation never affects results
498
+ // beyond which (equally valid) cluster holds a code.
499
+ static SUPER_G = 64;
500
+ static SUPER_TOP = 4;
501
+ /** Majority super-pivot codes, one per group of SUPER_G clusters. */
502
+ superWords = new Uint32Array(0);
503
+ /** Group ids whose super-pivot is stale (member pivot changed). */
504
+ superStale = new Set();
505
+ /** Scratch word view of the code being routed (avoids per-call copies). */
506
+ routeScratch = new Uint32Array(0);
507
+ markSuperStale(cid) {
508
+ this.superStale.add(Math.floor(cid / IvfIndex.SUPER_G));
509
+ }
510
+ /** Recompute stale super-pivots (majority bit over member pivots). */
511
+ freshenSupers() {
512
+ if (this.superStale.size === 0)
513
+ return;
514
+ const words = this.codeBytes >> 2;
515
+ const G = IvfIndex.SUPER_G;
516
+ const groups = Math.ceil(this.K / G);
517
+ if (this.superWords.length < groups * words) {
518
+ const grown = new Uint32Array(Math.max(groups, 16) * 2 * words);
519
+ grown.set(this.superWords);
520
+ this.superWords = grown;
521
+ }
522
+ const cw = this.centWords;
523
+ for (const g of this.superStale) {
524
+ const lo = g * G;
525
+ const hi = Math.min(lo + G, this.K);
526
+ if (hi <= lo)
527
+ continue;
528
+ const m = hi - lo;
529
+ const half = m / 2;
530
+ const out = g * words;
531
+ for (let w = 0; w < words; w++) {
532
+ let word = 0;
533
+ for (let bit = 0; bit < 32; bit++) {
534
+ const mask = 1 << bit;
535
+ let ones = 0;
536
+ for (let cid = lo; cid < hi; cid++) {
537
+ if (cw[cid * words + w] & mask)
538
+ ones++;
539
+ }
540
+ if (ones > half)
541
+ word |= mask;
542
+ }
543
+ this.superWords[out + w] = word;
544
+ }
545
+ }
546
+ this.superStale.clear();
547
+ }
548
+ /** Word-popcount Hamming between the routing scratch and a words-row. */
549
+ hamAt(q, arr, base) {
550
+ const words = this.codeBytes >> 2;
551
+ let ham = 0;
552
+ for (let w = 0; w < words; w++) {
553
+ let x = q[w] ^ arr[base + w];
554
+ x -= (x >>> 1) & 0x55555555;
555
+ x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
556
+ x = (x + (x >>> 4)) & 0x0f0f0f0f;
557
+ ham += Math.imul(x, 0x01010101) >>> 24;
558
+ }
559
+ return ham;
560
+ }
561
+ /** Nearest cluster to a code — exact over member pivots of the SUPER_TOP
562
+ * nearest super-groups (or over everything while K is small). Pure RAM,
563
+ * no allocation, no storage reads. */
564
+ nearestCid(code) {
565
+ const words = this.codeBytes >> 2;
566
+ if (this.routeScratch.length < words) {
567
+ this.routeScratch = new Uint32Array(words);
568
+ }
569
+ const q = this.routeScratch;
570
+ new Uint8Array(q.buffer, 0, this.codeBytes).set(code);
571
+ const cw = this.centWords;
572
+ const G = IvfIndex.SUPER_G;
573
+ let best = -1;
574
+ let bestD = Infinity;
575
+ const scanGroup = (lo, hi) => {
576
+ for (let cid = lo; cid < hi; cid++) {
577
+ const d = this.hamAt(q, cw, cid * words);
578
+ if (d < bestD) {
579
+ bestD = d;
580
+ best = cid;
581
+ }
582
+ }
583
+ };
584
+ if (this.K <= 2 * G * IvfIndex.SUPER_TOP) {
585
+ scanGroup(0, this.K); // small collection — exact flat scan
586
+ return best;
587
+ }
588
+ this.freshenSupers();
589
+ const groups = Math.ceil(this.K / G);
590
+ // Top SUPER_TOP groups by super-pivot Hamming (tiny insertion "heap").
591
+ const T = IvfIndex.SUPER_TOP;
592
+ const gIds = new Int32Array(T).fill(-1);
593
+ const gDs = new Float64Array(T).fill(Infinity);
594
+ for (let g = 0; g < groups; g++) {
595
+ const d = this.hamAt(q, this.superWords, g * words);
596
+ if (d >= gDs[T - 1])
597
+ continue;
598
+ let i = T - 1;
599
+ while (i > 0 && gDs[i - 1] > d) {
600
+ gDs[i] = gDs[i - 1];
601
+ gIds[i] = gIds[i - 1];
602
+ i--;
603
+ }
604
+ gDs[i] = d;
605
+ gIds[i] = g;
606
+ }
607
+ for (let t = 0; t < T; t++) {
608
+ const g = gIds[t];
609
+ if (g < 0)
610
+ continue;
611
+ scanGroup(g * G, Math.min((g + 1) * G, this.K));
612
+ }
613
+ return best;
614
+ }
615
+ /** The nprobe nearest clusters to a QUERY (accurate estimator on pivots),
616
+ * as cids ordered nearest-first. */
617
+ probeOrder(q, nprobe) {
618
+ const top = new TopK(Math.min(nprobe, this.K));
619
+ const cb = this.codeBytes;
620
+ for (let cid = 0; cid < this.K; cid++) {
621
+ // Empty clusters can exist transiently after a compact; skip.
622
+ if (this.centEntries[cid] === 0)
623
+ continue;
624
+ const d = this.quantizer.estimate(this.centCodes, cid * cb, q);
625
+ top.push(d, cid);
626
+ }
627
+ return top.drain().map((h) => h.id);
628
+ }
629
+ probeOrderByCode(code, nprobe) {
630
+ const top = new TopK(Math.min(nprobe, this.K));
631
+ const cb = this.codeBytes;
632
+ const cc = this.centCodes;
633
+ for (let cid = 0; cid < this.K; cid++) {
634
+ if (this.centEntries[cid] === 0)
635
+ continue;
636
+ top.push(this.quantizer.codeDistanceBytes(code, cc.subarray(cid * cb, cid * cb + cb)), cid);
637
+ }
638
+ return top.drain().map((h) => h.id);
639
+ }
640
+ // ── insert / update / delete ────────────────────────────────────────────
641
+ /** Whether an external id is present (live). */
642
+ has(ext) {
643
+ return this.sSelVmap.get(ext) !== undefined;
644
+ }
645
+ /** The stored code for an ext (a copy), or null. */
646
+ codeOf(ext) {
647
+ const row = this.sSelVmap.get(ext);
648
+ if (!row)
649
+ return null;
650
+ const c = this.chunkAt(row[1], row[2], this.cacheEnabled);
651
+ const off = c.codeAt(row[3], this.codeBytes);
652
+ return c.buf.slice(off, off + this.codeBytes);
653
+ }
654
+ /** Insert a code under `ext`. Caller owns the transaction; buffered rows
655
+ * are flushed by {@link commitFlush}. Throws if ext is already live. */
656
+ insert(ext, code) {
657
+ if (this.sSelVmap.get(ext) !== undefined) {
658
+ throw new Error(`External id already exists: ${ext} (use update())`);
659
+ }
660
+ this.insertNew(ext, code);
661
+ }
662
+ /** Insert-or-update with ONE vmap probe. The point probe is a real cost
663
+ * at bulk-load rates (statement dispatch + a B-tree descent per row —
664
+ * profiled as the single largest insert-path term), so the presence check
665
+ * and the update lookup share it. */
666
+ upsert(ext, code) {
667
+ const row = this.sSelVmap.get(ext);
668
+ if (row === undefined)
669
+ this.insertNew(ext, code);
670
+ else
671
+ this.updateAt(row, ext, code);
672
+ }
673
+ /** The insert core — presence already established by the caller. */
674
+ insertNew(ext, code) {
675
+ const cid = this.K === 0 ? this.newCluster(code) : this.nearestCid(code);
676
+ this.appendToCluster(cid, ext, code, this.nextId++);
677
+ this.live++;
678
+ this.totalSlots++;
679
+ this.metaDirty = true;
680
+ if (this.centEntries[cid] >= SPLIT_MAX)
681
+ this.split(cid);
682
+ }
683
+ newCluster(pivot) {
684
+ const cid = this.K++;
685
+ this.growCents();
686
+ this.centCodes.set(pivot, cid * this.codeBytes);
687
+ this.centEntries[cid] = 0;
688
+ this.centChunks[cid] = 0;
689
+ this.dirtyCents.add(cid);
690
+ this.markSuperStale(cid);
691
+ return cid;
692
+ }
693
+ appendToCluster(cid, ext, code, id) {
694
+ let seq = this.centChunks[cid] - 1;
695
+ let c;
696
+ if (seq < 0 || (c = this.chunkAt(cid, seq)).n >= CHUNK_CAP) {
697
+ seq = this.centChunks[cid]++;
698
+ c = new Chunk(new Uint8Array(this.chunkBytes));
699
+ this.cacheChunk(cid * SEQ_SPAN + seq, c);
700
+ }
701
+ const slot = c.n;
702
+ c.ids[slot] = id;
703
+ c.exts[slot] = ext;
704
+ c.buf.set(code, c.codeAt(slot, this.codeBytes));
705
+ c.n = slot + 1;
706
+ this.markDirty(cid, seq, c);
707
+ this.centEntries[cid]++;
708
+ this.dirtyCents.add(cid);
709
+ this.sInsVmap.run(ext, id, cid, seq, slot);
710
+ }
711
+ /** Tombstone a live ext. Returns false when absent. */
712
+ remove(ext) {
713
+ const row = this.sSelVmap.get(ext);
714
+ if (!row)
715
+ return false;
716
+ const [, cid, seq, slot] = row;
717
+ const c = this.chunkAt(cid, seq);
718
+ c.markDead(slot);
719
+ this.markDirty(cid, seq, c);
720
+ this.sDelVmap.run(ext);
721
+ this.live--;
722
+ this.metaDirty = true;
723
+ return true;
724
+ }
725
+ /** Update the code bound to a live ext. A byte-identical code is a no-op
726
+ * (content-addressed callers re-upsert unchanged vectors wholesale after a
727
+ * restart; each no-op otherwise costs a tombstone + reinsert). */
728
+ update(ext, code) {
729
+ const row = this.sSelVmap.get(ext);
730
+ if (!row)
731
+ throw new Error(`Unknown external id: ${ext}`);
732
+ this.updateAt(row, ext, code);
733
+ }
734
+ updateAt(row, ext, code) {
735
+ const [id, cid, seq, slot] = row;
736
+ const c = this.chunkAt(cid, seq);
737
+ const off = c.codeAt(slot, this.codeBytes);
738
+ let same = true;
739
+ for (let i = 0; i < this.codeBytes; i++) {
740
+ if (c.buf[off + i] !== code[i]) {
741
+ same = false;
742
+ break;
743
+ }
744
+ }
745
+ if (same)
746
+ return;
747
+ c.markDead(slot);
748
+ this.markDirty(cid, seq, c);
749
+ const ncid = this.nearestCid(code);
750
+ this.sDelVmap.run(ext);
751
+ this.appendToCluster(ncid, ext, code, id); // internal id is preserved
752
+ this.totalSlots++;
753
+ this.metaDirty = true;
754
+ if (this.centEntries[ncid] >= SPLIT_MAX)
755
+ this.split(ncid);
756
+ }
757
+ /** Flush buffered chunk/cent/meta rows. Call before COMMIT. */
758
+ commitFlush() {
759
+ this.flushAll();
760
+ }
761
+ // ── split ───────────────────────────────────────────────────────────────
762
+ /** Split a full cluster in two, deterministically. Dead slots are dropped
763
+ * (splits double as incremental compaction). Both halves get fresh
764
+ * majority-bit pivots, so routing sharpens as the corpus grows. */
765
+ split(cid) {
766
+ const cb = this.codeBytes;
767
+ // Slots (live + dead) this cluster held before the rewrite — the exact
768
+ // amount the rewrite's dead-drop reclaims from the physical count.
769
+ const slotsBefore = this.centEntries[cid];
770
+ // 1. Gather live members.
771
+ const chunks = this.centChunks[cid];
772
+ const exts = [];
773
+ const ids = [];
774
+ const codes = [];
775
+ for (let seq = 0; seq < chunks; seq++) {
776
+ const c = this.chunkAt(cid, seq);
777
+ for (let s = 0; s < c.n; s++) {
778
+ if (c.isDead(s))
779
+ continue;
780
+ exts.push(c.exts[s]);
781
+ ids.push(c.ids[s]);
782
+ const off = c.codeAt(s, cb);
783
+ codes.push(c.buf.slice(off, off + cb));
784
+ }
785
+ }
786
+ const n = codes.length;
787
+ if (n < 2) {
788
+ // Degenerate (a cluster emptied by removes): rewrite in place, dropping
789
+ // dead slots; no second cluster is minted.
790
+ const oc = this.centChunks[cid];
791
+ this.rebuildCluster(cid, oc, codes, exts, ids, null, 0);
792
+ this.totalSlots -= slotsBefore - n;
793
+ this.metaDirty = true;
794
+ return;
795
+ }
796
+ const pivot = this.centCodes.subarray(cid * cb, cid * cb + cb);
797
+ // 2. Deterministic 2-seed choice: farthest from the old pivot, then
798
+ // farthest from the first seed (ties → lowest index).
799
+ const dist = (a, b) => this.quantizer.codeDistanceBytes(a, b);
800
+ let ai = 0, bd = -1;
801
+ for (let i = 0; i < n; i++) {
802
+ const d = dist(codes[i], pivot);
803
+ if (d > bd) {
804
+ bd = d;
805
+ ai = i;
806
+ }
807
+ }
808
+ let bi = ai === 0 ? 1 % Math.max(n, 1) : 0;
809
+ bd = -1;
810
+ for (let i = 0; i < n; i++) {
811
+ if (i === ai)
812
+ continue;
813
+ const d = dist(codes[i], codes[ai]);
814
+ if (d > bd) {
815
+ bd = d;
816
+ bi = i;
817
+ }
818
+ }
819
+ // 3. MEDIAN split on the two-seed margin. Assigning each member to its
820
+ // nearer seed degenerates on tie-heavy code distributions (everything
821
+ // lands on one side, the full side immediately re-splits, and cluster
822
+ // count cascades) — the median cut guarantees two n/2 halves whatever
823
+ // the geometry, so a split always halves the cluster. Ties break by
824
+ // insertion index: deterministic.
825
+ const seedA0 = codes[ai], seedB0 = codes[bi];
826
+ const margin = new Float64Array(n);
827
+ for (let i = 0; i < n; i++) {
828
+ margin[i] = dist(codes[i], seedA0) - dist(codes[i], seedB0);
829
+ }
830
+ const order = new Array(n);
831
+ for (let i = 0; i < n; i++)
832
+ order[i] = i;
833
+ order.sort((x, y) => margin[x] - margin[y] || x - y);
834
+ const side = new Uint8Array(n);
835
+ for (let s = n >> 1; s < n; s++)
836
+ side[order[s]] = 1;
837
+ // Fresh majority pivots for both halves — routing sharpens with the data.
838
+ const seedA = this.majorityCode(codes, side, 0) ?? seedA0;
839
+ const seedB = this.majorityCode(codes, side, 1) ?? seedB0;
840
+ // 4. Rewrite: side-0 stays in cid, side-1 becomes a fresh cluster.
841
+ const oldChunks = this.centChunks[cid];
842
+ const ncid = this.newCluster(seedB);
843
+ this.centCodes.set(seedA, cid * cb);
844
+ this.markSuperStale(cid);
845
+ this.rebuildCluster(cid, oldChunks, codes, exts, ids, side, 0);
846
+ this.rebuildCluster(ncid, 0, codes, exts, ids, side, 1);
847
+ // Dead slots vanished in the rewrite.
848
+ this.totalSlots -= slotsBefore - n;
849
+ this.metaDirty = true;
850
+ this.dirtyCents.add(cid);
851
+ this.dirtyCents.add(ncid);
852
+ }
853
+ /** Bit-majority sample size per split side. A pivot is a routing aid, not
854
+ * a stored value: the majority bit of a 512-member deterministic sample
855
+ * agrees with the full-population majority except on near-tied bits,
856
+ * where either choice routes equally well — and counting every member of
857
+ * a 4096-entry cluster was 25% of bulk-load CPU (profiled). */
858
+ static MAJORITY_SAMPLE = 512;
859
+ /** Majority-bit code of the side's members (the binary centroid), or null
860
+ * when the side is empty. Members are sampled on a deterministic stride
861
+ * when the side exceeds {@link MAJORITY_SAMPLE}. */
862
+ majorityCode(codes, side, which) {
863
+ const cb = this.codeBytes;
864
+ let total = 0;
865
+ for (let i = 0; i < codes.length; i++)
866
+ if (side[i] === which)
867
+ total++;
868
+ if (total === 0)
869
+ return null;
870
+ const stride = Math.max(1, Math.ceil(total / IvfIndex.MAJORITY_SAMPLE));
871
+ const counts = new Int32Array(cb * 8);
872
+ let m = 0;
873
+ let seen = 0;
874
+ for (let i = 0; i < codes.length; i++) {
875
+ if (side[i] !== which)
876
+ continue;
877
+ if (seen++ % stride !== 0)
878
+ continue;
879
+ m++;
880
+ const c = codes[i];
881
+ for (let p = 0; p < cb; p++) {
882
+ const byte = c[p];
883
+ if (byte === 0)
884
+ continue;
885
+ const base = p * 8;
886
+ for (let k = 0; k < 8; k++)
887
+ if (byte & (1 << k))
888
+ counts[base + k]++;
889
+ }
890
+ }
891
+ const out = new Uint8Array(cb);
892
+ const half = m / 2;
893
+ for (let p = 0; p < cb; p++) {
894
+ let byte = 0;
895
+ const base = p * 8;
896
+ for (let k = 0; k < 8; k++)
897
+ if (counts[base + k] > half)
898
+ byte |= 1 << k;
899
+ out[p] = byte;
900
+ }
901
+ return out;
902
+ }
903
+ /** Rewrite a cluster's chunks from a member list (one side of a split or a
904
+ * compaction survivor set), updating vmap rows. `dropFrom` deletes any
905
+ * leftover chunk rows beyond the new count. */
906
+ rebuildCluster(cid, dropFrom, codes, exts, ids, side, which) {
907
+ const cb = this.codeBytes;
908
+ let seq = 0;
909
+ let c = new Chunk(new Uint8Array(this.chunkBytes));
910
+ let slot = 0;
911
+ let entries = 0;
912
+ const vrows = []; // flat (ext, id, cid, seq, slot) tuples
913
+ for (let i = 0; i < codes.length; i++) {
914
+ if (side !== null && side[i] !== which)
915
+ continue;
916
+ if (slot === CHUNK_CAP) {
917
+ this.sUpsChunk.run(cid * SEQ_SPAN + seq, c.buf);
918
+ this.chunkCache.delete(cid * SEQ_SPAN + seq);
919
+ this.dirtyChunks.delete(cid * SEQ_SPAN + seq);
920
+ seq++;
921
+ c = new Chunk(new Uint8Array(this.chunkBytes));
922
+ slot = 0;
923
+ }
924
+ c.ids[slot] = ids[i];
925
+ c.exts[slot] = exts[i];
926
+ c.buf.set(codes[i], c.codeAt(slot, cb));
927
+ c.n = slot + 1;
928
+ // (ext, id, cid, seq, slot) — flushed below as multi-row replaces; one
929
+ // statement dispatch per member made vmap rewriting the dominant split
930
+ // cost (an entry is rewritten at every split it lives through).
931
+ vrows.push(exts[i], ids[i], cid, seq, slot);
932
+ slot++;
933
+ entries++;
934
+ }
935
+ this.flushVmapRows(vrows);
936
+ const usedChunks = entries === 0 ? 0 : seq + 1;
937
+ if (entries > 0) {
938
+ this.sUpsChunk.run(cid * SEQ_SPAN + seq, c.buf);
939
+ this.chunkCache.delete(cid * SEQ_SPAN + seq);
940
+ this.dirtyChunks.delete(cid * SEQ_SPAN + seq);
941
+ }
942
+ for (let s = usedChunks; s < dropFrom; s++) {
943
+ this.sDelChunk.run(cid * SEQ_SPAN + s);
944
+ this.chunkCache.delete(cid * SEQ_SPAN + s);
945
+ this.dirtyChunks.delete(cid * SEQ_SPAN + s);
946
+ }
947
+ this.centChunks[cid] = usedChunks;
948
+ this.centEntries[cid] = entries;
949
+ this.dirtyCents.add(cid);
950
+ }
951
+ /** Multi-row vmap statements, prepared per row count (powers of a fixed
952
+ * batch width) — REPLACE semantics on the ext primary key. */
953
+ vmapBatchStmts = new Map();
954
+ // 5 params/row → 320 params per dispatch. Wider batches lose more to the
955
+ // per-argument spread/bind overhead than they save in dispatches (measured
956
+ // at 256 rows: the spread itself became 15% of a bulk load).
957
+ static VMAP_BATCH = 64;
958
+ vmapBatchStmt(rows) {
959
+ let s = this.vmapBatchStmts.get(rows);
960
+ if (s === undefined) {
961
+ s = this.db.prepare("INSERT OR REPLACE INTO vmap (ext, id, cid, seq, slot) VALUES " +
962
+ "(?,?,?,?,?),".repeat(rows - 1) + "(?,?,?,?,?)");
963
+ this.vmapBatchStmts.set(rows, s);
964
+ }
965
+ return s;
966
+ }
967
+ /** Write flat (ext, id, cid, seq, slot) tuples in wide batches. */
968
+ flushVmapRows(vrows) {
969
+ const W = IvfIndex.VMAP_BATCH * 5;
970
+ let o = 0;
971
+ for (; o + W <= vrows.length; o += W) {
972
+ this.vmapBatchStmt(IvfIndex.VMAP_BATCH).run(...vrows.slice(o, o + W));
973
+ }
974
+ const rest = (vrows.length - o) / 5;
975
+ if (rest > 0)
976
+ this.vmapBatchStmt(rest).run(...vrows.slice(o));
977
+ }
978
+ // ── query ───────────────────────────────────────────────────────────────
979
+ /** ef → clusters probed. ef is the familiar "candidate breadth" knob; a
980
+ * probe scans one whole cluster, so nprobe = ef/4 keeps the default
981
+ * (ef 64 → 16 probes) both accurate on trained stores and bounded. */
982
+ nprobeOf(ef) {
983
+ return Math.max(1, Math.ceil(ef / 4));
984
+ }
985
+ /** k-NN with a full-precision query (accurate RaBitQ estimator). */
986
+ searchKnn(vec, k, ef) {
987
+ this.lastQueryDistComps = 0;
988
+ this.resetReads();
989
+ if (this.live === 0 || k <= 0) {
990
+ this.lastQueryStorageReads = 0;
991
+ return [];
992
+ }
993
+ const q = this.quantizer.prepareQuery(vec);
994
+ const hits = this.scanClusters(this.probeOrder(q, this.nprobeOf(ef ?? this._efSearch)), k, (buf, off) => {
995
+ this.lastQueryDistComps++;
996
+ return this.quantizer.estimate(buf, off, q);
997
+ });
998
+ this.lastQueryStorageReads = this.reads;
999
+ return hits;
1000
+ }
1001
+ /** k-NN with an already-quantized code (sign-bit Hamming distance). */
1002
+ searchKnnByCode(code, k, ef) {
1003
+ this.lastQueryDistComps = 0;
1004
+ this.resetReads();
1005
+ if (this.live === 0 || k <= 0) {
1006
+ this.lastQueryStorageReads = 0;
1007
+ return [];
1008
+ }
1009
+ const cb = this.codeBytes;
1010
+ const hits = this.scanClusters(this.probeOrderByCode(code, this.nprobeOf(ef ?? this._efSearch)), k, (buf, off) => {
1011
+ this.lastQueryDistComps++;
1012
+ return this.quantizer.codeDistanceBytes(code, buf.subarray(off, off + cb));
1013
+ });
1014
+ this.lastQueryStorageReads = this.reads;
1015
+ return hits;
1016
+ }
1017
+ scanClusters(cids, k, distAt) {
1018
+ const cb = this.codeBytes;
1019
+ const top = new TopK(k);
1020
+ const retain = this.cacheEnabled; // reads stay honest with the budget off
1021
+ for (const cid of cids) {
1022
+ const chunks = this.centChunks[cid];
1023
+ for (let seq = 0; seq < chunks; seq++) {
1024
+ const c = this.chunkAt(cid, seq, retain);
1025
+ const n = c.n;
1026
+ // Tombstones are rare (updates/deletes only); when the bitmap is
1027
+ // clear the whole chunk scans without a per-slot dead test.
1028
+ const anyDead = c.dead[0] | c.dead[1] | c.dead[2] | c.dead[3] |
1029
+ c.dead[4] | c.dead[5] | c.dead[6] | c.dead[7];
1030
+ let off = CODES_OFF;
1031
+ if (anyDead === 0) {
1032
+ for (let s = 0; s < n; s++, off += cb) {
1033
+ const d = distAt(c.buf, off);
1034
+ if (d < top.worst)
1035
+ top.push(d, c.exts[s]);
1036
+ }
1037
+ }
1038
+ else {
1039
+ for (let s = 0; s < n; s++, off += cb) {
1040
+ if (c.isDead(s))
1041
+ continue;
1042
+ const d = distAt(c.buf, off);
1043
+ if (d < top.worst)
1044
+ top.push(d, c.exts[s]);
1045
+ }
1046
+ }
1047
+ }
1048
+ }
1049
+ return top.drain();
1050
+ }
1051
+ // ── maintenance ─────────────────────────────────────────────────────────
1052
+ /** Stream live entries whose internal id is > `after`, in id order.
1053
+ * Internal ids are monotone at insert and PRESERVED by update/compact, so
1054
+ * the largest id a caller has seen is a durable incremental watermark. */
1055
+ *keysSince(after, batch = 1024) {
1056
+ const q = this.db.prepare("SELECT ext, id FROM vmap WHERE id > ? ORDER BY id LIMIT ?");
1057
+ let cursor = after;
1058
+ for (;;) {
1059
+ const rows = q.all(cursor, batch);
1060
+ if (rows.length === 0)
1061
+ break;
1062
+ for (const r of rows)
1063
+ yield { ext: r.ext, internal: r.id };
1064
+ cursor = rows[rows.length - 1].id;
1065
+ }
1066
+ }
1067
+ /** Stream every live external id. */
1068
+ *keys() {
1069
+ for (const { ext } of this.keysSince(0))
1070
+ yield ext;
1071
+ }
1072
+ /** Drop tombstoned slots by rewriting each cluster that carries any, then
1073
+ * VACUUM. Internal ids and cluster assignment are preserved — routing
1074
+ * quality is untouched, only dead space is reclaimed. */
1075
+ compact() {
1076
+ this.db.exec("BEGIN");
1077
+ try {
1078
+ const cb = this.codeBytes;
1079
+ for (let cid = 0; cid < this.K; cid++) {
1080
+ const chunks = this.centChunks[cid];
1081
+ if (chunks === 0)
1082
+ continue;
1083
+ let hasDead = false;
1084
+ const exts = [];
1085
+ const ids = [];
1086
+ const codes = [];
1087
+ for (let seq = 0; seq < chunks; seq++) {
1088
+ const c = this.chunkAt(cid, seq);
1089
+ for (let s = 0; s < c.n; s++) {
1090
+ if (c.isDead(s)) {
1091
+ hasDead = true;
1092
+ continue;
1093
+ }
1094
+ exts.push(c.exts[s]);
1095
+ ids.push(c.ids[s]);
1096
+ const off = c.codeAt(s, cb);
1097
+ codes.push(c.buf.slice(off, off + cb));
1098
+ }
1099
+ }
1100
+ if (!hasDead)
1101
+ continue;
1102
+ this.rebuildCluster(cid, chunks, codes, exts, ids, null, 0);
1103
+ }
1104
+ this.totalSlots = this.live;
1105
+ this.metaDirty = true;
1106
+ this.flushAll();
1107
+ this.db.exec("COMMIT");
1108
+ }
1109
+ catch (e) {
1110
+ this.db.exec("ROLLBACK");
1111
+ throw e;
1112
+ }
1113
+ this.chunkCache.clear();
1114
+ this.db.exec("VACUUM");
1115
+ }
1116
+ /** Heat SQLite's page cache with sequential scans of the chunk and vmap
1117
+ * tables — a cold session otherwise warms it through random point reads.
1118
+ * Purely a latency optimisation; returns rows touched. */
1119
+ warmCache() {
1120
+ let n = 0;
1121
+ // Selecting the blob itself (not just the key) pulls the overflow pages
1122
+ // through the page cache — a key-only scan touches only the B-tree.
1123
+ for (const _ of this.db.prepare("SELECT blob FROM chunk").iterate())
1124
+ n++;
1125
+ for (const _ of this.db.prepare("SELECT ext, id FROM vmap").iterate())
1126
+ n++;
1127
+ return n;
1128
+ }
1129
+ begin() {
1130
+ this.db.exec("BEGIN");
1131
+ }
1132
+ commit() {
1133
+ this.flushAll();
1134
+ this.db.exec("COMMIT");
1135
+ }
1136
+ rollback() {
1137
+ // Buffered state may be ahead of the rolled-back rows — reload from disk.
1138
+ this.chunkCache.clear();
1139
+ this.dirtyChunks.clear();
1140
+ this.dirtyCents.clear();
1141
+ this.metaDirty = false;
1142
+ this.db.exec("ROLLBACK");
1143
+ const row = this.db.prepare("SELECT live, total, next_id FROM meta WHERE id = 0").get();
1144
+ this.live = row.live;
1145
+ this.totalSlots = row.total;
1146
+ this.nextId = row.next_id;
1147
+ this.loadCents();
1148
+ }
1149
+ /** Encode a raw vector to its 1-bit code bytes. */
1150
+ encodeToBytes(vec) {
1151
+ return this.quantizer.codeToBytes(this.quantizer.encode(vec));
1152
+ }
1153
+ codeToBytes(code) {
1154
+ return this.quantizer.codeToBytes(code);
1155
+ }
1156
+ bytesToCode(bytes) {
1157
+ return this.quantizer.bytesToCode(bytes);
1158
+ }
1159
+ get codeWords() {
1160
+ return this.quantizer.codeWords;
1161
+ }
1162
+ close() {
1163
+ this.db.close();
1164
+ }
1165
+ }