@hviana/sema 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/example/demo.d.ts +1 -0
  2. package/dist/example/demo.js +39 -0
  3. package/dist/example/train_base.d.ts +87 -0
  4. package/dist/example/train_base.js +2216 -0
  5. package/dist/src/alphabet.d.ts +7 -0
  6. package/dist/src/alphabet.js +33 -0
  7. package/dist/src/alu/src/alu.d.ts +185 -0
  8. package/dist/src/alu/src/alu.js +440 -0
  9. package/dist/src/alu/src/expr.d.ts +61 -0
  10. package/dist/src/alu/src/expr.js +318 -0
  11. package/dist/src/alu/src/index.d.ts +11 -0
  12. package/dist/src/alu/src/index.js +19 -0
  13. package/dist/src/alu/src/kernel-arith.d.ts +16 -0
  14. package/dist/src/alu/src/kernel-arith.js +264 -0
  15. package/dist/src/alu/src/kernel-bits.d.ts +19 -0
  16. package/dist/src/alu/src/kernel-bits.js +152 -0
  17. package/dist/src/alu/src/kernel-logic.d.ts +4 -0
  18. package/dist/src/alu/src/kernel-logic.js +60 -0
  19. package/dist/src/alu/src/kernel-nd.d.ts +3 -0
  20. package/dist/src/alu/src/kernel-nd.js +208 -0
  21. package/dist/src/alu/src/kernel-numeric.d.ts +54 -0
  22. package/dist/src/alu/src/kernel-numeric.js +366 -0
  23. package/dist/src/alu/src/operation.d.ts +168 -0
  24. package/dist/src/alu/src/operation.js +189 -0
  25. package/dist/src/alu/src/parser.d.ts +212 -0
  26. package/dist/src/alu/src/parser.js +469 -0
  27. package/dist/src/alu/src/resonance.d.ts +55 -0
  28. package/dist/src/alu/src/resonance.js +126 -0
  29. package/dist/src/alu/src/text.d.ts +31 -0
  30. package/dist/src/alu/src/text.js +73 -0
  31. package/dist/src/alu/src/value.d.ts +109 -0
  32. package/dist/src/alu/src/value.js +300 -0
  33. package/dist/src/alu/test/alu.test.d.ts +1 -0
  34. package/dist/src/alu/test/alu.test.js +764 -0
  35. package/dist/src/bytes.d.ts +14 -0
  36. package/dist/src/bytes.js +59 -0
  37. package/dist/src/config.d.ts +114 -0
  38. package/dist/src/config.js +96 -0
  39. package/dist/src/derive/src/deduction.d.ts +125 -0
  40. package/dist/src/derive/src/deduction.js +155 -0
  41. package/dist/src/derive/src/index.d.ts +7 -0
  42. package/dist/src/derive/src/index.js +11 -0
  43. package/dist/src/derive/src/priority-queue.d.ts +20 -0
  44. package/dist/src/derive/src/priority-queue.js +73 -0
  45. package/dist/src/derive/src/rewrite.d.ts +56 -0
  46. package/dist/src/derive/src/rewrite.js +100 -0
  47. package/dist/src/derive/src/trie.d.ts +90 -0
  48. package/dist/src/derive/src/trie.js +217 -0
  49. package/dist/src/derive/test/derive.test.d.ts +1 -0
  50. package/dist/src/derive/test/derive.test.js +122 -0
  51. package/dist/src/extension.d.ts +37 -0
  52. package/dist/src/extension.js +7 -0
  53. package/dist/src/geometry.d.ts +137 -0
  54. package/dist/src/geometry.js +430 -0
  55. package/dist/src/index.d.ts +15 -0
  56. package/dist/src/index.js +21 -0
  57. package/dist/src/ingest-cache.d.ts +41 -0
  58. package/dist/src/ingest-cache.js +161 -0
  59. package/dist/src/mind/articulation.d.ts +6 -0
  60. package/dist/src/mind/articulation.js +99 -0
  61. package/dist/src/mind/attention.d.ts +72 -0
  62. package/dist/src/mind/attention.js +894 -0
  63. package/dist/src/mind/canonical.d.ts +29 -0
  64. package/dist/src/mind/canonical.js +88 -0
  65. package/dist/src/mind/graph-search.d.ts +270 -0
  66. package/dist/src/mind/graph-search.js +847 -0
  67. package/dist/src/mind/index.d.ts +5 -0
  68. package/dist/src/mind/index.js +5 -0
  69. package/dist/src/mind/junction.d.ts +95 -0
  70. package/dist/src/mind/junction.js +262 -0
  71. package/dist/src/mind/learning.d.ts +47 -0
  72. package/dist/src/mind/learning.js +201 -0
  73. package/dist/src/mind/match.d.ts +111 -0
  74. package/dist/src/mind/match.js +422 -0
  75. package/dist/src/mind/mechanisms/alu.d.ts +4 -0
  76. package/dist/src/mind/mechanisms/alu.js +29 -0
  77. package/dist/src/mind/mechanisms/cast.d.ts +35 -0
  78. package/dist/src/mind/mechanisms/cast.js +447 -0
  79. package/dist/src/mind/mechanisms/confluence.d.ts +24 -0
  80. package/dist/src/mind/mechanisms/confluence.js +213 -0
  81. package/dist/src/mind/mechanisms/cover.d.ts +6 -0
  82. package/dist/src/mind/mechanisms/cover.js +179 -0
  83. package/dist/src/mind/mechanisms/extraction.d.ts +67 -0
  84. package/dist/src/mind/mechanisms/extraction.js +342 -0
  85. package/dist/src/mind/mechanisms/recall.d.ts +13 -0
  86. package/dist/src/mind/mechanisms/recall.js +151 -0
  87. package/dist/src/mind/mind.d.ts +147 -0
  88. package/dist/src/mind/mind.js +300 -0
  89. package/dist/src/mind/pipeline-mechanism.d.ts +142 -0
  90. package/dist/src/mind/pipeline-mechanism.js +213 -0
  91. package/dist/src/mind/pipeline.d.ts +20 -0
  92. package/dist/src/mind/pipeline.js +185 -0
  93. package/dist/src/mind/primitives.d.ts +43 -0
  94. package/dist/src/mind/primitives.js +162 -0
  95. package/dist/src/mind/rationale.d.ts +134 -0
  96. package/dist/src/mind/rationale.js +162 -0
  97. package/dist/src/mind/reasoning.d.ts +15 -0
  98. package/dist/src/mind/reasoning.js +162 -0
  99. package/dist/src/mind/recognition.d.ts +20 -0
  100. package/dist/src/mind/recognition.js +223 -0
  101. package/dist/src/mind/resonance.d.ts +23 -0
  102. package/dist/src/mind/resonance.js +0 -0
  103. package/dist/src/mind/trace.d.ts +15 -0
  104. package/dist/src/mind/trace.js +73 -0
  105. package/dist/src/mind/traverse.d.ts +100 -0
  106. package/dist/src/mind/traverse.js +447 -0
  107. package/dist/src/mind/types.d.ts +174 -0
  108. package/dist/src/mind/types.js +84 -0
  109. package/dist/src/rabitq-hnsw/src/database.d.ts +200 -0
  110. package/dist/src/rabitq-hnsw/src/database.js +388 -0
  111. package/dist/src/rabitq-hnsw/src/heap.d.ts +22 -0
  112. package/dist/src/rabitq-hnsw/src/heap.js +89 -0
  113. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +125 -0
  114. package/dist/src/rabitq-hnsw/src/hnsw.js +474 -0
  115. package/dist/src/rabitq-hnsw/src/index.d.ts +10 -0
  116. package/dist/src/rabitq-hnsw/src/index.js +6 -0
  117. package/dist/src/rabitq-hnsw/src/prng.d.ts +19 -0
  118. package/dist/src/rabitq-hnsw/src/prng.js +36 -0
  119. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +95 -0
  120. package/dist/src/rabitq-hnsw/src/rabitq.js +283 -0
  121. package/dist/src/rabitq-hnsw/src/store.d.ts +162 -0
  122. package/dist/src/rabitq-hnsw/src/store.js +825 -0
  123. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +1 -0
  124. package/dist/src/rabitq-hnsw/test/hnsw.test.js +948 -0
  125. package/dist/src/store-sqlite.d.ts +149 -0
  126. package/dist/src/store-sqlite.js +702 -0
  127. package/dist/src/store.d.ts +638 -0
  128. package/dist/src/store.js +1618 -0
  129. package/dist/src/vec.d.ts +31 -0
  130. package/dist/src/vec.js +109 -0
  131. package/package.json +1 -1
@@ -0,0 +1,474 @@
1
+ import { Heap } from "./heap.js";
2
+ import { Prng } from "./prng.js";
3
+ /**
4
+ * Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
5
+ * 1-bit RaBitQ codes that live in SQLite (see Store) rather than in RAM. The
6
+ * algorithm is the textbook one; the only difference is that codes and adjacency
7
+ * lists are read and written through the store on demand, so the resident set is
8
+ * the working set of the current operation -- never the whole graph.
9
+ *
10
+ * Distances use the codes throughout:
11
+ * - searching with a full-precision query uses RaBitQ's accurate estimator;
12
+ * - building the graph and searching by code compare two codes by sign-bit
13
+ * Hamming distance.
14
+ * Building on codes alone is what lets `compact` rebuild an honest index with no
15
+ * access to the original vectors.
16
+ */
17
+ export class HnswIndex {
18
+ M;
19
+ Mmax0;
20
+ efConstruction;
21
+ mL;
22
+ quantizer;
23
+ store;
24
+ seed;
25
+ levelRng;
26
+ // tiny global scalars, cached from meta and written through on change
27
+ entry;
28
+ maxLevel;
29
+ live;
30
+ total;
31
+ _efSearch;
32
+ // Per-operation working set: the records the current insert/query is actively
33
+ // comparing. Cleared at the start of every op and bounded by efConstruction /
34
+ // efSearch (the nodes one operation can touch), never by the collection size.
35
+ // This is the algorithm's working memory -- it makes each touched node cost one
36
+ // storage read per op -- not a tunable cache. The only cache is SQLite's page
37
+ // cache (see Store), sized in MB.
38
+ working = new Map();
39
+ visited = new Set();
40
+ // Epoch-tagged visited marks — the Set above without per-candidate hashing:
41
+ // visitedTag[id] === visitedEpoch means "seen this operation", and bumping
42
+ // the epoch clears the whole set in O(1). Semantically IDENTICAL to the
43
+ // Set (results never differ); used only when the store has a cache budget,
44
+ // because the array is 4 B per internal id — amortized working memory that
45
+ // grows with the collection, which the cacheSizeMb:0 flat-memory mode must
46
+ // not pay. The epoch wrap (2³²−1 ops) refills the array once.
47
+ visitedTag = new Uint32Array(1024);
48
+ visitedEpoch = 0;
49
+ seenTag(id, epoch) {
50
+ return id < this.visitedTag.length && this.visitedTag[id] === epoch;
51
+ }
52
+ markVisited(id, epoch) {
53
+ if (id >= this.visitedTag.length) {
54
+ const grown = new Uint32Array(Math.max(id + 1, this.visitedTag.length * 2));
55
+ grown.set(this.visitedTag);
56
+ this.visitedTag = grown;
57
+ }
58
+ this.visitedTag[id] = epoch;
59
+ }
60
+ candHeap = new Heap(true); // min-heap: nearest first
61
+ resHeap = new Heap(false); // max-heap: farthest first
62
+ singleEp = [0];
63
+ idxScratch = [];
64
+ distScratch = [];
65
+ freshScratch = [];
66
+ // distance dispatch: a full-precision query sets qCtx; building or a code
67
+ // query sets refBytes (code<->code). `building` only suppresses the counter.
68
+ qCtx = null;
69
+ refBytes = null;
70
+ building = false;
71
+ lastQueryDistComps = 0;
72
+ /** Storage row reads issued by the most recent query (cache-independent). */
73
+ lastQueryStorageReads = 0;
74
+ constructor(quantizer, store, params) {
75
+ this.quantizer = quantizer;
76
+ this.store = store;
77
+ this.M = params.M;
78
+ this.Mmax0 = this.M * 2;
79
+ this.efConstruction = params.efConstruction;
80
+ this._efSearch = params.efSearch;
81
+ this.mL = 1 / Math.log(this.M);
82
+ this.seed = params.seed >>> 0;
83
+ const s = store.loadState();
84
+ this.entry = s.entryPoint;
85
+ this.maxLevel = s.maxLevel;
86
+ this.live = s.live;
87
+ this.total = s.total;
88
+ this.levelRng = new Prng(this.seed);
89
+ this.levelRng.restore(s.rng);
90
+ }
91
+ get size() {
92
+ return this.live;
93
+ }
94
+ get physicalSize() {
95
+ return this.total;
96
+ }
97
+ get bytesPerVector() {
98
+ return this.quantizer.codeWords * 4;
99
+ }
100
+ get efSearch() {
101
+ return this._efSearch;
102
+ }
103
+ set efSearch(v) {
104
+ this._efSearch = Math.max(1, v | 0);
105
+ this.store.setEfSearch(this._efSearch);
106
+ }
107
+ persistState() {
108
+ this.store.saveState({
109
+ entryPoint: this.entry,
110
+ maxLevel: this.maxLevel,
111
+ live: this.live,
112
+ total: this.total,
113
+ rng: this.levelRng.snapshot(),
114
+ });
115
+ }
116
+ randomLevel() {
117
+ let u = this.levelRng.next();
118
+ if (u < 1e-12)
119
+ u = 1e-12;
120
+ return Math.floor(-Math.log(u) * this.mL);
121
+ }
122
+ /**
123
+ * Fetch a node record into the operation's working set. The first touch in an
124
+ * op is a storage read; later touches in the same op reuse it. The set is
125
+ * cleared per op and bounded by efConstruction / efSearch, so storage reads
126
+ * per op equal the number of *distinct* nodes the op visits -- minimal, and
127
+ * independent of any cache.
128
+ */
129
+ node(id) {
130
+ const hit = this.working.get(id);
131
+ if (hit !== undefined)
132
+ return hit;
133
+ const rec = this.store.getNode(id);
134
+ if (rec === null)
135
+ throw new Error(`node ${id} not found`);
136
+ this.working.set(id, rec);
137
+ return rec;
138
+ }
139
+ /** Prefetch several nodes into the working set with ONE batched storage
140
+ * read for the misses. Point queries per neighbour made statement
141
+ * dispatch — not distance arithmetic — the dominant cost of a large
142
+ * build; the batch keeps `reads` accounting identical per row. */
143
+ fetchNodes(ids, count = ids.length) {
144
+ // getNodesInto skips ids already present in `working` itself — no
145
+ // second pre-filter pass over the same map here.
146
+ this.store.getNodesInto(ids, this.working, count);
147
+ }
148
+ /** Distance from the current source (query vector or reference code) to a node. */
149
+ distOf(rec) {
150
+ if (this.qCtx !== null) {
151
+ this.lastQueryDistComps++;
152
+ return this.quantizer.estimate(rec.code, 0, this.qCtx);
153
+ }
154
+ if (!this.building)
155
+ this.lastQueryDistComps++;
156
+ return this.quantizer.codeDistanceBytes(this.refBytes, rec.code);
157
+ }
158
+ /** Code-to-code distance between two stored nodes (graph wiring only). */
159
+ codeDist(a, b) {
160
+ return this.quantizer.codeDistanceBytes(this.node(a).code, this.node(b).code);
161
+ }
162
+ /**
163
+ * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
164
+ * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
165
+ * never returned. `visited` is a per-call set sized by the nodes seen here.
166
+ */
167
+ searchLayer(entryPoints, ef, layer) {
168
+ const cand = this.candHeap;
169
+ const res = this.resHeap;
170
+ cand.clear();
171
+ res.clear();
172
+ // Visited marks: epoch tags when RAM-for-speed is allowed, the plain Set
173
+ // in flat-memory mode. Identical semantics either way.
174
+ const useTags = this.store.cacheEnabled;
175
+ let epoch = 0;
176
+ const visited = this.visited;
177
+ if (useTags) {
178
+ epoch = ++this.visitedEpoch;
179
+ if (epoch === 0xffffffff) {
180
+ this.visitedTag.fill(0);
181
+ epoch = this.visitedEpoch = 1;
182
+ }
183
+ }
184
+ else {
185
+ visited.clear();
186
+ }
187
+ for (let k = 0; k < entryPoints.length; k++) {
188
+ const ep = entryPoints[k];
189
+ if (useTags) {
190
+ if (this.seenTag(ep, epoch))
191
+ continue;
192
+ this.markVisited(ep, epoch);
193
+ }
194
+ else {
195
+ if (visited.has(ep))
196
+ continue;
197
+ visited.add(ep);
198
+ }
199
+ const rec = this.node(ep);
200
+ const d = this.distOf(rec);
201
+ cand.push(d, ep);
202
+ if (rec.deleted === 0) {
203
+ res.push(d, ep);
204
+ if (res.size > ef)
205
+ res.pop();
206
+ }
207
+ }
208
+ while (cand.size > 0) {
209
+ const cd = cand.topKey();
210
+ if (res.size >= ef && cd > res.topKey())
211
+ break;
212
+ const c = cand.topVal();
213
+ cand.pop();
214
+ const nbrs = this.store.getNeighbors(c, layer);
215
+ if (nbrs === null)
216
+ continue;
217
+ // Batch: mark the unvisited neighbours, fetch their codes in one
218
+ // storage read, then score them in the original order.
219
+ const fresh = this.freshScratch;
220
+ fresh.length = 0;
221
+ for (let i = 0; i < nbrs.length; i++) {
222
+ const e = nbrs[i];
223
+ if (useTags) {
224
+ if (this.seenTag(e, epoch))
225
+ continue;
226
+ this.markVisited(e, epoch);
227
+ }
228
+ else {
229
+ if (visited.has(e))
230
+ continue;
231
+ visited.add(e);
232
+ }
233
+ fresh.push(e);
234
+ }
235
+ if (fresh.length > 1)
236
+ this.fetchNodes(fresh);
237
+ for (let i = 0; i < fresh.length; i++) {
238
+ const e = fresh[i];
239
+ const rec = this.node(e);
240
+ const d = this.distOf(rec);
241
+ const worst = res.size > 0 ? res.topKey() : Infinity;
242
+ if (res.size < ef || d < worst) {
243
+ cand.push(d, e);
244
+ if (rec.deleted === 0) {
245
+ res.push(d, e);
246
+ if (res.size > ef)
247
+ res.pop();
248
+ }
249
+ }
250
+ }
251
+ }
252
+ }
253
+ /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
254
+ greedyDescend(entry, fromLayer, toLayer) {
255
+ let cur = entry;
256
+ const ep = this.singleEp;
257
+ for (let layer = fromLayer; layer > toLayer; layer--) {
258
+ ep[0] = cur;
259
+ this.searchLayer(ep, 1, layer);
260
+ if (this.resHeap.size > 0)
261
+ cur = this.resHeap.vals[0];
262
+ }
263
+ return cur;
264
+ }
265
+ /**
266
+ * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
267
+ * neighbours from `candIds`, preferring those closer to `base` than to any
268
+ * already-chosen neighbour, then fills remaining slots with the closest
269
+ * leftovers (keep-pruned connections). Appends to `out`.
270
+ */
271
+ selectNeighbors(base, candIds, count, M, out) {
272
+ const idx = this.idxScratch;
273
+ const ds = this.distScratch;
274
+ this.fetchNodes(candIds, count); // one batched read for the misses
275
+ idx.length = count;
276
+ ds.length = count;
277
+ for (let i = 0; i < count; i++) {
278
+ idx[i] = i;
279
+ ds[i] = this.codeDist(base, candIds[i]);
280
+ }
281
+ idx.sort((a, b) => ds[a] - ds[b]);
282
+ for (let s = 0; s < count; s++) {
283
+ if (out.length >= M)
284
+ break;
285
+ const i = idx[s];
286
+ const cid = candIds[i];
287
+ if (cid === base)
288
+ continue;
289
+ const cd = ds[i];
290
+ let keep = true;
291
+ for (let j = 0; j < out.length; j++) {
292
+ if (this.codeDist(cid, out[j]) < cd) {
293
+ keep = false;
294
+ break;
295
+ }
296
+ }
297
+ if (keep)
298
+ out.push(cid);
299
+ }
300
+ if (out.length < M) {
301
+ for (let s = 0; s < count && out.length < M; s++) {
302
+ const cid = candIds[idx[s]];
303
+ if (cid === base)
304
+ continue;
305
+ let dup = false;
306
+ for (let j = 0; j < out.length; j++) {
307
+ if (out[j] === cid) {
308
+ dup = true;
309
+ break;
310
+ }
311
+ }
312
+ if (!dup)
313
+ out.push(cid);
314
+ }
315
+ }
316
+ }
317
+ /**
318
+ * Insert a vector's code under external id `ext`; returns the internal node id.
319
+ * All graph reads/writes go through the store; the caller owns the transaction.
320
+ */
321
+ insert(ext, code, efC) {
322
+ const ef = efC !== undefined && efC >= 1
323
+ ? Math.min(efC, this.efConstruction)
324
+ : this.efConstruction;
325
+ this.working.clear();
326
+ const level = this.randomLevel();
327
+ const id = this.store.addNode(ext, level, code);
328
+ this.total++;
329
+ this.working.set(id, { code, deleted: 0, ext });
330
+ if (this.entry === -1) {
331
+ this.entry = id;
332
+ this.maxLevel = level;
333
+ this.live++;
334
+ this.persistState();
335
+ return id;
336
+ }
337
+ this.qCtx = null;
338
+ this.refBytes = code;
339
+ this.building = true;
340
+ let entry = this.entry;
341
+ const topLayer = this.maxLevel;
342
+ if (topLayer > level)
343
+ entry = this.greedyDescend(entry, topLayer, level);
344
+ let entryPoints = [entry];
345
+ const startLayer = Math.min(topLayer, level);
346
+ for (let layer = startLayer; layer >= 0; layer--) {
347
+ this.searchLayer(entryPoints, ef, layer);
348
+ const res = this.resHeap;
349
+ const wCount = res.size;
350
+ const selected = [];
351
+ this.selectNeighbors(id, res.vals, wCount, this.M, selected);
352
+ if (selected.length > 0)
353
+ this.store.setNeighbors(id, layer, selected);
354
+ const cap = layer === 0 ? this.Mmax0 : this.M;
355
+ for (let s = 0; s < selected.length; s++) {
356
+ const nb = selected[s];
357
+ const cur = this.store.getNeighbors(nb, layer);
358
+ const list = cur ? Array.from(cur) : [];
359
+ list.push(id);
360
+ if (list.length > cap) {
361
+ const pruned = [];
362
+ this.selectNeighbors(nb, list, list.length, cap, pruned);
363
+ this.store.setNeighbors(nb, layer, pruned);
364
+ }
365
+ else {
366
+ this.store.setNeighbors(nb, layer, list);
367
+ }
368
+ }
369
+ if (wCount > 0) {
370
+ const next = new Array(wCount);
371
+ for (let i = 0; i < wCount; i++)
372
+ next[i] = res.vals[i];
373
+ entryPoints = next;
374
+ }
375
+ else {
376
+ entryPoints = [entry];
377
+ }
378
+ }
379
+ this.building = false;
380
+ this.refBytes = null;
381
+ this.live++;
382
+ if (level > this.maxLevel) {
383
+ this.maxLevel = level;
384
+ this.entry = id;
385
+ }
386
+ this.persistState();
387
+ return id;
388
+ }
389
+ /** k-NN with a full-precision query vector (accurate estimator). */
390
+ searchKnn(vec, k, ef) {
391
+ this.lastQueryDistComps = 0;
392
+ this.store.resetReads();
393
+ this.working.clear();
394
+ this.building = false;
395
+ this.refBytes = null;
396
+ if (this.entry === -1 || k <= 0) {
397
+ this.lastQueryStorageReads = 0;
398
+ return [];
399
+ }
400
+ this.qCtx = this.quantizer.prepareQuery(vec);
401
+ const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
402
+ this.qCtx = null;
403
+ this.lastQueryStorageReads = this.store.reads;
404
+ return hits;
405
+ }
406
+ /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
407
+ searchKnnByCode(codeBytes, k, ef) {
408
+ this.lastQueryDistComps = 0;
409
+ this.store.resetReads();
410
+ this.working.clear();
411
+ this.building = false;
412
+ this.qCtx = null;
413
+ if (this.entry === -1 || k <= 0) {
414
+ this.lastQueryStorageReads = 0;
415
+ return [];
416
+ }
417
+ this.refBytes = codeBytes;
418
+ const hits = this.collectKnn(k, Math.max(ef ?? this._efSearch, k));
419
+ this.refBytes = null;
420
+ this.lastQueryStorageReads = this.store.reads;
421
+ return hits;
422
+ }
423
+ collectKnn(k, efs) {
424
+ let entry = this.entry;
425
+ if (this.maxLevel > 0)
426
+ entry = this.greedyDescend(entry, this.maxLevel, 0);
427
+ this.singleEp[0] = entry;
428
+ this.searchLayer(this.singleEp, efs, 0);
429
+ const res = this.resHeap;
430
+ const n = res.size;
431
+ const hits = new Array(n);
432
+ for (let i = 0; i < n; i++) {
433
+ let d = res.keys[i];
434
+ if (d < 0)
435
+ d = 0;
436
+ hits[i] = { id: this.node(res.vals[i]).ext, distance: d };
437
+ }
438
+ hits.sort((a, b) => a.distance - b.distance);
439
+ if (hits.length > k)
440
+ hits.length = k;
441
+ return hits;
442
+ }
443
+ /** Tombstone a live node by internal id. Caller owns the transaction. */
444
+ remove(id) {
445
+ const rec = this.store.getNode(id);
446
+ if (rec === null || rec.deleted === 1)
447
+ return false;
448
+ this.store.tombstone(id);
449
+ this.live--;
450
+ this.persistState();
451
+ return true;
452
+ }
453
+ /**
454
+ * Reclaim tombstones with a graph-preserving splice (see
455
+ * {@link Store.spliceCompact}): live nodes keep their internal ids and their
456
+ * wiring; each dead neighbour is replaced by its own live neighbours. Codes
457
+ * are untouched, so the index loses nothing beyond what deletion itself
458
+ * removes. The previous implementation replayed every live code through a
459
+ * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
460
+ * multi-million-node store, inside one WAL-bloating transaction. The splice
461
+ * is a streaming pass with batched commits, then a VACUUM to return the
462
+ * freed pages.
463
+ */
464
+ compact() {
465
+ const r = this.store.spliceCompact(this.M, this.Mmax0, this.entry);
466
+ this.entry = r.entry;
467
+ this.maxLevel = r.maxLevel;
468
+ this.live = r.live;
469
+ this.total = r.live;
470
+ this.persistState();
471
+ this.working.clear();
472
+ this.store.vacuum();
473
+ }
474
+ }
@@ -0,0 +1,10 @@
1
+ export { VectorDatabase } from "./database.js";
2
+ export type { DatabaseOptions, ExternalId, QueryResult, StorageStats, } from "./database.js";
3
+ export { HnswIndex } from "./hnsw.js";
4
+ export type { HnswParams, KnnHit } from "./hnsw.js";
5
+ export { RaBitQuantizer } from "./rabitq.js";
6
+ export type { QueryContext, RaBitQOptions } from "./rabitq.js";
7
+ export { Store } from "./store.js";
8
+ export type { GlobalState, NodeRec, StoreConfig } from "./store.js";
9
+ export { Heap } from "./heap.js";
10
+ export { Prng } from "./prng.js";
@@ -0,0 +1,6 @@
1
+ export { VectorDatabase } from "./database.js";
2
+ export { HnswIndex } from "./hnsw.js";
3
+ export { RaBitQuantizer } from "./rabitq.js";
4
+ export { Store } from "./store.js";
5
+ export { Heap } from "./heap.js";
6
+ export { Prng } from "./prng.js";
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Small deterministic pseudo-random number generator (mulberry32).
3
+ *
4
+ * Pure ECMAScript. It is used so that the random orthogonal rotation of the
5
+ * RaBitQ quantizer and the HNSW level assignment are reproducible and can be
6
+ * regenerated after (de)serialization without storing large matrices.
7
+ */
8
+ export declare class Prng {
9
+ private state;
10
+ constructor(seed: number);
11
+ /** Uniform float in [0, 1). */
12
+ next(): number;
13
+ /** Uniform integer in [0, n). */
14
+ int(n: number): number;
15
+ /** Current internal state (one uint32), for persisting/resuming the stream. */
16
+ snapshot(): number;
17
+ /** Resume the stream from a previously snapshotted state. */
18
+ restore(state: number): void;
19
+ }
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Small deterministic pseudo-random number generator (mulberry32).
3
+ *
4
+ * Pure ECMAScript. It is used so that the random orthogonal rotation of the
5
+ * RaBitQ quantizer and the HNSW level assignment are reproducible and can be
6
+ * regenerated after (de)serialization without storing large matrices.
7
+ */
8
+ export class Prng {
9
+ state;
10
+ constructor(seed) {
11
+ this.state = seed >>> 0;
12
+ if (this.state === 0)
13
+ this.state = 0x9e3779b9;
14
+ }
15
+ /** Uniform float in [0, 1). */
16
+ next() {
17
+ let t = (this.state = (this.state + 0x6d2b79f5) | 0);
18
+ t = Math.imul(t ^ (t >>> 15), t | 1);
19
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
20
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
21
+ }
22
+ /** Uniform integer in [0, n). */
23
+ int(n) {
24
+ return (this.next() * n) | 0;
25
+ }
26
+ /** Current internal state (one uint32), for persisting/resuming the stream. */
27
+ snapshot() {
28
+ return this.state >>> 0;
29
+ }
30
+ /** Resume the stream from a previously snapshotted state. */
31
+ restore(state) {
32
+ this.state = state >>> 0;
33
+ if (this.state === 0)
34
+ this.state = 0x9e3779b9;
35
+ }
36
+ }
@@ -0,0 +1,95 @@
1
+ export interface QueryContext {
2
+ vmin: number;
3
+ delta: number;
4
+ sumQInt: number;
5
+ /**
6
+ * Per-query lookup table of length nbytes*256. `qlut[p*256 + b]` is the sum of
7
+ * the quantised query values at the (up to 8) coordinates whose code bits are
8
+ * set in byte value `b` at code-byte position `p`. The query/code inner product
9
+ * for a stored code is then sum over its bytes of qlut[p*256 + codeByte_p].
10
+ */
11
+ qlut: Uint8Array | Uint16Array;
12
+ /** number of bytes per code (paddedDim / 8). */
13
+ nbytes: number;
14
+ /** true when the (centered) query has zero norm. */
15
+ zero: boolean;
16
+ }
17
+ export interface RaBitQOptions {
18
+ queryBits?: number;
19
+ rounds?: number;
20
+ seed?: number;
21
+ centroid?: ArrayLike<number>;
22
+ }
23
+ /**
24
+ * 1-bit RaBitQ quantizer (cosine) -- the ONLY representation of a vector kept by
25
+ * the index. A D-dimensional vector collapses to ceil(D/32) 32-bit words of sign
26
+ * bits, e.g. a 256-d vector goes from 256*4 = 1024 bytes to 32 bytes of code.
27
+ *
28
+ * Each vector is centered by an optional centroid, normalised, rotated by a
29
+ * fast random orthogonal transform (random sign flips + Walsh-Hadamard,
30
+ * O(D log D)) and reduced to one sign bit per padded dimension. The random
31
+ * rotation makes the quantisation error essentially uniform across vectors, so
32
+ * the cosine estimate needs only a single fixed scale (`cosFactor`) rather than
33
+ * any per-vector correction.
34
+ *
35
+ * Two estimators are provided:
36
+ * - `estimate` : full-precision query vs stored code (accurate)
37
+ * - `codeDistanceBytes` : stored code vs stored code (Hamming based; used while
38
+ * building the graph, where neither side is full precision)
39
+ *
40
+ * Reference: Gao & Long, "RaBitQ: Quantizing High-Dimensional Vectors with a
41
+ * Theoretical Error Bound for Approximate Nearest Neighbor Search", SIGMOD 2024.
42
+ */
43
+ export declare class RaBitQuantizer {
44
+ readonly dim: number;
45
+ readonly paddedDim: number;
46
+ readonly codeWords: number;
47
+ readonly queryBits: number;
48
+ readonly rounds: number;
49
+ readonly seed: number;
50
+ readonly centroid: Float64Array;
51
+ private readonly sqrtD;
52
+ private readonly maxQInt;
53
+ private readonly signs;
54
+ private readonly scratch;
55
+ /** Fixed cosine inner-product scale (= 1 / E[L1] of a rotated unit vector). */
56
+ readonly cosFactor: number;
57
+ private readonly nbytes;
58
+ /** coordinate index sitting at byte position p, bit k -> bitCoord[p*8 + k]. */
59
+ private readonly bitCoord;
60
+ /** true when the largest possible LUT entry overflows a Uint8. */
61
+ private readonly lutWide;
62
+ constructor(dim: number, opts?: RaBitQOptions);
63
+ /** In-place fast Walsh-Hadamard transform; `a.length` must be a power of two. */
64
+ private fwht;
65
+ /** Apply the orthogonal rotation in place (a.length === paddedDim). */
66
+ private rotate;
67
+ /** Encode a raw vector into its 1-bit sign code (the whole representation). */
68
+ encode(vec: ArrayLike<number>): Uint32Array;
69
+ /** Pre-process a full-precision query into the structure consumed by `estimate`. */
70
+ prepareQuery(vec: ArrayLike<number>): QueryContext;
71
+ /**
72
+ * Estimate the cosine distance (1 - cosine) between a stored code and a
73
+ * full-precision query, reading the code's bytes against the query's byte LUT.
74
+ * The code's set-bit count is tallied in the same byte scan, so nothing beyond
75
+ * the code itself is needed.
76
+ *
77
+ * `codeBytes` is a Uint8 view of the packed code buffer and `byteOffset` is the
78
+ * code's start byte (id * paddedDim/8).
79
+ */
80
+ estimate(codeBytes: Uint8Array, byteOffset: number, q: QueryContext): number;
81
+ /**
82
+ * Cosine distance (1 - cosine, in [0, 2]) between two packed codes, computed
83
+ * directly from the BLOB bytes via their sign-bit Hamming distance — no word
84
+ * reinterpretation, so it is endianness-agnostic. Identical codes score 0.
85
+ *
86
+ * This is the distance used to build the graph (code vs code) and to answer a
87
+ * query given an already-quantized code. It is coarser than `estimate`, where
88
+ * one side is full precision.
89
+ */
90
+ codeDistanceBytes(a: Uint8Array, b: Uint8Array): number;
91
+ /** Pack a code (codeWords 32-bit words) into its little-endian BLOB bytes. */
92
+ codeToBytes(code: ArrayLike<number>): Uint8Array;
93
+ /** Reinterpret a code BLOB as a copy of codeWords 32-bit words. */
94
+ bytesToCode(bytes: Uint8Array): Uint32Array;
95
+ }