@hviana/sema 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1700 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +29 -43
  51. package/dist/src/geometry.js +324 -268
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +754 -872
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +141 -185
  82. package/dist/src/mind/mind.d.ts +194 -155
  83. package/dist/src/mind/mind.js +392 -289
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +20 -33
  89. package/dist/src/mind/primitives.js +124 -120
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +198 -210
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +29 -58
  101. package/dist/src/mind/traverse.js +344 -356
  102. package/dist/src/mind/types.d.ts +122 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +171 -166
  117. package/dist/src/store-sqlite.js +768 -662
  118. package/dist/src/store.d.ts +597 -630
  119. package/dist/src/store.js +1434 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +33 -15
  123. package/index.html +65 -0
  124. package/package.json +1 -1
  125. package/src/alu/README.md +1 -1
  126. package/src/alu/src/index.ts +1 -1
  127. package/src/config.ts +19 -27
  128. package/src/geometry.ts +93 -1
  129. package/src/index.ts +6 -11
  130. package/src/mind/attention.ts +11 -6
  131. package/src/mind/mechanisms/recall.ts +29 -13
  132. package/src/mind/mind.ts +250 -19
  133. package/src/mind/primitives.ts +43 -6
  134. package/src/mind/recognition.ts +26 -8
  135. package/src/mind/traverse.ts +47 -0
  136. package/src/mind/types.ts +20 -21
  137. package/src/rabitq-ivf/README.md +56 -0
  138. package/src/rabitq-ivf/src/database.ts +276 -0
  139. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  140. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  141. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  142. package/src/store-sqlite.ts +204 -9
  143. package/src/store.ts +27 -34
  144. package/test/08-storage.test.mjs +3 -3
  145. package/test/13-conversation.test.mjs +190 -20
  146. package/test/14-scaling.test.mjs +2 -2
  147. package/test/35-ivf.test.mjs +263 -0
  148. package/test/36-bloom.test.mjs +123 -0
  149. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  150. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  151. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  152. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  153. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  154. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  155. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  156. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  157. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  158. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  159. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  160. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  161. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  162. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  163. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  164. package/src/rabitq-hnsw/README.md +0 -303
  165. package/src/rabitq-hnsw/src/database.ts +0 -492
  166. package/src/rabitq-hnsw/src/heap.ts +0 -90
  167. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  168. package/src/rabitq-hnsw/src/store.ts +0 -994
  169. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  170. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -2,7 +2,7 @@
2
2
  * Small deterministic pseudo-random number generator (mulberry32).
3
3
  *
4
4
  * Pure ECMAScript. It is used so that the random orthogonal rotation of the
5
- * RaBitQ quantizer and the HNSW level assignment are reproducible and can be
5
+ * RaBitQ quantizer rotation is reproducible and can be
6
6
  * regenerated after (de)serialization without storing large matrices.
7
7
  */
8
8
  export class Prng {
@@ -1,4 +1,4 @@
1
- // store-sqlite.ts — SQLite + rabitq-hnsw persistence adapter.
1
+ // store-sqlite.ts — SQLite + rabitq-ivf persistence adapter.
2
2
  //
3
3
  // SQliteStore extends AbstractStore and implements only the bare essentials
4
4
  // required for database communication: SQL schema, prepared statements, and
@@ -39,7 +39,7 @@ import {
39
39
  unpackKids,
40
40
  } from "./store.js";
41
41
  import { DEFAULT_CONFIG, type StoreConfig } from "./config.js";
42
- import { VectorDatabase } from "./rabitq-hnsw/src/index.js";
42
+ import { VectorDatabase } from "./rabitq-ivf/src/index.js";
43
43
 
44
44
  export interface SQliteStoreOptions extends Partial<StoreConfig> {
45
45
  path?: string;
@@ -66,6 +66,21 @@ CREATE TABLE IF NOT EXISTS node (
66
66
  h INTEGER NOT NULL
67
67
  );
68
68
  CREATE INDEX IF NOT EXISTS idx_node_h ON node(h);
69
+ -- Negative-lookup BLOOM FILTER over node.h — the RAM structure that lets the
70
+ -- content-addressed dedup probes (findLeaf/findBranch, the training hot path)
71
+ -- answer "definitely new content" without an idx_node_h descent. Persisted
72
+ -- here at checkpoint/close with upto = the next node id NOT covered; on
73
+ -- open, rows with id >= upto (a crash's tail) are re-added by an incremental
74
+ -- rowid-range scan, so the filter can NEVER be missing an inserted hash — a
75
+ -- false negative would silently break hash-consing (duplicate mints), while
76
+ -- a false positive only costs the SQLite probe that was paid on every call
77
+ -- before the filter existed.
78
+ CREATE TABLE IF NOT EXISTS bloom (
79
+ id INTEGER PRIMARY KEY CHECK (id = 1),
80
+ bits BLOB NOT NULL,
81
+ n INTEGER NOT NULL,
82
+ upto INTEGER NOT NULL
83
+ );
69
84
  -- The reverse structural edge child→parent. A WITHOUT ROWID table clustered on
70
85
  -- (child, parent) IS the lookup index: parents(child) is one B-tree descent to a
71
86
  -- contiguous run, with no separate rowid heap and no secondary index to maintain
@@ -129,9 +144,66 @@ CREATE TABLE IF NOT EXISTS meta (
129
144
  );
130
145
  `;
131
146
 
147
+ /** Blocked Bloom filter over 32-bit content hashes. k=4 bit positions per
148
+ * key via double hashing; at the maintained ≥12 bits/key the false-positive
149
+ * rate stays under ~1%, so ≥99% of miss-probes skip SQLite entirely.
150
+ * No deletions (nodes are never deleted), so growth is a rebuild. */
151
+ class NodeBloom {
152
+ bits: Uint8Array;
153
+ mask: number;
154
+ n = 0;
155
+ constructor(log2bits: number) {
156
+ this.bits = new Uint8Array(1 << (log2bits - 3));
157
+ this.mask = (1 << log2bits) - 1 >>> 0;
158
+ }
159
+ add(h: number): void {
160
+ const h2 = (Math.imul(h, 0x9e3779b1) | 1) >>> 0;
161
+ let x = h >>> 0;
162
+ const bits = this.bits;
163
+ const mask = this.mask;
164
+ for (let i = 0; i < 4; i++) {
165
+ const b = x & mask;
166
+ bits[b >> 3] |= 1 << (b & 7);
167
+ x = (x + h2) >>> 0;
168
+ }
169
+ this.n++;
170
+ }
171
+ mightContain(h: number): boolean {
172
+ const h2 = (Math.imul(h, 0x9e3779b1) | 1) >>> 0;
173
+ let x = h >>> 0;
174
+ const bits = this.bits;
175
+ const mask = this.mask;
176
+ for (let i = 0; i < 4; i++) {
177
+ const b = x & mask;
178
+ if ((bits[b >> 3] & (1 << (b & 7))) === 0) return false;
179
+ x = (x + h2) >>> 0;
180
+ }
181
+ return true;
182
+ }
183
+ /** Whether the filter is due for a bigger rebuild (< 12 bits/key). */
184
+ get saturated(): boolean {
185
+ return this.n * 12 > this.bits.length * 8;
186
+ }
187
+ }
188
+
189
+ /** log2 of the bloom bit count sized for `n` keys at 16 bits/key, clamped to
190
+ * [17, 30] — 16 KiB floor, 128 MiB ceiling (past ~90M nodes the FPR is
191
+ * allowed to drift up rather than RAM). */
192
+ function bloomLog2For(n: number): number {
193
+ let log2 = 17;
194
+ while (log2 < 30 && (1 << log2) < n * 16) log2++;
195
+ return log2;
196
+ }
197
+
198
+ /** Below this node count the filter is NOT persisted: a rebuild scan at open
199
+ * is instant at this size, and the blob would dominate a small store's
200
+ * on-disk footprint. */
201
+ const BLOOM_PERSIST_MIN_NODES = 65_536;
202
+
132
203
  export class SQliteStore extends AbstractStore implements Store {
133
204
  private readonly opts: SQliteStoreOptions;
134
205
  private readonly vectorCacheMb: number;
206
+ private readonly sqliteCacheMb: number;
135
207
  private readonly vectorSeed: number;
136
208
 
137
209
  // ── the two SEPARATE vector indices ───────────────────────────────────
@@ -143,6 +215,17 @@ export class SQliteStore extends AbstractStore implements Store {
143
215
  // Deferred write transaction guard.
144
216
  private _inTx = false;
145
217
 
218
+ // ── negative dedup filter ──────────────────────────────────────────────
219
+ /** Bloom filter over every node row's content hash `h`. The dedup probes
220
+ * consult it first: "definitely absent" answers the training hot path's
221
+ * dominant miss case in RAM (profiled: the SQLite probes were 38.7% of
222
+ * ingest wall at 2.9M nodes). INVARIANT: every inserted h is added
223
+ * before the probe that could see it — a false negative would mint a
224
+ * duplicate node and silently break hash-consing. */
225
+ private _bloom: NodeBloom | null = null;
226
+ /** Dedup probes answered by the filter alone this session (observability). */
227
+ bloomSkips = 0;
228
+
146
229
  private _insertNode: any = null;
147
230
  private _insertKid: any = null;
148
231
  private _selContain: any = null;
@@ -167,10 +250,6 @@ export class SQliteStore extends AbstractStore implements Store {
167
250
  // Resolve config from opts, falling back to defaults.
168
251
  const config: StoreConfig = {
169
252
  minHaloMass: opts.minHaloMass ?? d.minHaloMass,
170
- m: opts.m ?? d.m,
171
- efConstruction: opts.efConstruction ?? d.efConstruction,
172
- efConstructionInterior: opts.efConstructionInterior ??
173
- d.efConstructionInterior,
174
253
  efSearch: opts.efSearch ?? d.efSearch,
175
254
  compactEveryNWrites: opts.compactEveryNWrites ?? d.compactEveryNWrites,
176
255
  overfetch: opts.overfetch ?? d.overfetch,
@@ -182,6 +261,7 @@ export class SQliteStore extends AbstractStore implements Store {
182
261
  pendingGistBytes: opts.pendingGistBytes ?? d.pendingGistBytes,
183
262
  haloCacheBytes: opts.haloCacheBytes ?? d.haloCacheBytes,
184
263
  vectorCacheMb: opts.vectorCacheMb ?? d.vectorCacheMb,
264
+ sqliteCacheMb: opts.sqliteCacheMb ?? d.sqliteCacheMb,
185
265
  coveredIdsMax: opts.coveredIdsMax ?? d.coveredIdsMax,
186
266
  chainCacheBytes: opts.chainCacheBytes ?? d.chainCacheBytes,
187
267
  };
@@ -191,6 +271,7 @@ export class SQliteStore extends AbstractStore implements Store {
191
271
 
192
272
  this.opts = opts;
193
273
  this.vectorCacheMb = opts.vectorCacheMb ?? d.vectorCacheMb;
274
+ this.sqliteCacheMb = opts.sqliteCacheMb ?? d.sqliteCacheMb;
194
275
  this.vectorSeed = (0x51f15e ^ 0x9e3779b9) >>> 0;
195
276
  this._ready = this._dbOpen();
196
277
  }
@@ -207,8 +288,6 @@ export class SQliteStore extends AbstractStore implements Store {
207
288
  return new VectorDatabase({
208
289
  dbPath: this.vectorDbPath(name),
209
290
  dim: this.D,
210
- M: this.m,
211
- efConstruction: this.efConstruction,
212
291
  efSearch: this.efSearch,
213
292
  // Query-side estimator precision. 8 bits: 4 bits measurably misranks
214
293
  // tight gist clusters (mixture recall@10 37.5% vs 39.0%, self-recall
@@ -242,6 +321,25 @@ export class SQliteStore extends AbstractStore implements Store {
242
321
  // WAL + NORMAL sync: crash-safe (WAL commits are atomic) without a full
243
322
  // fsync per commit — the same discipline the vector databases use.
244
323
  this.sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;");
324
+ // SQLite's autocheckpoint default is 1000 PAGES — 1 MiB at this store's
325
+ // 1 KiB page size — so at training write rates nearly every batch commit
326
+ // crossed the threshold and paid a synchronous checkpoint: random writes
327
+ // into a GB-scale main file, with the hottest B-tree pages copied out on
328
+ // every commit instead of being coalesced across commits in the WAL.
329
+ // Profiled on a 7.5M-node store: ~114 ms PER COMMIT, ~19% of ingest
330
+ // wall-clock. 16 MiB of WAL between checkpoints coalesces hot-page
331
+ // rewrites and amortizes the checkpoint's fixed cost; crash-safety is
332
+ // unchanged (WAL recovery replays it), the only trade is a bounded,
333
+ // fixed-size WAL file.
334
+ this.sqlite.exec("PRAGMA wal_autocheckpoint = 65536;");
335
+ // Page cache for the DAG tables. Content-addressed dedup (findLeaf /
336
+ // findBranch), parent probes and contain appends are millions of point
337
+ // queries per session against a file that outgrows SQLite's ~2 MiB
338
+ // default cache almost immediately; every miss is a file read on the
339
+ // ingest hot path. Negative = KiB budget (size-based, page-size
340
+ // independent). Latency only — results identical at any value.
341
+ const cacheKb = Math.max(0, Math.floor(this.sqliteCacheMb * 1024));
342
+ if (cacheKb > 0) this.sqlite.exec(`PRAGMA cache_size = -${cacheKb};`);
245
343
  this.sqlite.exec(SCHEMA);
246
344
 
247
345
  // Recover D from a previous run so loadFromStore() can bootstrap with any
@@ -290,9 +388,77 @@ export class SQliteStore extends AbstractStore implements Store {
290
388
  this._nextId = ((this.sqlite.prepare(
291
389
  "SELECT MAX(id) AS m FROM node",
292
390
  ).get() as { m: number | null }).m ?? -1) + 1;
391
+
392
+ this._bloomLoad();
393
+ }
394
+
395
+ // ── negative dedup filter: load / rebuild / persist ────────────────────
396
+
397
+ /** Restore the persisted filter and top up the rows a crash left uncovered
398
+ * (id ≥ stored `upto`); a missing/undersized blob forces a full rebuild.
399
+ * Clean-close sessions restart with a 0-row top-up. */
400
+ private _bloomLoad(): void {
401
+ const row = this.sqlite!.prepare(
402
+ "SELECT bits, n, upto FROM bloom WHERE id = 1",
403
+ ).get() as { bits: Uint8Array; n: number; upto: number } | undefined;
404
+ const wantLog2 = bloomLog2For(this._nextId);
405
+ if (row && row.bits.length * 8 >= (1 << wantLog2)) {
406
+ const log2 = 31 - Math.clz32(row.bits.length * 8);
407
+ const b = new NodeBloom(log2);
408
+ b.bits.set(row.bits);
409
+ b.n = row.n;
410
+ this._bloom = b;
411
+ this._bloomTopUp(row.upto);
412
+ } else {
413
+ this._bloomRebuild();
414
+ }
415
+ }
416
+
417
+ /** Re-add the hashes of rows with id ≥ `from` (the tail a crash cut off).
418
+ * Re-adding a hash twice only re-sets its bits — never a correctness
419
+ * issue — so overlap with the persisted coverage is harmless. */
420
+ private _bloomTopUp(from: number): void {
421
+ if (from >= this._nextId) return;
422
+ const b = this._bloom!;
423
+ const scan = this.sqlite!.prepare("SELECT h FROM node WHERE id >= ?");
424
+ scan.setReturnArrays(true);
425
+ for (const r of scan.iterate(from) as IterableIterator<[number]>) {
426
+ b.add(r[0]);
427
+ }
428
+ }
429
+
430
+ /** Build a fresh filter sized for the current node count from one
431
+ * sequential scan of the h column. Called at open when no usable blob is
432
+ * stored, and again whenever growth saturates the bits-per-key budget. */
433
+ private _bloomRebuild(): void {
434
+ const b = new NodeBloom(bloomLog2For(this._nextId));
435
+ b.n = 0;
436
+ if (this._nextId > 0) {
437
+ const scan = this.sqlite!.prepare("SELECT h FROM node");
438
+ scan.setReturnArrays(true);
439
+ for (const r of scan.iterate() as IterableIterator<[number]>) {
440
+ // add() counts n itself.
441
+ b.add(r[0]);
442
+ }
443
+ }
444
+ this._bloom = b;
445
+ }
446
+
447
+ /** Persist the filter with its coverage watermark. Runs inside the
448
+ * caller's transaction when one is open (checkpoint/close both flush
449
+ * first). */
450
+ private _bloomPersist(): void {
451
+ if (!this._bloom || !this.sqlite) return;
452
+ if (this._nextId < BLOOM_PERSIST_MIN_NODES) return;
453
+ this.sqlite.prepare(
454
+ "INSERT INTO bloom (id, bits, n, upto) VALUES (1, ?, ?, ?) " +
455
+ "ON CONFLICT(id) DO UPDATE SET bits = excluded.bits, " +
456
+ "n = excluded.n, upto = excluded.upto",
457
+ ).run(this._bloom.bits, this._bloom.n, this._nextId);
293
458
  }
294
459
 
295
460
  protected _dbClose(): void {
461
+ if (this.sqlite) this._bloomPersist();
296
462
  if (this.content) {
297
463
  this.content.close();
298
464
  this.content = null;
@@ -335,6 +501,11 @@ export class SQliteStore extends AbstractStore implements Store {
335
501
  );
336
502
  }
337
503
  this._insertNode.run(id, leaf, kids, h);
504
+ // The filter must see every inserted hash BEFORE any probe could — the
505
+ // add sits in the same synchronous call as the row insert.
506
+ const b = this._bloom!;
507
+ b.add(h);
508
+ if (b.saturated) this._bloomRebuild();
338
509
  }
339
510
 
340
511
  protected _dbGetNode(id: NodeId): NodeRec | null {
@@ -363,6 +534,10 @@ export class SQliteStore extends AbstractStore implements Store {
363
534
  h: number,
364
535
  bytes: Uint8Array,
365
536
  ): NodeId | null {
537
+ if (this._bloom && !this._bloom.mightContain(h)) {
538
+ this.bloomSkips++;
539
+ return null;
540
+ }
366
541
  if (!this._selLeaf) {
367
542
  this._selLeaf = this.sqlite!.prepare(
368
543
  "SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NULL LIMIT 1",
@@ -376,6 +551,10 @@ export class SQliteStore extends AbstractStore implements Store {
376
551
  h: number,
377
552
  bytes: Uint8Array,
378
553
  ): NodeId | null {
554
+ if (this._bloom && !this._bloom.mightContain(h)) {
555
+ this.bloomSkips++;
556
+ return null;
557
+ }
379
558
  if (!this._selFlat) {
380
559
  this._selFlat = this.sqlite!.prepare(
381
560
  "SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NOT NULL LIMIT 1",
@@ -389,6 +568,10 @@ export class SQliteStore extends AbstractStore implements Store {
389
568
  h: number,
390
569
  packed: Uint8Array,
391
570
  ): NodeId | null {
571
+ if (this._bloom && !this._bloom.mightContain(h)) {
572
+ this.bloomSkips++;
573
+ return null;
574
+ }
392
575
  if (!this._selKids) {
393
576
  this._selKids = this.sqlite!.prepare(
394
577
  "SELECT id FROM node WHERE h = ? AND kids = ? LIMIT 1",
@@ -817,6 +1000,10 @@ export class SQliteStore extends AbstractStore implements Store {
817
1000
  );
818
1001
  }
819
1002
  this._insSnapshot.run(bytes);
1003
+ // Checkpoint cadence: persisting the dedup filter here keeps the crash
1004
+ // top-up (the id ≥ upto rescan at next open) bounded by one checkpoint
1005
+ // interval instead of the whole session.
1006
+ this._bloomPersist();
820
1007
  }
821
1008
 
822
1009
  protected _dbLoadSnapshot(): Uint8Array | null {
@@ -832,7 +1019,7 @@ export class SQliteStore extends AbstractStore implements Store {
832
1019
  // -- Vector DB: content (gist) index --
833
1020
 
834
1021
  protected _vecContentUpsert(
835
- entries: Array<{ id: NodeId; vector: Float32Array; ef?: number }>,
1022
+ entries: Array<{ id: NodeId; vector: Float32Array }>,
836
1023
  ): void {
837
1024
  this.content!.upsertMany(entries);
838
1025
  }
@@ -861,6 +1048,10 @@ export class SQliteStore extends AbstractStore implements Store {
861
1048
  return this.content ? this.content.physicalSize : 0;
862
1049
  }
863
1050
 
1051
+ protected _vecContentClusterCount(): number {
1052
+ return this.content ? this.content.clusterCount : 0;
1053
+ }
1054
+
864
1055
  protected _vecContentCompact(): void {
865
1056
  this.content!.compact();
866
1057
  }
@@ -912,6 +1103,10 @@ export class SQliteStore extends AbstractStore implements Store {
912
1103
  return this.halos ? this.halos.physicalSize : 0;
913
1104
  }
914
1105
 
1106
+ protected _vecHaloClusterCount(): number {
1107
+ return this.halos ? this.halos.clusterCount : 0;
1108
+ }
1109
+
915
1110
  protected _vecHaloCompact(): void {
916
1111
  this.halos!.compact();
917
1112
  }
package/src/store.ts CHANGED
@@ -710,7 +710,7 @@ export abstract class AbstractStore implements Store {
710
710
 
711
711
  // -- Vector DB: content (gist) index --
712
712
  protected abstract _vecContentUpsert(
713
- entries: Array<{ id: NodeId; vector: Float32Array; ef?: number }>,
713
+ entries: Array<{ id: NodeId; vector: Float32Array }>,
714
714
  ): void;
715
715
  protected abstract _vecContentQuery(
716
716
  v: Float32Array,
@@ -724,6 +724,7 @@ export abstract class AbstractStore implements Store {
724
724
  protected abstract _vecContentSize(): number;
725
725
  protected abstract _vecContentLastReads(): number;
726
726
  protected abstract _vecContentPhysicalSize(): number;
727
+ protected abstract _vecContentClusterCount(): number;
727
728
  protected abstract _vecContentCompact(): void;
728
729
  /** Live content-index entries whose INTERNAL id is > `after`, as
729
730
  * {ext, internal} pairs in internal-id order. Internal ids are monotone
@@ -759,17 +760,30 @@ export abstract class AbstractStore implements Store {
759
760
  * the tombstone-ratio compaction trigger compares physical size against. */
760
761
  protected abstract _vecHaloSize(): number;
761
762
  protected abstract _vecHaloPhysicalSize(): number;
763
+ protected abstract _vecHaloClusterCount(): number;
762
764
  protected abstract _vecHaloCompact(): void;
763
765
 
766
+ /** Derived query breadth for a partitioned index of C clusters: probe √C
767
+ * of them (the same √-of-the-population convention as the hub bound √N).
768
+ * The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
769
+ * ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
770
+ * collection outgrows it: at 4,270 clusters the default 64 probed 16
771
+ * clusters (0.4%), and an exact stored match of a query routinely sat in
772
+ * an unprobed cluster — recall silently degraded as the store grew. The
773
+ * configured efSearch remains the floor for small collections. */
774
+ protected efFor(clusterCount: number): number {
775
+ return Math.max(
776
+ this.efSearch,
777
+ 4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))),
778
+ );
779
+ }
780
+
764
781
  // ── Config ─────────────────────────────────────────────────────────────
765
782
 
766
783
  protected _D: number;
767
784
  protected _maxGroup: number;
768
785
  protected readonly minHaloMass: number;
769
786
  protected readonly efSearch: number;
770
- protected readonly m: number;
771
- protected readonly efConstruction: number;
772
- protected readonly efConstructionInterior: number;
773
787
  protected readonly overfetch: number;
774
788
  protected readonly batchSize: number;
775
789
  protected readonly compactEveryNWrites: number;
@@ -829,7 +843,7 @@ export abstract class AbstractStore implements Store {
829
843
  protected _coveredIds!: BoundedMap<NodeId, true>;
830
844
  /** Live content-index id set, LRU-bounded so a massive ingest never leaks
831
845
  * memory; an evicted entry is still indexed (the row is durable), so the
832
- * only cost of an eviction is a duplicate HNSW probe on next visit. */
846
+ * only cost of an eviction is a duplicate index probe on next visit. */
833
847
  protected _indexedIds!: BoundedMap<NodeId, true>;
834
848
 
835
849
  // ── ANN read cache ─────────────────────────────────────────────────────
@@ -846,12 +860,8 @@ export abstract class AbstractStore implements Store {
846
860
 
847
861
  // ── Write buffers ──────────────────────────────────────────────────────
848
862
 
849
- /** Content (gist) index write buffer. `ef` is the per-entry HNSW
850
- * construction budget: reach-only interiors carry the reduced
851
- * `efConstructionInterior`; dedup targets omit it (full budget). */
852
- protected _contentBuffer: Array<
853
- { id: NodeId; vector: Float32Array; ef?: number }
854
- > = [];
863
+ /** Content (gist) index write buffer. */
864
+ protected _contentBuffer: Array<{ id: NodeId; vector: Float32Array }> = [];
855
865
  /** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
856
866
  protected _haloBuffer = new Map<NodeId, Float32Array>();
857
867
  /** Containment write buffer: child → new parents, merged on flush cadence. */
@@ -889,12 +899,6 @@ export abstract class AbstractStore implements Store {
889
899
  this._maxGroup = maxGroup;
890
900
  this.minHaloMass = config.minHaloMass;
891
901
  this.efSearch = config.efSearch;
892
- this.m = config.m;
893
- this.efConstruction = config.efConstruction;
894
- this.efConstructionInterior = Math.max(
895
- 1,
896
- Math.min(config.efConstructionInterior, config.efConstruction),
897
- );
898
902
  this.overfetch = config.overfetch;
899
903
  this.batchSize = config.batchSize;
900
904
  this.compactEveryNWrites = config.compactEveryNWrites;
@@ -1339,7 +1343,7 @@ export abstract class AbstractStore implements Store {
1339
1343
  // leaf, and near-merging distinct leaves only corrupts bytes for no real
1340
1344
  // saving. Real near-dedup compression lives in subtree (branch) fusion.
1341
1345
  //
1342
- // There is deliberately NO HNSW probe of the FLUSHED index here. It used
1346
+ // There is deliberately NO ANN probe of the FLUSHED index here. It used
1343
1347
  // to fire for EVERY new branch that the buffer scan didn't settle — i.e.
1344
1348
  // ~every interior branch, since interiors are never dedup targets —
1345
1349
  // making one ANN query per branch the dominant training cost (it dwarfed
@@ -1447,7 +1451,7 @@ export abstract class AbstractStore implements Store {
1447
1451
  // Capture the gist; it is pushed into the content index lazily, the first
1448
1452
  // time this node becomes a resonance target (link / pourHalo). A node that
1449
1453
  // never does (a pure intermediate DAG node — ~99.5% of them) is never
1450
- // indexed: it costs one persistence row, no HNSW slot and no merge probe.
1454
+ // indexed: it costs one persistence row, no vector-index slot and no merge probe.
1451
1455
  this._pendingGist.set(id, normalize(copy(gist)));
1452
1456
  await this.maybeFlush();
1453
1457
  return id;
@@ -1527,14 +1531,7 @@ export abstract class AbstractStore implements Store {
1527
1531
  return;
1528
1532
  }
1529
1533
  this._indexedIds.set(id, true);
1530
- // Reach-only interiors build with the reduced construction budget — the
1531
- // one deliberate speed-for-quality trade of ingestion (see
1532
- // {@link StoreConfig.efConstructionInterior}); targets keep the full one.
1533
- this._contentBuffer.push(
1534
- dedupTarget
1535
- ? { id, vector: v }
1536
- : { id, vector: v, ef: this.efConstructionInterior },
1537
- );
1534
+ this._contentBuffer.push({ id, vector: v });
1538
1535
  this._bufferedIds.add(id);
1539
1536
  // A node indexed AS a dedup target enters the candidate set immediately.
1540
1537
  if (dedupTarget) this._nearDedupBuf.set(id, v);
@@ -1635,7 +1632,7 @@ export abstract class AbstractStore implements Store {
1635
1632
  const results = this._vecContentQuery(
1636
1633
  normalize(copy(v)),
1637
1634
  k * this.overfetch,
1638
- this.efSearch,
1635
+ this.efFor(this._vecContentClusterCount()),
1639
1636
  );
1640
1637
  const out: Hit[] = [];
1641
1638
  for (const r of results) {
@@ -1840,11 +1837,7 @@ export abstract class AbstractStore implements Store {
1840
1837
  // Index it — same code path as indexGist, but the vector is injected
1841
1838
  // directly rather than read from the (empty) pending-gist cache.
1842
1839
  this._indexedIds.set(id, true);
1843
- this._contentBuffer.push({
1844
- id,
1845
- vector: gist,
1846
- ef: this.efConstructionInterior,
1847
- });
1840
+ this._contentBuffer.push({ id, vector: gist });
1848
1841
  this._bufferedIds.add(id);
1849
1842
  // Repaired nodes are reach-indexed, never dedup targets: their gist is
1850
1843
  // regenerated (may differ numerically from the original) and they are
@@ -1990,7 +1983,7 @@ export abstract class AbstractStore implements Store {
1990
1983
  const results = this._vecHaloQuery(
1991
1984
  normalize(copy(v)),
1992
1985
  k * this.overfetch,
1993
- this.efSearch,
1986
+ this.efFor(this._vecHaloClusterCount()),
1994
1987
  );
1995
1988
  const out: Hit[] = [];
1996
1989
  for (const r of results) {
@@ -277,7 +277,7 @@ test("a node resonates to itself once it is a resonance TARGET", async () => {
277
277
  // never-linked node is intentionally NOT indexed (it is reached only by the
278
278
  // structural DAG climb, never by direct resonance), so it costs no index slot.
279
279
  // This is the store's core compression: the exploded intermediate DAG (~99.5%
280
- // of nodes) is never poured into the HNSW. So before any edge/halo, resonate
280
+ // of nodes) is never poured into the vector index. So before any edge/halo, resonate
281
281
  // finds nothing.
282
282
  assert.equal(
283
283
  (await s.resonate(v, 1)).length,
@@ -809,7 +809,7 @@ test("compression: on-disk footprint is the three files, bounded in input", asyn
809
809
  // STORAGE-AMPLIFICATION GUARD (the regression this whole change fixes). Folding
810
810
  // text into a Merkle DAG explodes it into ~0.2 nodes/byte, most of which are
811
811
  // intermediate branches that recall reaches by CLIMBING the structural graph,
812
- // never by direct resonance. Indexing every one into the content HNSW made the
812
+ // never by direct resonance. Indexing every one into the content index made the
813
813
  // on-disk store ~50× the learned content and training crawl at a few KB/s. The
814
814
  // fix indexes a form's gist only when it becomes a resonance target AND it is
815
815
  // not already covered by the content index — the index adds a
@@ -868,7 +868,7 @@ test("compression: post-hoc compaction bounds content-index footprint", async ()
868
868
  // only nodes that bridge multiple experiences (2+ parents) or are
869
869
  // experience roots (1 parent + edges). This is a small minority of
870
870
  // the exploded DAG. Vector-similarity gating previously tried to achieve this during
871
- // training via expensive HNSW probes, but was rigid, dimensionally
871
+ // training via expensive index probes, but was rigid, dimensionally
872
872
  // fragile, and a no-op at typical D. Post-hoc structural compaction
873
873
  // is adaptive (the data decides what's shared) and doesn't slow training.
874
874
  assert.ok(