@hviana/sema 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CITATION.cff +49 -0
- package/HOW_IT_WORKS.md +11 -12
- package/README.md +7 -5
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +19 -21
- package/dist/example/train_base.js +1700 -1952
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +24 -27
- package/dist/src/alu/src/alu.d.ts +169 -194
- package/dist/src/alu/src/alu.js +374 -414
- package/dist/src/alu/src/expr.d.ts +36 -43
- package/dist/src/alu/src/expr.js +252 -278
- package/dist/src/alu/src/index.d.ts +9 -98
- package/dist/src/alu/src/index.js +10 -64
- package/dist/src/alu/src/kernel-arith.d.ts +1 -5
- package/dist/src/alu/src/kernel-arith.js +195 -285
- package/dist/src/alu/src/kernel-bits.d.ts +1 -5
- package/dist/src/alu/src/kernel-bits.js +81 -115
- package/dist/src/alu/src/kernel-logic.js +33 -63
- package/dist/src/alu/src/kernel-nd.js +155 -195
- package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
- package/dist/src/alu/src/kernel-numeric.js +259 -331
- package/dist/src/alu/src/operation.d.ts +112 -132
- package/dist/src/alu/src/operation.js +149 -163
- package/dist/src/alu/src/parser.d.ts +168 -175
- package/dist/src/alu/src/parser.js +382 -421
- package/dist/src/alu/src/resonance.d.ts +16 -25
- package/dist/src/alu/src/resonance.js +49 -56
- package/dist/src/alu/src/text.d.ts +3 -7
- package/dist/src/alu/src/text.js +31 -37
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +150 -160
- package/dist/src/alu/test/alu.test.js +543 -656
- package/dist/src/bytes.d.ts +1 -5
- package/dist/src/bytes.js +33 -40
- package/dist/src/config.d.ts +99 -102
- package/dist/src/config.js +78 -83
- package/dist/src/derive/src/deduction.d.ts +60 -62
- package/dist/src/derive/src/deduction.js +103 -109
- package/dist/src/derive/src/index.d.ts +1 -7
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +57 -61
- package/dist/src/derive/src/rewrite.d.ts +15 -18
- package/dist/src/derive/src/rewrite.js +59 -67
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +175 -179
- package/dist/src/derive/test/derive.test.js +96 -100
- package/dist/src/extension.d.ts +13 -16
- package/dist/src/geometry.d.ts +29 -43
- package/dist/src/geometry.js +324 -268
- package/dist/src/index.d.ts +2 -17
- package/dist/src/index.js +2 -12
- package/dist/src/ingest-cache.d.ts +25 -30
- package/dist/src/ingest-cache.js +108 -127
- package/dist/src/mind/articulation.d.ts +1 -5
- package/dist/src/mind/articulation.js +77 -112
- package/dist/src/mind/attention.d.ts +40 -108
- package/dist/src/mind/attention.js +754 -872
- package/dist/src/mind/canonical.d.ts +4 -19
- package/dist/src/mind/canonical.js +27 -31
- package/dist/src/mind/graph-search.d.ts +201 -225
- package/dist/src/mind/graph-search.js +742 -821
- package/dist/src/mind/index.d.ts +2 -10
- package/dist/src/mind/junction.d.ts +31 -58
- package/dist/src/mind/junction.js +172 -237
- package/dist/src/mind/learning.d.ts +16 -52
- package/dist/src/mind/learning.js +143 -165
- package/dist/src/mind/match.d.ts +20 -69
- package/dist/src/mind/match.js +259 -318
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +9 -13
- package/dist/src/mind/mechanisms/cast.js +363 -456
- package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
- package/dist/src/mind/mechanisms/confluence.js +152 -183
- package/dist/src/mind/mechanisms/cover.d.ts +2 -8
- package/dist/src/mind/mechanisms/cover.js +148 -210
- package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
- package/dist/src/mind/mechanisms/extraction.js +234 -288
- package/dist/src/mind/mechanisms/recall.d.ts +6 -10
- package/dist/src/mind/mechanisms/recall.js +141 -185
- package/dist/src/mind/mind.d.ts +194 -155
- package/dist/src/mind/mind.js +392 -289
- package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
- package/dist/src/mind/pipeline-mechanism.js +161 -172
- package/dist/src/mind/pipeline.d.ts +4 -14
- package/dist/src/mind/pipeline.js +125 -189
- package/dist/src/mind/primitives.d.ts +20 -33
- package/dist/src/mind/primitives.js +124 -120
- package/dist/src/mind/rationale.d.ts +86 -98
- package/dist/src/mind/rationale.js +113 -121
- package/dist/src/mind/reasoning.d.ts +2 -13
- package/dist/src/mind/reasoning.js +129 -166
- package/dist/src/mind/recognition.d.ts +1 -4
- package/dist/src/mind/recognition.js +198 -210
- package/dist/src/mind/resonance.d.ts +5 -22
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +6 -25
- package/dist/src/mind/trace.js +50 -58
- package/dist/src/mind/traverse.d.ts +29 -58
- package/dist/src/mind/traverse.js +344 -356
- package/dist/src/mind/types.d.ts +122 -127
- package/dist/src/mind/types.js +60 -69
- package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
- package/dist/src/rabitq-ivf/src/database.js +201 -0
- package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
- package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
- package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
- package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
- package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
- package/dist/src/rabitq-ivf/src/prng.js +36 -0
- package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
- package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
- package/dist/src/sema.d.ts +9 -13
- package/dist/src/sema.js +26 -40
- package/dist/src/store-sqlite.d.ts +171 -166
- package/dist/src/store-sqlite.js +768 -662
- package/dist/src/store.d.ts +597 -630
- package/dist/src/store.js +1434 -1581
- package/dist/src/vec.d.ts +5 -9
- package/dist/src/vec.js +61 -73
- package/example/train_base.ts +33 -15
- package/index.html +65 -0
- package/package.json +1 -1
- package/src/alu/README.md +1 -1
- package/src/alu/src/index.ts +1 -1
- package/src/config.ts +19 -27
- package/src/geometry.ts +93 -1
- package/src/index.ts +6 -11
- package/src/mind/attention.ts +11 -6
- package/src/mind/mechanisms/recall.ts +29 -13
- package/src/mind/mind.ts +250 -19
- package/src/mind/primitives.ts +43 -6
- package/src/mind/recognition.ts +26 -8
- package/src/mind/traverse.ts +47 -0
- package/src/mind/types.ts +20 -21
- package/src/rabitq-ivf/README.md +56 -0
- package/src/rabitq-ivf/src/database.ts +276 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
- package/src/rabitq-ivf/src/ivf.ts +1330 -0
- package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
- package/src/store-sqlite.ts +204 -9
- package/src/store.ts +27 -34
- package/test/08-storage.test.mjs +3 -3
- package/test/13-conversation.test.mjs +190 -20
- package/test/14-scaling.test.mjs +2 -2
- package/test/35-ivf.test.mjs +263 -0
- package/test/36-bloom.test.mjs +123 -0
- package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
- package/dist/src/rabitq-hnsw/src/database.js +0 -405
- package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
- package/dist/src/rabitq-hnsw/src/heap.js +0 -94
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
- package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
- package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
- package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
- package/dist/src/rabitq-hnsw/src/prng.js +0 -38
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
- package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
- package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
- package/dist/src/rabitq-hnsw/src/store.js +0 -916
- package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
- package/src/rabitq-hnsw/README.md +0 -303
- package/src/rabitq-hnsw/src/database.ts +0 -492
- package/src/rabitq-hnsw/src/heap.ts +0 -90
- package/src/rabitq-hnsw/src/hnsw.ts +0 -514
- package/src/rabitq-hnsw/src/store.ts +0 -994
- package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
- /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Small deterministic pseudo-random number generator (mulberry32).
|
|
3
3
|
*
|
|
4
4
|
* Pure ECMAScript. It is used so that the random orthogonal rotation of the
|
|
5
|
-
* RaBitQ quantizer
|
|
5
|
+
* RaBitQ quantizer rotation is reproducible and can be
|
|
6
6
|
* regenerated after (de)serialization without storing large matrices.
|
|
7
7
|
*/
|
|
8
8
|
export class Prng {
|
package/src/store-sqlite.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// store-sqlite.ts — SQLite + rabitq-
|
|
1
|
+
// store-sqlite.ts — SQLite + rabitq-ivf persistence adapter.
|
|
2
2
|
//
|
|
3
3
|
// SQliteStore extends AbstractStore and implements only the bare essentials
|
|
4
4
|
// required for database communication: SQL schema, prepared statements, and
|
|
@@ -39,7 +39,7 @@ import {
|
|
|
39
39
|
unpackKids,
|
|
40
40
|
} from "./store.js";
|
|
41
41
|
import { DEFAULT_CONFIG, type StoreConfig } from "./config.js";
|
|
42
|
-
import { VectorDatabase } from "./rabitq-
|
|
42
|
+
import { VectorDatabase } from "./rabitq-ivf/src/index.js";
|
|
43
43
|
|
|
44
44
|
export interface SQliteStoreOptions extends Partial<StoreConfig> {
|
|
45
45
|
path?: string;
|
|
@@ -66,6 +66,21 @@ CREATE TABLE IF NOT EXISTS node (
|
|
|
66
66
|
h INTEGER NOT NULL
|
|
67
67
|
);
|
|
68
68
|
CREATE INDEX IF NOT EXISTS idx_node_h ON node(h);
|
|
69
|
+
-- Negative-lookup BLOOM FILTER over node.h — the RAM structure that lets the
|
|
70
|
+
-- content-addressed dedup probes (findLeaf/findBranch, the training hot path)
|
|
71
|
+
-- answer "definitely new content" without an idx_node_h descent. Persisted
|
|
72
|
+
-- here at checkpoint/close with upto = the next node id NOT covered; on
|
|
73
|
+
-- open, rows with id >= upto (a crash's tail) are re-added by an incremental
|
|
74
|
+
-- rowid-range scan, so the filter can NEVER be missing an inserted hash — a
|
|
75
|
+
-- false negative would silently break hash-consing (duplicate mints), while
|
|
76
|
+
-- a false positive only costs the SQLite probe that was paid on every call
|
|
77
|
+
-- before the filter existed.
|
|
78
|
+
CREATE TABLE IF NOT EXISTS bloom (
|
|
79
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
80
|
+
bits BLOB NOT NULL,
|
|
81
|
+
n INTEGER NOT NULL,
|
|
82
|
+
upto INTEGER NOT NULL
|
|
83
|
+
);
|
|
69
84
|
-- The reverse structural edge child→parent. A WITHOUT ROWID table clustered on
|
|
70
85
|
-- (child, parent) IS the lookup index: parents(child) is one B-tree descent to a
|
|
71
86
|
-- contiguous run, with no separate rowid heap and no secondary index to maintain
|
|
@@ -129,9 +144,66 @@ CREATE TABLE IF NOT EXISTS meta (
|
|
|
129
144
|
);
|
|
130
145
|
`;
|
|
131
146
|
|
|
147
|
+
/** Blocked Bloom filter over 32-bit content hashes. k=4 bit positions per
|
|
148
|
+
* key via double hashing; at the maintained ≥12 bits/key the false-positive
|
|
149
|
+
* rate stays under ~1%, so ≥99% of miss-probes skip SQLite entirely.
|
|
150
|
+
* No deletions (nodes are never deleted), so growth is a rebuild. */
|
|
151
|
+
class NodeBloom {
|
|
152
|
+
bits: Uint8Array;
|
|
153
|
+
mask: number;
|
|
154
|
+
n = 0;
|
|
155
|
+
constructor(log2bits: number) {
|
|
156
|
+
this.bits = new Uint8Array(1 << (log2bits - 3));
|
|
157
|
+
this.mask = (1 << log2bits) - 1 >>> 0;
|
|
158
|
+
}
|
|
159
|
+
add(h: number): void {
|
|
160
|
+
const h2 = (Math.imul(h, 0x9e3779b1) | 1) >>> 0;
|
|
161
|
+
let x = h >>> 0;
|
|
162
|
+
const bits = this.bits;
|
|
163
|
+
const mask = this.mask;
|
|
164
|
+
for (let i = 0; i < 4; i++) {
|
|
165
|
+
const b = x & mask;
|
|
166
|
+
bits[b >> 3] |= 1 << (b & 7);
|
|
167
|
+
x = (x + h2) >>> 0;
|
|
168
|
+
}
|
|
169
|
+
this.n++;
|
|
170
|
+
}
|
|
171
|
+
mightContain(h: number): boolean {
|
|
172
|
+
const h2 = (Math.imul(h, 0x9e3779b1) | 1) >>> 0;
|
|
173
|
+
let x = h >>> 0;
|
|
174
|
+
const bits = this.bits;
|
|
175
|
+
const mask = this.mask;
|
|
176
|
+
for (let i = 0; i < 4; i++) {
|
|
177
|
+
const b = x & mask;
|
|
178
|
+
if ((bits[b >> 3] & (1 << (b & 7))) === 0) return false;
|
|
179
|
+
x = (x + h2) >>> 0;
|
|
180
|
+
}
|
|
181
|
+
return true;
|
|
182
|
+
}
|
|
183
|
+
/** Whether the filter is due for a bigger rebuild (< 12 bits/key). */
|
|
184
|
+
get saturated(): boolean {
|
|
185
|
+
return this.n * 12 > this.bits.length * 8;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** log2 of the bloom bit count sized for `n` keys at 16 bits/key, clamped to
|
|
190
|
+
* [17, 30] — 16 KiB floor, 128 MiB ceiling (past ~90M nodes the FPR is
|
|
191
|
+
* allowed to drift up rather than RAM). */
|
|
192
|
+
function bloomLog2For(n: number): number {
|
|
193
|
+
let log2 = 17;
|
|
194
|
+
while (log2 < 30 && (1 << log2) < n * 16) log2++;
|
|
195
|
+
return log2;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Below this node count the filter is NOT persisted: a rebuild scan at open
|
|
199
|
+
* is instant at this size, and the blob would dominate a small store's
|
|
200
|
+
* on-disk footprint. */
|
|
201
|
+
const BLOOM_PERSIST_MIN_NODES = 65_536;
|
|
202
|
+
|
|
132
203
|
export class SQliteStore extends AbstractStore implements Store {
|
|
133
204
|
private readonly opts: SQliteStoreOptions;
|
|
134
205
|
private readonly vectorCacheMb: number;
|
|
206
|
+
private readonly sqliteCacheMb: number;
|
|
135
207
|
private readonly vectorSeed: number;
|
|
136
208
|
|
|
137
209
|
// ── the two SEPARATE vector indices ───────────────────────────────────
|
|
@@ -143,6 +215,17 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
143
215
|
// Deferred write transaction guard.
|
|
144
216
|
private _inTx = false;
|
|
145
217
|
|
|
218
|
+
// ── negative dedup filter ──────────────────────────────────────────────
|
|
219
|
+
/** Bloom filter over every node row's content hash `h`. The dedup probes
|
|
220
|
+
* consult it first: "definitely absent" answers the training hot path's
|
|
221
|
+
* dominant miss case in RAM (profiled: the SQLite probes were 38.7% of
|
|
222
|
+
* ingest wall at 2.9M nodes). INVARIANT: every inserted h is added
|
|
223
|
+
* before the probe that could see it — a false negative would mint a
|
|
224
|
+
* duplicate node and silently break hash-consing. */
|
|
225
|
+
private _bloom: NodeBloom | null = null;
|
|
226
|
+
/** Dedup probes answered by the filter alone this session (observability). */
|
|
227
|
+
bloomSkips = 0;
|
|
228
|
+
|
|
146
229
|
private _insertNode: any = null;
|
|
147
230
|
private _insertKid: any = null;
|
|
148
231
|
private _selContain: any = null;
|
|
@@ -167,10 +250,6 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
167
250
|
// Resolve config from opts, falling back to defaults.
|
|
168
251
|
const config: StoreConfig = {
|
|
169
252
|
minHaloMass: opts.minHaloMass ?? d.minHaloMass,
|
|
170
|
-
m: opts.m ?? d.m,
|
|
171
|
-
efConstruction: opts.efConstruction ?? d.efConstruction,
|
|
172
|
-
efConstructionInterior: opts.efConstructionInterior ??
|
|
173
|
-
d.efConstructionInterior,
|
|
174
253
|
efSearch: opts.efSearch ?? d.efSearch,
|
|
175
254
|
compactEveryNWrites: opts.compactEveryNWrites ?? d.compactEveryNWrites,
|
|
176
255
|
overfetch: opts.overfetch ?? d.overfetch,
|
|
@@ -182,6 +261,7 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
182
261
|
pendingGistBytes: opts.pendingGistBytes ?? d.pendingGistBytes,
|
|
183
262
|
haloCacheBytes: opts.haloCacheBytes ?? d.haloCacheBytes,
|
|
184
263
|
vectorCacheMb: opts.vectorCacheMb ?? d.vectorCacheMb,
|
|
264
|
+
sqliteCacheMb: opts.sqliteCacheMb ?? d.sqliteCacheMb,
|
|
185
265
|
coveredIdsMax: opts.coveredIdsMax ?? d.coveredIdsMax,
|
|
186
266
|
chainCacheBytes: opts.chainCacheBytes ?? d.chainCacheBytes,
|
|
187
267
|
};
|
|
@@ -191,6 +271,7 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
191
271
|
|
|
192
272
|
this.opts = opts;
|
|
193
273
|
this.vectorCacheMb = opts.vectorCacheMb ?? d.vectorCacheMb;
|
|
274
|
+
this.sqliteCacheMb = opts.sqliteCacheMb ?? d.sqliteCacheMb;
|
|
194
275
|
this.vectorSeed = (0x51f15e ^ 0x9e3779b9) >>> 0;
|
|
195
276
|
this._ready = this._dbOpen();
|
|
196
277
|
}
|
|
@@ -207,8 +288,6 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
207
288
|
return new VectorDatabase({
|
|
208
289
|
dbPath: this.vectorDbPath(name),
|
|
209
290
|
dim: this.D,
|
|
210
|
-
M: this.m,
|
|
211
|
-
efConstruction: this.efConstruction,
|
|
212
291
|
efSearch: this.efSearch,
|
|
213
292
|
// Query-side estimator precision. 8 bits: 4 bits measurably misranks
|
|
214
293
|
// tight gist clusters (mixture recall@10 37.5% vs 39.0%, self-recall
|
|
@@ -242,6 +321,25 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
242
321
|
// WAL + NORMAL sync: crash-safe (WAL commits are atomic) without a full
|
|
243
322
|
// fsync per commit — the same discipline the vector databases use.
|
|
244
323
|
this.sqlite.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;");
|
|
324
|
+
// SQLite's autocheckpoint default is 1000 PAGES — 1 MiB at this store's
|
|
325
|
+
// 1 KiB page size — so at training write rates nearly every batch commit
|
|
326
|
+
// crossed the threshold and paid a synchronous checkpoint: random writes
|
|
327
|
+
// into a GB-scale main file, with the hottest B-tree pages copied out on
|
|
328
|
+
// every commit instead of being coalesced across commits in the WAL.
|
|
329
|
+
// Profiled on a 7.5M-node store: ~114 ms PER COMMIT, ~19% of ingest
|
|
330
|
+
// wall-clock. 16 MiB of WAL between checkpoints coalesces hot-page
|
|
331
|
+
// rewrites and amortizes the checkpoint's fixed cost; crash-safety is
|
|
332
|
+
// unchanged (WAL recovery replays it), the only trade is a bounded,
|
|
333
|
+
// fixed-size WAL file.
|
|
334
|
+
this.sqlite.exec("PRAGMA wal_autocheckpoint = 65536;");
|
|
335
|
+
// Page cache for the DAG tables. Content-addressed dedup (findLeaf /
|
|
336
|
+
// findBranch), parent probes and contain appends are millions of point
|
|
337
|
+
// queries per session against a file that outgrows SQLite's ~2 MiB
|
|
338
|
+
// default cache almost immediately; every miss is a file read on the
|
|
339
|
+
// ingest hot path. Negative = KiB budget (size-based, page-size
|
|
340
|
+
// independent). Latency only — results identical at any value.
|
|
341
|
+
const cacheKb = Math.max(0, Math.floor(this.sqliteCacheMb * 1024));
|
|
342
|
+
if (cacheKb > 0) this.sqlite.exec(`PRAGMA cache_size = -${cacheKb};`);
|
|
245
343
|
this.sqlite.exec(SCHEMA);
|
|
246
344
|
|
|
247
345
|
// Recover D from a previous run so loadFromStore() can bootstrap with any
|
|
@@ -290,9 +388,77 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
290
388
|
this._nextId = ((this.sqlite.prepare(
|
|
291
389
|
"SELECT MAX(id) AS m FROM node",
|
|
292
390
|
).get() as { m: number | null }).m ?? -1) + 1;
|
|
391
|
+
|
|
392
|
+
this._bloomLoad();
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// ── negative dedup filter: load / rebuild / persist ────────────────────
|
|
396
|
+
|
|
397
|
+
/** Restore the persisted filter and top up the rows a crash left uncovered
|
|
398
|
+
* (id ≥ stored `upto`); a missing/undersized blob forces a full rebuild.
|
|
399
|
+
* Clean-close sessions restart with a 0-row top-up. */
|
|
400
|
+
private _bloomLoad(): void {
|
|
401
|
+
const row = this.sqlite!.prepare(
|
|
402
|
+
"SELECT bits, n, upto FROM bloom WHERE id = 1",
|
|
403
|
+
).get() as { bits: Uint8Array; n: number; upto: number } | undefined;
|
|
404
|
+
const wantLog2 = bloomLog2For(this._nextId);
|
|
405
|
+
if (row && row.bits.length * 8 >= (1 << wantLog2)) {
|
|
406
|
+
const log2 = 31 - Math.clz32(row.bits.length * 8);
|
|
407
|
+
const b = new NodeBloom(log2);
|
|
408
|
+
b.bits.set(row.bits);
|
|
409
|
+
b.n = row.n;
|
|
410
|
+
this._bloom = b;
|
|
411
|
+
this._bloomTopUp(row.upto);
|
|
412
|
+
} else {
|
|
413
|
+
this._bloomRebuild();
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** Re-add the hashes of rows with id ≥ `from` (the tail a crash cut off).
|
|
418
|
+
* Re-adding a hash twice only re-sets its bits — never a correctness
|
|
419
|
+
* issue — so overlap with the persisted coverage is harmless. */
|
|
420
|
+
private _bloomTopUp(from: number): void {
|
|
421
|
+
if (from >= this._nextId) return;
|
|
422
|
+
const b = this._bloom!;
|
|
423
|
+
const scan = this.sqlite!.prepare("SELECT h FROM node WHERE id >= ?");
|
|
424
|
+
scan.setReturnArrays(true);
|
|
425
|
+
for (const r of scan.iterate(from) as IterableIterator<[number]>) {
|
|
426
|
+
b.add(r[0]);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** Build a fresh filter sized for the current node count from one
|
|
431
|
+
* sequential scan of the h column. Called at open when no usable blob is
|
|
432
|
+
* stored, and again whenever growth saturates the bits-per-key budget. */
|
|
433
|
+
private _bloomRebuild(): void {
|
|
434
|
+
const b = new NodeBloom(bloomLog2For(this._nextId));
|
|
435
|
+
b.n = 0;
|
|
436
|
+
if (this._nextId > 0) {
|
|
437
|
+
const scan = this.sqlite!.prepare("SELECT h FROM node");
|
|
438
|
+
scan.setReturnArrays(true);
|
|
439
|
+
for (const r of scan.iterate() as IterableIterator<[number]>) {
|
|
440
|
+
// add() counts n itself.
|
|
441
|
+
b.add(r[0]);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
this._bloom = b;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** Persist the filter with its coverage watermark. Runs inside the
|
|
448
|
+
* caller's transaction when one is open (checkpoint/close both flush
|
|
449
|
+
* first). */
|
|
450
|
+
private _bloomPersist(): void {
|
|
451
|
+
if (!this._bloom || !this.sqlite) return;
|
|
452
|
+
if (this._nextId < BLOOM_PERSIST_MIN_NODES) return;
|
|
453
|
+
this.sqlite.prepare(
|
|
454
|
+
"INSERT INTO bloom (id, bits, n, upto) VALUES (1, ?, ?, ?) " +
|
|
455
|
+
"ON CONFLICT(id) DO UPDATE SET bits = excluded.bits, " +
|
|
456
|
+
"n = excluded.n, upto = excluded.upto",
|
|
457
|
+
).run(this._bloom.bits, this._bloom.n, this._nextId);
|
|
293
458
|
}
|
|
294
459
|
|
|
295
460
|
protected _dbClose(): void {
|
|
461
|
+
if (this.sqlite) this._bloomPersist();
|
|
296
462
|
if (this.content) {
|
|
297
463
|
this.content.close();
|
|
298
464
|
this.content = null;
|
|
@@ -335,6 +501,11 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
335
501
|
);
|
|
336
502
|
}
|
|
337
503
|
this._insertNode.run(id, leaf, kids, h);
|
|
504
|
+
// The filter must see every inserted hash BEFORE any probe could — the
|
|
505
|
+
// add sits in the same synchronous call as the row insert.
|
|
506
|
+
const b = this._bloom!;
|
|
507
|
+
b.add(h);
|
|
508
|
+
if (b.saturated) this._bloomRebuild();
|
|
338
509
|
}
|
|
339
510
|
|
|
340
511
|
protected _dbGetNode(id: NodeId): NodeRec | null {
|
|
@@ -363,6 +534,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
363
534
|
h: number,
|
|
364
535
|
bytes: Uint8Array,
|
|
365
536
|
): NodeId | null {
|
|
537
|
+
if (this._bloom && !this._bloom.mightContain(h)) {
|
|
538
|
+
this.bloomSkips++;
|
|
539
|
+
return null;
|
|
540
|
+
}
|
|
366
541
|
if (!this._selLeaf) {
|
|
367
542
|
this._selLeaf = this.sqlite!.prepare(
|
|
368
543
|
"SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NULL LIMIT 1",
|
|
@@ -376,6 +551,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
376
551
|
h: number,
|
|
377
552
|
bytes: Uint8Array,
|
|
378
553
|
): NodeId | null {
|
|
554
|
+
if (this._bloom && !this._bloom.mightContain(h)) {
|
|
555
|
+
this.bloomSkips++;
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
379
558
|
if (!this._selFlat) {
|
|
380
559
|
this._selFlat = this.sqlite!.prepare(
|
|
381
560
|
"SELECT id FROM node WHERE h = ? AND leaf = ? AND kids IS NOT NULL LIMIT 1",
|
|
@@ -389,6 +568,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
389
568
|
h: number,
|
|
390
569
|
packed: Uint8Array,
|
|
391
570
|
): NodeId | null {
|
|
571
|
+
if (this._bloom && !this._bloom.mightContain(h)) {
|
|
572
|
+
this.bloomSkips++;
|
|
573
|
+
return null;
|
|
574
|
+
}
|
|
392
575
|
if (!this._selKids) {
|
|
393
576
|
this._selKids = this.sqlite!.prepare(
|
|
394
577
|
"SELECT id FROM node WHERE h = ? AND kids = ? LIMIT 1",
|
|
@@ -817,6 +1000,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
817
1000
|
);
|
|
818
1001
|
}
|
|
819
1002
|
this._insSnapshot.run(bytes);
|
|
1003
|
+
// Checkpoint cadence: persisting the dedup filter here keeps the crash
|
|
1004
|
+
// top-up (the id ≥ upto rescan at next open) bounded by one checkpoint
|
|
1005
|
+
// interval instead of the whole session.
|
|
1006
|
+
this._bloomPersist();
|
|
820
1007
|
}
|
|
821
1008
|
|
|
822
1009
|
protected _dbLoadSnapshot(): Uint8Array | null {
|
|
@@ -832,7 +1019,7 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
832
1019
|
// -- Vector DB: content (gist) index --
|
|
833
1020
|
|
|
834
1021
|
protected _vecContentUpsert(
|
|
835
|
-
entries: Array<{ id: NodeId; vector: Float32Array
|
|
1022
|
+
entries: Array<{ id: NodeId; vector: Float32Array }>,
|
|
836
1023
|
): void {
|
|
837
1024
|
this.content!.upsertMany(entries);
|
|
838
1025
|
}
|
|
@@ -861,6 +1048,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
861
1048
|
return this.content ? this.content.physicalSize : 0;
|
|
862
1049
|
}
|
|
863
1050
|
|
|
1051
|
+
protected _vecContentClusterCount(): number {
|
|
1052
|
+
return this.content ? this.content.clusterCount : 0;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
864
1055
|
protected _vecContentCompact(): void {
|
|
865
1056
|
this.content!.compact();
|
|
866
1057
|
}
|
|
@@ -912,6 +1103,10 @@ export class SQliteStore extends AbstractStore implements Store {
|
|
|
912
1103
|
return this.halos ? this.halos.physicalSize : 0;
|
|
913
1104
|
}
|
|
914
1105
|
|
|
1106
|
+
protected _vecHaloClusterCount(): number {
|
|
1107
|
+
return this.halos ? this.halos.clusterCount : 0;
|
|
1108
|
+
}
|
|
1109
|
+
|
|
915
1110
|
protected _vecHaloCompact(): void {
|
|
916
1111
|
this.halos!.compact();
|
|
917
1112
|
}
|
package/src/store.ts
CHANGED
|
@@ -710,7 +710,7 @@ export abstract class AbstractStore implements Store {
|
|
|
710
710
|
|
|
711
711
|
// -- Vector DB: content (gist) index --
|
|
712
712
|
protected abstract _vecContentUpsert(
|
|
713
|
-
entries: Array<{ id: NodeId; vector: Float32Array
|
|
713
|
+
entries: Array<{ id: NodeId; vector: Float32Array }>,
|
|
714
714
|
): void;
|
|
715
715
|
protected abstract _vecContentQuery(
|
|
716
716
|
v: Float32Array,
|
|
@@ -724,6 +724,7 @@ export abstract class AbstractStore implements Store {
|
|
|
724
724
|
protected abstract _vecContentSize(): number;
|
|
725
725
|
protected abstract _vecContentLastReads(): number;
|
|
726
726
|
protected abstract _vecContentPhysicalSize(): number;
|
|
727
|
+
protected abstract _vecContentClusterCount(): number;
|
|
727
728
|
protected abstract _vecContentCompact(): void;
|
|
728
729
|
/** Live content-index entries whose INTERNAL id is > `after`, as
|
|
729
730
|
* {ext, internal} pairs in internal-id order. Internal ids are monotone
|
|
@@ -759,17 +760,30 @@ export abstract class AbstractStore implements Store {
|
|
|
759
760
|
* the tombstone-ratio compaction trigger compares physical size against. */
|
|
760
761
|
protected abstract _vecHaloSize(): number;
|
|
761
762
|
protected abstract _vecHaloPhysicalSize(): number;
|
|
763
|
+
protected abstract _vecHaloClusterCount(): number;
|
|
762
764
|
protected abstract _vecHaloCompact(): void;
|
|
763
765
|
|
|
766
|
+
/** Derived query breadth for a partitioned index of C clusters: probe √C
|
|
767
|
+
* of them (the same √-of-the-population convention as the hub bound √N).
|
|
768
|
+
* The IVF maps ef → nprobe as ceil(ef/4), so ef = 4·⌈√C⌉ probes exactly
|
|
769
|
+
* ⌈√C⌉ clusters. A FIXED efSearch stops scaling the moment the
|
|
770
|
+
* collection outgrows it: at 4,270 clusters the default 64 probed 16
|
|
771
|
+
* clusters (0.4%), and an exact stored match of a query routinely sat in
|
|
772
|
+
* an unprobed cluster — recall silently degraded as the store grew. The
|
|
773
|
+
* configured efSearch remains the floor for small collections. */
|
|
774
|
+
protected efFor(clusterCount: number): number {
|
|
775
|
+
return Math.max(
|
|
776
|
+
this.efSearch,
|
|
777
|
+
4 * Math.ceil(Math.sqrt(Math.max(1, clusterCount))),
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
|
|
764
781
|
// ── Config ─────────────────────────────────────────────────────────────
|
|
765
782
|
|
|
766
783
|
protected _D: number;
|
|
767
784
|
protected _maxGroup: number;
|
|
768
785
|
protected readonly minHaloMass: number;
|
|
769
786
|
protected readonly efSearch: number;
|
|
770
|
-
protected readonly m: number;
|
|
771
|
-
protected readonly efConstruction: number;
|
|
772
|
-
protected readonly efConstructionInterior: number;
|
|
773
787
|
protected readonly overfetch: number;
|
|
774
788
|
protected readonly batchSize: number;
|
|
775
789
|
protected readonly compactEveryNWrites: number;
|
|
@@ -829,7 +843,7 @@ export abstract class AbstractStore implements Store {
|
|
|
829
843
|
protected _coveredIds!: BoundedMap<NodeId, true>;
|
|
830
844
|
/** Live content-index id set, LRU-bounded so a massive ingest never leaks
|
|
831
845
|
* memory; an evicted entry is still indexed (the row is durable), so the
|
|
832
|
-
* only cost of an eviction is a duplicate
|
|
846
|
+
* only cost of an eviction is a duplicate index probe on next visit. */
|
|
833
847
|
protected _indexedIds!: BoundedMap<NodeId, true>;
|
|
834
848
|
|
|
835
849
|
// ── ANN read cache ─────────────────────────────────────────────────────
|
|
@@ -846,12 +860,8 @@ export abstract class AbstractStore implements Store {
|
|
|
846
860
|
|
|
847
861
|
// ── Write buffers ──────────────────────────────────────────────────────
|
|
848
862
|
|
|
849
|
-
/** Content (gist) index write buffer.
|
|
850
|
-
|
|
851
|
-
* `efConstructionInterior`; dedup targets omit it (full budget). */
|
|
852
|
-
protected _contentBuffer: Array<
|
|
853
|
-
{ id: NodeId; vector: Float32Array; ef?: number }
|
|
854
|
-
> = [];
|
|
863
|
+
/** Content (gist) index write buffer. */
|
|
864
|
+
protected _contentBuffer: Array<{ id: NodeId; vector: Float32Array }> = [];
|
|
855
865
|
/** Halo index write buffer — keyed by id so repeats within a batch coalesce. */
|
|
856
866
|
protected _haloBuffer = new Map<NodeId, Float32Array>();
|
|
857
867
|
/** Containment write buffer: child → new parents, merged on flush cadence. */
|
|
@@ -889,12 +899,6 @@ export abstract class AbstractStore implements Store {
|
|
|
889
899
|
this._maxGroup = maxGroup;
|
|
890
900
|
this.minHaloMass = config.minHaloMass;
|
|
891
901
|
this.efSearch = config.efSearch;
|
|
892
|
-
this.m = config.m;
|
|
893
|
-
this.efConstruction = config.efConstruction;
|
|
894
|
-
this.efConstructionInterior = Math.max(
|
|
895
|
-
1,
|
|
896
|
-
Math.min(config.efConstructionInterior, config.efConstruction),
|
|
897
|
-
);
|
|
898
902
|
this.overfetch = config.overfetch;
|
|
899
903
|
this.batchSize = config.batchSize;
|
|
900
904
|
this.compactEveryNWrites = config.compactEveryNWrites;
|
|
@@ -1339,7 +1343,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1339
1343
|
// leaf, and near-merging distinct leaves only corrupts bytes for no real
|
|
1340
1344
|
// saving. Real near-dedup compression lives in subtree (branch) fusion.
|
|
1341
1345
|
//
|
|
1342
|
-
// There is deliberately NO
|
|
1346
|
+
// There is deliberately NO ANN probe of the FLUSHED index here. It used
|
|
1343
1347
|
// to fire for EVERY new branch that the buffer scan didn't settle — i.e.
|
|
1344
1348
|
// ~every interior branch, since interiors are never dedup targets —
|
|
1345
1349
|
// making one ANN query per branch the dominant training cost (it dwarfed
|
|
@@ -1447,7 +1451,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1447
1451
|
// Capture the gist; it is pushed into the content index lazily, the first
|
|
1448
1452
|
// time this node becomes a resonance target (link / pourHalo). A node that
|
|
1449
1453
|
// never does (a pure intermediate DAG node — ~99.5% of them) is never
|
|
1450
|
-
// indexed: it costs one persistence row, no
|
|
1454
|
+
// indexed: it costs one persistence row, no vector-index slot and no merge probe.
|
|
1451
1455
|
this._pendingGist.set(id, normalize(copy(gist)));
|
|
1452
1456
|
await this.maybeFlush();
|
|
1453
1457
|
return id;
|
|
@@ -1527,14 +1531,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1527
1531
|
return;
|
|
1528
1532
|
}
|
|
1529
1533
|
this._indexedIds.set(id, true);
|
|
1530
|
-
|
|
1531
|
-
// one deliberate speed-for-quality trade of ingestion (see
|
|
1532
|
-
// {@link StoreConfig.efConstructionInterior}); targets keep the full one.
|
|
1533
|
-
this._contentBuffer.push(
|
|
1534
|
-
dedupTarget
|
|
1535
|
-
? { id, vector: v }
|
|
1536
|
-
: { id, vector: v, ef: this.efConstructionInterior },
|
|
1537
|
-
);
|
|
1534
|
+
this._contentBuffer.push({ id, vector: v });
|
|
1538
1535
|
this._bufferedIds.add(id);
|
|
1539
1536
|
// A node indexed AS a dedup target enters the candidate set immediately.
|
|
1540
1537
|
if (dedupTarget) this._nearDedupBuf.set(id, v);
|
|
@@ -1635,7 +1632,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1635
1632
|
const results = this._vecContentQuery(
|
|
1636
1633
|
normalize(copy(v)),
|
|
1637
1634
|
k * this.overfetch,
|
|
1638
|
-
this.
|
|
1635
|
+
this.efFor(this._vecContentClusterCount()),
|
|
1639
1636
|
);
|
|
1640
1637
|
const out: Hit[] = [];
|
|
1641
1638
|
for (const r of results) {
|
|
@@ -1840,11 +1837,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1840
1837
|
// Index it — same code path as indexGist, but the vector is injected
|
|
1841
1838
|
// directly rather than read from the (empty) pending-gist cache.
|
|
1842
1839
|
this._indexedIds.set(id, true);
|
|
1843
|
-
this._contentBuffer.push({
|
|
1844
|
-
id,
|
|
1845
|
-
vector: gist,
|
|
1846
|
-
ef: this.efConstructionInterior,
|
|
1847
|
-
});
|
|
1840
|
+
this._contentBuffer.push({ id, vector: gist });
|
|
1848
1841
|
this._bufferedIds.add(id);
|
|
1849
1842
|
// Repaired nodes are reach-indexed, never dedup targets: their gist is
|
|
1850
1843
|
// regenerated (may differ numerically from the original) and they are
|
|
@@ -1990,7 +1983,7 @@ export abstract class AbstractStore implements Store {
|
|
|
1990
1983
|
const results = this._vecHaloQuery(
|
|
1991
1984
|
normalize(copy(v)),
|
|
1992
1985
|
k * this.overfetch,
|
|
1993
|
-
this.
|
|
1986
|
+
this.efFor(this._vecHaloClusterCount()),
|
|
1994
1987
|
);
|
|
1995
1988
|
const out: Hit[] = [];
|
|
1996
1989
|
for (const r of results) {
|
package/test/08-storage.test.mjs
CHANGED
|
@@ -277,7 +277,7 @@ test("a node resonates to itself once it is a resonance TARGET", async () => {
|
|
|
277
277
|
// never-linked node is intentionally NOT indexed (it is reached only by the
|
|
278
278
|
// structural DAG climb, never by direct resonance), so it costs no index slot.
|
|
279
279
|
// This is the store's core compression: the exploded intermediate DAG (~99.5%
|
|
280
|
-
// of nodes) is never poured into the
|
|
280
|
+
// of nodes) is never poured into the vector index. So before any edge/halo, resonate
|
|
281
281
|
// finds nothing.
|
|
282
282
|
assert.equal(
|
|
283
283
|
(await s.resonate(v, 1)).length,
|
|
@@ -809,7 +809,7 @@ test("compression: on-disk footprint is the three files, bounded in input", asyn
|
|
|
809
809
|
// STORAGE-AMPLIFICATION GUARD (the regression this whole change fixes). Folding
|
|
810
810
|
// text into a Merkle DAG explodes it into ~0.2 nodes/byte, most of which are
|
|
811
811
|
// intermediate branches that recall reaches by CLIMBING the structural graph,
|
|
812
|
-
// never by direct resonance. Indexing every one into the content
|
|
812
|
+
// never by direct resonance. Indexing every one into the content index made the
|
|
813
813
|
// on-disk store ~50× the learned content and training crawl at a few KB/s. The
|
|
814
814
|
// fix indexes a form's gist only when it becomes a resonance target AND it is
|
|
815
815
|
// not already covered by the content index — the index adds a
|
|
@@ -868,7 +868,7 @@ test("compression: post-hoc compaction bounds content-index footprint", async ()
|
|
|
868
868
|
// only nodes that bridge multiple experiences (2+ parents) or are
|
|
869
869
|
// experience roots (1 parent + edges). This is a small minority of
|
|
870
870
|
// the exploded DAG. Vector-similarity gating previously tried to achieve this during
|
|
871
|
-
// training via expensive
|
|
871
|
+
// training via expensive index probes, but was rigid, dimensionally
|
|
872
872
|
// fragile, and a no-op at typical D. Post-hoc structural compaction
|
|
873
873
|
// is adaptive (the data decides what's shared) and doesn't slow training.
|
|
874
874
|
assert.ok(
|