@hviana/sema 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/example/demo.js +25 -25
- package/dist/example/train_base.d.ts +21 -19
- package/dist/example/train_base.js +1952 -1684
- package/dist/src/alphabet.d.ts +3 -3
- package/dist/src/alphabet.js +27 -24
- package/dist/src/alu/src/alu.d.ts +194 -169
- package/dist/src/alu/src/alu.js +414 -374
- package/dist/src/alu/src/expr.d.ts +43 -36
- package/dist/src/alu/src/expr.js +278 -252
- package/dist/src/alu/src/index.d.ts +98 -9
- package/dist/src/alu/src/index.js +63 -9
- package/dist/src/alu/src/kernel-arith.d.ts +5 -1
- package/dist/src/alu/src/kernel-arith.js +285 -195
- package/dist/src/alu/src/kernel-bits.d.ts +5 -1
- package/dist/src/alu/src/kernel-bits.js +115 -81
- package/dist/src/alu/src/kernel-logic.js +63 -33
- package/dist/src/alu/src/kernel-nd.js +195 -155
- package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
- package/dist/src/alu/src/kernel-numeric.js +331 -259
- package/dist/src/alu/src/operation.d.ts +132 -112
- package/dist/src/alu/src/operation.js +163 -149
- package/dist/src/alu/src/parser.d.ts +175 -168
- package/dist/src/alu/src/parser.js +421 -382
- package/dist/src/alu/src/resonance.d.ts +25 -16
- package/dist/src/alu/src/resonance.js +56 -49
- package/dist/src/alu/src/text.d.ts +7 -3
- package/dist/src/alu/src/text.js +37 -31
- package/dist/src/alu/src/value.d.ts +14 -14
- package/dist/src/alu/src/value.js +160 -150
- package/dist/src/alu/test/alu.test.js +656 -543
- package/dist/src/bytes.d.ts +5 -1
- package/dist/src/bytes.js +40 -33
- package/dist/src/config.d.ts +102 -102
- package/dist/src/config.js +83 -83
- package/dist/src/derive/src/deduction.d.ts +62 -60
- package/dist/src/derive/src/deduction.js +109 -103
- package/dist/src/derive/src/index.d.ts +7 -1
- package/dist/src/derive/src/priority-queue.d.ts +8 -8
- package/dist/src/derive/src/priority-queue.js +61 -57
- package/dist/src/derive/src/rewrite.d.ts +18 -15
- package/dist/src/derive/src/rewrite.js +67 -59
- package/dist/src/derive/src/trie.d.ts +56 -56
- package/dist/src/derive/src/trie.js +179 -175
- package/dist/src/derive/test/derive.test.js +100 -96
- package/dist/src/extension.d.ts +16 -13
- package/dist/src/geometry.d.ts +42 -17
- package/dist/src/geometry.js +266 -235
- package/dist/src/index.d.ts +17 -2
- package/dist/src/index.js +7 -1
- package/dist/src/ingest-cache.d.ts +30 -25
- package/dist/src/ingest-cache.js +127 -108
- package/dist/src/mind/articulation.d.ts +5 -1
- package/dist/src/mind/articulation.js +112 -77
- package/dist/src/mind/attention.d.ts +108 -40
- package/dist/src/mind/attention.js +871 -749
- package/dist/src/mind/canonical.d.ts +19 -4
- package/dist/src/mind/canonical.js +31 -27
- package/dist/src/mind/graph-search.d.ts +225 -201
- package/dist/src/mind/graph-search.js +821 -742
- package/dist/src/mind/index.d.ts +10 -2
- package/dist/src/mind/junction.d.ts +58 -31
- package/dist/src/mind/junction.js +237 -172
- package/dist/src/mind/learning.d.ts +52 -16
- package/dist/src/mind/learning.js +165 -143
- package/dist/src/mind/match.d.ts +69 -20
- package/dist/src/mind/match.js +318 -259
- package/dist/src/mind/mechanisms/alu.js +16 -16
- package/dist/src/mind/mechanisms/cast.d.ts +13 -9
- package/dist/src/mind/mechanisms/cast.js +456 -363
- package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
- package/dist/src/mind/mechanisms/confluence.js +183 -152
- package/dist/src/mind/mechanisms/cover.d.ts +8 -2
- package/dist/src/mind/mechanisms/cover.js +210 -148
- package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
- package/dist/src/mind/mechanisms/extraction.js +288 -234
- package/dist/src/mind/mechanisms/recall.d.ts +10 -6
- package/dist/src/mind/mechanisms/recall.js +185 -126
- package/dist/src/mind/mind.d.ts +154 -129
- package/dist/src/mind/mind.js +288 -258
- package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
- package/dist/src/mind/pipeline-mechanism.js +172 -161
- package/dist/src/mind/pipeline.d.ts +14 -4
- package/dist/src/mind/pipeline.js +189 -125
- package/dist/src/mind/primitives.d.ts +32 -8
- package/dist/src/mind/primitives.js +117 -99
- package/dist/src/mind/rationale.d.ts +98 -86
- package/dist/src/mind/rationale.js +121 -113
- package/dist/src/mind/reasoning.d.ts +13 -2
- package/dist/src/mind/reasoning.js +166 -129
- package/dist/src/mind/recognition.d.ts +4 -1
- package/dist/src/mind/recognition.js +208 -183
- package/dist/src/mind/resonance.d.ts +22 -5
- package/dist/src/mind/resonance.js +0 -0
- package/dist/src/mind/trace.d.ts +25 -6
- package/dist/src/mind/trace.js +58 -50
- package/dist/src/mind/traverse.d.ts +58 -16
- package/dist/src/mind/traverse.js +357 -304
- package/dist/src/mind/types.d.ts +127 -120
- package/dist/src/mind/types.js +69 -60
- package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
- package/dist/src/rabitq-hnsw/src/database.js +359 -342
- package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
- package/dist/src/rabitq-hnsw/src/heap.js +80 -75
- package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
- package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
- package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
- package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
- package/dist/src/rabitq-hnsw/src/prng.js +27 -25
- package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
- package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
- package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
- package/dist/src/rabitq-hnsw/src/store.js +843 -752
- package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
- package/dist/src/sema.d.ts +35 -0
- package/dist/src/sema.js +77 -0
- package/dist/src/store-sqlite.d.ts +166 -144
- package/dist/src/store-sqlite.js +660 -578
- package/dist/src/store.d.ts +630 -593
- package/dist/src/store.js +1581 -1440
- package/dist/src/vec.d.ts +9 -5
- package/dist/src/vec.js +73 -61
- package/package.json +2 -2
- package/src/sema.ts +102 -0
|
@@ -2,12 +2,12 @@ import { HnswIndex } from "./hnsw.js";
|
|
|
2
2
|
import { RaBitQuantizer } from "./rabitq.js";
|
|
3
3
|
import { Store } from "./store.js";
|
|
4
4
|
const DEFAULTS = {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
M: 16,
|
|
6
|
+
efConstruction: 200,
|
|
7
|
+
efSearch: 100,
|
|
8
|
+
queryBits: 8,
|
|
9
|
+
rotationRounds: 3,
|
|
10
|
+
seed: 0x1234abcd,
|
|
11
11
|
};
|
|
12
12
|
/**
|
|
13
13
|
* A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
|
|
@@ -23,366 +23,383 @@ const DEFAULTS = {
|
|
|
23
23
|
* `insert`/`update`/`query`.
|
|
24
24
|
*/
|
|
25
25
|
export class VectorDatabase {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
let cfg = this.store.loadConfig();
|
|
38
|
-
if (cfg === null) {
|
|
39
|
-
// Fresh database: derive the configuration from the options and persist it.
|
|
40
|
-
if (!Number.isInteger(options.dim) || options.dim <= 0) {
|
|
41
|
-
throw new Error("DatabaseOptions.dim must be a positive integer for a new database");
|
|
42
|
-
}
|
|
43
|
-
const dim = options.dim;
|
|
44
|
-
const centroid = options.centroid
|
|
45
|
-
? Float64Array.from(options.centroid)
|
|
46
|
-
: null;
|
|
47
|
-
const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
|
|
48
|
-
cfg = {
|
|
49
|
-
dim,
|
|
50
|
-
m: options.M ?? DEFAULTS.M,
|
|
51
|
-
efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
|
|
52
|
-
efSearch: options.efSearch ?? DEFAULTS.efSearch,
|
|
53
|
-
queryBits: options.queryBits ?? DEFAULTS.queryBits,
|
|
54
|
-
rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
|
|
55
|
-
seed: (options.seed ?? DEFAULTS.seed) >>> 0,
|
|
56
|
-
centroid,
|
|
57
|
-
codeWords: probe.codeWords,
|
|
58
|
-
paddedDim: probe.paddedDim,
|
|
59
|
-
};
|
|
60
|
-
this.store.initConfig(cfg);
|
|
61
|
-
}
|
|
62
|
-
else {
|
|
63
|
-
// Reopened: keep the stored STRUCTURAL config, but allow tuning the two
|
|
64
|
-
// query-side knobs — efSearch and queryBits shape only how a query is
|
|
65
|
-
// executed, never what is stored, so honouring an explicit option here
|
|
66
|
-
// is safe and lets an existing database adopt better query settings.
|
|
67
|
-
if (options.efSearch !== undefined) {
|
|
68
|
-
cfg.efSearch = options.efSearch;
|
|
69
|
-
this.store.setEfSearch(options.efSearch);
|
|
70
|
-
}
|
|
71
|
-
if (options.queryBits !== undefined && options.queryBits !== cfg.queryBits) {
|
|
72
|
-
cfg.queryBits = options.queryBits;
|
|
73
|
-
this.store.setQueryBits(options.queryBits);
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
this.dim = cfg.dim;
|
|
77
|
-
this.quantizer = new RaBitQuantizer(cfg.dim, {
|
|
78
|
-
queryBits: cfg.queryBits,
|
|
79
|
-
rounds: cfg.rotationRounds,
|
|
80
|
-
seed: cfg.seed,
|
|
81
|
-
centroid: cfg.centroid ?? undefined,
|
|
82
|
-
});
|
|
83
|
-
if (this.quantizer.codeWords !== cfg.codeWords) {
|
|
84
|
-
throw new Error("Stored code geometry does not match the quantizer (corrupt database?)");
|
|
85
|
-
}
|
|
86
|
-
this.codeWords = this.quantizer.codeWords;
|
|
87
|
-
this.index = new HnswIndex(this.quantizer, this.store, {
|
|
88
|
-
M: cfg.m,
|
|
89
|
-
efConstruction: cfg.efConstruction,
|
|
90
|
-
efSearch: cfg.efSearch,
|
|
91
|
-
seed: cfg.seed,
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
/** Number of live (non-deleted) vectors. */
|
|
95
|
-
get size() {
|
|
96
|
-
return this.index.size;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Physical node count including tombstones from deletes/updates. When it grows
|
|
100
|
-
* well beyond `size`, call `compact()` to reclaim the space on disk.
|
|
101
|
-
*/
|
|
102
|
-
get physicalSize() {
|
|
103
|
-
return this.index.physicalSize;
|
|
104
|
-
}
|
|
105
|
-
get efSearch() {
|
|
106
|
-
return this.index.efSearch;
|
|
26
|
+
dim;
|
|
27
|
+
store;
|
|
28
|
+
quantizer;
|
|
29
|
+
index;
|
|
30
|
+
codeWords;
|
|
31
|
+
constructor(options) {
|
|
32
|
+
if (
|
|
33
|
+
!options || typeof options.dbPath !== "string" ||
|
|
34
|
+
options.dbPath.length === 0
|
|
35
|
+
) {
|
|
36
|
+
throw new Error("DatabaseOptions.dbPath (string) is required");
|
|
107
37
|
}
|
|
108
|
-
|
|
109
|
-
|
|
38
|
+
this.store = new Store(options.dbPath, options.cacheSizeMb ?? 64);
|
|
39
|
+
let cfg = this.store.loadConfig();
|
|
40
|
+
if (cfg === null) {
|
|
41
|
+
// Fresh database: derive the configuration from the options and persist it.
|
|
42
|
+
if (!Number.isInteger(options.dim) || options.dim <= 0) {
|
|
43
|
+
throw new Error(
|
|
44
|
+
"DatabaseOptions.dim must be a positive integer for a new database",
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
const dim = options.dim;
|
|
48
|
+
const centroid = options.centroid
|
|
49
|
+
? Float64Array.from(options.centroid)
|
|
50
|
+
: null;
|
|
51
|
+
const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
|
|
52
|
+
cfg = {
|
|
53
|
+
dim,
|
|
54
|
+
m: options.M ?? DEFAULTS.M,
|
|
55
|
+
efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
|
|
56
|
+
efSearch: options.efSearch ?? DEFAULTS.efSearch,
|
|
57
|
+
queryBits: options.queryBits ?? DEFAULTS.queryBits,
|
|
58
|
+
rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
|
|
59
|
+
seed: (options.seed ?? DEFAULTS.seed) >>> 0,
|
|
60
|
+
centroid,
|
|
61
|
+
codeWords: probe.codeWords,
|
|
62
|
+
paddedDim: probe.paddedDim,
|
|
63
|
+
};
|
|
64
|
+
this.store.initConfig(cfg);
|
|
65
|
+
} else {
|
|
66
|
+
// Reopened: keep the stored STRUCTURAL config, but allow tuning the two
|
|
67
|
+
// query-side knobs — efSearch and queryBits shape only how a query is
|
|
68
|
+
// executed, never what is stored, so honouring an explicit option here
|
|
69
|
+
// is safe and lets an existing database adopt better query settings.
|
|
70
|
+
if (options.efSearch !== undefined) {
|
|
71
|
+
cfg.efSearch = options.efSearch;
|
|
72
|
+
this.store.setEfSearch(options.efSearch);
|
|
73
|
+
}
|
|
74
|
+
if (
|
|
75
|
+
options.queryBits !== undefined && options.queryBits !== cfg.queryBits
|
|
76
|
+
) {
|
|
77
|
+
cfg.queryBits = options.queryBits;
|
|
78
|
+
this.store.setQueryBits(options.queryBits);
|
|
79
|
+
}
|
|
110
80
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
81
|
+
this.dim = cfg.dim;
|
|
82
|
+
this.quantizer = new RaBitQuantizer(cfg.dim, {
|
|
83
|
+
queryBits: cfg.queryBits,
|
|
84
|
+
rounds: cfg.rotationRounds,
|
|
85
|
+
seed: cfg.seed,
|
|
86
|
+
centroid: cfg.centroid ?? undefined,
|
|
87
|
+
});
|
|
88
|
+
if (this.quantizer.codeWords !== cfg.codeWords) {
|
|
89
|
+
throw new Error(
|
|
90
|
+
"Stored code geometry does not match the quantizer (corrupt database?)",
|
|
91
|
+
);
|
|
114
92
|
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
93
|
+
this.codeWords = this.quantizer.codeWords;
|
|
94
|
+
this.index = new HnswIndex(this.quantizer, this.store, {
|
|
95
|
+
M: cfg.m,
|
|
96
|
+
efConstruction: cfg.efConstruction,
|
|
97
|
+
efSearch: cfg.efSearch,
|
|
98
|
+
seed: cfg.seed,
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
/** Number of live (non-deleted) vectors. */
|
|
102
|
+
get size() {
|
|
103
|
+
return this.index.size;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Physical node count including tombstones from deletes/updates. When it grows
|
|
107
|
+
* well beyond `size`, call `compact()` to reclaim the space on disk.
|
|
108
|
+
*/
|
|
109
|
+
get physicalSize() {
|
|
110
|
+
return this.index.physicalSize;
|
|
111
|
+
}
|
|
112
|
+
get efSearch() {
|
|
113
|
+
return this.index.efSearch;
|
|
114
|
+
}
|
|
115
|
+
set efSearch(value) {
|
|
116
|
+
this.index.efSearch = value;
|
|
117
|
+
}
|
|
118
|
+
/** Distance computations performed during the most recent query. */
|
|
119
|
+
get lastQueryDistanceComputations() {
|
|
120
|
+
return this.index.lastQueryDistComps;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Storage row reads issued by the most recent query. This is the honest,
|
|
124
|
+
* cache-independent scalability metric: it counts every node/neighbour fetch
|
|
125
|
+
* that hit the database, so disabling the cache cannot hide a bad access pattern.
|
|
126
|
+
*/
|
|
127
|
+
get lastQueryStorageReads() {
|
|
128
|
+
return this.index.lastQueryStorageReads;
|
|
129
|
+
}
|
|
130
|
+
/** Per-vector storage cost of the index versus a Float32 baseline. */
|
|
131
|
+
get storage() {
|
|
132
|
+
const f32 = this.dim * 4;
|
|
133
|
+
const bpv = this.index.bytesPerVector;
|
|
134
|
+
return {
|
|
135
|
+
float32BytesPerVector: f32,
|
|
136
|
+
codeBytesPerVector: bpv,
|
|
137
|
+
bytesPerVector: bpv,
|
|
138
|
+
compressionRatio: f32 / bpv,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
has(id) {
|
|
142
|
+
return this.store.idByExt(this.checkId(id)) !== null;
|
|
143
|
+
}
|
|
144
|
+
/** Stream every live external id (bounded memory). */
|
|
145
|
+
*keys() {
|
|
146
|
+
yield* this.store.liveExts();
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Stream live entries whose INTERNAL id is > `after`, as
|
|
150
|
+
* {ext, internal} pairs in internal-id order. Internal ids are assigned
|
|
151
|
+
* monotonically at insert and preserved by {@link compact}, so the largest
|
|
152
|
+
* internal id a caller has seen is a durable incremental watermark: a later
|
|
153
|
+
* call with it yields exactly the entries added since.
|
|
154
|
+
*/
|
|
155
|
+
*keysSince(after) {
|
|
156
|
+
yield* this.store.liveExtsSince(after);
|
|
157
|
+
}
|
|
158
|
+
checkId(id) {
|
|
159
|
+
if (!Number.isInteger(id)) {
|
|
160
|
+
throw new Error(`External id must be an integer, got ${id}`);
|
|
122
161
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
162
|
+
return id;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Convert a value to code bytes, selecting by length:
|
|
166
|
+
* - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
|
|
167
|
+
* - otherwise -> a raw `dim`-vector, encoded first
|
|
168
|
+
* `codeWords < dim` for any dim >= 2, so the two never collide.
|
|
169
|
+
*/
|
|
170
|
+
toCodeBytes(value) {
|
|
171
|
+
if (value.length === this.codeWords) {
|
|
172
|
+
return this.quantizer.codeToBytes(value);
|
|
133
173
|
}
|
|
134
|
-
|
|
135
|
-
|
|
174
|
+
if (value.length !== this.dim) {
|
|
175
|
+
throw new Error(
|
|
176
|
+
`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`,
|
|
177
|
+
);
|
|
136
178
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
179
|
+
return this.quantizer.codeToBytes(this.quantizer.encode(value));
|
|
180
|
+
}
|
|
181
|
+
// ------------------------------- CRUD ------------------------------------
|
|
182
|
+
/**
|
|
183
|
+
* Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
|
|
184
|
+
* detected by length. Throws if the id already exists (use `update`/`upsert`).
|
|
185
|
+
*/
|
|
186
|
+
insert(id, value) {
|
|
187
|
+
this.store.begin();
|
|
188
|
+
try {
|
|
189
|
+
this.insertCore(id, value);
|
|
190
|
+
this.store.commit();
|
|
191
|
+
} catch (e) {
|
|
192
|
+
this.store.rollback();
|
|
193
|
+
throw e;
|
|
140
194
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
195
|
+
}
|
|
196
|
+
/** Insert with the caller owning the transaction (used by {@link upsertMany}).
|
|
197
|
+
* `ef` optionally narrows the construction beam for this one vector (a
|
|
198
|
+
* caller-declared cheap entry, e.g. a reach-only interior); omitted means
|
|
199
|
+
* the index's configured efConstruction. */
|
|
200
|
+
insertCore(id, value, ef) {
|
|
201
|
+
this.checkId(id);
|
|
202
|
+
if (this.store.idByExt(id) !== null) {
|
|
203
|
+
throw new Error(`External id already exists: ${id} (use update())`);
|
|
150
204
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
* Convert a value to code bytes, selecting by length:
|
|
159
|
-
* - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
|
|
160
|
-
* - otherwise -> a raw `dim`-vector, encoded first
|
|
161
|
-
* `codeWords < dim` for any dim >= 2, so the two never collide.
|
|
162
|
-
*/
|
|
163
|
-
toCodeBytes(value) {
|
|
164
|
-
if (value.length === this.codeWords) {
|
|
165
|
-
return this.quantizer.codeToBytes(value);
|
|
166
|
-
}
|
|
167
|
-
if (value.length !== this.dim) {
|
|
168
|
-
throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
|
|
169
|
-
}
|
|
170
|
-
return this.quantizer.codeToBytes(this.quantizer.encode(value));
|
|
171
|
-
}
|
|
172
|
-
// ------------------------------- CRUD ------------------------------------
|
|
173
|
-
/**
|
|
174
|
-
* Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
|
|
175
|
-
* detected by length. Throws if the id already exists (use `update`/`upsert`).
|
|
176
|
-
*/
|
|
177
|
-
insert(id, value) {
|
|
178
|
-
this.store.begin();
|
|
179
|
-
try {
|
|
180
|
-
this.insertCore(id, value);
|
|
181
|
-
this.store.commit();
|
|
182
|
-
}
|
|
183
|
-
catch (e) {
|
|
184
|
-
this.store.rollback();
|
|
185
|
-
throw e;
|
|
186
|
-
}
|
|
205
|
+
this.index.insert(id, this.toCodeBytes(value), ef);
|
|
206
|
+
}
|
|
207
|
+
upsert(id, value) {
|
|
208
|
+
const nodeId = this.store.idByExt(this.checkId(id));
|
|
209
|
+
if (nodeId === null) {
|
|
210
|
+
this.insert(id, value);
|
|
211
|
+
return;
|
|
187
212
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
throw new Error(`External id already exists: ${id} (use update())`);
|
|
196
|
-
}
|
|
197
|
-
this.index.insert(id, this.toCodeBytes(value), ef);
|
|
213
|
+
this.store.begin();
|
|
214
|
+
try {
|
|
215
|
+
this.updateAt(nodeId, id, value);
|
|
216
|
+
this.store.commit();
|
|
217
|
+
} catch (e) {
|
|
218
|
+
this.store.rollback();
|
|
219
|
+
throw e;
|
|
198
220
|
}
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Upsert many vectors under ONE transaction. The HNSW build touches the store
|
|
224
|
+
* on every wired edge, so a transaction per vector is one WAL commit per vector
|
|
225
|
+
* — which dominates a bulk load on disk. Wrapping the whole batch in a single
|
|
226
|
+
* transaction coalesces those commits into one while leaving the graph and its
|
|
227
|
+
* result identical (reads on the connection still see the uncommitted rows).
|
|
228
|
+
*
|
|
229
|
+
* This is purely about commit batching; it changes nothing about the storage
|
|
230
|
+
* model. Codes still live only in SQLite and are read on demand — the
|
|
231
|
+
* cache-independent per-operation read count is unchanged — so a batched load
|
|
232
|
+
* scales exactly as the per-item path does, just with far fewer fsyncs. A throw
|
|
233
|
+
* rolls the whole batch back, so the caller treats it as the per-item path on
|
|
234
|
+
* failure.
|
|
235
|
+
*/
|
|
236
|
+
upsertMany(entries) {
|
|
237
|
+
if (entries.length === 0) {
|
|
238
|
+
return;
|
|
214
239
|
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
* model. Codes still live only in SQLite and are read on demand — the
|
|
224
|
-
* cache-independent per-operation read count is unchanged — so a batched load
|
|
225
|
-
* scales exactly as the per-item path does, just with far fewer fsyncs. A throw
|
|
226
|
-
* rolls the whole batch back, so the caller treats it as the per-item path on
|
|
227
|
-
* failure.
|
|
228
|
-
*/
|
|
229
|
-
upsertMany(entries) {
|
|
230
|
-
if (entries.length === 0)
|
|
231
|
-
return;
|
|
232
|
-
this.store.begin();
|
|
233
|
-
try {
|
|
234
|
-
for (const e of entries) {
|
|
235
|
-
const nodeId = this.store.idByExt(this.checkId(e.id));
|
|
236
|
-
if (nodeId !== null)
|
|
237
|
-
this.updateAt(nodeId, e.id, e.vector, e.ef);
|
|
238
|
-
else
|
|
239
|
-
this.insertCore(e.id, e.vector, e.ef);
|
|
240
|
-
}
|
|
241
|
-
this.store.commit();
|
|
242
|
-
}
|
|
243
|
-
catch (e) {
|
|
244
|
-
this.store.rollback();
|
|
245
|
-
throw e;
|
|
240
|
+
this.store.begin();
|
|
241
|
+
try {
|
|
242
|
+
for (const e of entries) {
|
|
243
|
+
const nodeId = this.store.idByExt(this.checkId(e.id));
|
|
244
|
+
if (nodeId !== null) {
|
|
245
|
+
this.updateAt(nodeId, e.id, e.vector, e.ef);
|
|
246
|
+
} else {
|
|
247
|
+
this.insertCore(e.id, e.vector, e.ef);
|
|
246
248
|
}
|
|
249
|
+
}
|
|
250
|
+
this.store.commit();
|
|
251
|
+
} catch (e) {
|
|
252
|
+
this.store.rollback();
|
|
253
|
+
throw e;
|
|
247
254
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
return rec ? this.quantizer.bytesToCode(rec.code) : null;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
|
|
258
|
+
* original vector is not retained; the code round-trips into insert/update/query.
|
|
259
|
+
*/
|
|
260
|
+
get(id) {
|
|
261
|
+
const nodeId = this.store.idByExt(this.checkId(id));
|
|
262
|
+
if (nodeId === null) {
|
|
263
|
+
return null;
|
|
258
264
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
265
|
+
const rec = this.store.getNode(nodeId);
|
|
266
|
+
return rec ? this.quantizer.bytesToCode(rec.code) : null;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Update the vector bound to an id (raw vector or code, detected by length).
|
|
270
|
+
* The previous node is tombstoned and a fresh one inserted. Throws if absent.
|
|
271
|
+
*/
|
|
272
|
+
update(id, value) {
|
|
273
|
+
this.store.begin();
|
|
274
|
+
try {
|
|
275
|
+
this.updateCore(id, value);
|
|
276
|
+
this.store.commit();
|
|
277
|
+
} catch (e) {
|
|
278
|
+
this.store.rollback();
|
|
279
|
+
throw e;
|
|
273
280
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
281
|
+
}
|
|
282
|
+
/** Update with the caller owning the transaction (used by {@link upsertMany}). */
|
|
283
|
+
updateCore(id, value) {
|
|
284
|
+
const oldId = this.store.idByExt(this.checkId(id));
|
|
285
|
+
if (oldId === null) {
|
|
286
|
+
throw new Error(`Unknown external id: ${id}`);
|
|
280
287
|
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
if (same)
|
|
302
|
-
return; // identical code — the update is a no-op
|
|
288
|
+
this.updateAt(oldId, id, value);
|
|
289
|
+
}
|
|
290
|
+
/** Update when the live internal node id is already known.
|
|
291
|
+
*
|
|
292
|
+
* Skips the write entirely when the new code equals the stored one — the
|
|
293
|
+
* update would tombstone the node and replay a full graph insert for a
|
|
294
|
+
* byte-identical result. This is not a corner case: content-addressed
|
|
295
|
+
* callers re-upsert unchanged vectors wholesale after a restart (the same
|
|
296
|
+
* content always encodes to the same code), and each such no-op otherwise
|
|
297
|
+
* costs a tombstone (permanent routing/disk overhead until compaction)
|
|
298
|
+
* plus an O(ef·log N) reinsert. */
|
|
299
|
+
updateAt(nodeId, ext, value, ef) {
|
|
300
|
+
const bytes = this.toCodeBytes(value);
|
|
301
|
+
const rec = this.store.getNode(nodeId);
|
|
302
|
+
if (rec !== null && rec.code.length === bytes.length) {
|
|
303
|
+
let same = true;
|
|
304
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
305
|
+
if (rec.code[i] !== bytes[i]) {
|
|
306
|
+
same = false;
|
|
307
|
+
break;
|
|
303
308
|
}
|
|
304
|
-
|
|
305
|
-
|
|
309
|
+
}
|
|
310
|
+
if (same) {
|
|
311
|
+
return; // identical code — the update is a no-op
|
|
312
|
+
}
|
|
306
313
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
for (const id of ids) {
|
|
320
|
-
const nodeId = this.store.idByExt(this.checkId(id));
|
|
321
|
-
if (nodeId !== null && this.index.remove(nodeId))
|
|
322
|
-
removed++;
|
|
323
|
-
}
|
|
324
|
-
this.store.commit();
|
|
325
|
-
}
|
|
326
|
-
catch (e) {
|
|
327
|
-
this.store.rollback();
|
|
328
|
-
throw e;
|
|
329
|
-
}
|
|
330
|
-
return removed;
|
|
314
|
+
this.index.remove(nodeId); // frees the ext id, then re-insert under it
|
|
315
|
+
this.index.insert(ext, bytes, ef);
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Delete many ids under ONE transaction — same commit-coalescing rationale
|
|
319
|
+
* as {@link upsertMany}: a tombstone per implicit transaction is one WAL
|
|
320
|
+
* commit per id, which dominates a bulk prune. Absent ids are skipped.
|
|
321
|
+
* Returns the number of vectors actually removed.
|
|
322
|
+
*/
|
|
323
|
+
deleteMany(ids) {
|
|
324
|
+
if (ids.length === 0) {
|
|
325
|
+
return 0;
|
|
331
326
|
}
|
|
332
|
-
|
|
333
|
-
|
|
327
|
+
let removed = 0;
|
|
328
|
+
this.store.begin();
|
|
329
|
+
try {
|
|
330
|
+
for (const id of ids) {
|
|
334
331
|
const nodeId = this.store.idByExt(this.checkId(id));
|
|
335
|
-
if (nodeId
|
|
336
|
-
|
|
337
|
-
this.store.begin();
|
|
338
|
-
try {
|
|
339
|
-
this.index.remove(nodeId);
|
|
340
|
-
this.store.commit();
|
|
332
|
+
if (nodeId !== null && this.index.remove(nodeId)) {
|
|
333
|
+
removed++;
|
|
341
334
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
335
|
+
}
|
|
336
|
+
this.store.commit();
|
|
337
|
+
} catch (e) {
|
|
338
|
+
this.store.rollback();
|
|
339
|
+
throw e;
|
|
347
340
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
*/
|
|
356
|
-
warmCache() {
|
|
357
|
-
return this.store.warmCache();
|
|
341
|
+
return removed;
|
|
342
|
+
}
|
|
343
|
+
/** Delete the vector bound to an id. Returns false if absent. */
|
|
344
|
+
delete(id) {
|
|
345
|
+
const nodeId = this.store.idByExt(this.checkId(id));
|
|
346
|
+
if (nodeId === null) {
|
|
347
|
+
return false;
|
|
358
348
|
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
349
|
+
this.store.begin();
|
|
350
|
+
try {
|
|
351
|
+
this.index.remove(nodeId);
|
|
352
|
+
this.store.commit();
|
|
353
|
+
} catch (e) {
|
|
354
|
+
this.store.rollback();
|
|
355
|
+
throw e;
|
|
366
356
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Pre-fill the RAM caches (codes + neighbour lists) with sequential table
|
|
361
|
+
* scans, up to their budget-derived caps. Optional and purely a latency
|
|
362
|
+
* optimisation: a cold session otherwise pays the same warming through
|
|
363
|
+
* random point reads over its first minutes. Call once after open on a
|
|
364
|
+
* session that will do sustained inserts/queries. Returns rows warmed;
|
|
365
|
+
* a 0-budget database returns 0 immediately.
|
|
366
|
+
*/
|
|
367
|
+
warmCache() {
|
|
368
|
+
return this.store.warmCache();
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Rebuild the graph from the live codes only, dropping tombstones and returning
|
|
372
|
+
* the freed pages to the filesystem. Lossless -- equivalent to the original
|
|
373
|
+
* build -- and needs no original vectors.
|
|
374
|
+
*/
|
|
375
|
+
compact() {
|
|
376
|
+
this.index.compact();
|
|
377
|
+
}
|
|
378
|
+
// ------------------------------ search -----------------------------------
|
|
379
|
+
/**
|
|
380
|
+
* k-NN search. The argument's length selects the mode:
|
|
381
|
+
* - `dim` elements -> a raw vector (accurate 4-bit-query estimator)
|
|
382
|
+
* - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
|
|
383
|
+
* Detection is by length, so a code is recognised whether it is a Uint32Array
|
|
384
|
+
* or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
|
|
385
|
+
*/
|
|
386
|
+
query(query, k = 10, opts) {
|
|
387
|
+
if (query.length === this.codeWords) {
|
|
388
|
+
return this.index.searchKnnByCode(
|
|
389
|
+
this.quantizer.codeToBytes(query),
|
|
390
|
+
k,
|
|
391
|
+
opts?.ef,
|
|
392
|
+
);
|
|
383
393
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
this.
|
|
394
|
+
if (query.length !== this.dim) {
|
|
395
|
+
throw new Error(
|
|
396
|
+
`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`,
|
|
397
|
+
);
|
|
387
398
|
}
|
|
399
|
+
return this.index.searchKnn(query, k, opts?.ef);
|
|
400
|
+
}
|
|
401
|
+
/** Close the underlying database. The instance must not be used afterwards. */
|
|
402
|
+
close() {
|
|
403
|
+
this.store.close();
|
|
404
|
+
}
|
|
388
405
|
}
|