@hviana/sema 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/dist/example/demo.js +25 -25
  2. package/dist/example/train_base.d.ts +21 -19
  3. package/dist/example/train_base.js +1952 -1684
  4. package/dist/src/alphabet.d.ts +3 -3
  5. package/dist/src/alphabet.js +27 -24
  6. package/dist/src/alu/src/alu.d.ts +194 -169
  7. package/dist/src/alu/src/alu.js +414 -374
  8. package/dist/src/alu/src/expr.d.ts +43 -36
  9. package/dist/src/alu/src/expr.js +278 -252
  10. package/dist/src/alu/src/index.d.ts +98 -9
  11. package/dist/src/alu/src/index.js +63 -9
  12. package/dist/src/alu/src/kernel-arith.d.ts +5 -1
  13. package/dist/src/alu/src/kernel-arith.js +285 -195
  14. package/dist/src/alu/src/kernel-bits.d.ts +5 -1
  15. package/dist/src/alu/src/kernel-bits.js +115 -81
  16. package/dist/src/alu/src/kernel-logic.js +63 -33
  17. package/dist/src/alu/src/kernel-nd.js +195 -155
  18. package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
  19. package/dist/src/alu/src/kernel-numeric.js +331 -259
  20. package/dist/src/alu/src/operation.d.ts +132 -112
  21. package/dist/src/alu/src/operation.js +163 -149
  22. package/dist/src/alu/src/parser.d.ts +175 -168
  23. package/dist/src/alu/src/parser.js +421 -382
  24. package/dist/src/alu/src/resonance.d.ts +25 -16
  25. package/dist/src/alu/src/resonance.js +56 -49
  26. package/dist/src/alu/src/text.d.ts +7 -3
  27. package/dist/src/alu/src/text.js +37 -31
  28. package/dist/src/alu/src/value.d.ts +14 -14
  29. package/dist/src/alu/src/value.js +160 -150
  30. package/dist/src/alu/test/alu.test.js +656 -543
  31. package/dist/src/bytes.d.ts +5 -1
  32. package/dist/src/bytes.js +40 -33
  33. package/dist/src/config.d.ts +102 -102
  34. package/dist/src/config.js +83 -83
  35. package/dist/src/derive/src/deduction.d.ts +62 -60
  36. package/dist/src/derive/src/deduction.js +109 -103
  37. package/dist/src/derive/src/index.d.ts +7 -1
  38. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  39. package/dist/src/derive/src/priority-queue.js +61 -57
  40. package/dist/src/derive/src/rewrite.d.ts +18 -15
  41. package/dist/src/derive/src/rewrite.js +67 -59
  42. package/dist/src/derive/src/trie.d.ts +56 -56
  43. package/dist/src/derive/src/trie.js +179 -175
  44. package/dist/src/derive/test/derive.test.js +100 -96
  45. package/dist/src/extension.d.ts +16 -13
  46. package/dist/src/geometry.d.ts +42 -17
  47. package/dist/src/geometry.js +266 -235
  48. package/dist/src/index.d.ts +17 -2
  49. package/dist/src/index.js +7 -1
  50. package/dist/src/ingest-cache.d.ts +30 -25
  51. package/dist/src/ingest-cache.js +127 -108
  52. package/dist/src/mind/articulation.d.ts +5 -1
  53. package/dist/src/mind/articulation.js +112 -77
  54. package/dist/src/mind/attention.d.ts +108 -40
  55. package/dist/src/mind/attention.js +871 -749
  56. package/dist/src/mind/canonical.d.ts +19 -4
  57. package/dist/src/mind/canonical.js +31 -27
  58. package/dist/src/mind/graph-search.d.ts +225 -201
  59. package/dist/src/mind/graph-search.js +821 -742
  60. package/dist/src/mind/index.d.ts +10 -2
  61. package/dist/src/mind/junction.d.ts +58 -31
  62. package/dist/src/mind/junction.js +237 -172
  63. package/dist/src/mind/learning.d.ts +52 -16
  64. package/dist/src/mind/learning.js +165 -143
  65. package/dist/src/mind/match.d.ts +69 -20
  66. package/dist/src/mind/match.js +318 -259
  67. package/dist/src/mind/mechanisms/alu.js +16 -16
  68. package/dist/src/mind/mechanisms/cast.d.ts +13 -9
  69. package/dist/src/mind/mechanisms/cast.js +456 -363
  70. package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
  71. package/dist/src/mind/mechanisms/confluence.js +183 -152
  72. package/dist/src/mind/mechanisms/cover.d.ts +8 -2
  73. package/dist/src/mind/mechanisms/cover.js +210 -148
  74. package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
  75. package/dist/src/mind/mechanisms/extraction.js +288 -234
  76. package/dist/src/mind/mechanisms/recall.d.ts +10 -6
  77. package/dist/src/mind/mechanisms/recall.js +185 -126
  78. package/dist/src/mind/mind.d.ts +154 -129
  79. package/dist/src/mind/mind.js +288 -258
  80. package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
  81. package/dist/src/mind/pipeline-mechanism.js +172 -161
  82. package/dist/src/mind/pipeline.d.ts +14 -4
  83. package/dist/src/mind/pipeline.js +189 -125
  84. package/dist/src/mind/primitives.d.ts +32 -8
  85. package/dist/src/mind/primitives.js +117 -99
  86. package/dist/src/mind/rationale.d.ts +98 -86
  87. package/dist/src/mind/rationale.js +121 -113
  88. package/dist/src/mind/reasoning.d.ts +13 -2
  89. package/dist/src/mind/reasoning.js +166 -129
  90. package/dist/src/mind/recognition.d.ts +4 -1
  91. package/dist/src/mind/recognition.js +208 -183
  92. package/dist/src/mind/resonance.d.ts +22 -5
  93. package/dist/src/mind/resonance.js +0 -0
  94. package/dist/src/mind/trace.d.ts +25 -6
  95. package/dist/src/mind/trace.js +58 -50
  96. package/dist/src/mind/traverse.d.ts +58 -16
  97. package/dist/src/mind/traverse.js +357 -304
  98. package/dist/src/mind/types.d.ts +127 -120
  99. package/dist/src/mind/types.js +69 -60
  100. package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
  101. package/dist/src/rabitq-hnsw/src/database.js +359 -342
  102. package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
  103. package/dist/src/rabitq-hnsw/src/heap.js +80 -75
  104. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
  105. package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
  106. package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
  107. package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
  108. package/dist/src/rabitq-hnsw/src/prng.js +27 -25
  109. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
  110. package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
  111. package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
  112. package/dist/src/rabitq-hnsw/src/store.js +843 -752
  113. package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
  114. package/dist/src/sema.d.ts +13 -9
  115. package/dist/src/sema.js +40 -26
  116. package/dist/src/store-sqlite.d.ts +166 -144
  117. package/dist/src/store-sqlite.js +660 -578
  118. package/dist/src/store.d.ts +630 -593
  119. package/dist/src/store.js +1581 -1440
  120. package/dist/src/vec.d.ts +9 -5
  121. package/dist/src/vec.js +73 -61
  122. package/package.json +2 -2
@@ -2,12 +2,12 @@ import { HnswIndex } from "./hnsw.js";
2
2
  import { RaBitQuantizer } from "./rabitq.js";
3
3
  import { Store } from "./store.js";
4
4
  const DEFAULTS = {
5
- M: 16,
6
- efConstruction: 200,
7
- efSearch: 100,
8
- queryBits: 8,
9
- rotationRounds: 3,
10
- seed: 0x1234abcd,
5
+ M: 16,
6
+ efConstruction: 200,
7
+ efSearch: 100,
8
+ queryBits: 8,
9
+ rotationRounds: 3,
10
+ seed: 0x1234abcd,
11
11
  };
12
12
  /**
13
13
  * A persistent vector database: an HNSW graph over 1-bit RaBitQ codes (cosine),
@@ -23,366 +23,383 @@ const DEFAULTS = {
23
23
  * `insert`/`update`/`query`.
24
24
  */
25
25
  export class VectorDatabase {
26
- dim;
27
- store;
28
- quantizer;
29
- index;
30
- codeWords;
31
- constructor(options) {
32
- if (!options || typeof options.dbPath !== "string" ||
33
- options.dbPath.length === 0) {
34
- throw new Error("DatabaseOptions.dbPath (string) is required");
35
- }
36
- this.store = new Store(options.dbPath, options.cacheSizeMb ?? 64);
37
- let cfg = this.store.loadConfig();
38
- if (cfg === null) {
39
- // Fresh database: derive the configuration from the options and persist it.
40
- if (!Number.isInteger(options.dim) || options.dim <= 0) {
41
- throw new Error("DatabaseOptions.dim must be a positive integer for a new database");
42
- }
43
- const dim = options.dim;
44
- const centroid = options.centroid
45
- ? Float64Array.from(options.centroid)
46
- : null;
47
- const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
48
- cfg = {
49
- dim,
50
- m: options.M ?? DEFAULTS.M,
51
- efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
52
- efSearch: options.efSearch ?? DEFAULTS.efSearch,
53
- queryBits: options.queryBits ?? DEFAULTS.queryBits,
54
- rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
55
- seed: (options.seed ?? DEFAULTS.seed) >>> 0,
56
- centroid,
57
- codeWords: probe.codeWords,
58
- paddedDim: probe.paddedDim,
59
- };
60
- this.store.initConfig(cfg);
61
- }
62
- else {
63
- // Reopened: keep the stored STRUCTURAL config, but allow tuning the two
64
- // query-side knobs — efSearch and queryBits shape only how a query is
65
- // executed, never what is stored, so honouring an explicit option here
66
- // is safe and lets an existing database adopt better query settings.
67
- if (options.efSearch !== undefined) {
68
- cfg.efSearch = options.efSearch;
69
- this.store.setEfSearch(options.efSearch);
70
- }
71
- if (options.queryBits !== undefined && options.queryBits !== cfg.queryBits) {
72
- cfg.queryBits = options.queryBits;
73
- this.store.setQueryBits(options.queryBits);
74
- }
75
- }
76
- this.dim = cfg.dim;
77
- this.quantizer = new RaBitQuantizer(cfg.dim, {
78
- queryBits: cfg.queryBits,
79
- rounds: cfg.rotationRounds,
80
- seed: cfg.seed,
81
- centroid: cfg.centroid ?? undefined,
82
- });
83
- if (this.quantizer.codeWords !== cfg.codeWords) {
84
- throw new Error("Stored code geometry does not match the quantizer (corrupt database?)");
85
- }
86
- this.codeWords = this.quantizer.codeWords;
87
- this.index = new HnswIndex(this.quantizer, this.store, {
88
- M: cfg.m,
89
- efConstruction: cfg.efConstruction,
90
- efSearch: cfg.efSearch,
91
- seed: cfg.seed,
92
- });
93
- }
94
- /** Number of live (non-deleted) vectors. */
95
- get size() {
96
- return this.index.size;
97
- }
98
- /**
99
- * Physical node count including tombstones from deletes/updates. When it grows
100
- * well beyond `size`, call `compact()` to reclaim the space on disk.
101
- */
102
- get physicalSize() {
103
- return this.index.physicalSize;
104
- }
105
- get efSearch() {
106
- return this.index.efSearch;
26
+ dim;
27
+ store;
28
+ quantizer;
29
+ index;
30
+ codeWords;
31
+ constructor(options) {
32
+ if (
33
+ !options || typeof options.dbPath !== "string" ||
34
+ options.dbPath.length === 0
35
+ ) {
36
+ throw new Error("DatabaseOptions.dbPath (string) is required");
107
37
  }
108
- set efSearch(value) {
109
- this.index.efSearch = value;
38
+ this.store = new Store(options.dbPath, options.cacheSizeMb ?? 64);
39
+ let cfg = this.store.loadConfig();
40
+ if (cfg === null) {
41
+ // Fresh database: derive the configuration from the options and persist it.
42
+ if (!Number.isInteger(options.dim) || options.dim <= 0) {
43
+ throw new Error(
44
+ "DatabaseOptions.dim must be a positive integer for a new database",
45
+ );
46
+ }
47
+ const dim = options.dim;
48
+ const centroid = options.centroid
49
+ ? Float64Array.from(options.centroid)
50
+ : null;
51
+ const probe = new RaBitQuantizer(dim, { rounds: 1, seed: 0 });
52
+ cfg = {
53
+ dim,
54
+ m: options.M ?? DEFAULTS.M,
55
+ efConstruction: options.efConstruction ?? DEFAULTS.efConstruction,
56
+ efSearch: options.efSearch ?? DEFAULTS.efSearch,
57
+ queryBits: options.queryBits ?? DEFAULTS.queryBits,
58
+ rotationRounds: options.rotationRounds ?? DEFAULTS.rotationRounds,
59
+ seed: (options.seed ?? DEFAULTS.seed) >>> 0,
60
+ centroid,
61
+ codeWords: probe.codeWords,
62
+ paddedDim: probe.paddedDim,
63
+ };
64
+ this.store.initConfig(cfg);
65
+ } else {
66
+ // Reopened: keep the stored STRUCTURAL config, but allow tuning the two
67
+ // query-side knobs — efSearch and queryBits shape only how a query is
68
+ // executed, never what is stored, so honouring an explicit option here
69
+ // is safe and lets an existing database adopt better query settings.
70
+ if (options.efSearch !== undefined) {
71
+ cfg.efSearch = options.efSearch;
72
+ this.store.setEfSearch(options.efSearch);
73
+ }
74
+ if (
75
+ options.queryBits !== undefined && options.queryBits !== cfg.queryBits
76
+ ) {
77
+ cfg.queryBits = options.queryBits;
78
+ this.store.setQueryBits(options.queryBits);
79
+ }
110
80
  }
111
- /** Distance computations performed during the most recent query. */
112
- get lastQueryDistanceComputations() {
113
- return this.index.lastQueryDistComps;
81
+ this.dim = cfg.dim;
82
+ this.quantizer = new RaBitQuantizer(cfg.dim, {
83
+ queryBits: cfg.queryBits,
84
+ rounds: cfg.rotationRounds,
85
+ seed: cfg.seed,
86
+ centroid: cfg.centroid ?? undefined,
87
+ });
88
+ if (this.quantizer.codeWords !== cfg.codeWords) {
89
+ throw new Error(
90
+ "Stored code geometry does not match the quantizer (corrupt database?)",
91
+ );
114
92
  }
115
- /**
116
- * Storage row reads issued by the most recent query. This is the honest,
117
- * cache-independent scalability metric: it counts every node/neighbour fetch
118
- * that hit the database, so disabling the cache cannot hide a bad access pattern.
119
- */
120
- get lastQueryStorageReads() {
121
- return this.index.lastQueryStorageReads;
93
+ this.codeWords = this.quantizer.codeWords;
94
+ this.index = new HnswIndex(this.quantizer, this.store, {
95
+ M: cfg.m,
96
+ efConstruction: cfg.efConstruction,
97
+ efSearch: cfg.efSearch,
98
+ seed: cfg.seed,
99
+ });
100
+ }
101
+ /** Number of live (non-deleted) vectors. */
102
+ get size() {
103
+ return this.index.size;
104
+ }
105
+ /**
106
+ * Physical node count including tombstones from deletes/updates. When it grows
107
+ * well beyond `size`, call `compact()` to reclaim the space on disk.
108
+ */
109
+ get physicalSize() {
110
+ return this.index.physicalSize;
111
+ }
112
+ get efSearch() {
113
+ return this.index.efSearch;
114
+ }
115
+ set efSearch(value) {
116
+ this.index.efSearch = value;
117
+ }
118
+ /** Distance computations performed during the most recent query. */
119
+ get lastQueryDistanceComputations() {
120
+ return this.index.lastQueryDistComps;
121
+ }
122
+ /**
123
+ * Storage row reads issued by the most recent query. This is the honest,
124
+ * cache-independent scalability metric: it counts every node/neighbour fetch
125
+ * that hit the database, so disabling the cache cannot hide a bad access pattern.
126
+ */
127
+ get lastQueryStorageReads() {
128
+ return this.index.lastQueryStorageReads;
129
+ }
130
+ /** Per-vector storage cost of the index versus a Float32 baseline. */
131
+ get storage() {
132
+ const f32 = this.dim * 4;
133
+ const bpv = this.index.bytesPerVector;
134
+ return {
135
+ float32BytesPerVector: f32,
136
+ codeBytesPerVector: bpv,
137
+ bytesPerVector: bpv,
138
+ compressionRatio: f32 / bpv,
139
+ };
140
+ }
141
+ has(id) {
142
+ return this.store.idByExt(this.checkId(id)) !== null;
143
+ }
144
+ /** Stream every live external id (bounded memory). */
145
+ *keys() {
146
+ yield* this.store.liveExts();
147
+ }
148
+ /**
149
+ * Stream live entries whose INTERNAL id is > `after`, as
150
+ * {ext, internal} pairs in internal-id order. Internal ids are assigned
151
+ * monotonically at insert and preserved by {@link compact}, so the largest
152
+ * internal id a caller has seen is a durable incremental watermark: a later
153
+ * call with it yields exactly the entries added since.
154
+ */
155
+ *keysSince(after) {
156
+ yield* this.store.liveExtsSince(after);
157
+ }
158
+ checkId(id) {
159
+ if (!Number.isInteger(id)) {
160
+ throw new Error(`External id must be an integer, got ${id}`);
122
161
  }
123
- /** Per-vector storage cost of the index versus a Float32 baseline. */
124
- get storage() {
125
- const f32 = this.dim * 4;
126
- const bpv = this.index.bytesPerVector;
127
- return {
128
- float32BytesPerVector: f32,
129
- codeBytesPerVector: bpv,
130
- bytesPerVector: bpv,
131
- compressionRatio: f32 / bpv,
132
- };
162
+ return id;
163
+ }
164
+ /**
165
+ * Convert a value to code bytes, selecting by length:
166
+ * - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
167
+ * - otherwise -> a raw `dim`-vector, encoded first
168
+ * `codeWords < dim` for any dim >= 2, so the two never collide.
169
+ */
170
+ toCodeBytes(value) {
171
+ if (value.length === this.codeWords) {
172
+ return this.quantizer.codeToBytes(value);
133
173
  }
134
- has(id) {
135
- return this.store.idByExt(this.checkId(id)) !== null;
174
+ if (value.length !== this.dim) {
175
+ throw new Error(
176
+ `Vector dimension mismatch: expected ${this.dim}, got ${value.length}`,
177
+ );
136
178
  }
137
- /** Stream every live external id (bounded memory). */
138
- *keys() {
139
- yield* this.store.liveExts();
179
+ return this.quantizer.codeToBytes(this.quantizer.encode(value));
180
+ }
181
+ // ------------------------------- CRUD ------------------------------------
182
+ /**
183
+ * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
184
+ * detected by length. Throws if the id already exists (use `update`/`upsert`).
185
+ */
186
+ insert(id, value) {
187
+ this.store.begin();
188
+ try {
189
+ this.insertCore(id, value);
190
+ this.store.commit();
191
+ } catch (e) {
192
+ this.store.rollback();
193
+ throw e;
140
194
  }
141
- /**
142
- * Stream live entries whose INTERNAL id is > `after`, as
143
- * {ext, internal} pairs in internal-id order. Internal ids are assigned
144
- * monotonically at insert and preserved by {@link compact}, so the largest
145
- * internal id a caller has seen is a durable incremental watermark: a later
146
- * call with it yields exactly the entries added since.
147
- */
148
- *keysSince(after) {
149
- yield* this.store.liveExtsSince(after);
195
+ }
196
+ /** Insert with the caller owning the transaction (used by {@link upsertMany}).
197
+ * `ef` optionally narrows the construction beam for this one vector (a
198
+ * caller-declared cheap entry, e.g. a reach-only interior); omitted means
199
+ * the index's configured efConstruction. */
200
+ insertCore(id, value, ef) {
201
+ this.checkId(id);
202
+ if (this.store.idByExt(id) !== null) {
203
+ throw new Error(`External id already exists: ${id} (use update())`);
150
204
  }
151
- checkId(id) {
152
- if (!Number.isInteger(id)) {
153
- throw new Error(`External id must be an integer, got ${id}`);
154
- }
155
- return id;
156
- }
157
- /**
158
- * Convert a value to code bytes, selecting by length:
159
- * - `codeWords` elements -> an existing 1-bit code (e.g. from `get()`)
160
- * - otherwise -> a raw `dim`-vector, encoded first
161
- * `codeWords < dim` for any dim >= 2, so the two never collide.
162
- */
163
- toCodeBytes(value) {
164
- if (value.length === this.codeWords) {
165
- return this.quantizer.codeToBytes(value);
166
- }
167
- if (value.length !== this.dim) {
168
- throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${value.length}`);
169
- }
170
- return this.quantizer.codeToBytes(this.quantizer.encode(value));
171
- }
172
- // ------------------------------- CRUD ------------------------------------
173
- /**
174
- * Create. Accepts a raw `dim`-vector or a 1-bit code (`codeWords` words),
175
- * detected by length. Throws if the id already exists (use `update`/`upsert`).
176
- */
177
- insert(id, value) {
178
- this.store.begin();
179
- try {
180
- this.insertCore(id, value);
181
- this.store.commit();
182
- }
183
- catch (e) {
184
- this.store.rollback();
185
- throw e;
186
- }
205
+ this.index.insert(id, this.toCodeBytes(value), ef);
206
+ }
207
+ upsert(id, value) {
208
+ const nodeId = this.store.idByExt(this.checkId(id));
209
+ if (nodeId === null) {
210
+ this.insert(id, value);
211
+ return;
187
212
  }
188
- /** Insert with the caller owning the transaction (used by {@link upsertMany}).
189
- * `ef` optionally narrows the construction beam for this one vector (a
190
- * caller-declared cheap entry, e.g. a reach-only interior); omitted means
191
- * the index's configured efConstruction. */
192
- insertCore(id, value, ef) {
193
- this.checkId(id);
194
- if (this.store.idByExt(id) !== null) {
195
- throw new Error(`External id already exists: ${id} (use update())`);
196
- }
197
- this.index.insert(id, this.toCodeBytes(value), ef);
213
+ this.store.begin();
214
+ try {
215
+ this.updateAt(nodeId, id, value);
216
+ this.store.commit();
217
+ } catch (e) {
218
+ this.store.rollback();
219
+ throw e;
198
220
  }
199
- upsert(id, value) {
200
- const nodeId = this.store.idByExt(this.checkId(id));
201
- if (nodeId === null) {
202
- this.insert(id, value);
203
- return;
204
- }
205
- this.store.begin();
206
- try {
207
- this.updateAt(nodeId, id, value);
208
- this.store.commit();
209
- }
210
- catch (e) {
211
- this.store.rollback();
212
- throw e;
213
- }
221
+ }
222
+ /**
223
+ * Upsert many vectors under ONE transaction. The HNSW build touches the store
224
+ * on every wired edge, so a transaction per vector is one WAL commit per vector
225
+ * — which dominates a bulk load on disk. Wrapping the whole batch in a single
226
+ * transaction coalesces those commits into one while leaving the graph and its
227
+ * result identical (reads on the connection still see the uncommitted rows).
228
+ *
229
+ * This is purely about commit batching; it changes nothing about the storage
230
+ * model. Codes still live only in SQLite and are read on demand — the
231
+ * cache-independent per-operation read count is unchanged — so a batched load
232
+ * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
233
+ * rolls the whole batch back, so the caller treats it as the per-item path on
234
+ * failure.
235
+ */
236
+ upsertMany(entries) {
237
+ if (entries.length === 0) {
238
+ return;
214
239
  }
215
- /**
216
- * Upsert many vectors under ONE transaction. The HNSW build touches the store
217
- * on every wired edge, so a transaction per vector is one WAL commit per vector
218
- * which dominates a bulk load on disk. Wrapping the whole batch in a single
219
- * transaction coalesces those commits into one while leaving the graph and its
220
- * result identical (reads on the connection still see the uncommitted rows).
221
- *
222
- * This is purely about commit batching; it changes nothing about the storage
223
- * model. Codes still live only in SQLite and are read on demand — the
224
- * cache-independent per-operation read count is unchanged — so a batched load
225
- * scales exactly as the per-item path does, just with far fewer fsyncs. A throw
226
- * rolls the whole batch back, so the caller treats it as the per-item path on
227
- * failure.
228
- */
229
- upsertMany(entries) {
230
- if (entries.length === 0)
231
- return;
232
- this.store.begin();
233
- try {
234
- for (const e of entries) {
235
- const nodeId = this.store.idByExt(this.checkId(e.id));
236
- if (nodeId !== null)
237
- this.updateAt(nodeId, e.id, e.vector, e.ef);
238
- else
239
- this.insertCore(e.id, e.vector, e.ef);
240
- }
241
- this.store.commit();
242
- }
243
- catch (e) {
244
- this.store.rollback();
245
- throw e;
240
+ this.store.begin();
241
+ try {
242
+ for (const e of entries) {
243
+ const nodeId = this.store.idByExt(this.checkId(e.id));
244
+ if (nodeId !== null) {
245
+ this.updateAt(nodeId, e.id, e.vector, e.ef);
246
+ } else {
247
+ this.insertCore(e.id, e.vector, e.ef);
246
248
  }
249
+ }
250
+ this.store.commit();
251
+ } catch (e) {
252
+ this.store.rollback();
253
+ throw e;
247
254
  }
248
- /**
249
- * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
250
- * original vector is not retained; the code round-trips into insert/update/query.
251
- */
252
- get(id) {
253
- const nodeId = this.store.idByExt(this.checkId(id));
254
- if (nodeId === null)
255
- return null;
256
- const rec = this.store.getNode(nodeId);
257
- return rec ? this.quantizer.bytesToCode(rec.code) : null;
255
+ }
256
+ /**
257
+ * Read the stored 1-bit code for an id (a copy as a Uint32Array), or null. The
258
+ * original vector is not retained; the code round-trips into insert/update/query.
259
+ */
260
+ get(id) {
261
+ const nodeId = this.store.idByExt(this.checkId(id));
262
+ if (nodeId === null) {
263
+ return null;
258
264
  }
259
- /**
260
- * Update the vector bound to an id (raw vector or code, detected by length).
261
- * The previous node is tombstoned and a fresh one inserted. Throws if absent.
262
- */
263
- update(id, value) {
264
- this.store.begin();
265
- try {
266
- this.updateCore(id, value);
267
- this.store.commit();
268
- }
269
- catch (e) {
270
- this.store.rollback();
271
- throw e;
272
- }
265
+ const rec = this.store.getNode(nodeId);
266
+ return rec ? this.quantizer.bytesToCode(rec.code) : null;
267
+ }
268
+ /**
269
+ * Update the vector bound to an id (raw vector or code, detected by length).
270
+ * The previous node is tombstoned and a fresh one inserted. Throws if absent.
271
+ */
272
+ update(id, value) {
273
+ this.store.begin();
274
+ try {
275
+ this.updateCore(id, value);
276
+ this.store.commit();
277
+ } catch (e) {
278
+ this.store.rollback();
279
+ throw e;
273
280
  }
274
- /** Update with the caller owning the transaction (used by {@link upsertMany}). */
275
- updateCore(id, value) {
276
- const oldId = this.store.idByExt(this.checkId(id));
277
- if (oldId === null)
278
- throw new Error(`Unknown external id: ${id}`);
279
- this.updateAt(oldId, id, value);
281
+ }
282
+ /** Update with the caller owning the transaction (used by {@link upsertMany}). */
283
+ updateCore(id, value) {
284
+ const oldId = this.store.idByExt(this.checkId(id));
285
+ if (oldId === null) {
286
+ throw new Error(`Unknown external id: ${id}`);
280
287
  }
281
- /** Update when the live internal node id is already known.
282
- *
283
- * Skips the write entirely when the new code equals the stored one — the
284
- * update would tombstone the node and replay a full graph insert for a
285
- * byte-identical result. This is not a corner case: content-addressed
286
- * callers re-upsert unchanged vectors wholesale after a restart (the same
287
- * content always encodes to the same code), and each such no-op otherwise
288
- * costs a tombstone (permanent routing/disk overhead until compaction)
289
- * plus an O(ef·log N) reinsert. */
290
- updateAt(nodeId, ext, value, ef) {
291
- const bytes = this.toCodeBytes(value);
292
- const rec = this.store.getNode(nodeId);
293
- if (rec !== null && rec.code.length === bytes.length) {
294
- let same = true;
295
- for (let i = 0; i < bytes.length; i++) {
296
- if (rec.code[i] !== bytes[i]) {
297
- same = false;
298
- break;
299
- }
300
- }
301
- if (same)
302
- return; // identical code — the update is a no-op
288
+ this.updateAt(oldId, id, value);
289
+ }
290
+ /** Update when the live internal node id is already known.
291
+ *
292
+ * Skips the write entirely when the new code equals the stored one — the
293
+ * update would tombstone the node and replay a full graph insert for a
294
+ * byte-identical result. This is not a corner case: content-addressed
295
+ * callers re-upsert unchanged vectors wholesale after a restart (the same
296
+ * content always encodes to the same code), and each such no-op otherwise
297
+ * costs a tombstone (permanent routing/disk overhead until compaction)
298
+ * plus an O(ef·log N) reinsert. */
299
+ updateAt(nodeId, ext, value, ef) {
300
+ const bytes = this.toCodeBytes(value);
301
+ const rec = this.store.getNode(nodeId);
302
+ if (rec !== null && rec.code.length === bytes.length) {
303
+ let same = true;
304
+ for (let i = 0; i < bytes.length; i++) {
305
+ if (rec.code[i] !== bytes[i]) {
306
+ same = false;
307
+ break;
303
308
  }
304
- this.index.remove(nodeId); // frees the ext id, then re-insert under it
305
- this.index.insert(ext, bytes, ef);
309
+ }
310
+ if (same) {
311
+ return; // identical code — the update is a no-op
312
+ }
306
313
  }
307
- /**
308
- * Delete many ids under ONE transaction — same commit-coalescing rationale
309
- * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
310
- * commit per id, which dominates a bulk prune. Absent ids are skipped.
311
- * Returns the number of vectors actually removed.
312
- */
313
- deleteMany(ids) {
314
- if (ids.length === 0)
315
- return 0;
316
- let removed = 0;
317
- this.store.begin();
318
- try {
319
- for (const id of ids) {
320
- const nodeId = this.store.idByExt(this.checkId(id));
321
- if (nodeId !== null && this.index.remove(nodeId))
322
- removed++;
323
- }
324
- this.store.commit();
325
- }
326
- catch (e) {
327
- this.store.rollback();
328
- throw e;
329
- }
330
- return removed;
314
+ this.index.remove(nodeId); // frees the ext id, then re-insert under it
315
+ this.index.insert(ext, bytes, ef);
316
+ }
317
+ /**
318
+ * Delete many ids under ONE transaction — same commit-coalescing rationale
319
+ * as {@link upsertMany}: a tombstone per implicit transaction is one WAL
320
+ * commit per id, which dominates a bulk prune. Absent ids are skipped.
321
+ * Returns the number of vectors actually removed.
322
+ */
323
+ deleteMany(ids) {
324
+ if (ids.length === 0) {
325
+ return 0;
331
326
  }
332
- /** Delete the vector bound to an id. Returns false if absent. */
333
- delete(id) {
327
+ let removed = 0;
328
+ this.store.begin();
329
+ try {
330
+ for (const id of ids) {
334
331
  const nodeId = this.store.idByExt(this.checkId(id));
335
- if (nodeId === null)
336
- return false;
337
- this.store.begin();
338
- try {
339
- this.index.remove(nodeId);
340
- this.store.commit();
332
+ if (nodeId !== null && this.index.remove(nodeId)) {
333
+ removed++;
341
334
  }
342
- catch (e) {
343
- this.store.rollback();
344
- throw e;
345
- }
346
- return true;
335
+ }
336
+ this.store.commit();
337
+ } catch (e) {
338
+ this.store.rollback();
339
+ throw e;
347
340
  }
348
- /**
349
- * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
350
- * scans, up to their budget-derived caps. Optional and purely a latency
351
- * optimisation: a cold session otherwise pays the same warming through
352
- * random point reads over its first minutes. Call once after open on a
353
- * session that will do sustained inserts/queries. Returns rows warmed;
354
- * a 0-budget database returns 0 immediately.
355
- */
356
- warmCache() {
357
- return this.store.warmCache();
341
+ return removed;
342
+ }
343
+ /** Delete the vector bound to an id. Returns false if absent. */
344
+ delete(id) {
345
+ const nodeId = this.store.idByExt(this.checkId(id));
346
+ if (nodeId === null) {
347
+ return false;
358
348
  }
359
- /**
360
- * Rebuild the graph from the live codes only, dropping tombstones and returning
361
- * the freed pages to the filesystem. Lossless -- equivalent to the original
362
- * build -- and needs no original vectors.
363
- */
364
- compact() {
365
- this.index.compact();
349
+ this.store.begin();
350
+ try {
351
+ this.index.remove(nodeId);
352
+ this.store.commit();
353
+ } catch (e) {
354
+ this.store.rollback();
355
+ throw e;
366
356
  }
367
- // ------------------------------ search -----------------------------------
368
- /**
369
- * k-NN search. The argument's length selects the mode:
370
- * - `dim` elements -> a raw vector (accurate 4-bit-query estimator)
371
- * - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
372
- * Detection is by length, so a code is recognised whether it is a Uint32Array
373
- * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
374
- */
375
- query(query, k = 10, opts) {
376
- if (query.length === this.codeWords) {
377
- return this.index.searchKnnByCode(this.quantizer.codeToBytes(query), k, opts?.ef);
378
- }
379
- if (query.length !== this.dim) {
380
- throw new Error(`Vector dimension mismatch: expected ${this.dim}, got ${query.length}`);
381
- }
382
- return this.index.searchKnn(query, k, opts?.ef);
357
+ return true;
358
+ }
359
+ /**
360
+ * Pre-fill the RAM caches (codes + neighbour lists) with sequential table
361
+ * scans, up to their budget-derived caps. Optional and purely a latency
362
+ * optimisation: a cold session otherwise pays the same warming through
363
+ * random point reads over its first minutes. Call once after open on a
364
+ * session that will do sustained inserts/queries. Returns rows warmed;
365
+ * a 0-budget database returns 0 immediately.
366
+ */
367
+ warmCache() {
368
+ return this.store.warmCache();
369
+ }
370
+ /**
371
+ * Rebuild the graph from the live codes only, dropping tombstones and returning
372
+ * the freed pages to the filesystem. Lossless -- equivalent to the original
373
+ * build -- and needs no original vectors.
374
+ */
375
+ compact() {
376
+ this.index.compact();
377
+ }
378
+ // ------------------------------ search -----------------------------------
379
+ /**
380
+ * k-NN search. The argument's length selects the mode:
381
+ * - `dim` elements -> a raw vector (accurate 4-bit-query estimator)
382
+ * - `codeWords` elements -> a 1-bit code (e.g. `get(id)`), by sign-bit Hamming
383
+ * Detection is by length, so a code is recognised whether it is a Uint32Array
384
+ * or a plain number[]. `codeWords < dim` for any dim >= 2, so they never collide.
385
+ */
386
+ query(query, k = 10, opts) {
387
+ if (query.length === this.codeWords) {
388
+ return this.index.searchKnnByCode(
389
+ this.quantizer.codeToBytes(query),
390
+ k,
391
+ opts?.ef,
392
+ );
383
393
  }
384
- /** Close the underlying database. The instance must not be used afterwards. */
385
- close() {
386
- this.store.close();
394
+ if (query.length !== this.dim) {
395
+ throw new Error(
396
+ `Vector dimension mismatch: expected ${this.dim}, got ${query.length}`,
397
+ );
387
398
  }
399
+ return this.index.searchKnn(query, k, opts?.ef);
400
+ }
401
+ /** Close the underlying database. The instance must not be used afterwards. */
402
+ close() {
403
+ this.store.close();
404
+ }
388
405
  }