eigen-db 4.3.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +41 -3
- package/dist/compute.d.ts +20 -0
- package/dist/eigen-db.js +165 -113
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/dist/errors.d.ts +7 -0
- package/dist/index.d.ts +12 -0
- package/dist/lexicon.d.ts +28 -0
- package/dist/memory-manager.d.ts +68 -0
- package/dist/result-set.d.ts +35 -0
- package/dist/simd-binary.d.ts +1 -0
- package/dist/storage.d.ts +38 -0
- package/dist/types.d.ts +44 -0
- package/dist/vector-db.d.ts +131 -0
- package/dist/wasm-compute.d.ts +13 -0
- package/package.json +4 -4
- package/src/lib/__tests__/vector-db.test.ts +288 -0
- package/src/lib/vector-db.ts +91 -16
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -56,7 +56,34 @@ Notes:
|
|
|
56
56
|
- Each vector must be a `number[]` (or `Float32Array`) with exactly `dimensions` elements.
|
|
57
57
|
- Duplicate keys use last-write-wins semantics.
|
|
58
58
|
|
|
59
|
-
### 3)
|
|
59
|
+
### 3) Look up, check, and remove vectors
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
db.get("doc:1"); // number[] | undefined
|
|
63
|
+
db.has("doc:1"); // true
|
|
64
|
+
db.delete("doc:1"); // true (removed), false (not found)
|
|
65
|
+
db.dimensions; // configured vector dimensions
|
|
66
|
+
db.size; // number of entries
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### 4) Iterate over the database
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
// Iterate over all keys
|
|
73
|
+
for (const key of db.keys()) {
|
|
74
|
+
console.log(key);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Iterate over all [key, value] pairs
|
|
78
|
+
for (const [key, vector] of db.entries()) {
|
|
79
|
+
console.log(key, vector);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Spread into an array (uses Symbol.iterator, same as entries())
|
|
83
|
+
const all = [...db];
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### 5) Query nearest vectors
|
|
60
87
|
|
|
61
88
|
```ts
|
|
62
89
|
const queryVector = embeddingQuery;
|
|
@@ -94,7 +121,7 @@ const results = db.query(queryVector, { minSimilarity: 0.7 });
|
|
|
94
121
|
const results = db.query(queryVector, { minSimilarity: 0.7, iterable: true });
|
|
95
122
|
```
|
|
96
123
|
|
|
97
|
-
###
|
|
124
|
+
### 6) Persist and lifecycle
|
|
98
125
|
|
|
99
126
|
```ts
|
|
100
127
|
await db.flush(); // persist current state
|
|
@@ -107,7 +134,7 @@ To delete all vectors and storage:
|
|
|
107
134
|
await db.clear();
|
|
108
135
|
```
|
|
109
136
|
|
|
110
|
-
###
|
|
137
|
+
### 7) Export and import
|
|
111
138
|
|
|
112
139
|
Export the entire database as a streaming binary file:
|
|
113
140
|
|
|
@@ -183,6 +210,7 @@ Opens (or creates) a database instance and loads persisted data.
|
|
|
183
210
|
#### Properties
|
|
184
211
|
|
|
185
212
|
- `size: number` — current number of key-vector pairs
|
|
213
|
+
- `dimensions: number` — number of dimensions per vector
|
|
186
214
|
|
|
187
215
|
#### Methods
|
|
188
216
|
|
|
@@ -191,10 +219,20 @@ Opens (or creates) a database instance and loads persisted data.
|
|
|
191
219
|
- Throws on dimension mismatch.
|
|
192
220
|
- `get(key: string): number[] | undefined`
|
|
193
221
|
- Returns a copy of the stored vector.
|
|
222
|
+
- `has(key: string): boolean`
|
|
223
|
+
- Returns `true` if the key exists, `false` otherwise. O(1) lookup.
|
|
224
|
+
- `delete(key: string): boolean`
|
|
225
|
+
- Removes the entry for the given key. Returns `true` if the key existed, `false` otherwise.
|
|
194
226
|
- `setMany(entries: [string, VectorInput][]): void`
|
|
195
227
|
- Batch insert/update.
|
|
196
228
|
- `getMany(keys: string[]): (number[] | undefined)[]`
|
|
197
229
|
- Batch lookup.
|
|
230
|
+
- `keys(): IterableIterator<string>`
|
|
231
|
+
- Returns an iterable of all keys.
|
|
232
|
+
- `entries(): IterableIterator<[string, number[]]>`
|
|
233
|
+
- Returns an iterable of `[key, value]` pairs. Values are plain number array copies.
|
|
234
|
+
- `[Symbol.iterator](): IterableIterator<[string, number[]]>`
|
|
235
|
+
- Same as `entries()`. Enables `[...db]` and `for...of` iteration.
|
|
198
236
|
- `query(value: VectorInput, options?: QueryOptions): ResultItem[]`
|
|
199
237
|
- Returns results sorted by descending similarity as a plain array.
|
|
200
238
|
- Throws on dimension mismatch.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure JavaScript compute functions for vector operations.
|
|
3
|
+
* These serve as the reference implementation and fallback when WASM SIMD is unavailable.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Normalizes a vector in-place to unit length.
|
|
7
|
+
* After normalization, cosine similarity reduces to a simple dot product.
|
|
8
|
+
*/
|
|
9
|
+
export declare function normalize(vec: Float32Array): void;
|
|
10
|
+
/**
|
|
11
|
+
* Computes dot products of query against all vectors in the database.
|
|
12
|
+
* Writes scores to the output array.
|
|
13
|
+
*
|
|
14
|
+
* @param query - Normalized query vector (length = dimensions)
|
|
15
|
+
* @param db - Contiguous flat array of normalized vectors (length = dbSize * dimensions)
|
|
16
|
+
* @param scores - Output array for dot product scores (length = dbSize)
|
|
17
|
+
* @param dbSize - Number of vectors in the database
|
|
18
|
+
* @param dimensions - Dimensionality of each vector
|
|
19
|
+
*/
|
|
20
|
+
export declare function searchAll(query: Float32Array, db: Float32Array, scores: Float32Array, dbSize: number, dimensions: number): void;
|
package/dist/eigen-db.js
CHANGED
|
@@ -3,7 +3,7 @@ class k extends Error {
|
|
|
3
3
|
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class K {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
9
|
constructor(e) {
|
|
@@ -25,8 +25,8 @@ class _ {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
async append(e, t) {
|
|
28
|
-
const s = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await s.createWritable({ keepExistingData: !0 }),
|
|
29
|
-
await n.seek(
|
|
28
|
+
const s = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await s.createWritable({ keepExistingData: !0 }), o = await s.getFile();
|
|
29
|
+
await n.seek(o.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
31
|
async write(e, t) {
|
|
32
32
|
const n = await (await (await this.getDir()).getFileHandle(e, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
@@ -36,15 +36,15 @@ class _ {
|
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class S {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
41
|
async readAll(e) {
|
|
42
42
|
const t = this.files.get(e);
|
|
43
43
|
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
-
const r = t.reduce((
|
|
44
|
+
const r = t.reduce((o, a) => o + a.byteLength, 0), s = new Uint8Array(r);
|
|
45
45
|
let n = 0;
|
|
46
|
-
for (const
|
|
47
|
-
s.set(
|
|
46
|
+
for (const o of t)
|
|
47
|
+
s.set(o, n), n += o.byteLength;
|
|
48
48
|
return s;
|
|
49
49
|
}
|
|
50
50
|
async append(e, t) {
|
|
@@ -57,7 +57,7 @@ class v {
|
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
60
|
+
function v(c) {
|
|
61
61
|
let e = 0;
|
|
62
62
|
for (let s = 0; s < c.length; s++)
|
|
63
63
|
e += c[s] * c[s];
|
|
@@ -69,19 +69,19 @@ function S(c) {
|
|
|
69
69
|
}
|
|
70
70
|
function O(c, e, t, r, s) {
|
|
71
71
|
for (let n = 0; n < r; n++) {
|
|
72
|
-
let
|
|
72
|
+
let o = 0;
|
|
73
73
|
const a = n * s;
|
|
74
|
-
for (let
|
|
75
|
-
|
|
76
|
-
t[n] =
|
|
74
|
+
for (let i = 0; i < s; i++)
|
|
75
|
+
o += c[i] * e[a + i];
|
|
76
|
+
t[n] = o;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
79
|
const U = new TextEncoder(), x = new TextDecoder();
|
|
80
80
|
function d(c) {
|
|
81
|
-
const e = c.map((a) => U.encode(a)), t = e.reduce((a,
|
|
82
|
-
let
|
|
81
|
+
const e = c.map((a) => U.encode(a)), t = e.reduce((a, i) => a + 4 + i.byteLength, 0), r = new ArrayBuffer(t), s = new DataView(r), n = new Uint8Array(r);
|
|
82
|
+
let o = 0;
|
|
83
83
|
for (const a of e)
|
|
84
|
-
s.setUint32(
|
|
84
|
+
s.setUint32(o, a.byteLength, !0), o += 4, n.set(a, o), o += a.byteLength;
|
|
85
85
|
return n;
|
|
86
86
|
}
|
|
87
87
|
function g(c) {
|
|
@@ -96,7 +96,7 @@ function g(c) {
|
|
|
96
96
|
return e;
|
|
97
97
|
}
|
|
98
98
|
const m = 65536, w = 65536;
|
|
99
|
-
class
|
|
99
|
+
class T {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
@@ -106,8 +106,8 @@ class Q {
|
|
|
106
106
|
this.dimensions = e, this.queryOffset = 0;
|
|
107
107
|
const r = e * 4;
|
|
108
108
|
this.dbOffset = Math.ceil(r / m) * m;
|
|
109
|
-
const s = t * e * 4, n = this.dbOffset + s,
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
109
|
+
const s = t * e * 4, n = this.dbOffset + s, o = Math.max(1, Math.ceil(n / m));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: o }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -201,59 +201,59 @@ class Q {
|
|
|
201
201
|
this._vectorCount = e;
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
-
function
|
|
204
|
+
function Q(c, e, t, r) {
|
|
205
205
|
const s = c.length;
|
|
206
206
|
if (s === 0) return [];
|
|
207
207
|
const n = new Uint32Array(s);
|
|
208
|
-
for (let
|
|
209
|
-
n.sort((
|
|
210
|
-
const
|
|
211
|
-
for (let
|
|
212
|
-
const h = n[
|
|
213
|
-
if (r !== void 0 &&
|
|
214
|
-
a.push({ key: e(h), similarity:
|
|
208
|
+
for (let i = 0; i < s; i++) n[i] = i;
|
|
209
|
+
n.sort((i, h) => c[h] - c[i]);
|
|
210
|
+
const o = Math.min(t, s), a = [];
|
|
211
|
+
for (let i = 0; i < o; i++) {
|
|
212
|
+
const h = n[i], y = c[h];
|
|
213
|
+
if (r !== void 0 && y < r) break;
|
|
214
|
+
a.push({ key: e(h), similarity: y });
|
|
215
215
|
}
|
|
216
216
|
return a;
|
|
217
217
|
}
|
|
218
|
-
function
|
|
218
|
+
function V(c, e, t, r) {
|
|
219
219
|
const s = c.length;
|
|
220
220
|
if (s === 0) return [];
|
|
221
221
|
const n = new Uint32Array(s);
|
|
222
222
|
for (let a = 0; a < s; a++) n[a] = a;
|
|
223
|
-
n.sort((a,
|
|
224
|
-
const
|
|
223
|
+
n.sort((a, i) => c[i] - c[a]);
|
|
224
|
+
const o = Math.min(t, s);
|
|
225
225
|
return {
|
|
226
226
|
[Symbol.iterator]() {
|
|
227
227
|
let a = 0;
|
|
228
228
|
return {
|
|
229
229
|
next() {
|
|
230
|
-
if (a >=
|
|
231
|
-
const
|
|
230
|
+
if (a >= o) return { done: !0, value: void 0 };
|
|
231
|
+
const i = n[a++], h = c[i];
|
|
232
232
|
return r !== void 0 && h < r ? { done: !0, value: void 0 } : {
|
|
233
233
|
done: !1,
|
|
234
|
-
value: { key: e(
|
|
234
|
+
value: { key: e(i), similarity: h }
|
|
235
235
|
};
|
|
236
236
|
}
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
};
|
|
240
240
|
}
|
|
241
|
-
const
|
|
242
|
-
function
|
|
243
|
-
const c = atob(
|
|
241
|
+
const _ = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
242
|
+
function D() {
|
|
243
|
+
const c = atob(_), e = new Uint8Array(c.length);
|
|
244
244
|
for (let t = 0; t < c.length; t++)
|
|
245
245
|
e[t] = c.charCodeAt(t);
|
|
246
246
|
return e;
|
|
247
247
|
}
|
|
248
|
-
async function
|
|
248
|
+
async function L(c, e) {
|
|
249
249
|
const t = { env: { memory: e } };
|
|
250
250
|
return (await WebAssembly.instantiate(c, t)).instance.exports;
|
|
251
251
|
}
|
|
252
|
-
const b = "vectors.bin",
|
|
253
|
-
class
|
|
252
|
+
const b = "vectors.bin", C = "keys.bin", E = 1111770949, u = 1, M = 24, F = 65536;
|
|
253
|
+
class I {
|
|
254
254
|
memoryManager;
|
|
255
255
|
storage;
|
|
256
|
-
|
|
256
|
+
_dimensions;
|
|
257
257
|
shouldNormalize;
|
|
258
258
|
wasmExports;
|
|
259
259
|
/** Maps key to its slot index in the vector array */
|
|
@@ -262,46 +262,98 @@ class M {
|
|
|
262
262
|
slotToKey;
|
|
263
263
|
/** Whether this instance has been closed */
|
|
264
264
|
closed = !1;
|
|
265
|
-
constructor(e, t, r, s, n,
|
|
266
|
-
this.memoryManager = e, this.storage = t, this.
|
|
265
|
+
constructor(e, t, r, s, n, o, a) {
|
|
266
|
+
this.memoryManager = e, this.storage = t, this._dimensions = r, this.shouldNormalize = s, this.wasmExports = n, this.keyToSlot = o, this.slotToKey = a;
|
|
267
267
|
}
|
|
268
268
|
static async open(e) {
|
|
269
|
-
const t = e.storage ?? new
|
|
270
|
-
for (let A = 0; A <
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
s.byteLength > 0 &&
|
|
269
|
+
const t = e.storage ?? new S(), r = e.normalize !== !1, [s, n] = await Promise.all([t.readAll(b), t.readAll(C)]), o = n.byteLength > 0 ? g(n) : [], a = s.byteLength / (e.dimensions * 4), i = /* @__PURE__ */ new Map(), h = [];
|
|
270
|
+
for (let A = 0; A < o.length; A++)
|
|
271
|
+
i.set(o[A], A), h[A] = o[A];
|
|
272
|
+
const y = new T(e.dimensions, a);
|
|
273
|
+
s.byteLength > 0 && y.loadVectorBytes(s, a);
|
|
274
274
|
let l = null;
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
275
|
+
const f = e.wasmBinary !== void 0 ? e.wasmBinary : D();
|
|
276
|
+
if (f !== null)
|
|
277
277
|
try {
|
|
278
|
-
l = await
|
|
278
|
+
l = await L(f, y.memory);
|
|
279
279
|
} catch {
|
|
280
280
|
}
|
|
281
|
-
return new
|
|
281
|
+
return new I(y, t, e.dimensions, r, l, i, h);
|
|
282
282
|
}
|
|
283
283
|
/** Total number of key-value pairs in the database */
|
|
284
284
|
get size() {
|
|
285
285
|
return this.keyToSlot.size;
|
|
286
286
|
}
|
|
287
|
+
/** Number of dimensions per vector */
|
|
288
|
+
get dimensions() {
|
|
289
|
+
return this._dimensions;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Check whether a key exists in the database.
|
|
293
|
+
* Uses the internal key-to-slot map for O(1) lookup.
|
|
294
|
+
*/
|
|
295
|
+
has(e) {
|
|
296
|
+
return this.assertOpen(), this.keyToSlot.has(e);
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Delete an entry by key. Returns true if the key existed, false otherwise.
|
|
300
|
+
* Uses swap-and-pop to avoid gaps in the underlying vector array.
|
|
301
|
+
*/
|
|
302
|
+
delete(e) {
|
|
303
|
+
this.assertOpen();
|
|
304
|
+
const t = this.keyToSlot.get(e);
|
|
305
|
+
if (t === void 0) return !1;
|
|
306
|
+
const r = this.memoryManager.vectorCount - 1;
|
|
307
|
+
if (t !== r) {
|
|
308
|
+
const s = new Float32Array(this.memoryManager.readVector(r));
|
|
309
|
+
this.memoryManager.writeVector(t, s);
|
|
310
|
+
const n = this.slotToKey[r];
|
|
311
|
+
this.keyToSlot.set(n, t), this.slotToKey[t] = n;
|
|
312
|
+
}
|
|
313
|
+
return this.keyToSlot.delete(e), this.slotToKey.length = r, this.memoryManager.setVectorCount(r), !0;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Returns an iterable of all keys in the database.
|
|
317
|
+
*/
|
|
318
|
+
keys() {
|
|
319
|
+
return this.assertOpen(), this.keyToSlot.keys();
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Returns an iterable of [key, value] pairs.
|
|
323
|
+
* Values are returned as plain number array copies.
|
|
324
|
+
*/
|
|
325
|
+
entries() {
|
|
326
|
+
this.assertOpen();
|
|
327
|
+
const e = this.keyToSlot, t = this.memoryManager;
|
|
328
|
+
return (function* () {
|
|
329
|
+
for (const [r, s] of e)
|
|
330
|
+
yield [r, Array.from(t.readVector(s))];
|
|
331
|
+
})();
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Implements the iterable protocol. Same as entries().
|
|
335
|
+
*/
|
|
336
|
+
[Symbol.iterator]() {
|
|
337
|
+
return this.entries();
|
|
338
|
+
}
|
|
287
339
|
/**
|
|
288
340
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
289
341
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
290
342
|
*/
|
|
291
343
|
set(e, t, r) {
|
|
292
|
-
if (this.assertOpen(), t.length !== this.
|
|
293
|
-
throw new Error(`Vector dimension mismatch: expected ${this.
|
|
344
|
+
if (this.assertOpen(), t.length !== this._dimensions)
|
|
345
|
+
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${t.length}`);
|
|
294
346
|
const s = new Float32Array(t);
|
|
295
347
|
(r?.normalize ?? this.shouldNormalize) && this.normalizeVector(s);
|
|
296
|
-
const
|
|
297
|
-
if (
|
|
298
|
-
this.memoryManager.writeVector(
|
|
348
|
+
const o = this.keyToSlot.get(e);
|
|
349
|
+
if (o !== void 0)
|
|
350
|
+
this.memoryManager.writeVector(o, s);
|
|
299
351
|
else {
|
|
300
352
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
301
353
|
throw new k(this.memoryManager.maxVectors);
|
|
302
354
|
this.memoryManager.ensureCapacity(1);
|
|
303
|
-
const
|
|
304
|
-
this.memoryManager.appendVectors([s]), this.keyToSlot.set(e,
|
|
355
|
+
const i = this.memoryManager.vectorCount;
|
|
356
|
+
this.memoryManager.appendVectors([s]), this.keyToSlot.set(e, i), this.slotToKey[i] = e;
|
|
305
357
|
}
|
|
306
358
|
}
|
|
307
359
|
/**
|
|
@@ -332,50 +384,50 @@ class M {
|
|
|
332
384
|
const r = t?.topK ?? 1 / 0, s = t?.minSimilarity, n = t && "iterable" in t && t.iterable;
|
|
333
385
|
if (this.size === 0)
|
|
334
386
|
return [];
|
|
335
|
-
if (e.length !== this.
|
|
336
|
-
throw new Error(`Query vector dimension mismatch: expected ${this.
|
|
337
|
-
const
|
|
338
|
-
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
339
|
-
const
|
|
387
|
+
if (e.length !== this._dimensions)
|
|
388
|
+
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${e.length}`);
|
|
389
|
+
const o = new Float32Array(e);
|
|
390
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(o), this.memoryManager.writeQuery(o), this.memoryManager.ensureCapacity(0);
|
|
391
|
+
const i = this.memoryManager.vectorCount, h = this.memoryManager.scoresOffset;
|
|
340
392
|
if (this.wasmExports)
|
|
341
393
|
this.wasmExports.search_all(
|
|
342
394
|
this.memoryManager.queryOffset,
|
|
343
395
|
this.memoryManager.dbOffset,
|
|
344
396
|
h,
|
|
345
|
-
|
|
346
|
-
this.
|
|
397
|
+
i,
|
|
398
|
+
this._dimensions
|
|
347
399
|
);
|
|
348
400
|
else {
|
|
349
401
|
const A = new Float32Array(
|
|
350
402
|
this.memoryManager.memory.buffer,
|
|
351
403
|
this.memoryManager.queryOffset,
|
|
352
|
-
this.
|
|
353
|
-
),
|
|
404
|
+
this._dimensions
|
|
405
|
+
), p = new Float32Array(
|
|
354
406
|
this.memoryManager.memory.buffer,
|
|
355
407
|
this.memoryManager.dbOffset,
|
|
356
|
-
|
|
357
|
-
),
|
|
358
|
-
O(A,
|
|
408
|
+
i * this._dimensions
|
|
409
|
+
), B = new Float32Array(this.memoryManager.memory.buffer, h, i);
|
|
410
|
+
O(A, p, B, i, this._dimensions);
|
|
359
411
|
}
|
|
360
|
-
const
|
|
361
|
-
return n ?
|
|
412
|
+
const y = new Float32Array(this.memoryManager.readScores()), l = this.slotToKey, f = (A) => l[A];
|
|
413
|
+
return n ? V(y, f, r, s) : Q(y, f, r, s);
|
|
362
414
|
}
|
|
363
415
|
/**
|
|
364
416
|
* Persist the current in-memory state to storage.
|
|
365
417
|
*/
|
|
366
418
|
async flush() {
|
|
367
419
|
this.assertOpen();
|
|
368
|
-
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this.
|
|
420
|
+
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this._dimensions * 4);
|
|
369
421
|
if (e > 0) {
|
|
370
422
|
const s = new Uint8Array(
|
|
371
423
|
this.memoryManager.memory.buffer,
|
|
372
424
|
this.memoryManager.dbOffset,
|
|
373
|
-
e * this.
|
|
425
|
+
e * this._dimensions * 4
|
|
374
426
|
);
|
|
375
427
|
t.set(s);
|
|
376
428
|
}
|
|
377
429
|
const r = d(this.slotToKey);
|
|
378
|
-
await Promise.all([this.storage.write(b, t), this.storage.write(
|
|
430
|
+
await Promise.all([this.storage.write(b, t), this.storage.write(C, r)]);
|
|
379
431
|
}
|
|
380
432
|
/**
|
|
381
433
|
* Flush data to storage and release the instance.
|
|
@@ -401,23 +453,23 @@ class M {
|
|
|
401
453
|
*/
|
|
402
454
|
async export() {
|
|
403
455
|
this.assertOpen();
|
|
404
|
-
const e = this.memoryManager.vectorCount, t = e * this.
|
|
405
|
-
|
|
456
|
+
const e = this.memoryManager.vectorCount, t = e * this._dimensions * 4, r = d(this.slotToKey), s = r.byteLength, n = new ArrayBuffer(M), o = new DataView(n);
|
|
457
|
+
o.setUint32(0, E, !0), o.setUint32(4, u, !0), o.setUint32(8, this._dimensions, !0), o.setUint32(12, e, !0), o.setUint32(16, t, !0), o.setUint32(20, s, !0);
|
|
406
458
|
const a = this.memoryManager;
|
|
407
|
-
let
|
|
459
|
+
let i = "header", h = 0;
|
|
408
460
|
return new ReadableStream({
|
|
409
|
-
pull(
|
|
410
|
-
switch (
|
|
461
|
+
pull(y) {
|
|
462
|
+
switch (i) {
|
|
411
463
|
case "header":
|
|
412
|
-
|
|
464
|
+
y.enqueue(new Uint8Array(n)), i = t > 0 ? "vectors" : s > 0 ? "keys" : "done", i === "done" && y.close();
|
|
413
465
|
break;
|
|
414
466
|
case "vectors": {
|
|
415
|
-
const l = t - h,
|
|
416
|
-
A.set(new Uint8Array(a.memory.buffer, a.dbOffset + h,
|
|
467
|
+
const l = t - h, f = Math.min(l, F), A = new Uint8Array(f);
|
|
468
|
+
A.set(new Uint8Array(a.memory.buffer, a.dbOffset + h, f)), y.enqueue(A), h += f, h >= t && (i = s > 0 ? "keys" : "done", i === "done" && y.close());
|
|
417
469
|
break;
|
|
418
470
|
}
|
|
419
471
|
case "keys":
|
|
420
|
-
|
|
472
|
+
y.enqueue(r), i = "done", y.close();
|
|
421
473
|
break;
|
|
422
474
|
}
|
|
423
475
|
}
|
|
@@ -432,29 +484,29 @@ class M {
|
|
|
432
484
|
*/
|
|
433
485
|
async import(e) {
|
|
434
486
|
this.assertOpen();
|
|
435
|
-
const t = new
|
|
436
|
-
if (s.getUint32(0, !0) !==
|
|
487
|
+
const t = new z(e), r = await t.readExact(M), s = new DataView(r.buffer, r.byteOffset, r.byteLength);
|
|
488
|
+
if (s.getUint32(0, !0) !== E)
|
|
437
489
|
throw new Error("Invalid import data: unrecognized format");
|
|
438
|
-
const
|
|
439
|
-
if (
|
|
440
|
-
throw new Error(`Unsupported import version: expected ${u}, got ${
|
|
490
|
+
const o = s.getUint32(4, !0);
|
|
491
|
+
if (o !== u)
|
|
492
|
+
throw new Error(`Unsupported import version: expected ${u}, got ${o}`);
|
|
441
493
|
const a = s.getUint32(8, !0);
|
|
442
|
-
if (a !== this.
|
|
443
|
-
throw new Error(`Import dimension mismatch: expected ${this.
|
|
444
|
-
const
|
|
445
|
-
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(),
|
|
446
|
-
this.memoryManager.ensureCapacity(
|
|
494
|
+
if (a !== this._dimensions)
|
|
495
|
+
throw new Error(`Import dimension mismatch: expected ${this._dimensions}, got ${a}`);
|
|
496
|
+
const i = s.getUint32(12, !0), h = s.getUint32(16, !0), y = s.getUint32(20, !0);
|
|
497
|
+
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), i > 0) {
|
|
498
|
+
this.memoryManager.ensureCapacity(i);
|
|
447
499
|
let l = 0;
|
|
448
|
-
await t.readChunked(h, (
|
|
449
|
-
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l,
|
|
450
|
-
|
|
451
|
-
), l +=
|
|
452
|
-
}), this.memoryManager.setVectorCount(
|
|
500
|
+
await t.readChunked(h, (f) => {
|
|
501
|
+
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, f.byteLength).set(
|
|
502
|
+
f
|
|
503
|
+
), l += f.byteLength;
|
|
504
|
+
}), this.memoryManager.setVectorCount(i);
|
|
453
505
|
}
|
|
454
|
-
if (
|
|
455
|
-
const l = await t.readExact(
|
|
456
|
-
for (let A = 0; A <
|
|
457
|
-
this.keyToSlot.set(
|
|
506
|
+
if (y > 0) {
|
|
507
|
+
const l = await t.readExact(y), f = g(l);
|
|
508
|
+
for (let A = 0; A < f.length; A++)
|
|
509
|
+
this.keyToSlot.set(f[A], A), this.slotToKey[A] = f[A];
|
|
458
510
|
}
|
|
459
511
|
t.release();
|
|
460
512
|
}
|
|
@@ -468,14 +520,14 @@ class M {
|
|
|
468
520
|
const r = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
469
521
|
e.set(r);
|
|
470
522
|
} else
|
|
471
|
-
|
|
523
|
+
v(e);
|
|
472
524
|
}
|
|
473
525
|
assertOpen() {
|
|
474
526
|
if (this.closed)
|
|
475
527
|
throw new Error("VectorDB instance has been closed");
|
|
476
528
|
}
|
|
477
529
|
}
|
|
478
|
-
class
|
|
530
|
+
class z {
|
|
479
531
|
reader;
|
|
480
532
|
buffer = new Uint8Array(0);
|
|
481
533
|
constructor(e) {
|
|
@@ -485,20 +537,20 @@ class q {
|
|
|
485
537
|
async readExact(e) {
|
|
486
538
|
if (e === 0) return new Uint8Array(0);
|
|
487
539
|
if (this.buffer.byteLength >= e) {
|
|
488
|
-
const
|
|
489
|
-
return this.buffer = this.buffer.subarray(e),
|
|
540
|
+
const o = this.buffer.subarray(0, e);
|
|
541
|
+
return this.buffer = this.buffer.subarray(e), o;
|
|
490
542
|
}
|
|
491
543
|
const t = [];
|
|
492
544
|
let r = this.buffer.byteLength;
|
|
493
545
|
for (r > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); r < e; ) {
|
|
494
|
-
const { done:
|
|
495
|
-
if (
|
|
546
|
+
const { done: o, value: a } = await this.reader.read();
|
|
547
|
+
if (o) throw new Error("Invalid import data: unexpected end of stream");
|
|
496
548
|
t.push(a), r += a.byteLength;
|
|
497
549
|
}
|
|
498
550
|
const s = new Uint8Array(r);
|
|
499
551
|
let n = 0;
|
|
500
|
-
for (const
|
|
501
|
-
s.set(
|
|
552
|
+
for (const o of t)
|
|
553
|
+
s.set(o, n), n += o.byteLength;
|
|
502
554
|
return r > e ? (this.buffer = s.subarray(e), s.subarray(0, e)) : s;
|
|
503
555
|
}
|
|
504
556
|
/**
|
|
@@ -523,9 +575,9 @@ class q {
|
|
|
523
575
|
}
|
|
524
576
|
}
|
|
525
577
|
export {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
578
|
+
I as DB,
|
|
579
|
+
S as InMemoryStorageProvider,
|
|
580
|
+
K as OPFSStorageProvider,
|
|
529
581
|
k as VectorCapacityExceededError
|
|
530
582
|
};
|
|
531
583
|
//# sourceMappingURL=eigen-db.js.map
|