eigen-db 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +55 -8
- package/dist/compute.d.ts +20 -0
- package/dist/eigen-db.js +242 -190
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/dist/errors.d.ts +7 -0
- package/dist/index.d.ts +12 -0
- package/dist/lexicon.d.ts +28 -0
- package/dist/memory-manager.d.ts +68 -0
- package/dist/result-set.d.ts +35 -0
- package/dist/simd-binary.d.ts +1 -0
- package/dist/storage.d.ts +38 -0
- package/dist/types.d.ts +44 -0
- package/dist/vector-db.d.ts +131 -0
- package/dist/wasm-compute.d.ts +13 -0
- package/package.json +4 -4
- package/src/lib/__tests__/vector-db.test.ts +328 -0
- package/src/lib/simd-binary.ts +1 -1
- package/src/lib/simd-optimized.wat +362 -0
- package/src/lib/simd.wat +42 -248
- package/src/lib/types.ts +2 -4
- package/src/lib/vector-db.ts +93 -19
package/dist/eigen-db.js
CHANGED
|
@@ -3,7 +3,7 @@ class k extends Error {
|
|
|
3
3
|
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class K {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
9
|
constructor(e) {
|
|
@@ -18,34 +18,34 @@ class D {
|
|
|
18
18
|
}
|
|
19
19
|
async readAll(e) {
|
|
20
20
|
try {
|
|
21
|
-
const
|
|
22
|
-
return new Uint8Array(
|
|
21
|
+
const n = await (await (await (await this.getDir()).getFileHandle(e)).getFile()).arrayBuffer();
|
|
22
|
+
return new Uint8Array(n);
|
|
23
23
|
} catch {
|
|
24
24
|
return new Uint8Array(0);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
async append(e, t) {
|
|
28
|
-
const
|
|
29
|
-
await
|
|
28
|
+
const s = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await s.createWritable({ keepExistingData: !0 }), o = await s.getFile();
|
|
29
|
+
await n.seek(o.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
31
|
async write(e, t) {
|
|
32
|
-
const
|
|
33
|
-
await
|
|
32
|
+
const n = await (await (await this.getDir()).getFileHandle(e, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
33
|
+
await n.write(t), await n.close();
|
|
34
34
|
}
|
|
35
35
|
async destroy() {
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class S {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
41
|
async readAll(e) {
|
|
42
42
|
const t = this.files.get(e);
|
|
43
43
|
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
-
const
|
|
45
|
-
let
|
|
46
|
-
for (const
|
|
47
|
-
|
|
48
|
-
return
|
|
44
|
+
const r = t.reduce((o, a) => o + a.byteLength, 0), s = new Uint8Array(r);
|
|
45
|
+
let n = 0;
|
|
46
|
+
for (const o of t)
|
|
47
|
+
s.set(o, n), n += o.byteLength;
|
|
48
|
+
return s;
|
|
49
49
|
}
|
|
50
50
|
async append(e, t) {
|
|
51
51
|
this.files.has(e) || this.files.set(e, []), this.files.get(e).push(new Uint8Array(t));
|
|
@@ -57,46 +57,46 @@ class F {
|
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
60
|
+
function v(c) {
|
|
61
61
|
let e = 0;
|
|
62
|
-
for (let
|
|
63
|
-
e +=
|
|
62
|
+
for (let s = 0; s < c.length; s++)
|
|
63
|
+
e += c[s] * c[s];
|
|
64
64
|
const t = Math.sqrt(e);
|
|
65
65
|
if (t === 0) return;
|
|
66
|
-
const
|
|
67
|
-
for (let
|
|
68
|
-
|
|
66
|
+
const r = 1 / t;
|
|
67
|
+
for (let s = 0; s < c.length; s++)
|
|
68
|
+
c[s] *= r;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let
|
|
72
|
-
let
|
|
73
|
-
const
|
|
74
|
-
for (let i = 0; i <
|
|
75
|
-
|
|
76
|
-
t[
|
|
70
|
+
function O(c, e, t, r, s) {
|
|
71
|
+
for (let n = 0; n < r; n++) {
|
|
72
|
+
let o = 0;
|
|
73
|
+
const a = n * s;
|
|
74
|
+
for (let i = 0; i < s; i++)
|
|
75
|
+
o += c[i] * e[a + i];
|
|
76
|
+
t[n] = o;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function
|
|
81
|
-
const e =
|
|
82
|
-
let
|
|
83
|
-
for (const
|
|
84
|
-
|
|
85
|
-
return
|
|
79
|
+
const U = new TextEncoder(), x = new TextDecoder();
|
|
80
|
+
function d(c) {
|
|
81
|
+
const e = c.map((a) => U.encode(a)), t = e.reduce((a, i) => a + 4 + i.byteLength, 0), r = new ArrayBuffer(t), s = new DataView(r), n = new Uint8Array(r);
|
|
82
|
+
let o = 0;
|
|
83
|
+
for (const a of e)
|
|
84
|
+
s.setUint32(o, a.byteLength, !0), o += 4, n.set(a, o), o += a.byteLength;
|
|
85
|
+
return n;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const e = [], t = new DataView(
|
|
89
|
-
let
|
|
90
|
-
for (;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
e.push(
|
|
87
|
+
function g(c) {
|
|
88
|
+
const e = [], t = new DataView(c.buffer, c.byteOffset, c.byteLength);
|
|
89
|
+
let r = 0;
|
|
90
|
+
for (; r < c.byteLength; ) {
|
|
91
|
+
const s = t.getUint32(r, !0);
|
|
92
|
+
r += 4;
|
|
93
|
+
const n = x.decode(c.subarray(r, r + s));
|
|
94
|
+
e.push(n), r += s;
|
|
95
95
|
}
|
|
96
96
|
return e;
|
|
97
97
|
}
|
|
98
|
-
const
|
|
99
|
-
class
|
|
98
|
+
const m = 65536, w = 65536;
|
|
99
|
+
class T {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
@@ -104,10 +104,10 @@ class L {
|
|
|
104
104
|
_vectorCount;
|
|
105
105
|
constructor(e, t = 0) {
|
|
106
106
|
this.dimensions = e, this.queryOffset = 0;
|
|
107
|
-
const
|
|
108
|
-
this.dbOffset = Math.ceil(
|
|
109
|
-
const
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
107
|
+
const r = e * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(r / m) * m;
|
|
109
|
+
const s = t * e * 4, n = this.dbOffset + s, o = Math.max(1, Math.ceil(n / m));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: o }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -126,7 +126,7 @@ class L {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const e =
|
|
129
|
+
const e = w * m - this.dbOffset, t = this.dimensions * 4 + 4;
|
|
130
130
|
return Math.floor(e / t);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
@@ -134,12 +134,12 @@ class L {
|
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
136
|
ensureCapacity(e) {
|
|
137
|
-
const t = this._vectorCount + e,
|
|
138
|
-
if (
|
|
139
|
-
const
|
|
140
|
-
if (
|
|
137
|
+
const t = this._vectorCount + e, r = this.dbOffset + t * this.dimensions * 4 + t * 4, s = this.memory.buffer.byteLength;
|
|
138
|
+
if (r > s) {
|
|
139
|
+
const n = Math.ceil((r - s) / m);
|
|
140
|
+
if (s / m + n > w)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
|
-
this.memory.grow(
|
|
142
|
+
this.memory.grow(n);
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
/**
|
|
@@ -154,9 +154,9 @@ class L {
|
|
|
154
154
|
*/
|
|
155
155
|
appendVectors(e) {
|
|
156
156
|
const t = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
-
let
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer,
|
|
157
|
+
let r = t;
|
|
158
|
+
for (const s of e)
|
|
159
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(s), r += this.dimensions * 4;
|
|
160
160
|
return this._vectorCount += e.length, t;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
@@ -183,8 +183,8 @@ class L {
|
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
185
|
writeVector(e, t) {
|
|
186
|
-
const
|
|
187
|
-
new Float32Array(this.memory.buffer,
|
|
186
|
+
const r = this.dbOffset + e * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(t);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -201,59 +201,59 @@ class L {
|
|
|
201
201
|
this._vectorCount = e;
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
-
function
|
|
205
|
-
const
|
|
206
|
-
if (
|
|
207
|
-
const
|
|
208
|
-
for (let i = 0; i <
|
|
209
|
-
|
|
210
|
-
const
|
|
211
|
-
for (let i = 0; i <
|
|
212
|
-
const
|
|
213
|
-
if (
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
return
|
|
204
|
+
function Q(c, e, t, r) {
|
|
205
|
+
const s = c.length;
|
|
206
|
+
if (s === 0) return [];
|
|
207
|
+
const n = new Uint32Array(s);
|
|
208
|
+
for (let i = 0; i < s; i++) n[i] = i;
|
|
209
|
+
n.sort((i, h) => c[h] - c[i]);
|
|
210
|
+
const o = Math.min(t, s), a = [];
|
|
211
|
+
for (let i = 0; i < o; i++) {
|
|
212
|
+
const h = n[i], y = c[h];
|
|
213
|
+
if (r !== void 0 && y < r) break;
|
|
214
|
+
a.push({ key: e(h), similarity: y });
|
|
215
|
+
}
|
|
216
|
+
return a;
|
|
217
217
|
}
|
|
218
|
-
function
|
|
219
|
-
const
|
|
220
|
-
if (
|
|
221
|
-
const
|
|
222
|
-
for (let
|
|
223
|
-
|
|
224
|
-
const
|
|
218
|
+
function V(c, e, t, r) {
|
|
219
|
+
const s = c.length;
|
|
220
|
+
if (s === 0) return [];
|
|
221
|
+
const n = new Uint32Array(s);
|
|
222
|
+
for (let a = 0; a < s; a++) n[a] = a;
|
|
223
|
+
n.sort((a, i) => c[i] - c[a]);
|
|
224
|
+
const o = Math.min(t, s);
|
|
225
225
|
return {
|
|
226
226
|
[Symbol.iterator]() {
|
|
227
|
-
let
|
|
227
|
+
let a = 0;
|
|
228
228
|
return {
|
|
229
229
|
next() {
|
|
230
|
-
if (
|
|
231
|
-
const i =
|
|
232
|
-
return
|
|
230
|
+
if (a >= o) return { done: !0, value: void 0 };
|
|
231
|
+
const i = n[a++], h = c[i];
|
|
232
|
+
return r !== void 0 && h < r ? { done: !0, value: void 0 } : {
|
|
233
233
|
done: !1,
|
|
234
|
-
value: { key: e(i), similarity:
|
|
234
|
+
value: { key: e(i), similarity: h }
|
|
235
235
|
};
|
|
236
236
|
}
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
};
|
|
240
240
|
}
|
|
241
|
-
const
|
|
242
|
-
function
|
|
243
|
-
const
|
|
244
|
-
for (let t = 0; t <
|
|
245
|
-
e[t] =
|
|
241
|
+
const _ = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
242
|
+
function D() {
|
|
243
|
+
const c = atob(_), e = new Uint8Array(c.length);
|
|
244
|
+
for (let t = 0; t < c.length; t++)
|
|
245
|
+
e[t] = c.charCodeAt(t);
|
|
246
246
|
return e;
|
|
247
247
|
}
|
|
248
|
-
async function
|
|
248
|
+
async function L(c, e) {
|
|
249
249
|
const t = { env: { memory: e } };
|
|
250
|
-
return (await WebAssembly.instantiate(
|
|
250
|
+
return (await WebAssembly.instantiate(c, t)).instance.exports;
|
|
251
251
|
}
|
|
252
|
-
const
|
|
253
|
-
class
|
|
252
|
+
const b = "vectors.bin", C = "keys.bin", E = 1111770949, u = 1, M = 24, F = 65536;
|
|
253
|
+
class I {
|
|
254
254
|
memoryManager;
|
|
255
255
|
storage;
|
|
256
|
-
|
|
256
|
+
_dimensions;
|
|
257
257
|
shouldNormalize;
|
|
258
258
|
wasmExports;
|
|
259
259
|
/** Maps key to its slot index in the vector array */
|
|
@@ -262,46 +262,98 @@ class b {
|
|
|
262
262
|
slotToKey;
|
|
263
263
|
/** Whether this instance has been closed */
|
|
264
264
|
closed = !1;
|
|
265
|
-
constructor(e, t,
|
|
266
|
-
this.memoryManager = e, this.storage = t, this.
|
|
265
|
+
constructor(e, t, r, s, n, o, a) {
|
|
266
|
+
this.memoryManager = e, this.storage = t, this._dimensions = r, this.shouldNormalize = s, this.wasmExports = n, this.keyToSlot = o, this.slotToKey = a;
|
|
267
267
|
}
|
|
268
268
|
static async open(e) {
|
|
269
|
-
const t = e.
|
|
270
|
-
for (let
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
let
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
269
|
+
const t = e.storage ?? new S(), r = e.normalize !== !1, [s, n] = await Promise.all([t.readAll(b), t.readAll(C)]), o = n.byteLength > 0 ? g(n) : [], a = s.byteLength / (e.dimensions * 4), i = /* @__PURE__ */ new Map(), h = [];
|
|
270
|
+
for (let A = 0; A < o.length; A++)
|
|
271
|
+
i.set(o[A], A), h[A] = o[A];
|
|
272
|
+
const y = new T(e.dimensions, a);
|
|
273
|
+
s.byteLength > 0 && y.loadVectorBytes(s, a);
|
|
274
|
+
let l = null;
|
|
275
|
+
const f = e.wasmBinary !== void 0 ? e.wasmBinary : D();
|
|
276
|
+
if (f !== null)
|
|
277
277
|
try {
|
|
278
|
-
|
|
278
|
+
l = await L(f, y.memory);
|
|
279
279
|
} catch {
|
|
280
280
|
}
|
|
281
|
-
return new
|
|
281
|
+
return new I(y, t, e.dimensions, r, l, i, h);
|
|
282
282
|
}
|
|
283
283
|
/** Total number of key-value pairs in the database */
|
|
284
284
|
get size() {
|
|
285
285
|
return this.keyToSlot.size;
|
|
286
286
|
}
|
|
287
|
+
/** Number of dimensions per vector */
|
|
288
|
+
get dimensions() {
|
|
289
|
+
return this._dimensions;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Check whether a key exists in the database.
|
|
293
|
+
* Uses the internal key-to-slot map for O(1) lookup.
|
|
294
|
+
*/
|
|
295
|
+
has(e) {
|
|
296
|
+
return this.assertOpen(), this.keyToSlot.has(e);
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Delete an entry by key. Returns true if the key existed, false otherwise.
|
|
300
|
+
* Uses swap-and-pop to avoid gaps in the underlying vector array.
|
|
301
|
+
*/
|
|
302
|
+
delete(e) {
|
|
303
|
+
this.assertOpen();
|
|
304
|
+
const t = this.keyToSlot.get(e);
|
|
305
|
+
if (t === void 0) return !1;
|
|
306
|
+
const r = this.memoryManager.vectorCount - 1;
|
|
307
|
+
if (t !== r) {
|
|
308
|
+
const s = new Float32Array(this.memoryManager.readVector(r));
|
|
309
|
+
this.memoryManager.writeVector(t, s);
|
|
310
|
+
const n = this.slotToKey[r];
|
|
311
|
+
this.keyToSlot.set(n, t), this.slotToKey[t] = n;
|
|
312
|
+
}
|
|
313
|
+
return this.keyToSlot.delete(e), this.slotToKey.length = r, this.memoryManager.setVectorCount(r), !0;
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Returns an iterable of all keys in the database.
|
|
317
|
+
*/
|
|
318
|
+
keys() {
|
|
319
|
+
return this.assertOpen(), this.keyToSlot.keys();
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Returns an iterable of [key, value] pairs.
|
|
323
|
+
* Values are returned as plain number array copies.
|
|
324
|
+
*/
|
|
325
|
+
entries() {
|
|
326
|
+
this.assertOpen();
|
|
327
|
+
const e = this.keyToSlot, t = this.memoryManager;
|
|
328
|
+
return (function* () {
|
|
329
|
+
for (const [r, s] of e)
|
|
330
|
+
yield [r, Array.from(t.readVector(s))];
|
|
331
|
+
})();
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Implements the iterable protocol. Same as entries().
|
|
335
|
+
*/
|
|
336
|
+
[Symbol.iterator]() {
|
|
337
|
+
return this.entries();
|
|
338
|
+
}
|
|
287
339
|
/**
|
|
288
340
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
289
341
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
290
342
|
*/
|
|
291
|
-
set(e, t,
|
|
292
|
-
if (this.assertOpen(), t.length !== this.
|
|
293
|
-
throw new Error(`Vector dimension mismatch: expected ${this.
|
|
294
|
-
const
|
|
295
|
-
(
|
|
296
|
-
const
|
|
297
|
-
if (
|
|
298
|
-
this.memoryManager.writeVector(
|
|
343
|
+
set(e, t, r) {
|
|
344
|
+
if (this.assertOpen(), t.length !== this._dimensions)
|
|
345
|
+
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${t.length}`);
|
|
346
|
+
const s = new Float32Array(t);
|
|
347
|
+
(r?.normalize ?? this.shouldNormalize) && this.normalizeVector(s);
|
|
348
|
+
const o = this.keyToSlot.get(e);
|
|
349
|
+
if (o !== void 0)
|
|
350
|
+
this.memoryManager.writeVector(o, s);
|
|
299
351
|
else {
|
|
300
352
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
301
353
|
throw new k(this.memoryManager.maxVectors);
|
|
302
354
|
this.memoryManager.ensureCapacity(1);
|
|
303
355
|
const i = this.memoryManager.vectorCount;
|
|
304
|
-
this.memoryManager.appendVectors([
|
|
356
|
+
this.memoryManager.appendVectors([s]), this.keyToSlot.set(e, i), this.slotToKey[i] = e;
|
|
305
357
|
}
|
|
306
358
|
}
|
|
307
359
|
/**
|
|
@@ -318,8 +370,8 @@ class b {
|
|
|
318
370
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
319
371
|
*/
|
|
320
372
|
setMany(e) {
|
|
321
|
-
for (const [t,
|
|
322
|
-
this.set(t,
|
|
373
|
+
for (const [t, r] of e)
|
|
374
|
+
this.set(t, r);
|
|
323
375
|
}
|
|
324
376
|
/**
|
|
325
377
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
@@ -329,53 +381,53 @@ class b {
|
|
|
329
381
|
}
|
|
330
382
|
query(e, t) {
|
|
331
383
|
this.assertOpen();
|
|
332
|
-
const
|
|
384
|
+
const r = t?.topK ?? 1 / 0, s = t?.minSimilarity, n = t && "iterable" in t && t.iterable;
|
|
333
385
|
if (this.size === 0)
|
|
334
386
|
return [];
|
|
335
|
-
if (e.length !== this.
|
|
336
|
-
throw new Error(`Query vector dimension mismatch: expected ${this.
|
|
337
|
-
const
|
|
338
|
-
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
339
|
-
const i = this.memoryManager.vectorCount,
|
|
387
|
+
if (e.length !== this._dimensions)
|
|
388
|
+
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${e.length}`);
|
|
389
|
+
const o = new Float32Array(e);
|
|
390
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(o), this.memoryManager.writeQuery(o), this.memoryManager.ensureCapacity(0);
|
|
391
|
+
const i = this.memoryManager.vectorCount, h = this.memoryManager.scoresOffset;
|
|
340
392
|
if (this.wasmExports)
|
|
341
393
|
this.wasmExports.search_all(
|
|
342
394
|
this.memoryManager.queryOffset,
|
|
343
395
|
this.memoryManager.dbOffset,
|
|
344
|
-
|
|
396
|
+
h,
|
|
345
397
|
i,
|
|
346
|
-
this.
|
|
398
|
+
this._dimensions
|
|
347
399
|
);
|
|
348
400
|
else {
|
|
349
|
-
const
|
|
401
|
+
const A = new Float32Array(
|
|
350
402
|
this.memoryManager.memory.buffer,
|
|
351
403
|
this.memoryManager.queryOffset,
|
|
352
|
-
this.
|
|
353
|
-
),
|
|
404
|
+
this._dimensions
|
|
405
|
+
), p = new Float32Array(
|
|
354
406
|
this.memoryManager.memory.buffer,
|
|
355
407
|
this.memoryManager.dbOffset,
|
|
356
|
-
i * this.
|
|
357
|
-
),
|
|
358
|
-
|
|
408
|
+
i * this._dimensions
|
|
409
|
+
), B = new Float32Array(this.memoryManager.memory.buffer, h, i);
|
|
410
|
+
O(A, p, B, i, this._dimensions);
|
|
359
411
|
}
|
|
360
|
-
const
|
|
361
|
-
return
|
|
412
|
+
const y = new Float32Array(this.memoryManager.readScores()), l = this.slotToKey, f = (A) => l[A];
|
|
413
|
+
return n ? V(y, f, r, s) : Q(y, f, r, s);
|
|
362
414
|
}
|
|
363
415
|
/**
|
|
364
416
|
* Persist the current in-memory state to storage.
|
|
365
417
|
*/
|
|
366
418
|
async flush() {
|
|
367
419
|
this.assertOpen();
|
|
368
|
-
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this.
|
|
420
|
+
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this._dimensions * 4);
|
|
369
421
|
if (e > 0) {
|
|
370
|
-
const
|
|
422
|
+
const s = new Uint8Array(
|
|
371
423
|
this.memoryManager.memory.buffer,
|
|
372
424
|
this.memoryManager.dbOffset,
|
|
373
|
-
e * this.
|
|
425
|
+
e * this._dimensions * 4
|
|
374
426
|
);
|
|
375
|
-
t.set(
|
|
427
|
+
t.set(s);
|
|
376
428
|
}
|
|
377
|
-
const
|
|
378
|
-
await Promise.all([this.storage.write(
|
|
429
|
+
const r = d(this.slotToKey);
|
|
430
|
+
await Promise.all([this.storage.write(b, t), this.storage.write(C, r)]);
|
|
379
431
|
}
|
|
380
432
|
/**
|
|
381
433
|
* Flush data to storage and release the instance.
|
|
@@ -401,23 +453,23 @@ class b {
|
|
|
401
453
|
*/
|
|
402
454
|
async export() {
|
|
403
455
|
this.assertOpen();
|
|
404
|
-
const e = this.memoryManager.vectorCount, t = e * this.
|
|
405
|
-
|
|
406
|
-
const
|
|
407
|
-
let i = "header",
|
|
456
|
+
const e = this.memoryManager.vectorCount, t = e * this._dimensions * 4, r = d(this.slotToKey), s = r.byteLength, n = new ArrayBuffer(M), o = new DataView(n);
|
|
457
|
+
o.setUint32(0, E, !0), o.setUint32(4, u, !0), o.setUint32(8, this._dimensions, !0), o.setUint32(12, e, !0), o.setUint32(16, t, !0), o.setUint32(20, s, !0);
|
|
458
|
+
const a = this.memoryManager;
|
|
459
|
+
let i = "header", h = 0;
|
|
408
460
|
return new ReadableStream({
|
|
409
|
-
pull(
|
|
461
|
+
pull(y) {
|
|
410
462
|
switch (i) {
|
|
411
463
|
case "header":
|
|
412
|
-
|
|
464
|
+
y.enqueue(new Uint8Array(n)), i = t > 0 ? "vectors" : s > 0 ? "keys" : "done", i === "done" && y.close();
|
|
413
465
|
break;
|
|
414
466
|
case "vectors": {
|
|
415
|
-
const
|
|
416
|
-
|
|
467
|
+
const l = t - h, f = Math.min(l, F), A = new Uint8Array(f);
|
|
468
|
+
A.set(new Uint8Array(a.memory.buffer, a.dbOffset + h, f)), y.enqueue(A), h += f, h >= t && (i = s > 0 ? "keys" : "done", i === "done" && y.close());
|
|
417
469
|
break;
|
|
418
470
|
}
|
|
419
471
|
case "keys":
|
|
420
|
-
|
|
472
|
+
y.enqueue(r), i = "done", y.close();
|
|
421
473
|
break;
|
|
422
474
|
}
|
|
423
475
|
}
|
|
@@ -432,29 +484,29 @@ class b {
|
|
|
432
484
|
*/
|
|
433
485
|
async import(e) {
|
|
434
486
|
this.assertOpen();
|
|
435
|
-
const t = new
|
|
436
|
-
if (
|
|
487
|
+
const t = new z(e), r = await t.readExact(M), s = new DataView(r.buffer, r.byteOffset, r.byteLength);
|
|
488
|
+
if (s.getUint32(0, !0) !== E)
|
|
437
489
|
throw new Error("Invalid import data: unrecognized format");
|
|
438
|
-
const
|
|
439
|
-
if (
|
|
440
|
-
throw new Error(`Unsupported import version: expected ${
|
|
441
|
-
const
|
|
442
|
-
if (
|
|
443
|
-
throw new Error(`Import dimension mismatch: expected ${this.
|
|
444
|
-
const i =
|
|
490
|
+
const o = s.getUint32(4, !0);
|
|
491
|
+
if (o !== u)
|
|
492
|
+
throw new Error(`Unsupported import version: expected ${u}, got ${o}`);
|
|
493
|
+
const a = s.getUint32(8, !0);
|
|
494
|
+
if (a !== this._dimensions)
|
|
495
|
+
throw new Error(`Import dimension mismatch: expected ${this._dimensions}, got ${a}`);
|
|
496
|
+
const i = s.getUint32(12, !0), h = s.getUint32(16, !0), y = s.getUint32(20, !0);
|
|
445
497
|
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), i > 0) {
|
|
446
498
|
this.memoryManager.ensureCapacity(i);
|
|
447
|
-
let
|
|
448
|
-
await t.readChunked(
|
|
449
|
-
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset +
|
|
499
|
+
let l = 0;
|
|
500
|
+
await t.readChunked(h, (f) => {
|
|
501
|
+
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, f.byteLength).set(
|
|
450
502
|
f
|
|
451
|
-
),
|
|
503
|
+
), l += f.byteLength;
|
|
452
504
|
}), this.memoryManager.setVectorCount(i);
|
|
453
505
|
}
|
|
454
|
-
if (
|
|
455
|
-
const
|
|
456
|
-
for (let
|
|
457
|
-
this.keyToSlot.set(f[
|
|
506
|
+
if (y > 0) {
|
|
507
|
+
const l = await t.readExact(y), f = g(l);
|
|
508
|
+
for (let A = 0; A < f.length; A++)
|
|
509
|
+
this.keyToSlot.set(f[A], A), this.slotToKey[A] = f[A];
|
|
458
510
|
}
|
|
459
511
|
t.release();
|
|
460
512
|
}
|
|
@@ -465,17 +517,17 @@ class b {
|
|
|
465
517
|
if (this.wasmExports) {
|
|
466
518
|
const t = this.memoryManager.queryOffset;
|
|
467
519
|
new Float32Array(this.memoryManager.memory.buffer, t, e.length).set(e), this.wasmExports.normalize(t, e.length);
|
|
468
|
-
const
|
|
469
|
-
e.set(
|
|
520
|
+
const r = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
521
|
+
e.set(r);
|
|
470
522
|
} else
|
|
471
|
-
|
|
523
|
+
v(e);
|
|
472
524
|
}
|
|
473
525
|
assertOpen() {
|
|
474
526
|
if (this.closed)
|
|
475
527
|
throw new Error("VectorDB instance has been closed");
|
|
476
528
|
}
|
|
477
529
|
}
|
|
478
|
-
class
|
|
530
|
+
class z {
|
|
479
531
|
reader;
|
|
480
532
|
buffer = new Uint8Array(0);
|
|
481
533
|
constructor(e) {
|
|
@@ -485,36 +537,36 @@ class V {
|
|
|
485
537
|
async readExact(e) {
|
|
486
538
|
if (e === 0) return new Uint8Array(0);
|
|
487
539
|
if (this.buffer.byteLength >= e) {
|
|
488
|
-
const
|
|
489
|
-
return this.buffer = this.buffer.subarray(e),
|
|
540
|
+
const o = this.buffer.subarray(0, e);
|
|
541
|
+
return this.buffer = this.buffer.subarray(e), o;
|
|
490
542
|
}
|
|
491
543
|
const t = [];
|
|
492
|
-
let
|
|
493
|
-
for (
|
|
494
|
-
const { done:
|
|
495
|
-
if (
|
|
496
|
-
t.push(
|
|
544
|
+
let r = this.buffer.byteLength;
|
|
545
|
+
for (r > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); r < e; ) {
|
|
546
|
+
const { done: o, value: a } = await this.reader.read();
|
|
547
|
+
if (o) throw new Error("Invalid import data: unexpected end of stream");
|
|
548
|
+
t.push(a), r += a.byteLength;
|
|
497
549
|
}
|
|
498
|
-
const
|
|
499
|
-
let
|
|
500
|
-
for (const
|
|
501
|
-
|
|
502
|
-
return
|
|
550
|
+
const s = new Uint8Array(r);
|
|
551
|
+
let n = 0;
|
|
552
|
+
for (const o of t)
|
|
553
|
+
s.set(o, n), n += o.byteLength;
|
|
554
|
+
return r > e ? (this.buffer = s.subarray(e), s.subarray(0, e)) : s;
|
|
503
555
|
}
|
|
504
556
|
/**
|
|
505
557
|
* Read `totalBytes` from the stream in chunks, calling `onChunk` for each.
|
|
506
558
|
* Avoids allocating a single large buffer for the full data.
|
|
507
559
|
*/
|
|
508
560
|
async readChunked(e, t) {
|
|
509
|
-
let
|
|
561
|
+
let r = e;
|
|
510
562
|
if (this.buffer.byteLength > 0) {
|
|
511
|
-
const
|
|
512
|
-
t(this.buffer.subarray(0,
|
|
563
|
+
const s = Math.min(this.buffer.byteLength, r);
|
|
564
|
+
t(this.buffer.subarray(0, s)), r -= s, this.buffer = this.buffer.subarray(s);
|
|
513
565
|
}
|
|
514
|
-
for (;
|
|
515
|
-
const { done:
|
|
516
|
-
if (
|
|
517
|
-
|
|
566
|
+
for (; r > 0; ) {
|
|
567
|
+
const { done: s, value: n } = await this.reader.read();
|
|
568
|
+
if (s) throw new Error("Invalid import data: unexpected end of stream");
|
|
569
|
+
n.byteLength <= r ? (t(n), r -= n.byteLength) : (t(n.subarray(0, r)), this.buffer = n.slice(r), r = 0);
|
|
518
570
|
}
|
|
519
571
|
}
|
|
520
572
|
/** Release the stream reader lock. */
|
|
@@ -523,9 +575,9 @@ class V {
|
|
|
523
575
|
}
|
|
524
576
|
}
|
|
525
577
|
export {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
578
|
+
I as DB,
|
|
579
|
+
S as InMemoryStorageProvider,
|
|
580
|
+
K as OPFSStorageProvider,
|
|
529
581
|
k as VectorCapacityExceededError
|
|
530
582
|
};
|
|
531
583
|
//# sourceMappingURL=eigen-db.js.map
|