eigen-db 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +74 -23
- package/dist/compute.d.ts +20 -0
- package/dist/eigen-db.js +228 -173
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/dist/errors.d.ts +7 -0
- package/dist/index.d.ts +12 -0
- package/dist/lexicon.d.ts +28 -0
- package/dist/memory-manager.d.ts +68 -0
- package/dist/result-set.d.ts +46 -0
- package/dist/simd-binary.d.ts +1 -0
- package/dist/storage.d.ts +38 -0
- package/dist/types.d.ts +48 -0
- package/dist/vector-db.d.ts +131 -0
- package/dist/wasm-compute.d.ts +13 -0
- package/package.json +4 -4
- package/src/lib/__tests__/result-set.test.ts +146 -27
- package/src/lib/__tests__/vector-db.test.ts +476 -4
- package/src/lib/result-set.ts +55 -24
- package/src/lib/types.ts +5 -1
- package/src/lib/vector-db.ts +99 -20
package/dist/eigen-db.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
class
|
|
1
|
+
class x extends Error {
|
|
2
2
|
constructor(e) {
|
|
3
3
|
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class N {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
9
|
constructor(e) {
|
|
@@ -25,7 +25,7 @@ class _ {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
async append(e, t) {
|
|
28
|
-
const
|
|
28
|
+
const r = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await r.createWritable({ keepExistingData: !0 }), i = await r.getFile();
|
|
29
29
|
await n.seek(i.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
31
|
async write(e, t) {
|
|
@@ -36,16 +36,16 @@ class _ {
|
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class U {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
41
|
async readAll(e) {
|
|
42
42
|
const t = this.files.get(e);
|
|
43
43
|
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
-
const
|
|
44
|
+
const s = t.reduce((i, a) => i + a.byteLength, 0), r = new Uint8Array(s);
|
|
45
45
|
let n = 0;
|
|
46
46
|
for (const i of t)
|
|
47
|
-
|
|
48
|
-
return
|
|
47
|
+
r.set(i, n), n += i.byteLength;
|
|
48
|
+
return r;
|
|
49
49
|
}
|
|
50
50
|
async append(e, t) {
|
|
51
51
|
this.files.has(e) || this.files.set(e, []), this.files.get(e).push(new Uint8Array(t));
|
|
@@ -57,46 +57,46 @@ class v {
|
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
60
|
+
function T(o) {
|
|
61
61
|
let e = 0;
|
|
62
|
-
for (let
|
|
63
|
-
e +=
|
|
62
|
+
for (let r = 0; r < o.length; r++)
|
|
63
|
+
e += o[r] * o[r];
|
|
64
64
|
const t = Math.sqrt(e);
|
|
65
65
|
if (t === 0) return;
|
|
66
|
-
const
|
|
67
|
-
for (let
|
|
68
|
-
|
|
66
|
+
const s = 1 / t;
|
|
67
|
+
for (let r = 0; r < o.length; r++)
|
|
68
|
+
o[r] *= s;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let n = 0; n <
|
|
70
|
+
function Q(o, e, t, s, r) {
|
|
71
|
+
for (let n = 0; n < s; n++) {
|
|
72
72
|
let i = 0;
|
|
73
|
-
const a = n *
|
|
74
|
-
for (let
|
|
75
|
-
i += c
|
|
73
|
+
const a = n * r;
|
|
74
|
+
for (let c = 0; c < r; c++)
|
|
75
|
+
i += o[c] * e[a + c];
|
|
76
76
|
t[n] = i;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function
|
|
81
|
-
const e =
|
|
79
|
+
const V = new TextEncoder(), _ = new TextDecoder();
|
|
80
|
+
function b(o) {
|
|
81
|
+
const e = o.map((a) => V.encode(a)), t = e.reduce((a, c) => a + 4 + c.byteLength, 0), s = new ArrayBuffer(t), r = new DataView(s), n = new Uint8Array(s);
|
|
82
82
|
let i = 0;
|
|
83
83
|
for (const a of e)
|
|
84
|
-
|
|
84
|
+
r.setUint32(i, a.byteLength, !0), i += 4, n.set(a, i), i += a.byteLength;
|
|
85
85
|
return n;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const e = [], t = new DataView(
|
|
89
|
-
let
|
|
90
|
-
for (;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const n =
|
|
94
|
-
e.push(n),
|
|
87
|
+
function C(o) {
|
|
88
|
+
const e = [], t = new DataView(o.buffer, o.byteOffset, o.byteLength);
|
|
89
|
+
let s = 0;
|
|
90
|
+
for (; s < o.byteLength; ) {
|
|
91
|
+
const r = t.getUint32(s, !0);
|
|
92
|
+
s += 4;
|
|
93
|
+
const n = _.decode(o.subarray(s, s + r));
|
|
94
|
+
e.push(n), s += r;
|
|
95
95
|
}
|
|
96
96
|
return e;
|
|
97
97
|
}
|
|
98
|
-
const m = 65536,
|
|
99
|
-
class
|
|
98
|
+
const m = 65536, E = 65536;
|
|
99
|
+
class D {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
@@ -104,9 +104,9 @@ class Q {
|
|
|
104
104
|
_vectorCount;
|
|
105
105
|
constructor(e, t = 0) {
|
|
106
106
|
this.dimensions = e, this.queryOffset = 0;
|
|
107
|
-
const
|
|
108
|
-
this.dbOffset = Math.ceil(
|
|
109
|
-
const
|
|
107
|
+
const s = e * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(s / m) * m;
|
|
109
|
+
const r = t * e * 4, n = this.dbOffset + r, i = Math.max(1, Math.ceil(n / m));
|
|
110
110
|
this.memory = new WebAssembly.Memory({ initial: i }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
@@ -126,7 +126,7 @@ class Q {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const e =
|
|
129
|
+
const e = E * m - this.dbOffset, t = this.dimensions * 4 + 4;
|
|
130
130
|
return Math.floor(e / t);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
@@ -134,10 +134,10 @@ class Q {
|
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
136
|
ensureCapacity(e) {
|
|
137
|
-
const t = this._vectorCount + e,
|
|
138
|
-
if (
|
|
139
|
-
const n = Math.ceil((
|
|
140
|
-
if (
|
|
137
|
+
const t = this._vectorCount + e, s = this.dbOffset + t * this.dimensions * 4 + t * 4, r = this.memory.buffer.byteLength;
|
|
138
|
+
if (s > r) {
|
|
139
|
+
const n = Math.ceil((s - r) / m);
|
|
140
|
+
if (r / m + n > E)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
142
|
this.memory.grow(n);
|
|
143
143
|
}
|
|
@@ -154,9 +154,9 @@ class Q {
|
|
|
154
154
|
*/
|
|
155
155
|
appendVectors(e) {
|
|
156
156
|
const t = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
-
let
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer,
|
|
157
|
+
let s = t;
|
|
158
|
+
for (const r of e)
|
|
159
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(r), s += this.dimensions * 4;
|
|
160
160
|
return this._vectorCount += e.length, t;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
@@ -183,8 +183,8 @@ class Q {
|
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
185
|
writeVector(e, t) {
|
|
186
|
-
const
|
|
187
|
-
new Float32Array(this.memory.buffer,
|
|
186
|
+
const s = this.dbOffset + e * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(t);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -201,59 +201,62 @@ class Q {
|
|
|
201
201
|
this._vectorCount = e;
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
-
function
|
|
205
|
-
|
|
206
|
-
if (s === 0) return [];
|
|
207
|
-
const n = new Uint32Array(s);
|
|
208
|
-
for (let o = 0; o < s; o++) n[o] = o;
|
|
209
|
-
n.sort((o, h) => c[h] - c[o]);
|
|
210
|
-
const i = Math.min(t, s), a = [];
|
|
211
|
-
for (let o = 0; o < i; o++) {
|
|
212
|
-
const h = n[o], f = c[h];
|
|
213
|
-
if (r !== void 0 && f < r) break;
|
|
214
|
-
a.push({ key: e(h), similarity: f });
|
|
215
|
-
}
|
|
216
|
-
return a;
|
|
204
|
+
function p(o, e, t) {
|
|
205
|
+
return !(e !== void 0 && o < e || t !== void 0 && o > t);
|
|
217
206
|
}
|
|
218
|
-
function L(
|
|
219
|
-
const s =
|
|
220
|
-
if (
|
|
221
|
-
const
|
|
222
|
-
for (let
|
|
223
|
-
|
|
224
|
-
const
|
|
225
|
-
|
|
207
|
+
function L(o, e, t) {
|
|
208
|
+
const { limit: s, order: r, minSimilarity: n, maxSimilarity: i } = t, a = o.length;
|
|
209
|
+
if (a === 0) return [];
|
|
210
|
+
const c = new Uint32Array(a);
|
|
211
|
+
for (let l = 0; l < a; l++) c[l] = l;
|
|
212
|
+
r === "ascend" ? c.sort((l, y) => o[l] - o[y]) : c.sort((l, y) => o[y] - o[l]);
|
|
213
|
+
const A = Math.min(s, a), h = [];
|
|
214
|
+
for (let l = 0; l < a && h.length < A; l++) {
|
|
215
|
+
const y = c[l], f = o[y];
|
|
216
|
+
p(f, n, i) && h.push({ key: e(y), similarity: f });
|
|
217
|
+
}
|
|
218
|
+
return h;
|
|
219
|
+
}
|
|
220
|
+
function F(o, e, t) {
|
|
221
|
+
const { limit: s, order: r, minSimilarity: n, maxSimilarity: i } = t, a = o.length;
|
|
222
|
+
if (a === 0) return [];
|
|
223
|
+
const c = new Uint32Array(a);
|
|
224
|
+
for (let A = 0; A < a; A++) c[A] = A;
|
|
225
|
+
return r === "ascend" ? c.sort((A, h) => o[A] - o[h]) : c.sort((A, h) => o[h] - o[A]), {
|
|
226
226
|
[Symbol.iterator]() {
|
|
227
|
-
let
|
|
227
|
+
let A = 0, h = 0;
|
|
228
228
|
return {
|
|
229
229
|
next() {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
230
|
+
for (; A < a && h < s; ) {
|
|
231
|
+
const l = c[A++], y = o[l];
|
|
232
|
+
if (p(y, n, i))
|
|
233
|
+
return h++, {
|
|
234
|
+
done: !1,
|
|
235
|
+
value: { key: e(l), similarity: y }
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
return { done: !0, value: void 0 };
|
|
236
239
|
}
|
|
237
240
|
};
|
|
238
241
|
}
|
|
239
242
|
};
|
|
240
243
|
}
|
|
241
|
-
const
|
|
242
|
-
function
|
|
243
|
-
const
|
|
244
|
-
for (let t = 0; t <
|
|
245
|
-
e[t] =
|
|
244
|
+
const z = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
245
|
+
function q() {
|
|
246
|
+
const o = atob(z), e = new Uint8Array(o.length);
|
|
247
|
+
for (let t = 0; t < o.length; t++)
|
|
248
|
+
e[t] = o.charCodeAt(t);
|
|
246
249
|
return e;
|
|
247
250
|
}
|
|
248
|
-
async function
|
|
251
|
+
async function R(o, e) {
|
|
249
252
|
const t = { env: { memory: e } };
|
|
250
|
-
return (await WebAssembly.instantiate(
|
|
253
|
+
return (await WebAssembly.instantiate(o, t)).instance.exports;
|
|
251
254
|
}
|
|
252
|
-
const
|
|
253
|
-
class
|
|
255
|
+
const I = "vectors.bin", M = "keys.bin", S = 1111770949, d = 1, B = 24, H = 65536;
|
|
256
|
+
class k {
|
|
254
257
|
memoryManager;
|
|
255
258
|
storage;
|
|
256
|
-
|
|
259
|
+
_dimensions;
|
|
257
260
|
shouldNormalize;
|
|
258
261
|
wasmExports;
|
|
259
262
|
/** Maps key to its slot index in the vector array */
|
|
@@ -262,46 +265,98 @@ class M {
|
|
|
262
265
|
slotToKey;
|
|
263
266
|
/** Whether this instance has been closed */
|
|
264
267
|
closed = !1;
|
|
265
|
-
constructor(e, t,
|
|
266
|
-
this.memoryManager = e, this.storage = t, this.
|
|
268
|
+
constructor(e, t, s, r, n, i, a) {
|
|
269
|
+
this.memoryManager = e, this.storage = t, this._dimensions = s, this.shouldNormalize = r, this.wasmExports = n, this.keyToSlot = i, this.slotToKey = a;
|
|
267
270
|
}
|
|
268
271
|
static async open(e) {
|
|
269
|
-
const t = e.storage ?? new
|
|
270
|
-
for (let
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
|
|
272
|
+
const t = e.storage ?? new U(), s = e.normalize !== !1, [r, n] = await Promise.all([t.readAll(I), t.readAll(M)]), i = n.byteLength > 0 ? C(n) : [], a = r.byteLength / (e.dimensions * 4), c = /* @__PURE__ */ new Map(), A = [];
|
|
273
|
+
for (let f = 0; f < i.length; f++)
|
|
274
|
+
c.set(i[f], f), A[f] = i[f];
|
|
275
|
+
const h = new D(e.dimensions, a);
|
|
276
|
+
r.byteLength > 0 && h.loadVectorBytes(r, a);
|
|
274
277
|
let l = null;
|
|
275
|
-
const y = e.wasmBinary !== void 0 ? e.wasmBinary :
|
|
278
|
+
const y = e.wasmBinary !== void 0 ? e.wasmBinary : q();
|
|
276
279
|
if (y !== null)
|
|
277
280
|
try {
|
|
278
|
-
l = await
|
|
281
|
+
l = await R(y, h.memory);
|
|
279
282
|
} catch {
|
|
280
283
|
}
|
|
281
|
-
return new
|
|
284
|
+
return new k(h, t, e.dimensions, s, l, c, A);
|
|
282
285
|
}
|
|
283
286
|
/** Total number of key-value pairs in the database */
|
|
284
287
|
get size() {
|
|
285
288
|
return this.keyToSlot.size;
|
|
286
289
|
}
|
|
290
|
+
/** Number of dimensions per vector */
|
|
291
|
+
get dimensions() {
|
|
292
|
+
return this._dimensions;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Check whether a key exists in the database.
|
|
296
|
+
* Uses the internal key-to-slot map for O(1) lookup.
|
|
297
|
+
*/
|
|
298
|
+
has(e) {
|
|
299
|
+
return this.assertOpen(), this.keyToSlot.has(e);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Delete an entry by key. Returns true if the key existed, false otherwise.
|
|
303
|
+
* Uses swap-and-pop to avoid gaps in the underlying vector array.
|
|
304
|
+
*/
|
|
305
|
+
delete(e) {
|
|
306
|
+
this.assertOpen();
|
|
307
|
+
const t = this.keyToSlot.get(e);
|
|
308
|
+
if (t === void 0) return !1;
|
|
309
|
+
const s = this.memoryManager.vectorCount - 1;
|
|
310
|
+
if (t !== s) {
|
|
311
|
+
const r = new Float32Array(this.memoryManager.readVector(s));
|
|
312
|
+
this.memoryManager.writeVector(t, r);
|
|
313
|
+
const n = this.slotToKey[s];
|
|
314
|
+
this.keyToSlot.set(n, t), this.slotToKey[t] = n;
|
|
315
|
+
}
|
|
316
|
+
return this.keyToSlot.delete(e), this.slotToKey.length = s, this.memoryManager.setVectorCount(s), !0;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Returns an iterable of all keys in the database.
|
|
320
|
+
*/
|
|
321
|
+
keys() {
|
|
322
|
+
return this.assertOpen(), this.keyToSlot.keys();
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Returns an iterable of [key, value] pairs.
|
|
326
|
+
* Values are returned as plain number array copies.
|
|
327
|
+
*/
|
|
328
|
+
entries() {
|
|
329
|
+
this.assertOpen();
|
|
330
|
+
const e = this.keyToSlot, t = this.memoryManager;
|
|
331
|
+
return (function* () {
|
|
332
|
+
for (const [s, r] of e)
|
|
333
|
+
yield [s, Array.from(t.readVector(r))];
|
|
334
|
+
})();
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Implements the iterable protocol. Same as entries().
|
|
338
|
+
*/
|
|
339
|
+
[Symbol.iterator]() {
|
|
340
|
+
return this.entries();
|
|
341
|
+
}
|
|
287
342
|
/**
|
|
288
343
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
289
344
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
290
345
|
*/
|
|
291
|
-
set(e, t,
|
|
292
|
-
if (this.assertOpen(), t.length !== this.
|
|
293
|
-
throw new Error(`Vector dimension mismatch: expected ${this.
|
|
294
|
-
const
|
|
295
|
-
(
|
|
346
|
+
set(e, t, s) {
|
|
347
|
+
if (this.assertOpen(), t.length !== this._dimensions)
|
|
348
|
+
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${t.length}`);
|
|
349
|
+
const r = new Float32Array(t);
|
|
350
|
+
(s?.normalize ?? this.shouldNormalize) && this.normalizeVector(r);
|
|
296
351
|
const i = this.keyToSlot.get(e);
|
|
297
352
|
if (i !== void 0)
|
|
298
|
-
this.memoryManager.writeVector(i,
|
|
353
|
+
this.memoryManager.writeVector(i, r);
|
|
299
354
|
else {
|
|
300
355
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
301
|
-
throw new
|
|
356
|
+
throw new x(this.memoryManager.maxVectors);
|
|
302
357
|
this.memoryManager.ensureCapacity(1);
|
|
303
|
-
const
|
|
304
|
-
this.memoryManager.appendVectors([
|
|
358
|
+
const c = this.memoryManager.vectorCount;
|
|
359
|
+
this.memoryManager.appendVectors([r]), this.keyToSlot.set(e, c), this.slotToKey[c] = e;
|
|
305
360
|
}
|
|
306
361
|
}
|
|
307
362
|
/**
|
|
@@ -318,8 +373,8 @@ class M {
|
|
|
318
373
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
319
374
|
*/
|
|
320
375
|
setMany(e) {
|
|
321
|
-
for (const [t,
|
|
322
|
-
this.set(t,
|
|
376
|
+
for (const [t, s] of e)
|
|
377
|
+
this.set(t, s);
|
|
323
378
|
}
|
|
324
379
|
/**
|
|
325
380
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
@@ -329,53 +384,53 @@ class M {
|
|
|
329
384
|
}
|
|
330
385
|
query(e, t) {
|
|
331
386
|
this.assertOpen();
|
|
332
|
-
const
|
|
387
|
+
const s = t?.limit ?? 1 / 0, r = t?.order ?? "descend", n = t?.minSimilarity, i = t?.maxSimilarity, a = t && "iterable" in t && t.iterable;
|
|
333
388
|
if (this.size === 0)
|
|
334
389
|
return [];
|
|
335
|
-
if (e.length !== this.
|
|
336
|
-
throw new Error(`Query vector dimension mismatch: expected ${this.
|
|
337
|
-
const
|
|
338
|
-
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
339
|
-
const
|
|
390
|
+
if (e.length !== this._dimensions)
|
|
391
|
+
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${e.length}`);
|
|
392
|
+
const c = new Float32Array(e);
|
|
393
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(c), this.memoryManager.writeQuery(c), this.memoryManager.ensureCapacity(0);
|
|
394
|
+
const h = this.memoryManager.vectorCount, l = this.memoryManager.scoresOffset;
|
|
340
395
|
if (this.wasmExports)
|
|
341
396
|
this.wasmExports.search_all(
|
|
342
397
|
this.memoryManager.queryOffset,
|
|
343
398
|
this.memoryManager.dbOffset,
|
|
399
|
+
l,
|
|
344
400
|
h,
|
|
345
|
-
|
|
346
|
-
this.dimensions
|
|
401
|
+
this._dimensions
|
|
347
402
|
);
|
|
348
403
|
else {
|
|
349
|
-
const
|
|
404
|
+
const u = new Float32Array(
|
|
350
405
|
this.memoryManager.memory.buffer,
|
|
351
406
|
this.memoryManager.queryOffset,
|
|
352
|
-
this.
|
|
353
|
-
),
|
|
407
|
+
this._dimensions
|
|
408
|
+
), O = new Float32Array(
|
|
354
409
|
this.memoryManager.memory.buffer,
|
|
355
410
|
this.memoryManager.dbOffset,
|
|
356
|
-
|
|
357
|
-
),
|
|
358
|
-
|
|
411
|
+
h * this._dimensions
|
|
412
|
+
), v = new Float32Array(this.memoryManager.memory.buffer, l, h);
|
|
413
|
+
Q(u, O, v, h, this._dimensions);
|
|
359
414
|
}
|
|
360
|
-
const
|
|
361
|
-
return
|
|
415
|
+
const y = new Float32Array(this.memoryManager.readScores()), f = this.slotToKey, g = (u) => f[u], w = { limit: s, order: r, minSimilarity: n, maxSimilarity: i };
|
|
416
|
+
return a ? F(y, g, w) : L(y, g, w);
|
|
362
417
|
}
|
|
363
418
|
/**
|
|
364
419
|
* Persist the current in-memory state to storage.
|
|
365
420
|
*/
|
|
366
421
|
async flush() {
|
|
367
422
|
this.assertOpen();
|
|
368
|
-
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this.
|
|
423
|
+
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this._dimensions * 4);
|
|
369
424
|
if (e > 0) {
|
|
370
|
-
const
|
|
425
|
+
const r = new Uint8Array(
|
|
371
426
|
this.memoryManager.memory.buffer,
|
|
372
427
|
this.memoryManager.dbOffset,
|
|
373
|
-
e * this.
|
|
428
|
+
e * this._dimensions * 4
|
|
374
429
|
);
|
|
375
|
-
t.set(
|
|
430
|
+
t.set(r);
|
|
376
431
|
}
|
|
377
|
-
const
|
|
378
|
-
await Promise.all([this.storage.write(
|
|
432
|
+
const s = b(this.slotToKey);
|
|
433
|
+
await Promise.all([this.storage.write(I, t), this.storage.write(M, s)]);
|
|
379
434
|
}
|
|
380
435
|
/**
|
|
381
436
|
* Flush data to storage and release the instance.
|
|
@@ -401,23 +456,23 @@ class M {
|
|
|
401
456
|
*/
|
|
402
457
|
async export() {
|
|
403
458
|
this.assertOpen();
|
|
404
|
-
const e = this.memoryManager.vectorCount, t = e * this.
|
|
405
|
-
i.setUint32(0,
|
|
459
|
+
const e = this.memoryManager.vectorCount, t = e * this._dimensions * 4, s = b(this.slotToKey), r = s.byteLength, n = new ArrayBuffer(B), i = new DataView(n);
|
|
460
|
+
i.setUint32(0, S, !0), i.setUint32(4, d, !0), i.setUint32(8, this._dimensions, !0), i.setUint32(12, e, !0), i.setUint32(16, t, !0), i.setUint32(20, r, !0);
|
|
406
461
|
const a = this.memoryManager;
|
|
407
|
-
let
|
|
462
|
+
let c = "header", A = 0;
|
|
408
463
|
return new ReadableStream({
|
|
409
|
-
pull(
|
|
410
|
-
switch (
|
|
464
|
+
pull(h) {
|
|
465
|
+
switch (c) {
|
|
411
466
|
case "header":
|
|
412
|
-
|
|
467
|
+
h.enqueue(new Uint8Array(n)), c = t > 0 ? "vectors" : r > 0 ? "keys" : "done", c === "done" && h.close();
|
|
413
468
|
break;
|
|
414
469
|
case "vectors": {
|
|
415
|
-
const l = t -
|
|
416
|
-
|
|
470
|
+
const l = t - A, y = Math.min(l, H), f = new Uint8Array(y);
|
|
471
|
+
f.set(new Uint8Array(a.memory.buffer, a.dbOffset + A, y)), h.enqueue(f), A += y, A >= t && (c = r > 0 ? "keys" : "done", c === "done" && h.close());
|
|
417
472
|
break;
|
|
418
473
|
}
|
|
419
474
|
case "keys":
|
|
420
|
-
|
|
475
|
+
h.enqueue(s), c = "done", h.close();
|
|
421
476
|
break;
|
|
422
477
|
}
|
|
423
478
|
}
|
|
@@ -432,29 +487,29 @@ class M {
|
|
|
432
487
|
*/
|
|
433
488
|
async import(e) {
|
|
434
489
|
this.assertOpen();
|
|
435
|
-
const t = new
|
|
436
|
-
if (
|
|
490
|
+
const t = new K(e), s = await t.readExact(B), r = new DataView(s.buffer, s.byteOffset, s.byteLength);
|
|
491
|
+
if (r.getUint32(0, !0) !== S)
|
|
437
492
|
throw new Error("Invalid import data: unrecognized format");
|
|
438
|
-
const i =
|
|
439
|
-
if (i !==
|
|
440
|
-
throw new Error(`Unsupported import version: expected ${
|
|
441
|
-
const a =
|
|
442
|
-
if (a !== this.
|
|
443
|
-
throw new Error(`Import dimension mismatch: expected ${this.
|
|
444
|
-
const
|
|
445
|
-
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(),
|
|
446
|
-
this.memoryManager.ensureCapacity(
|
|
493
|
+
const i = r.getUint32(4, !0);
|
|
494
|
+
if (i !== d)
|
|
495
|
+
throw new Error(`Unsupported import version: expected ${d}, got ${i}`);
|
|
496
|
+
const a = r.getUint32(8, !0);
|
|
497
|
+
if (a !== this._dimensions)
|
|
498
|
+
throw new Error(`Import dimension mismatch: expected ${this._dimensions}, got ${a}`);
|
|
499
|
+
const c = r.getUint32(12, !0), A = r.getUint32(16, !0), h = r.getUint32(20, !0);
|
|
500
|
+
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), c > 0) {
|
|
501
|
+
this.memoryManager.ensureCapacity(c);
|
|
447
502
|
let l = 0;
|
|
448
|
-
await t.readChunked(
|
|
503
|
+
await t.readChunked(A, (y) => {
|
|
449
504
|
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, y.byteLength).set(
|
|
450
505
|
y
|
|
451
506
|
), l += y.byteLength;
|
|
452
|
-
}), this.memoryManager.setVectorCount(
|
|
507
|
+
}), this.memoryManager.setVectorCount(c);
|
|
453
508
|
}
|
|
454
|
-
if (
|
|
455
|
-
const l = await t.readExact(
|
|
456
|
-
for (let
|
|
457
|
-
this.keyToSlot.set(y[
|
|
509
|
+
if (h > 0) {
|
|
510
|
+
const l = await t.readExact(h), y = C(l);
|
|
511
|
+
for (let f = 0; f < y.length; f++)
|
|
512
|
+
this.keyToSlot.set(y[f], f), this.slotToKey[f] = y[f];
|
|
458
513
|
}
|
|
459
514
|
t.release();
|
|
460
515
|
}
|
|
@@ -465,17 +520,17 @@ class M {
|
|
|
465
520
|
if (this.wasmExports) {
|
|
466
521
|
const t = this.memoryManager.queryOffset;
|
|
467
522
|
new Float32Array(this.memoryManager.memory.buffer, t, e.length).set(e), this.wasmExports.normalize(t, e.length);
|
|
468
|
-
const
|
|
469
|
-
e.set(
|
|
523
|
+
const s = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
524
|
+
e.set(s);
|
|
470
525
|
} else
|
|
471
|
-
|
|
526
|
+
T(e);
|
|
472
527
|
}
|
|
473
528
|
assertOpen() {
|
|
474
529
|
if (this.closed)
|
|
475
530
|
throw new Error("VectorDB instance has been closed");
|
|
476
531
|
}
|
|
477
532
|
}
|
|
478
|
-
class
|
|
533
|
+
class K {
|
|
479
534
|
reader;
|
|
480
535
|
buffer = new Uint8Array(0);
|
|
481
536
|
constructor(e) {
|
|
@@ -489,32 +544,32 @@ class q {
|
|
|
489
544
|
return this.buffer = this.buffer.subarray(e), i;
|
|
490
545
|
}
|
|
491
546
|
const t = [];
|
|
492
|
-
let
|
|
493
|
-
for (
|
|
547
|
+
let s = this.buffer.byteLength;
|
|
548
|
+
for (s > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); s < e; ) {
|
|
494
549
|
const { done: i, value: a } = await this.reader.read();
|
|
495
550
|
if (i) throw new Error("Invalid import data: unexpected end of stream");
|
|
496
|
-
t.push(a),
|
|
551
|
+
t.push(a), s += a.byteLength;
|
|
497
552
|
}
|
|
498
|
-
const
|
|
553
|
+
const r = new Uint8Array(s);
|
|
499
554
|
let n = 0;
|
|
500
555
|
for (const i of t)
|
|
501
|
-
|
|
502
|
-
return
|
|
556
|
+
r.set(i, n), n += i.byteLength;
|
|
557
|
+
return s > e ? (this.buffer = r.subarray(e), r.subarray(0, e)) : r;
|
|
503
558
|
}
|
|
504
559
|
/**
|
|
505
560
|
* Read `totalBytes` from the stream in chunks, calling `onChunk` for each.
|
|
506
561
|
* Avoids allocating a single large buffer for the full data.
|
|
507
562
|
*/
|
|
508
563
|
async readChunked(e, t) {
|
|
509
|
-
let
|
|
564
|
+
let s = e;
|
|
510
565
|
if (this.buffer.byteLength > 0) {
|
|
511
|
-
const
|
|
512
|
-
t(this.buffer.subarray(0,
|
|
566
|
+
const r = Math.min(this.buffer.byteLength, s);
|
|
567
|
+
t(this.buffer.subarray(0, r)), s -= r, this.buffer = this.buffer.subarray(r);
|
|
513
568
|
}
|
|
514
|
-
for (;
|
|
515
|
-
const { done:
|
|
516
|
-
if (
|
|
517
|
-
n.byteLength <=
|
|
569
|
+
for (; s > 0; ) {
|
|
570
|
+
const { done: r, value: n } = await this.reader.read();
|
|
571
|
+
if (r) throw new Error("Invalid import data: unexpected end of stream");
|
|
572
|
+
n.byteLength <= s ? (t(n), s -= n.byteLength) : (t(n.subarray(0, s)), this.buffer = n.slice(s), s = 0);
|
|
518
573
|
}
|
|
519
574
|
}
|
|
520
575
|
/** Release the stream reader lock. */
|
|
@@ -523,9 +578,9 @@ class q {
|
|
|
523
578
|
}
|
|
524
579
|
}
|
|
525
580
|
export {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
581
|
+
k as DB,
|
|
582
|
+
U as InMemoryStorageProvider,
|
|
583
|
+
N as OPFSStorageProvider,
|
|
584
|
+
x as VectorCapacityExceededError
|
|
530
585
|
};
|
|
531
586
|
//# sourceMappingURL=eigen-db.js.map
|