eigen-db 4.4.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +38 -25
- package/dist/eigen-db.js +190 -187
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/dist/result-set.d.ts +18 -7
- package/dist/types.d.ts +5 -1
- package/package.json +1 -1
- package/src/lib/__tests__/result-set.test.ts +146 -27
- package/src/lib/__tests__/vector-db.test.ts +188 -4
- package/src/lib/result-set.ts +55 -24
- package/src/lib/types.ts +5 -1
- package/src/lib/vector-db.ts +8 -4
package/dist/eigen-db.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
class
|
|
1
|
+
class x extends Error {
|
|
2
2
|
constructor(e) {
|
|
3
3
|
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class N {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
9
|
constructor(e) {
|
|
@@ -25,8 +25,8 @@ class K {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
async append(e, t) {
|
|
28
|
-
const
|
|
29
|
-
await n.seek(
|
|
28
|
+
const r = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await r.createWritable({ keepExistingData: !0 }), i = await r.getFile();
|
|
29
|
+
await n.seek(i.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
31
|
async write(e, t) {
|
|
32
32
|
const n = await (await (await this.getDir()).getFileHandle(e, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
@@ -36,16 +36,16 @@ class K {
|
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class U {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
41
|
async readAll(e) {
|
|
42
42
|
const t = this.files.get(e);
|
|
43
43
|
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
-
const
|
|
44
|
+
const s = t.reduce((i, a) => i + a.byteLength, 0), r = new Uint8Array(s);
|
|
45
45
|
let n = 0;
|
|
46
|
-
for (const
|
|
47
|
-
|
|
48
|
-
return
|
|
46
|
+
for (const i of t)
|
|
47
|
+
r.set(i, n), n += i.byteLength;
|
|
48
|
+
return r;
|
|
49
49
|
}
|
|
50
50
|
async append(e, t) {
|
|
51
51
|
this.files.has(e) || this.files.set(e, []), this.files.get(e).push(new Uint8Array(t));
|
|
@@ -57,46 +57,46 @@ class S {
|
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
60
|
+
function T(o) {
|
|
61
61
|
let e = 0;
|
|
62
|
-
for (let
|
|
63
|
-
e +=
|
|
62
|
+
for (let r = 0; r < o.length; r++)
|
|
63
|
+
e += o[r] * o[r];
|
|
64
64
|
const t = Math.sqrt(e);
|
|
65
65
|
if (t === 0) return;
|
|
66
|
-
const
|
|
67
|
-
for (let
|
|
68
|
-
|
|
66
|
+
const s = 1 / t;
|
|
67
|
+
for (let r = 0; r < o.length; r++)
|
|
68
|
+
o[r] *= s;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let n = 0; n <
|
|
72
|
-
let
|
|
73
|
-
const a = n *
|
|
74
|
-
for (let
|
|
75
|
-
|
|
76
|
-
t[n] =
|
|
70
|
+
function Q(o, e, t, s, r) {
|
|
71
|
+
for (let n = 0; n < s; n++) {
|
|
72
|
+
let i = 0;
|
|
73
|
+
const a = n * r;
|
|
74
|
+
for (let c = 0; c < r; c++)
|
|
75
|
+
i += o[c] * e[a + c];
|
|
76
|
+
t[n] = i;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function
|
|
81
|
-
const e =
|
|
82
|
-
let
|
|
79
|
+
const V = new TextEncoder(), _ = new TextDecoder();
|
|
80
|
+
function b(o) {
|
|
81
|
+
const e = o.map((a) => V.encode(a)), t = e.reduce((a, c) => a + 4 + c.byteLength, 0), s = new ArrayBuffer(t), r = new DataView(s), n = new Uint8Array(s);
|
|
82
|
+
let i = 0;
|
|
83
83
|
for (const a of e)
|
|
84
|
-
|
|
84
|
+
r.setUint32(i, a.byteLength, !0), i += 4, n.set(a, i), i += a.byteLength;
|
|
85
85
|
return n;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const e = [], t = new DataView(
|
|
89
|
-
let
|
|
90
|
-
for (;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const n =
|
|
94
|
-
e.push(n),
|
|
87
|
+
function C(o) {
|
|
88
|
+
const e = [], t = new DataView(o.buffer, o.byteOffset, o.byteLength);
|
|
89
|
+
let s = 0;
|
|
90
|
+
for (; s < o.byteLength; ) {
|
|
91
|
+
const r = t.getUint32(s, !0);
|
|
92
|
+
s += 4;
|
|
93
|
+
const n = _.decode(o.subarray(s, s + r));
|
|
94
|
+
e.push(n), s += r;
|
|
95
95
|
}
|
|
96
96
|
return e;
|
|
97
97
|
}
|
|
98
|
-
const m = 65536,
|
|
99
|
-
class
|
|
98
|
+
const m = 65536, E = 65536;
|
|
99
|
+
class D {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
@@ -104,10 +104,10 @@ class T {
|
|
|
104
104
|
_vectorCount;
|
|
105
105
|
constructor(e, t = 0) {
|
|
106
106
|
this.dimensions = e, this.queryOffset = 0;
|
|
107
|
-
const
|
|
108
|
-
this.dbOffset = Math.ceil(
|
|
109
|
-
const
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
107
|
+
const s = e * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(s / m) * m;
|
|
109
|
+
const r = t * e * 4, n = this.dbOffset + r, i = Math.max(1, Math.ceil(n / m));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: i }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -126,7 +126,7 @@ class T {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const e =
|
|
129
|
+
const e = E * m - this.dbOffset, t = this.dimensions * 4 + 4;
|
|
130
130
|
return Math.floor(e / t);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
@@ -134,10 +134,10 @@ class T {
|
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
136
|
ensureCapacity(e) {
|
|
137
|
-
const t = this._vectorCount + e,
|
|
138
|
-
if (
|
|
139
|
-
const n = Math.ceil((
|
|
140
|
-
if (
|
|
137
|
+
const t = this._vectorCount + e, s = this.dbOffset + t * this.dimensions * 4 + t * 4, r = this.memory.buffer.byteLength;
|
|
138
|
+
if (s > r) {
|
|
139
|
+
const n = Math.ceil((s - r) / m);
|
|
140
|
+
if (r / m + n > E)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
142
|
this.memory.grow(n);
|
|
143
143
|
}
|
|
@@ -154,9 +154,9 @@ class T {
|
|
|
154
154
|
*/
|
|
155
155
|
appendVectors(e) {
|
|
156
156
|
const t = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
-
let
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer,
|
|
157
|
+
let s = t;
|
|
158
|
+
for (const r of e)
|
|
159
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(r), s += this.dimensions * 4;
|
|
160
160
|
return this._vectorCount += e.length, t;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
@@ -183,8 +183,8 @@ class T {
|
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
185
|
writeVector(e, t) {
|
|
186
|
-
const
|
|
187
|
-
new Float32Array(this.memory.buffer,
|
|
186
|
+
const s = this.dbOffset + e * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(t);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -201,56 +201,59 @@ class T {
|
|
|
201
201
|
this._vectorCount = e;
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
-
function
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
const
|
|
211
|
-
for (let
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
204
|
+
function p(o, e, t) {
|
|
205
|
+
return !(e !== void 0 && o < e || t !== void 0 && o > t);
|
|
206
|
+
}
|
|
207
|
+
function L(o, e, t) {
|
|
208
|
+
const { limit: s, order: r, minSimilarity: n, maxSimilarity: i } = t, a = o.length;
|
|
209
|
+
if (a === 0) return [];
|
|
210
|
+
const c = new Uint32Array(a);
|
|
211
|
+
for (let l = 0; l < a; l++) c[l] = l;
|
|
212
|
+
r === "ascend" ? c.sort((l, y) => o[l] - o[y]) : c.sort((l, y) => o[y] - o[l]);
|
|
213
|
+
const A = Math.min(s, a), h = [];
|
|
214
|
+
for (let l = 0; l < a && h.length < A; l++) {
|
|
215
|
+
const y = c[l], f = o[y];
|
|
216
|
+
p(f, n, i) && h.push({ key: e(y), similarity: f });
|
|
217
|
+
}
|
|
218
|
+
return h;
|
|
217
219
|
}
|
|
218
|
-
function
|
|
219
|
-
const s =
|
|
220
|
-
if (
|
|
221
|
-
const
|
|
222
|
-
for (let
|
|
223
|
-
|
|
224
|
-
const o = Math.min(t, s);
|
|
225
|
-
return {
|
|
220
|
+
function F(o, e, t) {
|
|
221
|
+
const { limit: s, order: r, minSimilarity: n, maxSimilarity: i } = t, a = o.length;
|
|
222
|
+
if (a === 0) return [];
|
|
223
|
+
const c = new Uint32Array(a);
|
|
224
|
+
for (let A = 0; A < a; A++) c[A] = A;
|
|
225
|
+
return r === "ascend" ? c.sort((A, h) => o[A] - o[h]) : c.sort((A, h) => o[h] - o[A]), {
|
|
226
226
|
[Symbol.iterator]() {
|
|
227
|
-
let
|
|
227
|
+
let A = 0, h = 0;
|
|
228
228
|
return {
|
|
229
229
|
next() {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
230
|
+
for (; A < a && h < s; ) {
|
|
231
|
+
const l = c[A++], y = o[l];
|
|
232
|
+
if (p(y, n, i))
|
|
233
|
+
return h++, {
|
|
234
|
+
done: !1,
|
|
235
|
+
value: { key: e(l), similarity: y }
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
return { done: !0, value: void 0 };
|
|
236
239
|
}
|
|
237
240
|
};
|
|
238
241
|
}
|
|
239
242
|
};
|
|
240
243
|
}
|
|
241
|
-
const
|
|
242
|
-
function
|
|
243
|
-
const
|
|
244
|
-
for (let t = 0; t <
|
|
245
|
-
e[t] =
|
|
244
|
+
const z = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
245
|
+
function q() {
|
|
246
|
+
const o = atob(z), e = new Uint8Array(o.length);
|
|
247
|
+
for (let t = 0; t < o.length; t++)
|
|
248
|
+
e[t] = o.charCodeAt(t);
|
|
246
249
|
return e;
|
|
247
250
|
}
|
|
248
|
-
async function
|
|
251
|
+
async function R(o, e) {
|
|
249
252
|
const t = { env: { memory: e } };
|
|
250
|
-
return (await WebAssembly.instantiate(
|
|
253
|
+
return (await WebAssembly.instantiate(o, t)).instance.exports;
|
|
251
254
|
}
|
|
252
|
-
const
|
|
253
|
-
class
|
|
255
|
+
const I = "vectors.bin", M = "keys.bin", S = 1111770949, d = 1, B = 24, H = 65536;
|
|
256
|
+
class k {
|
|
254
257
|
memoryManager;
|
|
255
258
|
storage;
|
|
256
259
|
_dimensions;
|
|
@@ -262,23 +265,23 @@ class I {
|
|
|
262
265
|
slotToKey;
|
|
263
266
|
/** Whether this instance has been closed */
|
|
264
267
|
closed = !1;
|
|
265
|
-
constructor(e, t,
|
|
266
|
-
this.memoryManager = e, this.storage = t, this._dimensions =
|
|
268
|
+
constructor(e, t, s, r, n, i, a) {
|
|
269
|
+
this.memoryManager = e, this.storage = t, this._dimensions = s, this.shouldNormalize = r, this.wasmExports = n, this.keyToSlot = i, this.slotToKey = a;
|
|
267
270
|
}
|
|
268
271
|
static async open(e) {
|
|
269
|
-
const t = e.storage ?? new
|
|
270
|
-
for (let
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
|
|
272
|
+
const t = e.storage ?? new U(), s = e.normalize !== !1, [r, n] = await Promise.all([t.readAll(I), t.readAll(M)]), i = n.byteLength > 0 ? C(n) : [], a = r.byteLength / (e.dimensions * 4), c = /* @__PURE__ */ new Map(), A = [];
|
|
273
|
+
for (let f = 0; f < i.length; f++)
|
|
274
|
+
c.set(i[f], f), A[f] = i[f];
|
|
275
|
+
const h = new D(e.dimensions, a);
|
|
276
|
+
r.byteLength > 0 && h.loadVectorBytes(r, a);
|
|
274
277
|
let l = null;
|
|
275
|
-
const
|
|
276
|
-
if (
|
|
278
|
+
const y = e.wasmBinary !== void 0 ? e.wasmBinary : q();
|
|
279
|
+
if (y !== null)
|
|
277
280
|
try {
|
|
278
|
-
l = await
|
|
281
|
+
l = await R(y, h.memory);
|
|
279
282
|
} catch {
|
|
280
283
|
}
|
|
281
|
-
return new
|
|
284
|
+
return new k(h, t, e.dimensions, s, l, c, A);
|
|
282
285
|
}
|
|
283
286
|
/** Total number of key-value pairs in the database */
|
|
284
287
|
get size() {
|
|
@@ -303,14 +306,14 @@ class I {
|
|
|
303
306
|
this.assertOpen();
|
|
304
307
|
const t = this.keyToSlot.get(e);
|
|
305
308
|
if (t === void 0) return !1;
|
|
306
|
-
const
|
|
307
|
-
if (t !==
|
|
308
|
-
const
|
|
309
|
-
this.memoryManager.writeVector(t,
|
|
310
|
-
const n = this.slotToKey[
|
|
309
|
+
const s = this.memoryManager.vectorCount - 1;
|
|
310
|
+
if (t !== s) {
|
|
311
|
+
const r = new Float32Array(this.memoryManager.readVector(s));
|
|
312
|
+
this.memoryManager.writeVector(t, r);
|
|
313
|
+
const n = this.slotToKey[s];
|
|
311
314
|
this.keyToSlot.set(n, t), this.slotToKey[t] = n;
|
|
312
315
|
}
|
|
313
|
-
return this.keyToSlot.delete(e), this.slotToKey.length =
|
|
316
|
+
return this.keyToSlot.delete(e), this.slotToKey.length = s, this.memoryManager.setVectorCount(s), !0;
|
|
314
317
|
}
|
|
315
318
|
/**
|
|
316
319
|
* Returns an iterable of all keys in the database.
|
|
@@ -326,8 +329,8 @@ class I {
|
|
|
326
329
|
this.assertOpen();
|
|
327
330
|
const e = this.keyToSlot, t = this.memoryManager;
|
|
328
331
|
return (function* () {
|
|
329
|
-
for (const [
|
|
330
|
-
yield [
|
|
332
|
+
for (const [s, r] of e)
|
|
333
|
+
yield [s, Array.from(t.readVector(r))];
|
|
331
334
|
})();
|
|
332
335
|
}
|
|
333
336
|
/**
|
|
@@ -340,20 +343,20 @@ class I {
|
|
|
340
343
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
341
344
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
342
345
|
*/
|
|
343
|
-
set(e, t,
|
|
346
|
+
set(e, t, s) {
|
|
344
347
|
if (this.assertOpen(), t.length !== this._dimensions)
|
|
345
348
|
throw new Error(`Vector dimension mismatch: expected ${this._dimensions}, got ${t.length}`);
|
|
346
|
-
const
|
|
347
|
-
(
|
|
348
|
-
const
|
|
349
|
-
if (
|
|
350
|
-
this.memoryManager.writeVector(
|
|
349
|
+
const r = new Float32Array(t);
|
|
350
|
+
(s?.normalize ?? this.shouldNormalize) && this.normalizeVector(r);
|
|
351
|
+
const i = this.keyToSlot.get(e);
|
|
352
|
+
if (i !== void 0)
|
|
353
|
+
this.memoryManager.writeVector(i, r);
|
|
351
354
|
else {
|
|
352
355
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
353
|
-
throw new
|
|
356
|
+
throw new x(this.memoryManager.maxVectors);
|
|
354
357
|
this.memoryManager.ensureCapacity(1);
|
|
355
|
-
const
|
|
356
|
-
this.memoryManager.appendVectors([
|
|
358
|
+
const c = this.memoryManager.vectorCount;
|
|
359
|
+
this.memoryManager.appendVectors([r]), this.keyToSlot.set(e, c), this.slotToKey[c] = e;
|
|
357
360
|
}
|
|
358
361
|
}
|
|
359
362
|
/**
|
|
@@ -370,8 +373,8 @@ class I {
|
|
|
370
373
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
371
374
|
*/
|
|
372
375
|
setMany(e) {
|
|
373
|
-
for (const [t,
|
|
374
|
-
this.set(t,
|
|
376
|
+
for (const [t, s] of e)
|
|
377
|
+
this.set(t, s);
|
|
375
378
|
}
|
|
376
379
|
/**
|
|
377
380
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
@@ -381,36 +384,36 @@ class I {
|
|
|
381
384
|
}
|
|
382
385
|
query(e, t) {
|
|
383
386
|
this.assertOpen();
|
|
384
|
-
const
|
|
387
|
+
const s = t?.limit ?? 1 / 0, r = t?.order ?? "descend", n = t?.minSimilarity, i = t?.maxSimilarity, a = t && "iterable" in t && t.iterable;
|
|
385
388
|
if (this.size === 0)
|
|
386
389
|
return [];
|
|
387
390
|
if (e.length !== this._dimensions)
|
|
388
391
|
throw new Error(`Query vector dimension mismatch: expected ${this._dimensions}, got ${e.length}`);
|
|
389
|
-
const
|
|
390
|
-
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
391
|
-
const
|
|
392
|
+
const c = new Float32Array(e);
|
|
393
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(c), this.memoryManager.writeQuery(c), this.memoryManager.ensureCapacity(0);
|
|
394
|
+
const h = this.memoryManager.vectorCount, l = this.memoryManager.scoresOffset;
|
|
392
395
|
if (this.wasmExports)
|
|
393
396
|
this.wasmExports.search_all(
|
|
394
397
|
this.memoryManager.queryOffset,
|
|
395
398
|
this.memoryManager.dbOffset,
|
|
399
|
+
l,
|
|
396
400
|
h,
|
|
397
|
-
i,
|
|
398
401
|
this._dimensions
|
|
399
402
|
);
|
|
400
403
|
else {
|
|
401
|
-
const
|
|
404
|
+
const u = new Float32Array(
|
|
402
405
|
this.memoryManager.memory.buffer,
|
|
403
406
|
this.memoryManager.queryOffset,
|
|
404
407
|
this._dimensions
|
|
405
|
-
),
|
|
408
|
+
), O = new Float32Array(
|
|
406
409
|
this.memoryManager.memory.buffer,
|
|
407
410
|
this.memoryManager.dbOffset,
|
|
408
|
-
|
|
409
|
-
),
|
|
410
|
-
|
|
411
|
+
h * this._dimensions
|
|
412
|
+
), v = new Float32Array(this.memoryManager.memory.buffer, l, h);
|
|
413
|
+
Q(u, O, v, h, this._dimensions);
|
|
411
414
|
}
|
|
412
|
-
const y = new Float32Array(this.memoryManager.readScores()),
|
|
413
|
-
return
|
|
415
|
+
const y = new Float32Array(this.memoryManager.readScores()), f = this.slotToKey, g = (u) => f[u], w = { limit: s, order: r, minSimilarity: n, maxSimilarity: i };
|
|
416
|
+
return a ? F(y, g, w) : L(y, g, w);
|
|
414
417
|
}
|
|
415
418
|
/**
|
|
416
419
|
* Persist the current in-memory state to storage.
|
|
@@ -419,15 +422,15 @@ class I {
|
|
|
419
422
|
this.assertOpen();
|
|
420
423
|
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this._dimensions * 4);
|
|
421
424
|
if (e > 0) {
|
|
422
|
-
const
|
|
425
|
+
const r = new Uint8Array(
|
|
423
426
|
this.memoryManager.memory.buffer,
|
|
424
427
|
this.memoryManager.dbOffset,
|
|
425
428
|
e * this._dimensions * 4
|
|
426
429
|
);
|
|
427
|
-
t.set(
|
|
430
|
+
t.set(r);
|
|
428
431
|
}
|
|
429
|
-
const
|
|
430
|
-
await Promise.all([this.storage.write(
|
|
432
|
+
const s = b(this.slotToKey);
|
|
433
|
+
await Promise.all([this.storage.write(I, t), this.storage.write(M, s)]);
|
|
431
434
|
}
|
|
432
435
|
/**
|
|
433
436
|
* Flush data to storage and release the instance.
|
|
@@ -453,23 +456,23 @@ class I {
|
|
|
453
456
|
*/
|
|
454
457
|
async export() {
|
|
455
458
|
this.assertOpen();
|
|
456
|
-
const e = this.memoryManager.vectorCount, t = e * this._dimensions * 4,
|
|
457
|
-
|
|
459
|
+
const e = this.memoryManager.vectorCount, t = e * this._dimensions * 4, s = b(this.slotToKey), r = s.byteLength, n = new ArrayBuffer(B), i = new DataView(n);
|
|
460
|
+
i.setUint32(0, S, !0), i.setUint32(4, d, !0), i.setUint32(8, this._dimensions, !0), i.setUint32(12, e, !0), i.setUint32(16, t, !0), i.setUint32(20, r, !0);
|
|
458
461
|
const a = this.memoryManager;
|
|
459
|
-
let
|
|
462
|
+
let c = "header", A = 0;
|
|
460
463
|
return new ReadableStream({
|
|
461
|
-
pull(
|
|
462
|
-
switch (
|
|
464
|
+
pull(h) {
|
|
465
|
+
switch (c) {
|
|
463
466
|
case "header":
|
|
464
|
-
|
|
467
|
+
h.enqueue(new Uint8Array(n)), c = t > 0 ? "vectors" : r > 0 ? "keys" : "done", c === "done" && h.close();
|
|
465
468
|
break;
|
|
466
469
|
case "vectors": {
|
|
467
|
-
const l = t -
|
|
468
|
-
|
|
470
|
+
const l = t - A, y = Math.min(l, H), f = new Uint8Array(y);
|
|
471
|
+
f.set(new Uint8Array(a.memory.buffer, a.dbOffset + A, y)), h.enqueue(f), A += y, A >= t && (c = r > 0 ? "keys" : "done", c === "done" && h.close());
|
|
469
472
|
break;
|
|
470
473
|
}
|
|
471
474
|
case "keys":
|
|
472
|
-
|
|
475
|
+
h.enqueue(s), c = "done", h.close();
|
|
473
476
|
break;
|
|
474
477
|
}
|
|
475
478
|
}
|
|
@@ -484,29 +487,29 @@ class I {
|
|
|
484
487
|
*/
|
|
485
488
|
async import(e) {
|
|
486
489
|
this.assertOpen();
|
|
487
|
-
const t = new
|
|
488
|
-
if (
|
|
490
|
+
const t = new K(e), s = await t.readExact(B), r = new DataView(s.buffer, s.byteOffset, s.byteLength);
|
|
491
|
+
if (r.getUint32(0, !0) !== S)
|
|
489
492
|
throw new Error("Invalid import data: unrecognized format");
|
|
490
|
-
const
|
|
491
|
-
if (
|
|
492
|
-
throw new Error(`Unsupported import version: expected ${
|
|
493
|
-
const a =
|
|
493
|
+
const i = r.getUint32(4, !0);
|
|
494
|
+
if (i !== d)
|
|
495
|
+
throw new Error(`Unsupported import version: expected ${d}, got ${i}`);
|
|
496
|
+
const a = r.getUint32(8, !0);
|
|
494
497
|
if (a !== this._dimensions)
|
|
495
498
|
throw new Error(`Import dimension mismatch: expected ${this._dimensions}, got ${a}`);
|
|
496
|
-
const
|
|
497
|
-
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(),
|
|
498
|
-
this.memoryManager.ensureCapacity(
|
|
499
|
+
const c = r.getUint32(12, !0), A = r.getUint32(16, !0), h = r.getUint32(20, !0);
|
|
500
|
+
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), c > 0) {
|
|
501
|
+
this.memoryManager.ensureCapacity(c);
|
|
499
502
|
let l = 0;
|
|
500
|
-
await t.readChunked(
|
|
501
|
-
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l,
|
|
502
|
-
|
|
503
|
-
), l +=
|
|
504
|
-
}), this.memoryManager.setVectorCount(
|
|
503
|
+
await t.readChunked(A, (y) => {
|
|
504
|
+
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, y.byteLength).set(
|
|
505
|
+
y
|
|
506
|
+
), l += y.byteLength;
|
|
507
|
+
}), this.memoryManager.setVectorCount(c);
|
|
505
508
|
}
|
|
506
|
-
if (
|
|
507
|
-
const l = await t.readExact(
|
|
508
|
-
for (let
|
|
509
|
-
this.keyToSlot.set(f
|
|
509
|
+
if (h > 0) {
|
|
510
|
+
const l = await t.readExact(h), y = C(l);
|
|
511
|
+
for (let f = 0; f < y.length; f++)
|
|
512
|
+
this.keyToSlot.set(y[f], f), this.slotToKey[f] = y[f];
|
|
510
513
|
}
|
|
511
514
|
t.release();
|
|
512
515
|
}
|
|
@@ -517,17 +520,17 @@ class I {
|
|
|
517
520
|
if (this.wasmExports) {
|
|
518
521
|
const t = this.memoryManager.queryOffset;
|
|
519
522
|
new Float32Array(this.memoryManager.memory.buffer, t, e.length).set(e), this.wasmExports.normalize(t, e.length);
|
|
520
|
-
const
|
|
521
|
-
e.set(
|
|
523
|
+
const s = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
524
|
+
e.set(s);
|
|
522
525
|
} else
|
|
523
|
-
|
|
526
|
+
T(e);
|
|
524
527
|
}
|
|
525
528
|
assertOpen() {
|
|
526
529
|
if (this.closed)
|
|
527
530
|
throw new Error("VectorDB instance has been closed");
|
|
528
531
|
}
|
|
529
532
|
}
|
|
530
|
-
class
|
|
533
|
+
class K {
|
|
531
534
|
reader;
|
|
532
535
|
buffer = new Uint8Array(0);
|
|
533
536
|
constructor(e) {
|
|
@@ -537,36 +540,36 @@ class z {
|
|
|
537
540
|
async readExact(e) {
|
|
538
541
|
if (e === 0) return new Uint8Array(0);
|
|
539
542
|
if (this.buffer.byteLength >= e) {
|
|
540
|
-
const
|
|
541
|
-
return this.buffer = this.buffer.subarray(e),
|
|
543
|
+
const i = this.buffer.subarray(0, e);
|
|
544
|
+
return this.buffer = this.buffer.subarray(e), i;
|
|
542
545
|
}
|
|
543
546
|
const t = [];
|
|
544
|
-
let
|
|
545
|
-
for (
|
|
546
|
-
const { done:
|
|
547
|
-
if (
|
|
548
|
-
t.push(a),
|
|
547
|
+
let s = this.buffer.byteLength;
|
|
548
|
+
for (s > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); s < e; ) {
|
|
549
|
+
const { done: i, value: a } = await this.reader.read();
|
|
550
|
+
if (i) throw new Error("Invalid import data: unexpected end of stream");
|
|
551
|
+
t.push(a), s += a.byteLength;
|
|
549
552
|
}
|
|
550
|
-
const
|
|
553
|
+
const r = new Uint8Array(s);
|
|
551
554
|
let n = 0;
|
|
552
|
-
for (const
|
|
553
|
-
|
|
554
|
-
return
|
|
555
|
+
for (const i of t)
|
|
556
|
+
r.set(i, n), n += i.byteLength;
|
|
557
|
+
return s > e ? (this.buffer = r.subarray(e), r.subarray(0, e)) : r;
|
|
555
558
|
}
|
|
556
559
|
/**
|
|
557
560
|
* Read `totalBytes` from the stream in chunks, calling `onChunk` for each.
|
|
558
561
|
* Avoids allocating a single large buffer for the full data.
|
|
559
562
|
*/
|
|
560
563
|
async readChunked(e, t) {
|
|
561
|
-
let
|
|
564
|
+
let s = e;
|
|
562
565
|
if (this.buffer.byteLength > 0) {
|
|
563
|
-
const
|
|
564
|
-
t(this.buffer.subarray(0,
|
|
566
|
+
const r = Math.min(this.buffer.byteLength, s);
|
|
567
|
+
t(this.buffer.subarray(0, r)), s -= r, this.buffer = this.buffer.subarray(r);
|
|
565
568
|
}
|
|
566
|
-
for (;
|
|
567
|
-
const { done:
|
|
568
|
-
if (
|
|
569
|
-
n.byteLength <=
|
|
569
|
+
for (; s > 0; ) {
|
|
570
|
+
const { done: r, value: n } = await this.reader.read();
|
|
571
|
+
if (r) throw new Error("Invalid import data: unexpected end of stream");
|
|
572
|
+
n.byteLength <= s ? (t(n), s -= n.byteLength) : (t(n.subarray(0, s)), this.buffer = n.slice(s), s = 0);
|
|
570
573
|
}
|
|
571
574
|
}
|
|
572
575
|
/** Release the stream reader lock. */
|
|
@@ -575,9 +578,9 @@ class z {
|
|
|
575
578
|
}
|
|
576
579
|
}
|
|
577
580
|
export {
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
581
|
+
k as DB,
|
|
582
|
+
U as InMemoryStorageProvider,
|
|
583
|
+
N as OPFSStorageProvider,
|
|
584
|
+
x as VectorCapacityExceededError
|
|
582
585
|
};
|
|
583
586
|
//# sourceMappingURL=eigen-db.js.map
|