eigen-db 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +14 -5
- package/dist/eigen-db.js +186 -186
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/package.json +1 -1
- package/src/lib/__tests__/vector-db.test.ts +40 -0
- package/src/lib/simd-binary.ts +1 -1
- package/src/lib/simd-optimized.wat +362 -0
- package/src/lib/simd.wat +42 -248
- package/src/lib/types.ts +2 -4
- package/src/lib/vector-db.ts +2 -3
package/dist/eigen-db.js
CHANGED
|
@@ -3,7 +3,7 @@ class k extends Error {
|
|
|
3
3
|
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class _ {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
9
|
constructor(e) {
|
|
@@ -18,34 +18,34 @@ class D {
|
|
|
18
18
|
}
|
|
19
19
|
async readAll(e) {
|
|
20
20
|
try {
|
|
21
|
-
const
|
|
22
|
-
return new Uint8Array(
|
|
21
|
+
const n = await (await (await (await this.getDir()).getFileHandle(e)).getFile()).arrayBuffer();
|
|
22
|
+
return new Uint8Array(n);
|
|
23
23
|
} catch {
|
|
24
24
|
return new Uint8Array(0);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
async append(e, t) {
|
|
28
|
-
const
|
|
29
|
-
await
|
|
28
|
+
const s = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await s.createWritable({ keepExistingData: !0 }), i = await s.getFile();
|
|
29
|
+
await n.seek(i.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
31
|
async write(e, t) {
|
|
32
|
-
const
|
|
33
|
-
await
|
|
32
|
+
const n = await (await (await this.getDir()).getFileHandle(e, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
33
|
+
await n.write(t), await n.close();
|
|
34
34
|
}
|
|
35
35
|
async destroy() {
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class v {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
41
|
async readAll(e) {
|
|
42
42
|
const t = this.files.get(e);
|
|
43
43
|
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
-
const
|
|
45
|
-
let
|
|
46
|
-
for (const
|
|
47
|
-
|
|
48
|
-
return
|
|
44
|
+
const r = t.reduce((i, a) => i + a.byteLength, 0), s = new Uint8Array(r);
|
|
45
|
+
let n = 0;
|
|
46
|
+
for (const i of t)
|
|
47
|
+
s.set(i, n), n += i.byteLength;
|
|
48
|
+
return s;
|
|
49
49
|
}
|
|
50
50
|
async append(e, t) {
|
|
51
51
|
this.files.has(e) || this.files.set(e, []), this.files.get(e).push(new Uint8Array(t));
|
|
@@ -57,46 +57,46 @@ class F {
|
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function S(
|
|
60
|
+
function S(c) {
|
|
61
61
|
let e = 0;
|
|
62
|
-
for (let
|
|
63
|
-
e +=
|
|
62
|
+
for (let s = 0; s < c.length; s++)
|
|
63
|
+
e += c[s] * c[s];
|
|
64
64
|
const t = Math.sqrt(e);
|
|
65
65
|
if (t === 0) return;
|
|
66
|
-
const
|
|
67
|
-
for (let
|
|
68
|
-
|
|
66
|
+
const r = 1 / t;
|
|
67
|
+
for (let s = 0; s < c.length; s++)
|
|
68
|
+
c[s] *= r;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let
|
|
72
|
-
let
|
|
73
|
-
const
|
|
74
|
-
for (let
|
|
75
|
-
|
|
76
|
-
t[
|
|
70
|
+
function O(c, e, t, r, s) {
|
|
71
|
+
for (let n = 0; n < r; n++) {
|
|
72
|
+
let i = 0;
|
|
73
|
+
const a = n * s;
|
|
74
|
+
for (let o = 0; o < s; o++)
|
|
75
|
+
i += c[o] * e[a + o];
|
|
76
|
+
t[n] = i;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function
|
|
81
|
-
const e =
|
|
82
|
-
let
|
|
83
|
-
for (const
|
|
84
|
-
|
|
85
|
-
return
|
|
79
|
+
const U = new TextEncoder(), x = new TextDecoder();
|
|
80
|
+
function d(c) {
|
|
81
|
+
const e = c.map((a) => U.encode(a)), t = e.reduce((a, o) => a + 4 + o.byteLength, 0), r = new ArrayBuffer(t), s = new DataView(r), n = new Uint8Array(r);
|
|
82
|
+
let i = 0;
|
|
83
|
+
for (const a of e)
|
|
84
|
+
s.setUint32(i, a.byteLength, !0), i += 4, n.set(a, i), i += a.byteLength;
|
|
85
|
+
return n;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const e = [], t = new DataView(
|
|
89
|
-
let
|
|
90
|
-
for (;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
e.push(
|
|
87
|
+
function g(c) {
|
|
88
|
+
const e = [], t = new DataView(c.buffer, c.byteOffset, c.byteLength);
|
|
89
|
+
let r = 0;
|
|
90
|
+
for (; r < c.byteLength; ) {
|
|
91
|
+
const s = t.getUint32(r, !0);
|
|
92
|
+
r += 4;
|
|
93
|
+
const n = x.decode(c.subarray(r, r + s));
|
|
94
|
+
e.push(n), r += s;
|
|
95
95
|
}
|
|
96
96
|
return e;
|
|
97
97
|
}
|
|
98
|
-
const
|
|
99
|
-
class
|
|
98
|
+
const m = 65536, w = 65536;
|
|
99
|
+
class Q {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
@@ -104,10 +104,10 @@ class L {
|
|
|
104
104
|
_vectorCount;
|
|
105
105
|
constructor(e, t = 0) {
|
|
106
106
|
this.dimensions = e, this.queryOffset = 0;
|
|
107
|
-
const
|
|
108
|
-
this.dbOffset = Math.ceil(
|
|
109
|
-
const
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
107
|
+
const r = e * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(r / m) * m;
|
|
109
|
+
const s = t * e * 4, n = this.dbOffset + s, i = Math.max(1, Math.ceil(n / m));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: i }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -126,7 +126,7 @@ class L {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const e =
|
|
129
|
+
const e = w * m - this.dbOffset, t = this.dimensions * 4 + 4;
|
|
130
130
|
return Math.floor(e / t);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
@@ -134,12 +134,12 @@ class L {
|
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
136
|
ensureCapacity(e) {
|
|
137
|
-
const t = this._vectorCount + e,
|
|
138
|
-
if (
|
|
139
|
-
const
|
|
140
|
-
if (
|
|
137
|
+
const t = this._vectorCount + e, r = this.dbOffset + t * this.dimensions * 4 + t * 4, s = this.memory.buffer.byteLength;
|
|
138
|
+
if (r > s) {
|
|
139
|
+
const n = Math.ceil((r - s) / m);
|
|
140
|
+
if (s / m + n > w)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
|
-
this.memory.grow(
|
|
142
|
+
this.memory.grow(n);
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
/**
|
|
@@ -154,9 +154,9 @@ class L {
|
|
|
154
154
|
*/
|
|
155
155
|
appendVectors(e) {
|
|
156
156
|
const t = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
-
let
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer,
|
|
157
|
+
let r = t;
|
|
158
|
+
for (const s of e)
|
|
159
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(s), r += this.dimensions * 4;
|
|
160
160
|
return this._vectorCount += e.length, t;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
@@ -183,8 +183,8 @@ class L {
|
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
185
|
writeVector(e, t) {
|
|
186
|
-
const
|
|
187
|
-
new Float32Array(this.memory.buffer,
|
|
186
|
+
const r = this.dbOffset + e * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(t);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -201,56 +201,56 @@ class L {
|
|
|
201
201
|
this._vectorCount = e;
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
-
function
|
|
205
|
-
const
|
|
206
|
-
if (
|
|
207
|
-
const
|
|
208
|
-
for (let
|
|
209
|
-
|
|
210
|
-
const
|
|
211
|
-
for (let
|
|
212
|
-
const
|
|
213
|
-
if (
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
return
|
|
204
|
+
function D(c, e, t, r) {
|
|
205
|
+
const s = c.length;
|
|
206
|
+
if (s === 0) return [];
|
|
207
|
+
const n = new Uint32Array(s);
|
|
208
|
+
for (let o = 0; o < s; o++) n[o] = o;
|
|
209
|
+
n.sort((o, h) => c[h] - c[o]);
|
|
210
|
+
const i = Math.min(t, s), a = [];
|
|
211
|
+
for (let o = 0; o < i; o++) {
|
|
212
|
+
const h = n[o], f = c[h];
|
|
213
|
+
if (r !== void 0 && f < r) break;
|
|
214
|
+
a.push({ key: e(h), similarity: f });
|
|
215
|
+
}
|
|
216
|
+
return a;
|
|
217
217
|
}
|
|
218
|
-
function
|
|
219
|
-
const
|
|
220
|
-
if (
|
|
221
|
-
const
|
|
222
|
-
for (let
|
|
223
|
-
|
|
224
|
-
const
|
|
218
|
+
function L(c, e, t, r) {
|
|
219
|
+
const s = c.length;
|
|
220
|
+
if (s === 0) return [];
|
|
221
|
+
const n = new Uint32Array(s);
|
|
222
|
+
for (let a = 0; a < s; a++) n[a] = a;
|
|
223
|
+
n.sort((a, o) => c[o] - c[a]);
|
|
224
|
+
const i = Math.min(t, s);
|
|
225
225
|
return {
|
|
226
226
|
[Symbol.iterator]() {
|
|
227
|
-
let
|
|
227
|
+
let a = 0;
|
|
228
228
|
return {
|
|
229
229
|
next() {
|
|
230
|
-
if (
|
|
231
|
-
const
|
|
232
|
-
return
|
|
230
|
+
if (a >= i) return { done: !0, value: void 0 };
|
|
231
|
+
const o = n[a++], h = c[o];
|
|
232
|
+
return r !== void 0 && h < r ? { done: !0, value: void 0 } : {
|
|
233
233
|
done: !1,
|
|
234
|
-
value: { key: e(
|
|
234
|
+
value: { key: e(o), similarity: h }
|
|
235
235
|
};
|
|
236
236
|
}
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
};
|
|
240
240
|
}
|
|
241
|
-
const
|
|
242
|
-
function
|
|
243
|
-
const
|
|
244
|
-
for (let t = 0; t <
|
|
245
|
-
e[t] =
|
|
241
|
+
const V = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
242
|
+
function T() {
|
|
243
|
+
const c = atob(V), e = new Uint8Array(c.length);
|
|
244
|
+
for (let t = 0; t < c.length; t++)
|
|
245
|
+
e[t] = c.charCodeAt(t);
|
|
246
246
|
return e;
|
|
247
247
|
}
|
|
248
|
-
async function
|
|
248
|
+
async function F(c, e) {
|
|
249
249
|
const t = { env: { memory: e } };
|
|
250
|
-
return (await WebAssembly.instantiate(
|
|
250
|
+
return (await WebAssembly.instantiate(c, t)).instance.exports;
|
|
251
251
|
}
|
|
252
|
-
const
|
|
253
|
-
class
|
|
252
|
+
const b = "vectors.bin", E = "keys.bin", C = 1111770949, u = 1, I = 24, z = 65536;
|
|
253
|
+
class M {
|
|
254
254
|
memoryManager;
|
|
255
255
|
storage;
|
|
256
256
|
dimensions;
|
|
@@ -262,23 +262,23 @@ class b {
|
|
|
262
262
|
slotToKey;
|
|
263
263
|
/** Whether this instance has been closed */
|
|
264
264
|
closed = !1;
|
|
265
|
-
constructor(e, t,
|
|
266
|
-
this.memoryManager = e, this.storage = t, this.dimensions =
|
|
265
|
+
constructor(e, t, r, s, n, i, a) {
|
|
266
|
+
this.memoryManager = e, this.storage = t, this.dimensions = r, this.shouldNormalize = s, this.wasmExports = n, this.keyToSlot = i, this.slotToKey = a;
|
|
267
267
|
}
|
|
268
268
|
static async open(e) {
|
|
269
|
-
const t = e.
|
|
270
|
-
for (let
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
let
|
|
275
|
-
const y = e.wasmBinary !== void 0 ? e.wasmBinary :
|
|
269
|
+
const t = e.storage ?? new v(), r = e.normalize !== !1, [s, n] = await Promise.all([t.readAll(b), t.readAll(E)]), i = n.byteLength > 0 ? g(n) : [], a = s.byteLength / (e.dimensions * 4), o = /* @__PURE__ */ new Map(), h = [];
|
|
270
|
+
for (let A = 0; A < i.length; A++)
|
|
271
|
+
o.set(i[A], A), h[A] = i[A];
|
|
272
|
+
const f = new Q(e.dimensions, a);
|
|
273
|
+
s.byteLength > 0 && f.loadVectorBytes(s, a);
|
|
274
|
+
let l = null;
|
|
275
|
+
const y = e.wasmBinary !== void 0 ? e.wasmBinary : T();
|
|
276
276
|
if (y !== null)
|
|
277
277
|
try {
|
|
278
|
-
|
|
278
|
+
l = await F(y, f.memory);
|
|
279
279
|
} catch {
|
|
280
280
|
}
|
|
281
|
-
return new
|
|
281
|
+
return new M(f, t, e.dimensions, r, l, o, h);
|
|
282
282
|
}
|
|
283
283
|
/** Total number of key-value pairs in the database */
|
|
284
284
|
get size() {
|
|
@@ -288,20 +288,20 @@ class b {
|
|
|
288
288
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
289
289
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
290
290
|
*/
|
|
291
|
-
set(e, t,
|
|
291
|
+
set(e, t, r) {
|
|
292
292
|
if (this.assertOpen(), t.length !== this.dimensions)
|
|
293
293
|
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${t.length}`);
|
|
294
|
-
const
|
|
295
|
-
(
|
|
296
|
-
const
|
|
297
|
-
if (
|
|
298
|
-
this.memoryManager.writeVector(
|
|
294
|
+
const s = new Float32Array(t);
|
|
295
|
+
(r?.normalize ?? this.shouldNormalize) && this.normalizeVector(s);
|
|
296
|
+
const i = this.keyToSlot.get(e);
|
|
297
|
+
if (i !== void 0)
|
|
298
|
+
this.memoryManager.writeVector(i, s);
|
|
299
299
|
else {
|
|
300
300
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
301
301
|
throw new k(this.memoryManager.maxVectors);
|
|
302
302
|
this.memoryManager.ensureCapacity(1);
|
|
303
|
-
const
|
|
304
|
-
this.memoryManager.appendVectors([
|
|
303
|
+
const o = this.memoryManager.vectorCount;
|
|
304
|
+
this.memoryManager.appendVectors([s]), this.keyToSlot.set(e, o), this.slotToKey[o] = e;
|
|
305
305
|
}
|
|
306
306
|
}
|
|
307
307
|
/**
|
|
@@ -318,8 +318,8 @@ class b {
|
|
|
318
318
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
319
319
|
*/
|
|
320
320
|
setMany(e) {
|
|
321
|
-
for (const [t,
|
|
322
|
-
this.set(t,
|
|
321
|
+
for (const [t, r] of e)
|
|
322
|
+
this.set(t, r);
|
|
323
323
|
}
|
|
324
324
|
/**
|
|
325
325
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
@@ -329,36 +329,36 @@ class b {
|
|
|
329
329
|
}
|
|
330
330
|
query(e, t) {
|
|
331
331
|
this.assertOpen();
|
|
332
|
-
const
|
|
332
|
+
const r = t?.topK ?? 1 / 0, s = t?.minSimilarity, n = t && "iterable" in t && t.iterable;
|
|
333
333
|
if (this.size === 0)
|
|
334
334
|
return [];
|
|
335
335
|
if (e.length !== this.dimensions)
|
|
336
336
|
throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${e.length}`);
|
|
337
|
-
const
|
|
338
|
-
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
339
|
-
const
|
|
337
|
+
const i = new Float32Array(e);
|
|
338
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(i), this.memoryManager.writeQuery(i), this.memoryManager.ensureCapacity(0);
|
|
339
|
+
const o = this.memoryManager.vectorCount, h = this.memoryManager.scoresOffset;
|
|
340
340
|
if (this.wasmExports)
|
|
341
341
|
this.wasmExports.search_all(
|
|
342
342
|
this.memoryManager.queryOffset,
|
|
343
343
|
this.memoryManager.dbOffset,
|
|
344
|
-
|
|
345
|
-
|
|
344
|
+
h,
|
|
345
|
+
o,
|
|
346
346
|
this.dimensions
|
|
347
347
|
);
|
|
348
348
|
else {
|
|
349
|
-
const
|
|
349
|
+
const A = new Float32Array(
|
|
350
350
|
this.memoryManager.memory.buffer,
|
|
351
351
|
this.memoryManager.queryOffset,
|
|
352
352
|
this.dimensions
|
|
353
|
-
),
|
|
353
|
+
), B = new Float32Array(
|
|
354
354
|
this.memoryManager.memory.buffer,
|
|
355
355
|
this.memoryManager.dbOffset,
|
|
356
|
-
|
|
357
|
-
),
|
|
358
|
-
|
|
356
|
+
o * this.dimensions
|
|
357
|
+
), p = new Float32Array(this.memoryManager.memory.buffer, h, o);
|
|
358
|
+
O(A, B, p, o, this.dimensions);
|
|
359
359
|
}
|
|
360
|
-
const
|
|
361
|
-
return
|
|
360
|
+
const f = new Float32Array(this.memoryManager.readScores()), l = this.slotToKey, y = (A) => l[A];
|
|
361
|
+
return n ? L(f, y, r, s) : D(f, y, r, s);
|
|
362
362
|
}
|
|
363
363
|
/**
|
|
364
364
|
* Persist the current in-memory state to storage.
|
|
@@ -367,15 +367,15 @@ class b {
|
|
|
367
367
|
this.assertOpen();
|
|
368
368
|
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this.dimensions * 4);
|
|
369
369
|
if (e > 0) {
|
|
370
|
-
const
|
|
370
|
+
const s = new Uint8Array(
|
|
371
371
|
this.memoryManager.memory.buffer,
|
|
372
372
|
this.memoryManager.dbOffset,
|
|
373
373
|
e * this.dimensions * 4
|
|
374
374
|
);
|
|
375
|
-
t.set(
|
|
375
|
+
t.set(s);
|
|
376
376
|
}
|
|
377
|
-
const
|
|
378
|
-
await Promise.all([this.storage.write(
|
|
377
|
+
const r = d(this.slotToKey);
|
|
378
|
+
await Promise.all([this.storage.write(b, t), this.storage.write(E, r)]);
|
|
379
379
|
}
|
|
380
380
|
/**
|
|
381
381
|
* Flush data to storage and release the instance.
|
|
@@ -401,23 +401,23 @@ class b {
|
|
|
401
401
|
*/
|
|
402
402
|
async export() {
|
|
403
403
|
this.assertOpen();
|
|
404
|
-
const e = this.memoryManager.vectorCount, t = e * this.dimensions * 4,
|
|
405
|
-
|
|
406
|
-
const
|
|
407
|
-
let
|
|
404
|
+
const e = this.memoryManager.vectorCount, t = e * this.dimensions * 4, r = d(this.slotToKey), s = r.byteLength, n = new ArrayBuffer(I), i = new DataView(n);
|
|
405
|
+
i.setUint32(0, C, !0), i.setUint32(4, u, !0), i.setUint32(8, this.dimensions, !0), i.setUint32(12, e, !0), i.setUint32(16, t, !0), i.setUint32(20, s, !0);
|
|
406
|
+
const a = this.memoryManager;
|
|
407
|
+
let o = "header", h = 0;
|
|
408
408
|
return new ReadableStream({
|
|
409
|
-
pull(
|
|
410
|
-
switch (
|
|
409
|
+
pull(f) {
|
|
410
|
+
switch (o) {
|
|
411
411
|
case "header":
|
|
412
|
-
|
|
412
|
+
f.enqueue(new Uint8Array(n)), o = t > 0 ? "vectors" : s > 0 ? "keys" : "done", o === "done" && f.close();
|
|
413
413
|
break;
|
|
414
414
|
case "vectors": {
|
|
415
|
-
const
|
|
416
|
-
|
|
415
|
+
const l = t - h, y = Math.min(l, z), A = new Uint8Array(y);
|
|
416
|
+
A.set(new Uint8Array(a.memory.buffer, a.dbOffset + h, y)), f.enqueue(A), h += y, h >= t && (o = s > 0 ? "keys" : "done", o === "done" && f.close());
|
|
417
417
|
break;
|
|
418
418
|
}
|
|
419
419
|
case "keys":
|
|
420
|
-
|
|
420
|
+
f.enqueue(r), o = "done", f.close();
|
|
421
421
|
break;
|
|
422
422
|
}
|
|
423
423
|
}
|
|
@@ -432,29 +432,29 @@ class b {
|
|
|
432
432
|
*/
|
|
433
433
|
async import(e) {
|
|
434
434
|
this.assertOpen();
|
|
435
|
-
const t = new
|
|
436
|
-
if (
|
|
435
|
+
const t = new q(e), r = await t.readExact(I), s = new DataView(r.buffer, r.byteOffset, r.byteLength);
|
|
436
|
+
if (s.getUint32(0, !0) !== C)
|
|
437
437
|
throw new Error("Invalid import data: unrecognized format");
|
|
438
|
-
const
|
|
439
|
-
if (
|
|
440
|
-
throw new Error(`Unsupported import version: expected ${
|
|
441
|
-
const
|
|
442
|
-
if (
|
|
443
|
-
throw new Error(`Import dimension mismatch: expected ${this.dimensions}, got ${
|
|
444
|
-
const
|
|
445
|
-
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(),
|
|
446
|
-
this.memoryManager.ensureCapacity(
|
|
447
|
-
let
|
|
448
|
-
await t.readChunked(
|
|
449
|
-
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset +
|
|
450
|
-
|
|
451
|
-
),
|
|
452
|
-
}), this.memoryManager.setVectorCount(
|
|
438
|
+
const i = s.getUint32(4, !0);
|
|
439
|
+
if (i !== u)
|
|
440
|
+
throw new Error(`Unsupported import version: expected ${u}, got ${i}`);
|
|
441
|
+
const a = s.getUint32(8, !0);
|
|
442
|
+
if (a !== this.dimensions)
|
|
443
|
+
throw new Error(`Import dimension mismatch: expected ${this.dimensions}, got ${a}`);
|
|
444
|
+
const o = s.getUint32(12, !0), h = s.getUint32(16, !0), f = s.getUint32(20, !0);
|
|
445
|
+
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), o > 0) {
|
|
446
|
+
this.memoryManager.ensureCapacity(o);
|
|
447
|
+
let l = 0;
|
|
448
|
+
await t.readChunked(h, (y) => {
|
|
449
|
+
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, y.byteLength).set(
|
|
450
|
+
y
|
|
451
|
+
), l += y.byteLength;
|
|
452
|
+
}), this.memoryManager.setVectorCount(o);
|
|
453
453
|
}
|
|
454
|
-
if (
|
|
455
|
-
const
|
|
456
|
-
for (let
|
|
457
|
-
this.keyToSlot.set(
|
|
454
|
+
if (f > 0) {
|
|
455
|
+
const l = await t.readExact(f), y = g(l);
|
|
456
|
+
for (let A = 0; A < y.length; A++)
|
|
457
|
+
this.keyToSlot.set(y[A], A), this.slotToKey[A] = y[A];
|
|
458
458
|
}
|
|
459
459
|
t.release();
|
|
460
460
|
}
|
|
@@ -465,8 +465,8 @@ class b {
|
|
|
465
465
|
if (this.wasmExports) {
|
|
466
466
|
const t = this.memoryManager.queryOffset;
|
|
467
467
|
new Float32Array(this.memoryManager.memory.buffer, t, e.length).set(e), this.wasmExports.normalize(t, e.length);
|
|
468
|
-
const
|
|
469
|
-
e.set(
|
|
468
|
+
const r = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
469
|
+
e.set(r);
|
|
470
470
|
} else
|
|
471
471
|
S(e);
|
|
472
472
|
}
|
|
@@ -475,7 +475,7 @@ class b {
|
|
|
475
475
|
throw new Error("VectorDB instance has been closed");
|
|
476
476
|
}
|
|
477
477
|
}
|
|
478
|
-
class
|
|
478
|
+
class q {
|
|
479
479
|
reader;
|
|
480
480
|
buffer = new Uint8Array(0);
|
|
481
481
|
constructor(e) {
|
|
@@ -485,36 +485,36 @@ class V {
|
|
|
485
485
|
async readExact(e) {
|
|
486
486
|
if (e === 0) return new Uint8Array(0);
|
|
487
487
|
if (this.buffer.byteLength >= e) {
|
|
488
|
-
const
|
|
489
|
-
return this.buffer = this.buffer.subarray(e),
|
|
488
|
+
const i = this.buffer.subarray(0, e);
|
|
489
|
+
return this.buffer = this.buffer.subarray(e), i;
|
|
490
490
|
}
|
|
491
491
|
const t = [];
|
|
492
|
-
let
|
|
493
|
-
for (
|
|
494
|
-
const { done:
|
|
495
|
-
if (
|
|
496
|
-
t.push(
|
|
492
|
+
let r = this.buffer.byteLength;
|
|
493
|
+
for (r > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); r < e; ) {
|
|
494
|
+
const { done: i, value: a } = await this.reader.read();
|
|
495
|
+
if (i) throw new Error("Invalid import data: unexpected end of stream");
|
|
496
|
+
t.push(a), r += a.byteLength;
|
|
497
497
|
}
|
|
498
|
-
const
|
|
499
|
-
let
|
|
500
|
-
for (const
|
|
501
|
-
|
|
502
|
-
return
|
|
498
|
+
const s = new Uint8Array(r);
|
|
499
|
+
let n = 0;
|
|
500
|
+
for (const i of t)
|
|
501
|
+
s.set(i, n), n += i.byteLength;
|
|
502
|
+
return r > e ? (this.buffer = s.subarray(e), s.subarray(0, e)) : s;
|
|
503
503
|
}
|
|
504
504
|
/**
|
|
505
505
|
* Read `totalBytes` from the stream in chunks, calling `onChunk` for each.
|
|
506
506
|
* Avoids allocating a single large buffer for the full data.
|
|
507
507
|
*/
|
|
508
508
|
async readChunked(e, t) {
|
|
509
|
-
let
|
|
509
|
+
let r = e;
|
|
510
510
|
if (this.buffer.byteLength > 0) {
|
|
511
|
-
const
|
|
512
|
-
t(this.buffer.subarray(0,
|
|
511
|
+
const s = Math.min(this.buffer.byteLength, r);
|
|
512
|
+
t(this.buffer.subarray(0, s)), r -= s, this.buffer = this.buffer.subarray(s);
|
|
513
513
|
}
|
|
514
|
-
for (;
|
|
515
|
-
const { done:
|
|
516
|
-
if (
|
|
517
|
-
|
|
514
|
+
for (; r > 0; ) {
|
|
515
|
+
const { done: s, value: n } = await this.reader.read();
|
|
516
|
+
if (s) throw new Error("Invalid import data: unexpected end of stream");
|
|
517
|
+
n.byteLength <= r ? (t(n), r -= n.byteLength) : (t(n.subarray(0, r)), this.buffer = n.slice(r), r = 0);
|
|
518
518
|
}
|
|
519
519
|
}
|
|
520
520
|
/** Release the stream reader lock. */
|
|
@@ -523,9 +523,9 @@ class V {
|
|
|
523
523
|
}
|
|
524
524
|
}
|
|
525
525
|
export {
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
526
|
+
M as DB,
|
|
527
|
+
v as InMemoryStorageProvider,
|
|
528
|
+
_ as OPFSStorageProvider,
|
|
529
529
|
k as VectorCapacityExceededError
|
|
530
530
|
};
|
|
531
531
|
//# sourceMappingURL=eigen-db.js.map
|