eigen-db 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +79 -27
- package/dist/eigen-db.js +317 -195
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/package.json +1 -1
- package/src/lib/__tests__/result-set.test.ts +19 -19
- package/src/lib/__tests__/vector-db.test.ts +429 -16
- package/src/lib/memory-manager.ts +8 -0
- package/src/lib/result-set.ts +16 -15
- package/src/lib/simd-binary.ts +1 -1
- package/src/lib/simd-optimized.wat +362 -0
- package/src/lib/simd.wat +42 -248
- package/src/lib/types.ts +4 -6
- package/src/lib/vector-db.ts +241 -9
package/dist/eigen-db.js
CHANGED
|
@@ -1,113 +1,113 @@
|
|
|
1
|
-
class
|
|
2
|
-
constructor(
|
|
3
|
-
super(`Capacity exceeded. Max vectors for this dimension size is ~${
|
|
1
|
+
class k extends Error {
|
|
2
|
+
constructor(e) {
|
|
3
|
+
super(`Capacity exceeded. Max vectors for this dimension size is ~${e}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class _ {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
|
-
constructor(
|
|
10
|
-
this.dirName =
|
|
9
|
+
constructor(e) {
|
|
10
|
+
this.dirName = e;
|
|
11
11
|
}
|
|
12
12
|
async getDir() {
|
|
13
13
|
if (!this.dirHandle) {
|
|
14
|
-
const
|
|
15
|
-
this.dirHandle = await
|
|
14
|
+
const e = await navigator.storage.getDirectory();
|
|
15
|
+
this.dirHandle = await e.getDirectoryHandle(this.dirName, { create: !0 });
|
|
16
16
|
}
|
|
17
17
|
return this.dirHandle;
|
|
18
18
|
}
|
|
19
|
-
async readAll(
|
|
19
|
+
async readAll(e) {
|
|
20
20
|
try {
|
|
21
|
-
const
|
|
22
|
-
return new Uint8Array(
|
|
21
|
+
const n = await (await (await (await this.getDir()).getFileHandle(e)).getFile()).arrayBuffer();
|
|
22
|
+
return new Uint8Array(n);
|
|
23
23
|
} catch {
|
|
24
24
|
return new Uint8Array(0);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
|
-
async append(
|
|
28
|
-
const
|
|
29
|
-
await
|
|
27
|
+
async append(e, t) {
|
|
28
|
+
const s = await (await this.getDir()).getFileHandle(e, { create: !0 }), n = await s.createWritable({ keepExistingData: !0 }), i = await s.getFile();
|
|
29
|
+
await n.seek(i.size), await n.write(t), await n.close();
|
|
30
30
|
}
|
|
31
|
-
async write(
|
|
32
|
-
const
|
|
33
|
-
await
|
|
31
|
+
async write(e, t) {
|
|
32
|
+
const n = await (await (await this.getDir()).getFileHandle(e, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
33
|
+
await n.write(t), await n.close();
|
|
34
34
|
}
|
|
35
35
|
async destroy() {
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class v {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
|
-
async readAll(
|
|
42
|
-
const
|
|
43
|
-
if (!
|
|
44
|
-
const
|
|
45
|
-
let
|
|
46
|
-
for (const
|
|
47
|
-
|
|
48
|
-
return
|
|
41
|
+
async readAll(e) {
|
|
42
|
+
const t = this.files.get(e);
|
|
43
|
+
if (!t || t.length === 0) return new Uint8Array(0);
|
|
44
|
+
const r = t.reduce((i, a) => i + a.byteLength, 0), s = new Uint8Array(r);
|
|
45
|
+
let n = 0;
|
|
46
|
+
for (const i of t)
|
|
47
|
+
s.set(i, n), n += i.byteLength;
|
|
48
|
+
return s;
|
|
49
49
|
}
|
|
50
|
-
async append(
|
|
51
|
-
this.files.has(
|
|
50
|
+
async append(e, t) {
|
|
51
|
+
this.files.has(e) || this.files.set(e, []), this.files.get(e).push(new Uint8Array(t));
|
|
52
52
|
}
|
|
53
|
-
async write(
|
|
54
|
-
this.files.set(
|
|
53
|
+
async write(e, t) {
|
|
54
|
+
this.files.set(e, [new Uint8Array(t)]);
|
|
55
55
|
}
|
|
56
56
|
async destroy() {
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
61
|
-
let
|
|
62
|
-
for (let
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
if (
|
|
66
|
-
const
|
|
67
|
-
for (let
|
|
68
|
-
|
|
60
|
+
function S(c) {
|
|
61
|
+
let e = 0;
|
|
62
|
+
for (let s = 0; s < c.length; s++)
|
|
63
|
+
e += c[s] * c[s];
|
|
64
|
+
const t = Math.sqrt(e);
|
|
65
|
+
if (t === 0) return;
|
|
66
|
+
const r = 1 / t;
|
|
67
|
+
for (let s = 0; s < c.length; s++)
|
|
68
|
+
c[s] *= r;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let
|
|
72
|
-
let
|
|
73
|
-
const a =
|
|
74
|
-
for (let
|
|
75
|
-
|
|
76
|
-
|
|
70
|
+
function O(c, e, t, r, s) {
|
|
71
|
+
for (let n = 0; n < r; n++) {
|
|
72
|
+
let i = 0;
|
|
73
|
+
const a = n * s;
|
|
74
|
+
for (let o = 0; o < s; o++)
|
|
75
|
+
i += c[o] * e[a + o];
|
|
76
|
+
t[n] = i;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function
|
|
81
|
-
const
|
|
82
|
-
let
|
|
83
|
-
for (const a of
|
|
84
|
-
|
|
85
|
-
return
|
|
79
|
+
const U = new TextEncoder(), x = new TextDecoder();
|
|
80
|
+
function d(c) {
|
|
81
|
+
const e = c.map((a) => U.encode(a)), t = e.reduce((a, o) => a + 4 + o.byteLength, 0), r = new ArrayBuffer(t), s = new DataView(r), n = new Uint8Array(r);
|
|
82
|
+
let i = 0;
|
|
83
|
+
for (const a of e)
|
|
84
|
+
s.setUint32(i, a.byteLength, !0), i += 4, n.set(a, i), i += a.byteLength;
|
|
85
|
+
return n;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const
|
|
89
|
-
let
|
|
90
|
-
for (;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
return
|
|
87
|
+
function g(c) {
|
|
88
|
+
const e = [], t = new DataView(c.buffer, c.byteOffset, c.byteLength);
|
|
89
|
+
let r = 0;
|
|
90
|
+
for (; r < c.byteLength; ) {
|
|
91
|
+
const s = t.getUint32(r, !0);
|
|
92
|
+
r += 4;
|
|
93
|
+
const n = x.decode(c.subarray(r, r + s));
|
|
94
|
+
e.push(n), r += s;
|
|
95
|
+
}
|
|
96
|
+
return e;
|
|
97
97
|
}
|
|
98
|
-
const
|
|
99
|
-
class
|
|
98
|
+
const m = 65536, w = 65536;
|
|
99
|
+
class Q {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
103
103
|
dbOffset;
|
|
104
104
|
_vectorCount;
|
|
105
|
-
constructor(
|
|
106
|
-
this.dimensions =
|
|
107
|
-
const
|
|
108
|
-
this.dbOffset = Math.ceil(
|
|
109
|
-
const
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
105
|
+
constructor(e, t = 0) {
|
|
106
|
+
this.dimensions = e, this.queryOffset = 0;
|
|
107
|
+
const r = e * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(r / m) * m;
|
|
109
|
+
const s = t * e * 4, n = this.dbOffset + s, i = Math.max(1, Math.ceil(n / m));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: i }), this._vectorCount = t;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -126,45 +126,45 @@ class v {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const
|
|
130
|
-
return Math.floor(
|
|
129
|
+
const e = w * m - this.dbOffset, t = this.dimensions * 4 + 4;
|
|
130
|
+
return Math.floor(e / t);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
133
|
* Ensures memory is large enough for the current DB + scores buffer.
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
|
-
ensureCapacity(
|
|
137
|
-
const
|
|
138
|
-
if (
|
|
139
|
-
const
|
|
140
|
-
if (
|
|
136
|
+
ensureCapacity(e) {
|
|
137
|
+
const t = this._vectorCount + e, r = this.dbOffset + t * this.dimensions * 4 + t * 4, s = this.memory.buffer.byteLength;
|
|
138
|
+
if (r > s) {
|
|
139
|
+
const n = Math.ceil((r - s) / m);
|
|
140
|
+
if (s / m + n > w)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
|
-
this.memory.grow(
|
|
142
|
+
this.memory.grow(n);
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
/**
|
|
146
146
|
* Write a query vector into the query buffer region.
|
|
147
147
|
*/
|
|
148
|
-
writeQuery(
|
|
149
|
-
new Float32Array(this.memory.buffer, this.queryOffset, this.dimensions).set(
|
|
148
|
+
writeQuery(e) {
|
|
149
|
+
new Float32Array(this.memory.buffer, this.queryOffset, this.dimensions).set(e);
|
|
150
150
|
}
|
|
151
151
|
/**
|
|
152
152
|
* Append vectors to the database region.
|
|
153
153
|
* Returns the byte offset where the new vectors were written.
|
|
154
154
|
*/
|
|
155
|
-
appendVectors(
|
|
156
|
-
const
|
|
157
|
-
let
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer,
|
|
160
|
-
return this._vectorCount +=
|
|
155
|
+
appendVectors(e) {
|
|
156
|
+
const t = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
+
let r = t;
|
|
158
|
+
for (const s of e)
|
|
159
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(s), r += this.dimensions * 4;
|
|
160
|
+
return this._vectorCount += e.length, t;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
163
163
|
* Load raw vector bytes directly into the database region.
|
|
164
164
|
* Used for bulk loading from OPFS.
|
|
165
165
|
*/
|
|
166
|
-
loadVectorBytes(
|
|
167
|
-
new Uint8Array(this.memory.buffer, this.dbOffset,
|
|
166
|
+
loadVectorBytes(e, t) {
|
|
167
|
+
new Uint8Array(this.memory.buffer, this.dbOffset, e.byteLength).set(e), this._vectorCount = t;
|
|
168
168
|
}
|
|
169
169
|
/**
|
|
170
170
|
* Read the scores buffer as a Float32Array view.
|
|
@@ -175,16 +175,16 @@ class v {
|
|
|
175
175
|
/**
|
|
176
176
|
* Read the DB region for a specific vector index.
|
|
177
177
|
*/
|
|
178
|
-
readVector(
|
|
179
|
-
const
|
|
180
|
-
return new Float32Array(this.memory.buffer,
|
|
178
|
+
readVector(e) {
|
|
179
|
+
const t = this.dbOffset + e * this.dimensions * 4;
|
|
180
|
+
return new Float32Array(this.memory.buffer, t, this.dimensions);
|
|
181
181
|
}
|
|
182
182
|
/**
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
|
-
writeVector(
|
|
186
|
-
const
|
|
187
|
-
new Float32Array(this.memory.buffer,
|
|
185
|
+
writeVector(e, t) {
|
|
186
|
+
const r = this.dbOffset + e * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, r, this.dimensions).set(t);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -193,57 +193,64 @@ class v {
|
|
|
193
193
|
reset() {
|
|
194
194
|
this._vectorCount = 0;
|
|
195
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Set the vector count after bulk-loading data directly into the DB region.
|
|
198
|
+
* Use when writing raw bytes to the DB region externally (e.g., streaming import).
|
|
199
|
+
*/
|
|
200
|
+
setVectorCount(e) {
|
|
201
|
+
this._vectorCount = e;
|
|
202
|
+
}
|
|
196
203
|
}
|
|
197
|
-
function
|
|
198
|
-
const
|
|
199
|
-
if (
|
|
200
|
-
const
|
|
201
|
-
for (let
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
for (let
|
|
205
|
-
const
|
|
206
|
-
if (
|
|
207
|
-
a.push({ key:
|
|
204
|
+
function D(c, e, t, r) {
|
|
205
|
+
const s = c.length;
|
|
206
|
+
if (s === 0) return [];
|
|
207
|
+
const n = new Uint32Array(s);
|
|
208
|
+
for (let o = 0; o < s; o++) n[o] = o;
|
|
209
|
+
n.sort((o, h) => c[h] - c[o]);
|
|
210
|
+
const i = Math.min(t, s), a = [];
|
|
211
|
+
for (let o = 0; o < i; o++) {
|
|
212
|
+
const h = n[o], f = c[h];
|
|
213
|
+
if (r !== void 0 && f < r) break;
|
|
214
|
+
a.push({ key: e(h), similarity: f });
|
|
208
215
|
}
|
|
209
216
|
return a;
|
|
210
217
|
}
|
|
211
|
-
function
|
|
212
|
-
const
|
|
213
|
-
if (
|
|
214
|
-
const
|
|
215
|
-
for (let a = 0; a <
|
|
216
|
-
|
|
217
|
-
const
|
|
218
|
+
function L(c, e, t, r) {
|
|
219
|
+
const s = c.length;
|
|
220
|
+
if (s === 0) return [];
|
|
221
|
+
const n = new Uint32Array(s);
|
|
222
|
+
for (let a = 0; a < s; a++) n[a] = a;
|
|
223
|
+
n.sort((a, o) => c[o] - c[a]);
|
|
224
|
+
const i = Math.min(t, s);
|
|
218
225
|
return {
|
|
219
226
|
[Symbol.iterator]() {
|
|
220
227
|
let a = 0;
|
|
221
228
|
return {
|
|
222
229
|
next() {
|
|
223
|
-
if (a >=
|
|
224
|
-
const
|
|
225
|
-
return
|
|
230
|
+
if (a >= i) return { done: !0, value: void 0 };
|
|
231
|
+
const o = n[a++], h = c[o];
|
|
232
|
+
return r !== void 0 && h < r ? { done: !0, value: void 0 } : {
|
|
226
233
|
done: !1,
|
|
227
|
-
value: { key:
|
|
234
|
+
value: { key: e(o), similarity: h }
|
|
228
235
|
};
|
|
229
236
|
}
|
|
230
237
|
};
|
|
231
238
|
}
|
|
232
239
|
};
|
|
233
240
|
}
|
|
234
|
-
const
|
|
235
|
-
function
|
|
236
|
-
const
|
|
237
|
-
for (let
|
|
238
|
-
|
|
239
|
-
return
|
|
241
|
+
const V = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKpgQCrQIFAX8BewN9AXsCf/0MAAAAAAAAAAAAAAAAAAAAACEDIAFBfHEhCEEAIQICQANAIAIgCE8NASAAIAJBAnRqIQkgAyAJ/QAEACAJ/QAEAP3mAf3kASEDIAJBBGohAgwACwsgA/0fACAD/R8BkiAD/R8CIAP9HwOSkiEEAkADQCACIAFPDQEgACACQQJ0aiEJIAQgCSoCACAJKgIAlJIhBCACQQFqIQIMAAsLIASRIQUgBUMAAAAAWwRADwtDAACAPyAFlSEGIAb9EyEHQQAhAgJAA0AgAiAITw0BIAAgAkECdGohCSAJIAn9AAQAIAf95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQkgCSAJKgIAIAaUOAIAIAJBAWohAgwACwsL9AEEAn8BewF9BX8gBEF8cSEKIARBAnQhDUEAIQUCQANAIAUgA08NASABIAUgDWxqIQn9DAAAAAAAAAAAAAAAAAAAAAAhB0EAIQYCQANAIAYgCk8NASAAIAZBAnRqIQsgCSAGQQJ0aiEMIAcgC/0ABAAgDP0ABAD95gH95AEhByAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhCAJAA0AgBiAETw0BIAAgBkECdGohCyAJIAZBAnRqIQwgCCALKgIAIAwqAgCUkiEIIAZBAWohBgwACwsgAiAFQQJ0aiAIOAIAIAVBAWohBQwACwsL";
|
|
242
|
+
function T() {
|
|
243
|
+
const c = atob(V), e = new Uint8Array(c.length);
|
|
244
|
+
for (let t = 0; t < c.length; t++)
|
|
245
|
+
e[t] = c.charCodeAt(t);
|
|
246
|
+
return e;
|
|
240
247
|
}
|
|
241
|
-
async function F(
|
|
242
|
-
const
|
|
243
|
-
return (await WebAssembly.instantiate(
|
|
248
|
+
async function F(c, e) {
|
|
249
|
+
const t = { env: { memory: e } };
|
|
250
|
+
return (await WebAssembly.instantiate(c, t)).instance.exports;
|
|
244
251
|
}
|
|
245
|
-
const
|
|
246
|
-
class
|
|
252
|
+
const b = "vectors.bin", E = "keys.bin", C = 1111770949, u = 1, I = 24, z = 65536;
|
|
253
|
+
class M {
|
|
247
254
|
memoryManager;
|
|
248
255
|
storage;
|
|
249
256
|
dimensions;
|
|
@@ -255,23 +262,23 @@ class w {
|
|
|
255
262
|
slotToKey;
|
|
256
263
|
/** Whether this instance has been closed */
|
|
257
264
|
closed = !1;
|
|
258
|
-
constructor(
|
|
259
|
-
this.memoryManager =
|
|
260
|
-
}
|
|
261
|
-
static async open(
|
|
262
|
-
const
|
|
263
|
-
for (let
|
|
264
|
-
|
|
265
|
-
const
|
|
266
|
-
|
|
267
|
-
let
|
|
268
|
-
const
|
|
269
|
-
if (
|
|
265
|
+
constructor(e, t, r, s, n, i, a) {
|
|
266
|
+
this.memoryManager = e, this.storage = t, this.dimensions = r, this.shouldNormalize = s, this.wasmExports = n, this.keyToSlot = i, this.slotToKey = a;
|
|
267
|
+
}
|
|
268
|
+
static async open(e) {
|
|
269
|
+
const t = e.storage ?? new v(), r = e.normalize !== !1, [s, n] = await Promise.all([t.readAll(b), t.readAll(E)]), i = n.byteLength > 0 ? g(n) : [], a = s.byteLength / (e.dimensions * 4), o = /* @__PURE__ */ new Map(), h = [];
|
|
270
|
+
for (let A = 0; A < i.length; A++)
|
|
271
|
+
o.set(i[A], A), h[A] = i[A];
|
|
272
|
+
const f = new Q(e.dimensions, a);
|
|
273
|
+
s.byteLength > 0 && f.loadVectorBytes(s, a);
|
|
274
|
+
let l = null;
|
|
275
|
+
const y = e.wasmBinary !== void 0 ? e.wasmBinary : T();
|
|
276
|
+
if (y !== null)
|
|
270
277
|
try {
|
|
271
|
-
|
|
278
|
+
l = await F(y, f.memory);
|
|
272
279
|
} catch {
|
|
273
280
|
}
|
|
274
|
-
return new
|
|
281
|
+
return new M(f, t, e.dimensions, r, l, o, h);
|
|
275
282
|
}
|
|
276
283
|
/** Total number of key-value pairs in the database */
|
|
277
284
|
get size() {
|
|
@@ -281,94 +288,94 @@ class w {
|
|
|
281
288
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
282
289
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
283
290
|
*/
|
|
284
|
-
set(
|
|
285
|
-
if (this.assertOpen(),
|
|
286
|
-
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${
|
|
287
|
-
const
|
|
288
|
-
(
|
|
289
|
-
const
|
|
290
|
-
if (
|
|
291
|
-
this.memoryManager.writeVector(
|
|
291
|
+
set(e, t, r) {
|
|
292
|
+
if (this.assertOpen(), t.length !== this.dimensions)
|
|
293
|
+
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${t.length}`);
|
|
294
|
+
const s = new Float32Array(t);
|
|
295
|
+
(r?.normalize ?? this.shouldNormalize) && this.normalizeVector(s);
|
|
296
|
+
const i = this.keyToSlot.get(e);
|
|
297
|
+
if (i !== void 0)
|
|
298
|
+
this.memoryManager.writeVector(i, s);
|
|
292
299
|
else {
|
|
293
300
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
294
|
-
throw new
|
|
301
|
+
throw new k(this.memoryManager.maxVectors);
|
|
295
302
|
this.memoryManager.ensureCapacity(1);
|
|
296
|
-
const
|
|
297
|
-
this.memoryManager.appendVectors([
|
|
303
|
+
const o = this.memoryManager.vectorCount;
|
|
304
|
+
this.memoryManager.appendVectors([s]), this.keyToSlot.set(e, o), this.slotToKey[o] = e;
|
|
298
305
|
}
|
|
299
306
|
}
|
|
300
307
|
/**
|
|
301
308
|
* Get the stored vector for a key. Returns undefined if the key does not exist.
|
|
302
309
|
* Returns a copy of the stored vector as a plain number array.
|
|
303
310
|
*/
|
|
304
|
-
get(
|
|
311
|
+
get(e) {
|
|
305
312
|
this.assertOpen();
|
|
306
|
-
const
|
|
307
|
-
if (
|
|
308
|
-
return Array.from(this.memoryManager.readVector(
|
|
313
|
+
const t = this.keyToSlot.get(e);
|
|
314
|
+
if (t !== void 0)
|
|
315
|
+
return Array.from(this.memoryManager.readVector(t));
|
|
309
316
|
}
|
|
310
317
|
/**
|
|
311
318
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
312
319
|
*/
|
|
313
|
-
setMany(
|
|
314
|
-
for (const [
|
|
315
|
-
this.set(
|
|
320
|
+
setMany(e) {
|
|
321
|
+
for (const [t, r] of e)
|
|
322
|
+
this.set(t, r);
|
|
316
323
|
}
|
|
317
324
|
/**
|
|
318
325
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
319
326
|
*/
|
|
320
|
-
getMany(
|
|
321
|
-
return
|
|
327
|
+
getMany(e) {
|
|
328
|
+
return e.map((t) => this.get(t));
|
|
322
329
|
}
|
|
323
|
-
query(
|
|
330
|
+
query(e, t) {
|
|
324
331
|
this.assertOpen();
|
|
325
|
-
const
|
|
332
|
+
const r = t?.topK ?? 1 / 0, s = t?.minSimilarity, n = t && "iterable" in t && t.iterable;
|
|
326
333
|
if (this.size === 0)
|
|
327
334
|
return [];
|
|
328
|
-
if (
|
|
329
|
-
throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${
|
|
330
|
-
const
|
|
331
|
-
(
|
|
332
|
-
const
|
|
335
|
+
if (e.length !== this.dimensions)
|
|
336
|
+
throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${e.length}`);
|
|
337
|
+
const i = new Float32Array(e);
|
|
338
|
+
(t?.normalize ?? this.shouldNormalize) && this.normalizeVector(i), this.memoryManager.writeQuery(i), this.memoryManager.ensureCapacity(0);
|
|
339
|
+
const o = this.memoryManager.vectorCount, h = this.memoryManager.scoresOffset;
|
|
333
340
|
if (this.wasmExports)
|
|
334
341
|
this.wasmExports.search_all(
|
|
335
342
|
this.memoryManager.queryOffset,
|
|
336
343
|
this.memoryManager.dbOffset,
|
|
337
|
-
|
|
338
|
-
|
|
344
|
+
h,
|
|
345
|
+
o,
|
|
339
346
|
this.dimensions
|
|
340
347
|
);
|
|
341
348
|
else {
|
|
342
|
-
const
|
|
349
|
+
const A = new Float32Array(
|
|
343
350
|
this.memoryManager.memory.buffer,
|
|
344
351
|
this.memoryManager.queryOffset,
|
|
345
352
|
this.dimensions
|
|
346
|
-
),
|
|
353
|
+
), B = new Float32Array(
|
|
347
354
|
this.memoryManager.memory.buffer,
|
|
348
355
|
this.memoryManager.dbOffset,
|
|
349
|
-
|
|
350
|
-
),
|
|
351
|
-
|
|
356
|
+
o * this.dimensions
|
|
357
|
+
), p = new Float32Array(this.memoryManager.memory.buffer, h, o);
|
|
358
|
+
O(A, B, p, o, this.dimensions);
|
|
352
359
|
}
|
|
353
|
-
const
|
|
354
|
-
return
|
|
360
|
+
const f = new Float32Array(this.memoryManager.readScores()), l = this.slotToKey, y = (A) => l[A];
|
|
361
|
+
return n ? L(f, y, r, s) : D(f, y, r, s);
|
|
355
362
|
}
|
|
356
363
|
/**
|
|
357
364
|
* Persist the current in-memory state to storage.
|
|
358
365
|
*/
|
|
359
366
|
async flush() {
|
|
360
367
|
this.assertOpen();
|
|
361
|
-
const
|
|
362
|
-
if (
|
|
363
|
-
const
|
|
368
|
+
const e = this.memoryManager.vectorCount, t = new Uint8Array(e * this.dimensions * 4);
|
|
369
|
+
if (e > 0) {
|
|
370
|
+
const s = new Uint8Array(
|
|
364
371
|
this.memoryManager.memory.buffer,
|
|
365
372
|
this.memoryManager.dbOffset,
|
|
366
|
-
|
|
373
|
+
e * this.dimensions * 4
|
|
367
374
|
);
|
|
368
|
-
|
|
375
|
+
t.set(s);
|
|
369
376
|
}
|
|
370
|
-
const
|
|
371
|
-
await Promise.all([this.storage.write(
|
|
377
|
+
const r = d(this.slotToKey);
|
|
378
|
+
await Promise.all([this.storage.write(b, t), this.storage.write(E, r)]);
|
|
372
379
|
}
|
|
373
380
|
/**
|
|
374
381
|
* Flush data to storage and release the instance.
|
|
@@ -383,27 +390,142 @@ class w {
|
|
|
383
390
|
async clear() {
|
|
384
391
|
this.assertOpen(), this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), await this.storage.destroy();
|
|
385
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* Export the entire database as a ReadableStream of binary chunks.
|
|
395
|
+
*
|
|
396
|
+
* Format: [Header 24 bytes][Vector data][Keys data]
|
|
397
|
+
* Header: magic(4) + version(4) + dimensions(4) + vectorCount(4) + vectorDataLen(4) + keysDataLen(4)
|
|
398
|
+
*
|
|
399
|
+
* Vectors are streamed in 64KB chunks from WASM memory to avoid large
|
|
400
|
+
* heap allocations.
|
|
401
|
+
*/
|
|
402
|
+
async export() {
|
|
403
|
+
this.assertOpen();
|
|
404
|
+
const e = this.memoryManager.vectorCount, t = e * this.dimensions * 4, r = d(this.slotToKey), s = r.byteLength, n = new ArrayBuffer(I), i = new DataView(n);
|
|
405
|
+
i.setUint32(0, C, !0), i.setUint32(4, u, !0), i.setUint32(8, this.dimensions, !0), i.setUint32(12, e, !0), i.setUint32(16, t, !0), i.setUint32(20, s, !0);
|
|
406
|
+
const a = this.memoryManager;
|
|
407
|
+
let o = "header", h = 0;
|
|
408
|
+
return new ReadableStream({
|
|
409
|
+
pull(f) {
|
|
410
|
+
switch (o) {
|
|
411
|
+
case "header":
|
|
412
|
+
f.enqueue(new Uint8Array(n)), o = t > 0 ? "vectors" : s > 0 ? "keys" : "done", o === "done" && f.close();
|
|
413
|
+
break;
|
|
414
|
+
case "vectors": {
|
|
415
|
+
const l = t - h, y = Math.min(l, z), A = new Uint8Array(y);
|
|
416
|
+
A.set(new Uint8Array(a.memory.buffer, a.dbOffset + h, y)), f.enqueue(A), h += y, h >= t && (o = s > 0 ? "keys" : "done", o === "done" && f.close());
|
|
417
|
+
break;
|
|
418
|
+
}
|
|
419
|
+
case "keys":
|
|
420
|
+
f.enqueue(r), o = "done", f.close();
|
|
421
|
+
break;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Import data from a ReadableStream, replacing all existing data.
|
|
428
|
+
* Performs a dimension check against the configured dimensions.
|
|
429
|
+
*
|
|
430
|
+
* Vectors are streamed directly into WASM memory in chunks to avoid
|
|
431
|
+
* large heap allocations.
|
|
432
|
+
*/
|
|
433
|
+
async import(e) {
|
|
434
|
+
this.assertOpen();
|
|
435
|
+
const t = new q(e), r = await t.readExact(I), s = new DataView(r.buffer, r.byteOffset, r.byteLength);
|
|
436
|
+
if (s.getUint32(0, !0) !== C)
|
|
437
|
+
throw new Error("Invalid import data: unrecognized format");
|
|
438
|
+
const i = s.getUint32(4, !0);
|
|
439
|
+
if (i !== u)
|
|
440
|
+
throw new Error(`Unsupported import version: expected ${u}, got ${i}`);
|
|
441
|
+
const a = s.getUint32(8, !0);
|
|
442
|
+
if (a !== this.dimensions)
|
|
443
|
+
throw new Error(`Import dimension mismatch: expected ${this.dimensions}, got ${a}`);
|
|
444
|
+
const o = s.getUint32(12, !0), h = s.getUint32(16, !0), f = s.getUint32(20, !0);
|
|
445
|
+
if (this.keyToSlot.clear(), this.slotToKey.length = 0, this.memoryManager.reset(), o > 0) {
|
|
446
|
+
this.memoryManager.ensureCapacity(o);
|
|
447
|
+
let l = 0;
|
|
448
|
+
await t.readChunked(h, (y) => {
|
|
449
|
+
new Uint8Array(this.memoryManager.memory.buffer, this.memoryManager.dbOffset + l, y.byteLength).set(
|
|
450
|
+
y
|
|
451
|
+
), l += y.byteLength;
|
|
452
|
+
}), this.memoryManager.setVectorCount(o);
|
|
453
|
+
}
|
|
454
|
+
if (f > 0) {
|
|
455
|
+
const l = await t.readExact(f), y = g(l);
|
|
456
|
+
for (let A = 0; A < y.length; A++)
|
|
457
|
+
this.keyToSlot.set(y[A], A), this.slotToKey[A] = y[A];
|
|
458
|
+
}
|
|
459
|
+
t.release();
|
|
460
|
+
}
|
|
386
461
|
/**
|
|
387
462
|
* Normalize a vector using WASM (if available) or JS fallback.
|
|
388
463
|
*/
|
|
389
|
-
normalizeVector(
|
|
464
|
+
normalizeVector(e) {
|
|
390
465
|
if (this.wasmExports) {
|
|
391
|
-
const
|
|
392
|
-
new Float32Array(this.memoryManager.memory.buffer,
|
|
393
|
-
const
|
|
394
|
-
|
|
466
|
+
const t = this.memoryManager.queryOffset;
|
|
467
|
+
new Float32Array(this.memoryManager.memory.buffer, t, e.length).set(e), this.wasmExports.normalize(t, e.length);
|
|
468
|
+
const r = new Float32Array(this.memoryManager.memory.buffer, t, e.length);
|
|
469
|
+
e.set(r);
|
|
395
470
|
} else
|
|
396
|
-
|
|
471
|
+
S(e);
|
|
397
472
|
}
|
|
398
473
|
assertOpen() {
|
|
399
474
|
if (this.closed)
|
|
400
475
|
throw new Error("VectorDB instance has been closed");
|
|
401
476
|
}
|
|
402
477
|
}
|
|
478
|
+
class q {
|
|
479
|
+
reader;
|
|
480
|
+
buffer = new Uint8Array(0);
|
|
481
|
+
constructor(e) {
|
|
482
|
+
this.reader = e.getReader();
|
|
483
|
+
}
|
|
484
|
+
/** Read exactly `n` bytes, accumulating from stream chunks as needed. */
|
|
485
|
+
async readExact(e) {
|
|
486
|
+
if (e === 0) return new Uint8Array(0);
|
|
487
|
+
if (this.buffer.byteLength >= e) {
|
|
488
|
+
const i = this.buffer.subarray(0, e);
|
|
489
|
+
return this.buffer = this.buffer.subarray(e), i;
|
|
490
|
+
}
|
|
491
|
+
const t = [];
|
|
492
|
+
let r = this.buffer.byteLength;
|
|
493
|
+
for (r > 0 && (t.push(this.buffer), this.buffer = new Uint8Array(0)); r < e; ) {
|
|
494
|
+
const { done: i, value: a } = await this.reader.read();
|
|
495
|
+
if (i) throw new Error("Invalid import data: unexpected end of stream");
|
|
496
|
+
t.push(a), r += a.byteLength;
|
|
497
|
+
}
|
|
498
|
+
const s = new Uint8Array(r);
|
|
499
|
+
let n = 0;
|
|
500
|
+
for (const i of t)
|
|
501
|
+
s.set(i, n), n += i.byteLength;
|
|
502
|
+
return r > e ? (this.buffer = s.subarray(e), s.subarray(0, e)) : s;
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Read `totalBytes` from the stream in chunks, calling `onChunk` for each.
|
|
506
|
+
* Avoids allocating a single large buffer for the full data.
|
|
507
|
+
*/
|
|
508
|
+
async readChunked(e, t) {
|
|
509
|
+
let r = e;
|
|
510
|
+
if (this.buffer.byteLength > 0) {
|
|
511
|
+
const s = Math.min(this.buffer.byteLength, r);
|
|
512
|
+
t(this.buffer.subarray(0, s)), r -= s, this.buffer = this.buffer.subarray(s);
|
|
513
|
+
}
|
|
514
|
+
for (; r > 0; ) {
|
|
515
|
+
const { done: s, value: n } = await this.reader.read();
|
|
516
|
+
if (s) throw new Error("Invalid import data: unexpected end of stream");
|
|
517
|
+
n.byteLength <= r ? (t(n), r -= n.byteLength) : (t(n.subarray(0, r)), this.buffer = n.slice(r), r = 0);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
/** Release the stream reader lock. */
|
|
521
|
+
release() {
|
|
522
|
+
this.reader.releaseLock();
|
|
523
|
+
}
|
|
524
|
+
}
|
|
403
525
|
export {
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
526
|
+
M as DB,
|
|
527
|
+
v as InMemoryStorageProvider,
|
|
528
|
+
_ as OPFSStorageProvider,
|
|
529
|
+
k as VectorCapacityExceededError
|
|
408
530
|
};
|
|
409
531
|
//# sourceMappingURL=eigen-db.js.map
|