eigen-db 4.0.2 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +70 -20
- package/dist/eigen-db.js +192 -191
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/package.json +1 -1
- package/src/lib/__tests__/result-set.test.ts +53 -8
- package/src/lib/__tests__/vector-db.test.ts +64 -9
- package/src/lib/result-set.ts +28 -8
- package/src/lib/simd-binary.ts +1 -1
- package/src/lib/simd.wat +270 -128
- package/src/lib/types.ts +3 -1
- package/src/lib/vector-db.ts +8 -4
package/dist/eigen-db.js
CHANGED
|
@@ -1,113 +1,113 @@
|
|
|
1
|
-
class
|
|
2
|
-
constructor(
|
|
3
|
-
super(`Capacity exceeded. Max vectors for this dimension size is ~${
|
|
1
|
+
class Q extends Error {
|
|
2
|
+
constructor(A) {
|
|
3
|
+
super(`Capacity exceeded. Max vectors for this dimension size is ~${A}.`), this.name = "VectorCapacityExceededError";
|
|
4
4
|
}
|
|
5
5
|
}
|
|
6
|
-
class
|
|
6
|
+
class C {
|
|
7
7
|
dirHandle = null;
|
|
8
8
|
dirName;
|
|
9
|
-
constructor(
|
|
10
|
-
this.dirName =
|
|
9
|
+
constructor(A) {
|
|
10
|
+
this.dirName = A;
|
|
11
11
|
}
|
|
12
12
|
async getDir() {
|
|
13
13
|
if (!this.dirHandle) {
|
|
14
|
-
const
|
|
15
|
-
this.dirHandle = await
|
|
14
|
+
const A = await navigator.storage.getDirectory();
|
|
15
|
+
this.dirHandle = await A.getDirectoryHandle(this.dirName, { create: !0 });
|
|
16
16
|
}
|
|
17
17
|
return this.dirHandle;
|
|
18
18
|
}
|
|
19
|
-
async readAll(
|
|
19
|
+
async readAll(A) {
|
|
20
20
|
try {
|
|
21
|
-
const
|
|
22
|
-
return new Uint8Array(
|
|
21
|
+
const r = await (await (await (await this.getDir()).getFileHandle(A)).getFile()).arrayBuffer();
|
|
22
|
+
return new Uint8Array(r);
|
|
23
23
|
} catch {
|
|
24
24
|
return new Uint8Array(0);
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
|
-
async append(
|
|
28
|
-
const
|
|
29
|
-
await
|
|
27
|
+
async append(A, e) {
|
|
28
|
+
const t = await (await this.getDir()).getFileHandle(A, { create: !0 }), r = await t.createWritable({ keepExistingData: !0 }), o = await t.getFile();
|
|
29
|
+
await r.seek(o.size), await r.write(e), await r.close();
|
|
30
30
|
}
|
|
31
|
-
async write(
|
|
32
|
-
const
|
|
33
|
-
await
|
|
31
|
+
async write(A, e) {
|
|
32
|
+
const r = await (await (await this.getDir()).getFileHandle(A, { create: !0 })).createWritable({ keepExistingData: !1 });
|
|
33
|
+
await r.write(e), await r.close();
|
|
34
34
|
}
|
|
35
35
|
async destroy() {
|
|
36
36
|
await (await navigator.storage.getDirectory()).removeEntry(this.dirName, { recursive: !0 }), this.dirHandle = null;
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
class
|
|
39
|
+
class O {
|
|
40
40
|
files = /* @__PURE__ */ new Map();
|
|
41
|
-
async readAll(
|
|
42
|
-
const
|
|
43
|
-
if (!
|
|
44
|
-
const s =
|
|
45
|
-
let
|
|
46
|
-
for (const
|
|
47
|
-
|
|
48
|
-
return
|
|
41
|
+
async readAll(A) {
|
|
42
|
+
const e = this.files.get(A);
|
|
43
|
+
if (!e || e.length === 0) return new Uint8Array(0);
|
|
44
|
+
const s = e.reduce((o, a) => o + a.byteLength, 0), t = new Uint8Array(s);
|
|
45
|
+
let r = 0;
|
|
46
|
+
for (const o of e)
|
|
47
|
+
t.set(o, r), r += o.byteLength;
|
|
48
|
+
return t;
|
|
49
49
|
}
|
|
50
|
-
async append(
|
|
51
|
-
this.files.has(
|
|
50
|
+
async append(A, e) {
|
|
51
|
+
this.files.has(A) || this.files.set(A, []), this.files.get(A).push(new Uint8Array(e));
|
|
52
52
|
}
|
|
53
|
-
async write(
|
|
54
|
-
this.files.set(
|
|
53
|
+
async write(A, e) {
|
|
54
|
+
this.files.set(A, [new Uint8Array(e)]);
|
|
55
55
|
}
|
|
56
56
|
async destroy() {
|
|
57
57
|
this.files.clear();
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function
|
|
61
|
-
let
|
|
62
|
-
for (let
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
if (
|
|
66
|
-
const s = 1 /
|
|
67
|
-
for (let
|
|
68
|
-
|
|
60
|
+
function E(n) {
|
|
61
|
+
let A = 0;
|
|
62
|
+
for (let t = 0; t < n.length; t++)
|
|
63
|
+
A += n[t] * n[t];
|
|
64
|
+
const e = Math.sqrt(A);
|
|
65
|
+
if (e === 0) return;
|
|
66
|
+
const s = 1 / e;
|
|
67
|
+
for (let t = 0; t < n.length; t++)
|
|
68
|
+
n[t] *= s;
|
|
69
69
|
}
|
|
70
|
-
function
|
|
71
|
-
for (let
|
|
72
|
-
let
|
|
73
|
-
const
|
|
74
|
-
for (let
|
|
75
|
-
|
|
76
|
-
|
|
70
|
+
function b(n, A, e, s, t) {
|
|
71
|
+
for (let r = 0; r < s; r++) {
|
|
72
|
+
let o = 0;
|
|
73
|
+
const a = r * t;
|
|
74
|
+
for (let i = 0; i < t; i++)
|
|
75
|
+
o += n[i] * A[a + i];
|
|
76
|
+
e[r] = o;
|
|
77
77
|
}
|
|
78
78
|
}
|
|
79
|
-
const
|
|
80
|
-
function S(
|
|
81
|
-
const
|
|
82
|
-
let
|
|
83
|
-
for (const
|
|
84
|
-
|
|
85
|
-
return
|
|
79
|
+
const M = new TextEncoder(), D = new TextDecoder();
|
|
80
|
+
function S(n) {
|
|
81
|
+
const A = n.map((a) => M.encode(a)), e = A.reduce((a, i) => a + 4 + i.byteLength, 0), s = new ArrayBuffer(e), t = new DataView(s), r = new Uint8Array(s);
|
|
82
|
+
let o = 0;
|
|
83
|
+
for (const a of A)
|
|
84
|
+
t.setUint32(o, a.byteLength, !0), o += 4, r.set(a, o), o += a.byteLength;
|
|
85
|
+
return r;
|
|
86
86
|
}
|
|
87
|
-
function
|
|
88
|
-
const
|
|
87
|
+
function k(n) {
|
|
88
|
+
const A = [], e = new DataView(n.buffer, n.byteOffset, n.byteLength);
|
|
89
89
|
let s = 0;
|
|
90
|
-
for (; s <
|
|
91
|
-
const
|
|
90
|
+
for (; s < n.byteLength; ) {
|
|
91
|
+
const t = e.getUint32(s, !0);
|
|
92
92
|
s += 4;
|
|
93
|
-
const
|
|
94
|
-
|
|
93
|
+
const r = D.decode(n.subarray(s, s + t));
|
|
94
|
+
A.push(r), s += t;
|
|
95
95
|
}
|
|
96
|
-
return
|
|
96
|
+
return A;
|
|
97
97
|
}
|
|
98
|
-
const
|
|
99
|
-
class
|
|
98
|
+
const f = 65536, d = 65536;
|
|
99
|
+
class v {
|
|
100
100
|
memory;
|
|
101
101
|
dimensions;
|
|
102
102
|
queryOffset;
|
|
103
103
|
dbOffset;
|
|
104
104
|
_vectorCount;
|
|
105
|
-
constructor(
|
|
106
|
-
this.dimensions =
|
|
107
|
-
const s =
|
|
108
|
-
this.dbOffset = Math.ceil(s /
|
|
109
|
-
const
|
|
110
|
-
this.memory = new WebAssembly.Memory({ initial:
|
|
105
|
+
constructor(A, e = 0) {
|
|
106
|
+
this.dimensions = A, this.queryOffset = 0;
|
|
107
|
+
const s = A * 4;
|
|
108
|
+
this.dbOffset = Math.ceil(s / f) * f;
|
|
109
|
+
const t = e * A * 4, r = this.dbOffset + t, o = Math.max(1, Math.ceil(r / f));
|
|
110
|
+
this.memory = new WebAssembly.Memory({ initial: o }), this._vectorCount = e;
|
|
111
111
|
}
|
|
112
112
|
/** Current number of vectors stored */
|
|
113
113
|
get vectorCount() {
|
|
@@ -126,45 +126,45 @@ class p {
|
|
|
126
126
|
* Accounts for query buffer, DB space, and scores buffer.
|
|
127
127
|
*/
|
|
128
128
|
get maxVectors() {
|
|
129
|
-
const
|
|
130
|
-
return Math.floor(
|
|
129
|
+
const A = d * f - this.dbOffset, e = this.dimensions * 4 + 4;
|
|
130
|
+
return Math.floor(A / e);
|
|
131
131
|
}
|
|
132
132
|
/**
|
|
133
133
|
* Ensures memory is large enough for the current DB + scores buffer.
|
|
134
134
|
* Calls memory.grow() if needed.
|
|
135
135
|
*/
|
|
136
|
-
ensureCapacity(
|
|
137
|
-
const
|
|
138
|
-
if (s >
|
|
139
|
-
const
|
|
140
|
-
if (
|
|
136
|
+
ensureCapacity(A) {
|
|
137
|
+
const e = this._vectorCount + A, s = this.dbOffset + e * this.dimensions * 4 + e * 4, t = this.memory.buffer.byteLength;
|
|
138
|
+
if (s > t) {
|
|
139
|
+
const r = Math.ceil((s - t) / f);
|
|
140
|
+
if (t / f + r > d)
|
|
141
141
|
throw new Error("WASM memory limit exceeded");
|
|
142
|
-
this.memory.grow(
|
|
142
|
+
this.memory.grow(r);
|
|
143
143
|
}
|
|
144
144
|
}
|
|
145
145
|
/**
|
|
146
146
|
* Write a query vector into the query buffer region.
|
|
147
147
|
*/
|
|
148
|
-
writeQuery(
|
|
149
|
-
new Float32Array(this.memory.buffer, this.queryOffset, this.dimensions).set(
|
|
148
|
+
writeQuery(A) {
|
|
149
|
+
new Float32Array(this.memory.buffer, this.queryOffset, this.dimensions).set(A);
|
|
150
150
|
}
|
|
151
151
|
/**
|
|
152
152
|
* Append vectors to the database region.
|
|
153
153
|
* Returns the byte offset where the new vectors were written.
|
|
154
154
|
*/
|
|
155
|
-
appendVectors(
|
|
156
|
-
const
|
|
157
|
-
let s =
|
|
158
|
-
for (const
|
|
159
|
-
new Float32Array(this.memory.buffer, s, this.dimensions).set(
|
|
160
|
-
return this._vectorCount +=
|
|
155
|
+
appendVectors(A) {
|
|
156
|
+
const e = this.dbOffset + this._vectorCount * this.dimensions * 4;
|
|
157
|
+
let s = e;
|
|
158
|
+
for (const t of A)
|
|
159
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(t), s += this.dimensions * 4;
|
|
160
|
+
return this._vectorCount += A.length, e;
|
|
161
161
|
}
|
|
162
162
|
/**
|
|
163
163
|
* Load raw vector bytes directly into the database region.
|
|
164
164
|
* Used for bulk loading from OPFS.
|
|
165
165
|
*/
|
|
166
|
-
loadVectorBytes(
|
|
167
|
-
new Uint8Array(this.memory.buffer, this.dbOffset,
|
|
166
|
+
loadVectorBytes(A, e) {
|
|
167
|
+
new Uint8Array(this.memory.buffer, this.dbOffset, A.byteLength).set(A), this._vectorCount = e;
|
|
168
168
|
}
|
|
169
169
|
/**
|
|
170
170
|
* Read the scores buffer as a Float32Array view.
|
|
@@ -175,16 +175,16 @@ class p {
|
|
|
175
175
|
/**
|
|
176
176
|
* Read the DB region for a specific vector index.
|
|
177
177
|
*/
|
|
178
|
-
readVector(
|
|
179
|
-
const
|
|
180
|
-
return new Float32Array(this.memory.buffer,
|
|
178
|
+
readVector(A) {
|
|
179
|
+
const e = this.dbOffset + A * this.dimensions * 4;
|
|
180
|
+
return new Float32Array(this.memory.buffer, e, this.dimensions);
|
|
181
181
|
}
|
|
182
182
|
/**
|
|
183
183
|
* Write a vector to a specific slot in the database region.
|
|
184
184
|
*/
|
|
185
|
-
writeVector(
|
|
186
|
-
const s = this.dbOffset +
|
|
187
|
-
new Float32Array(this.memory.buffer, s, this.dimensions).set(
|
|
185
|
+
writeVector(A, e) {
|
|
186
|
+
const s = this.dbOffset + A * this.dimensions * 4;
|
|
187
|
+
new Float32Array(this.memory.buffer, s, this.dimensions).set(e);
|
|
188
188
|
}
|
|
189
189
|
/**
|
|
190
190
|
* Reset the vector count to zero, logically clearing the database.
|
|
@@ -194,55 +194,56 @@ class p {
|
|
|
194
194
|
this._vectorCount = 0;
|
|
195
195
|
}
|
|
196
196
|
}
|
|
197
|
-
function
|
|
198
|
-
const
|
|
199
|
-
if (
|
|
200
|
-
const r = new Uint32Array(
|
|
201
|
-
for (let
|
|
202
|
-
r.sort((
|
|
203
|
-
const
|
|
204
|
-
for (let
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
197
|
+
function G(n, A, e, s) {
|
|
198
|
+
const t = n.length;
|
|
199
|
+
if (t === 0) return [];
|
|
200
|
+
const r = new Uint32Array(t);
|
|
201
|
+
for (let i = 0; i < t; i++) r[i] = i;
|
|
202
|
+
r.sort((i, c) => n[c] - n[i]);
|
|
203
|
+
const o = Math.min(e, t), a = [];
|
|
204
|
+
for (let i = 0; i < o; i++) {
|
|
205
|
+
const c = r[i], h = 1 - n[c];
|
|
206
|
+
if (s !== void 0 && h > s) break;
|
|
207
|
+
a.push({ key: A(c), distance: h });
|
|
208
|
+
}
|
|
209
|
+
return a;
|
|
209
210
|
}
|
|
210
|
-
function
|
|
211
|
-
const
|
|
212
|
-
if (
|
|
213
|
-
const r = new Uint32Array(
|
|
214
|
-
for (let
|
|
215
|
-
r.sort((
|
|
216
|
-
const
|
|
211
|
+
function x(n, A, e, s) {
|
|
212
|
+
const t = n.length;
|
|
213
|
+
if (t === 0) return [];
|
|
214
|
+
const r = new Uint32Array(t);
|
|
215
|
+
for (let a = 0; a < t; a++) r[a] = a;
|
|
216
|
+
r.sort((a, i) => n[i] - n[a]);
|
|
217
|
+
const o = Math.min(e, t);
|
|
217
218
|
return {
|
|
218
219
|
[Symbol.iterator]() {
|
|
219
|
-
let
|
|
220
|
+
let a = 0;
|
|
220
221
|
return {
|
|
221
222
|
next() {
|
|
222
|
-
if (
|
|
223
|
-
const
|
|
224
|
-
return {
|
|
223
|
+
if (a >= o) return { done: !0, value: void 0 };
|
|
224
|
+
const i = r[a++], c = 1 - n[i];
|
|
225
|
+
return s !== void 0 && c > s ? { done: !0, value: void 0 } : {
|
|
225
226
|
done: !1,
|
|
226
|
-
value: { key:
|
|
227
|
+
value: { key: A(i), distance: c }
|
|
227
228
|
};
|
|
228
229
|
}
|
|
229
230
|
};
|
|
230
231
|
}
|
|
231
232
|
};
|
|
232
233
|
}
|
|
233
|
-
const
|
|
234
|
-
function
|
|
235
|
-
const
|
|
236
|
-
for (let
|
|
237
|
-
e
|
|
238
|
-
return
|
|
234
|
+
const H = "AGFzbQEAAAABDgJgAn9/AGAFf39/f38AAg8BA2VudgZtZW1vcnkCAAEDAwIAAQcaAglub3JtYWxpemUAAApzZWFyY2hfYWxsAAEKgQ4C3wQFAX8EewN9AXsDf/0MAAAAAAAAAAAAAAAAAAAAACED/QwAAAAAAAAAAAAAAAAAAAAAIQT9DAAAAAAAAAAAAAAAAAAAAAAhBf0MAAAAAAAAAAAAAAAAAAAAACEGIAFBcHEhCyABQXxxIQxBACECAkADQCACIAtPDQEgACACQQJ0aiENIAMgDf0ABAAgDf0ABAD95gH95AEhAyAEIA39AAQQIA39AAQQ/eYB/eQBIQQgBSAN/QAEICAN/QAEIP3mAf3kASEFIAYgDf0ABDAgDf0ABDD95gH95AEhBiACQRBqIQIMAAsLIAMgBP3kASAFIAb95AH95AEhAwJAA0AgAiAMTw0BIAAgAkECdGohDSADIA39AAQAIA39AAQA/eYB/eQBIQMgAkEEaiECDAALCyAD/R8AIAP9HwGSIAP9HwIgA/0fA5KSIQcCQANAIAIgAU8NASAAIAJBAnRqIQ0gByANKgIAIA0qAgCUkiEHIAJBAWohAgwACwsgB5EhCCAIQwAAAABbBEAPC0MAAIA/IAiVIQkgCf0TIQpBACECAkADQCACIAtPDQEgACACQQJ0aiENIA0gDf0ABAAgCv3mAf0LBAAgDSAN/QAEECAK/eYB/QsEECANIA39AAQgIAr95gH9CwQgIA0gDf0ABDAgCv3mAf0LBDAgAkEQaiECDAALCwJAA0AgAiAMTw0BIAAgAkECdGohDSANIA39AAQAIAr95gH9CwQAIAJBBGohAgwACwsCQANAIAIgAU8NASAAIAJBAnRqIQ0gDSANKgIAIAmUOAIAIAJBAWohAgwACwsLnQkEAn8MewJ9CX8gBEFwcSEXIARBfHEhGCAEQQJ0IRwgA0F+cSEdQQAhBQJAA0AgBSAdTw0BIAEgBSAcbGohFSAVIBxqIRb9DAAAAAAAAAAAAAAAAAAAAAAhB/0MAAAAAAAAAAAAAAAAAAAAACEI/QwAAAAAAAAAAAAAAAAAAAAAIQn9DAAAAAAAAAAAAAAAAAAAAAAhCv0MAAAAAAAAAAAAAAAAAAAAACEL/QwAAAAAAAAAAAAAAAAAAAAAIQz9DAAAAAAAAAAAAAAAAAAAAAAhDf0MAAAAAAAAAAAAAAAAAAAAACEOQQAhBgJAA0AgBiAXTw0BIAAgBkECdGohGSAVIAZBAnRqIRogFiAGQQJ0aiEbIBn9AAQAIQ8gGf0ABBAhECAZ/QAEICERIBn9AAQwIRIgByAPIBr9AAQA/eYB/eQBIQcgCCAQIBr9AAQQ/eYB/eQBIQggCSARIBr9AAQg/eYB/eQBIQkgCiASIBr9AAQw/eYB/eQBIQogCyAPIBv9AAQA/eYB/eQBIQsgDCAQIBv9AAQQ/eYB/eQBIQwgDSARIBv9AAQg/eYB/eQBIQ0gDiASIBv9AAQw/eYB/eQBIQ4gBkEQaiEGDAALCyAHIAj95AEgCSAK/eQB/eQBIQcgCyAM/eQBIA0gDv3kAf3kASELAkADQCAGIBhPDQEgACAGQQJ0aiEZIBUgBkECdGohGiAWIAZBAnRqIRsgGf0ABAAhDyAHIA8gGv0ABAD95gH95AEhByALIA8gG/0ABAD95gH95AEhCyAGQQRqIQYMAAsLIAf9HwAgB/0fAZIgB/0fAiAH/R8DkpIhEyAL/R8AIAv9HwGSIAv9HwIgC/0fA5KSIRQCQANAIAYgBE8NASAAIAZBAnRqIRkgFSAGQQJ0aiEaIBYgBkECdGohGyATIBkqAgAgGioCAJSSIRMgFCAZKgIAIBsqAgCUkiEUIAZBAWohBgwACwsgAiAFQQJ0aiATOAIAIAIgBUEBakECdGogFDgCACAFQQJqIQUMAAsLIAUgA0kEQCABIAUgHGxqIRX9DAAAAAAAAAAAAAAAAAAAAAAhB/0MAAAAAAAAAAAAAAAAAAAAACEI/QwAAAAAAAAAAAAAAAAAAAAAIQn9DAAAAAAAAAAAAAAAAAAAAAAhCkEAIQYCQANAIAYgF08NASAAIAZBAnRqIRkgFSAGQQJ0aiEaIAcgGf0ABAAgGv0ABAD95gH95AEhByAIIBn9AAQQIBr9AAQQ/eYB/eQBIQggCSAZ/QAEICAa/QAEIP3mAf3kASEJIAogGf0ABDAgGv0ABDD95gH95AEhCiAGQRBqIQYMAAsLIAcgCP3kASAJIAr95AH95AEhBwJAA0AgBiAYTw0BIAAgBkECdGohGSAVIAZBAnRqIRogByAZ/QAEACAa/QAEAP3mAf3kASEHIAZBBGohBgwACwsgB/0fACAH/R8BkiAH/R8CIAf9HwOSkiETAkADQCAGIARPDQEgACAGQQJ0aiEZIBUgBkECdGohGiATIBkqAgAgGioCAJSSIRMgBkEBaiEGDAALCyACIAVBAnRqIBM4AgALCw==";
|
|
235
|
+
function p() {
|
|
236
|
+
const n = atob(H), A = new Uint8Array(n.length);
|
|
237
|
+
for (let e = 0; e < n.length; e++)
|
|
238
|
+
A[e] = n.charCodeAt(e);
|
|
239
|
+
return A;
|
|
239
240
|
}
|
|
240
|
-
async function
|
|
241
|
-
const
|
|
242
|
-
return (await WebAssembly.instantiate(
|
|
241
|
+
async function F(n, A) {
|
|
242
|
+
const e = { env: { memory: A } };
|
|
243
|
+
return (await WebAssembly.instantiate(n, e)).instance.exports;
|
|
243
244
|
}
|
|
244
|
-
const
|
|
245
|
-
class
|
|
245
|
+
const B = "vectors.bin", I = "keys.bin";
|
|
246
|
+
class w {
|
|
246
247
|
memoryManager;
|
|
247
248
|
storage;
|
|
248
249
|
dimensions;
|
|
@@ -254,23 +255,23 @@ class C {
|
|
|
254
255
|
slotToKey;
|
|
255
256
|
/** Whether this instance has been closed */
|
|
256
257
|
closed = !1;
|
|
257
|
-
constructor(
|
|
258
|
-
this.memoryManager =
|
|
259
|
-
}
|
|
260
|
-
static async open(
|
|
261
|
-
const
|
|
262
|
-
for (let
|
|
263
|
-
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
let
|
|
267
|
-
const
|
|
268
|
-
if (
|
|
258
|
+
constructor(A, e, s, t, r, o, a) {
|
|
259
|
+
this.memoryManager = A, this.storage = e, this.dimensions = s, this.shouldNormalize = t, this.wasmExports = r, this.keyToSlot = o, this.slotToKey = a;
|
|
260
|
+
}
|
|
261
|
+
static async open(A) {
|
|
262
|
+
const e = A.name ?? "default", s = A.storage ?? new C(e), t = A.normalize !== !1, [r, o] = await Promise.all([s.readAll(B), s.readAll(I)]), a = o.byteLength > 0 ? k(o) : [], i = r.byteLength / (A.dimensions * 4), c = /* @__PURE__ */ new Map(), h = [];
|
|
263
|
+
for (let g = 0; g < a.length; g++)
|
|
264
|
+
c.set(a[g], g), h[g] = a[g];
|
|
265
|
+
const y = new v(A.dimensions, i);
|
|
266
|
+
r.byteLength > 0 && y.loadVectorBytes(r, i);
|
|
267
|
+
let m = null;
|
|
268
|
+
const l = A.wasmBinary !== void 0 ? A.wasmBinary : p();
|
|
269
|
+
if (l !== null)
|
|
269
270
|
try {
|
|
270
|
-
|
|
271
|
+
m = await F(l, y.memory);
|
|
271
272
|
} catch {
|
|
272
273
|
}
|
|
273
|
-
return new
|
|
274
|
+
return new w(y, s, A.dimensions, t, m, c, h);
|
|
274
275
|
}
|
|
275
276
|
/** Total number of key-value pairs in the database */
|
|
276
277
|
get size() {
|
|
@@ -280,61 +281,61 @@ class C {
|
|
|
280
281
|
* Set a key-value pair. If the key already exists, its vector is overwritten (last-write-wins).
|
|
281
282
|
* The value is a number[] or Float32Array of length equal to the configured dimensions.
|
|
282
283
|
*/
|
|
283
|
-
set(
|
|
284
|
-
if (this.assertOpen(),
|
|
285
|
-
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${
|
|
286
|
-
const
|
|
287
|
-
(s?.normalize ?? this.shouldNormalize) && this.normalizeVector(
|
|
288
|
-
const
|
|
289
|
-
if (
|
|
290
|
-
this.memoryManager.writeVector(
|
|
284
|
+
set(A, e, s) {
|
|
285
|
+
if (this.assertOpen(), e.length !== this.dimensions)
|
|
286
|
+
throw new Error(`Vector dimension mismatch: expected ${this.dimensions}, got ${e.length}`);
|
|
287
|
+
const t = new Float32Array(e);
|
|
288
|
+
(s?.normalize ?? this.shouldNormalize) && this.normalizeVector(t);
|
|
289
|
+
const o = this.keyToSlot.get(A);
|
|
290
|
+
if (o !== void 0)
|
|
291
|
+
this.memoryManager.writeVector(o, t);
|
|
291
292
|
else {
|
|
292
293
|
if (this.memoryManager.vectorCount + 1 > this.memoryManager.maxVectors)
|
|
293
|
-
throw new
|
|
294
|
+
throw new Q(this.memoryManager.maxVectors);
|
|
294
295
|
this.memoryManager.ensureCapacity(1);
|
|
295
|
-
const
|
|
296
|
-
this.memoryManager.appendVectors([
|
|
296
|
+
const i = this.memoryManager.vectorCount;
|
|
297
|
+
this.memoryManager.appendVectors([t]), this.keyToSlot.set(A, i), this.slotToKey[i] = A;
|
|
297
298
|
}
|
|
298
299
|
}
|
|
299
300
|
/**
|
|
300
301
|
* Get the stored vector for a key. Returns undefined if the key does not exist.
|
|
301
302
|
* Returns a copy of the stored vector as a plain number array.
|
|
302
303
|
*/
|
|
303
|
-
get(
|
|
304
|
+
get(A) {
|
|
304
305
|
this.assertOpen();
|
|
305
|
-
const
|
|
306
|
-
if (
|
|
307
|
-
return Array.from(this.memoryManager.readVector(
|
|
306
|
+
const e = this.keyToSlot.get(A);
|
|
307
|
+
if (e !== void 0)
|
|
308
|
+
return Array.from(this.memoryManager.readVector(e));
|
|
308
309
|
}
|
|
309
310
|
/**
|
|
310
311
|
* Set multiple key-value pairs at once. Last-write-wins applies within the batch.
|
|
311
312
|
*/
|
|
312
|
-
setMany(
|
|
313
|
-
for (const [
|
|
314
|
-
this.set(
|
|
313
|
+
setMany(A) {
|
|
314
|
+
for (const [e, s] of A)
|
|
315
|
+
this.set(e, s);
|
|
315
316
|
}
|
|
316
317
|
/**
|
|
317
318
|
* Get vectors for multiple keys. Returns undefined for keys that don't exist.
|
|
318
319
|
*/
|
|
319
|
-
getMany(
|
|
320
|
-
return
|
|
320
|
+
getMany(A) {
|
|
321
|
+
return A.map((e) => this.get(e));
|
|
321
322
|
}
|
|
322
|
-
query(
|
|
323
|
+
query(A, e) {
|
|
323
324
|
this.assertOpen();
|
|
324
|
-
const s =
|
|
325
|
+
const s = e?.topK ?? 1 / 0, t = e?.maxDistance, r = e && "iterable" in e && e.iterable;
|
|
325
326
|
if (this.size === 0)
|
|
326
327
|
return [];
|
|
327
|
-
if (
|
|
328
|
-
throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${
|
|
329
|
-
const
|
|
330
|
-
(
|
|
331
|
-
const
|
|
328
|
+
if (A.length !== this.dimensions)
|
|
329
|
+
throw new Error(`Query vector dimension mismatch: expected ${this.dimensions}, got ${A.length}`);
|
|
330
|
+
const o = new Float32Array(A);
|
|
331
|
+
(e?.normalize ?? this.shouldNormalize) && this.normalizeVector(o), this.memoryManager.writeQuery(o), this.memoryManager.ensureCapacity(0);
|
|
332
|
+
const i = this.memoryManager.vectorCount, c = this.memoryManager.scoresOffset;
|
|
332
333
|
if (this.wasmExports)
|
|
333
334
|
this.wasmExports.search_all(
|
|
334
335
|
this.memoryManager.queryOffset,
|
|
335
336
|
this.memoryManager.dbOffset,
|
|
336
|
-
|
|
337
|
-
|
|
337
|
+
c,
|
|
338
|
+
i,
|
|
338
339
|
this.dimensions
|
|
339
340
|
);
|
|
340
341
|
else {
|
|
@@ -345,29 +346,29 @@ class C {
|
|
|
345
346
|
), g = new Float32Array(
|
|
346
347
|
this.memoryManager.memory.buffer,
|
|
347
348
|
this.memoryManager.dbOffset,
|
|
348
|
-
|
|
349
|
-
),
|
|
350
|
-
|
|
349
|
+
i * this.dimensions
|
|
350
|
+
), u = new Float32Array(this.memoryManager.memory.buffer, c, i);
|
|
351
|
+
b(l, g, u, i, this.dimensions);
|
|
351
352
|
}
|
|
352
|
-
const
|
|
353
|
-
return r ?
|
|
353
|
+
const h = new Float32Array(this.memoryManager.readScores()), y = this.slotToKey, m = (l) => y[l];
|
|
354
|
+
return r ? x(h, m, s, t) : G(h, m, s, t);
|
|
354
355
|
}
|
|
355
356
|
/**
|
|
356
357
|
* Persist the current in-memory state to storage.
|
|
357
358
|
*/
|
|
358
359
|
async flush() {
|
|
359
360
|
this.assertOpen();
|
|
360
|
-
const
|
|
361
|
-
if (
|
|
362
|
-
const
|
|
361
|
+
const A = this.memoryManager.vectorCount, e = new Uint8Array(A * this.dimensions * 4);
|
|
362
|
+
if (A > 0) {
|
|
363
|
+
const t = new Uint8Array(
|
|
363
364
|
this.memoryManager.memory.buffer,
|
|
364
365
|
this.memoryManager.dbOffset,
|
|
365
|
-
|
|
366
|
+
A * this.dimensions * 4
|
|
366
367
|
);
|
|
367
|
-
|
|
368
|
+
e.set(t);
|
|
368
369
|
}
|
|
369
370
|
const s = S(this.slotToKey);
|
|
370
|
-
await Promise.all([this.storage.write(
|
|
371
|
+
await Promise.all([this.storage.write(B, e), this.storage.write(I, s)]);
|
|
371
372
|
}
|
|
372
373
|
/**
|
|
373
374
|
* Flush data to storage and release the instance.
|
|
@@ -385,14 +386,14 @@ class C {
|
|
|
385
386
|
/**
|
|
386
387
|
* Normalize a vector using WASM (if available) or JS fallback.
|
|
387
388
|
*/
|
|
388
|
-
normalizeVector(
|
|
389
|
+
normalizeVector(A) {
|
|
389
390
|
if (this.wasmExports) {
|
|
390
|
-
const
|
|
391
|
-
new Float32Array(this.memoryManager.memory.buffer,
|
|
392
|
-
const s = new Float32Array(this.memoryManager.memory.buffer,
|
|
393
|
-
|
|
391
|
+
const e = this.memoryManager.queryOffset;
|
|
392
|
+
new Float32Array(this.memoryManager.memory.buffer, e, A.length).set(A), this.wasmExports.normalize(e, A.length);
|
|
393
|
+
const s = new Float32Array(this.memoryManager.memory.buffer, e, A.length);
|
|
394
|
+
A.set(s);
|
|
394
395
|
} else
|
|
395
|
-
|
|
396
|
+
E(A);
|
|
396
397
|
}
|
|
397
398
|
assertOpen() {
|
|
398
399
|
if (this.closed)
|
|
@@ -400,9 +401,9 @@ class C {
|
|
|
400
401
|
}
|
|
401
402
|
}
|
|
402
403
|
export {
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
404
|
+
w as DB,
|
|
405
|
+
O as InMemoryStorageProvider,
|
|
406
|
+
C as OPFSStorageProvider,
|
|
407
|
+
Q as VectorCapacityExceededError
|
|
407
408
|
};
|
|
408
409
|
//# sourceMappingURL=eigen-db.js.map
|