eigen-db 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +55 -8
- package/dist/compute.d.ts +20 -0
- package/dist/eigen-db.js +242 -190
- package/dist/eigen-db.js.map +1 -1
- package/dist/eigen-db.umd.cjs +1 -1
- package/dist/eigen-db.umd.cjs.map +1 -1
- package/dist/errors.d.ts +7 -0
- package/dist/index.d.ts +12 -0
- package/dist/lexicon.d.ts +28 -0
- package/dist/memory-manager.d.ts +68 -0
- package/dist/result-set.d.ts +35 -0
- package/dist/simd-binary.d.ts +1 -0
- package/dist/storage.d.ts +38 -0
- package/dist/types.d.ts +44 -0
- package/dist/vector-db.d.ts +131 -0
- package/dist/wasm-compute.d.ts +13 -0
- package/package.json +4 -4
- package/src/lib/__tests__/vector-db.test.ts +328 -0
- package/src/lib/simd-binary.ts +1 -1
- package/src/lib/simd-optimized.wat +362 -0
- package/src/lib/simd.wat +42 -248
- package/src/lib/types.ts +2 -4
- package/src/lib/vector-db.ts +93 -19
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ High-performance vector database for the web.
|
|
|
4
4
|
|
|
5
5
|
`eigen-db` stores and queries embedding vectors in-browser, using:
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- Pluggable storage backends (in-memory by default, OPFS for browser persistence)
|
|
8
8
|
- WASM SIMD for fast compute when available
|
|
9
9
|
- JavaScript fallback when WASM SIMD is unavailable
|
|
10
10
|
|
|
@@ -21,13 +21,24 @@ npm install eigen-db
|
|
|
21
21
|
```ts
|
|
22
22
|
import { DB } from "eigen-db";
|
|
23
23
|
|
|
24
|
+
// In-memory (default) — no persistence, great for ephemeral sessions
|
|
24
25
|
const db = await DB.open({
|
|
25
|
-
name: "my-index", // optional, defaults to "default"
|
|
26
26
|
dimensions: 1536, // required
|
|
27
27
|
normalize: true, // optional, defaults to true
|
|
28
28
|
});
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
For browser persistence, mount an OPFS storage backend:
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
import { DB, OPFSStorageProvider } from "eigen-db";
|
|
35
|
+
|
|
36
|
+
const db = await DB.open({
|
|
37
|
+
dimensions: 1536,
|
|
38
|
+
storage: new OPFSStorageProvider("my-index"), // persistent OPFS directory
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
31
42
|
### 2) Insert vectors
|
|
32
43
|
|
|
33
44
|
```ts
|
|
@@ -45,7 +56,34 @@ Notes:
|
|
|
45
56
|
- Each vector must be a `number[]` (or `Float32Array`) with exactly `dimensions` elements.
|
|
46
57
|
- Duplicate keys use last-write-wins semantics.
|
|
47
58
|
|
|
48
|
-
### 3)
|
|
59
|
+
### 3) Look up, check, and remove vectors
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
db.get("doc:1"); // number[] | undefined
|
|
63
|
+
db.has("doc:1"); // true
|
|
64
|
+
db.delete("doc:1"); // true (removed), false (not found)
|
|
65
|
+
db.dimensions; // configured vector dimensions
|
|
66
|
+
db.size; // number of entries
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### 4) Iterate over the database
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
// Iterate over all keys
|
|
73
|
+
for (const key of db.keys()) {
|
|
74
|
+
console.log(key);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Iterate over all [key, value] pairs
|
|
78
|
+
for (const [key, vector] of db.entries()) {
|
|
79
|
+
console.log(key, vector);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Spread into an array (uses Symbol.iterator, same as entries())
|
|
83
|
+
const all = [...db];
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### 5) Query nearest vectors
|
|
49
87
|
|
|
50
88
|
```ts
|
|
51
89
|
const queryVector = embeddingQuery;
|
|
@@ -83,7 +121,7 @@ const results = db.query(queryVector, { minSimilarity: 0.7 });
|
|
|
83
121
|
const results = db.query(queryVector, { minSimilarity: 0.7, iterable: true });
|
|
84
122
|
```
|
|
85
123
|
|
|
86
|
-
###
|
|
124
|
+
### 6) Persist and lifecycle
|
|
87
125
|
|
|
88
126
|
```ts
|
|
89
127
|
await db.flush(); // persist current state
|
|
@@ -96,7 +134,7 @@ To delete all vectors and storage:
|
|
|
96
134
|
await db.clear();
|
|
97
135
|
```
|
|
98
136
|
|
|
99
|
-
###
|
|
137
|
+
### 7) Export and import
|
|
100
138
|
|
|
101
139
|
Export the entire database as a streaming binary file:
|
|
102
140
|
|
|
@@ -172,6 +210,7 @@ Opens (or creates) a database instance and loads persisted data.
|
|
|
172
210
|
#### Properties
|
|
173
211
|
|
|
174
212
|
- `size: number` — current number of key-vector pairs
|
|
213
|
+
- `dimensions: number` — number of dimensions per vector
|
|
175
214
|
|
|
176
215
|
#### Methods
|
|
177
216
|
|
|
@@ -180,10 +219,20 @@ Opens (or creates) a database instance and loads persisted data.
|
|
|
180
219
|
- Throws on dimension mismatch.
|
|
181
220
|
- `get(key: string): number[] | undefined`
|
|
182
221
|
- Returns a copy of the stored vector.
|
|
222
|
+
- `has(key: string): boolean`
|
|
223
|
+
- Returns `true` if the key exists, `false` otherwise. O(1) lookup.
|
|
224
|
+
- `delete(key: string): boolean`
|
|
225
|
+
- Removes the entry for the given key. Returns `true` if the key existed, `false` otherwise.
|
|
183
226
|
- `setMany(entries: [string, VectorInput][]): void`
|
|
184
227
|
- Batch insert/update.
|
|
185
228
|
- `getMany(keys: string[]): (number[] | undefined)[]`
|
|
186
229
|
- Batch lookup.
|
|
230
|
+
- `keys(): IterableIterator<string>`
|
|
231
|
+
- Returns an iterable of all keys.
|
|
232
|
+
- `entries(): IterableIterator<[string, number[]]>`
|
|
233
|
+
- Returns an iterable of `[key, value]` pairs. Values are plain number array copies.
|
|
234
|
+
- `[Symbol.iterator](): IterableIterator<[string, number[]]>`
|
|
235
|
+
- Same as `entries()`. Enables `[...db]` and `for...of` iteration.
|
|
187
236
|
- `query(value: VectorInput, options?: QueryOptions): ResultItem[]`
|
|
188
237
|
- Returns results sorted by descending similarity as a plain array.
|
|
189
238
|
- Throws on dimension mismatch.
|
|
@@ -221,9 +270,9 @@ interface ResultItem {
|
|
|
221
270
|
|
|
222
271
|
```ts
|
|
223
272
|
interface OpenOptions {
|
|
224
|
-
name?: string; // OPFS directory name, default: "default"
|
|
225
273
|
dimensions: number; // vector size
|
|
226
274
|
normalize?: boolean; // default: true
|
|
275
|
+
storage?: StorageProvider; // default: InMemoryStorageProvider
|
|
227
276
|
}
|
|
228
277
|
```
|
|
229
278
|
|
|
@@ -233,12 +282,10 @@ Advanced/testing override options.
|
|
|
233
282
|
|
|
234
283
|
```ts
|
|
235
284
|
interface OpenOptionsInternal extends OpenOptions {
|
|
236
|
-
storage?: StorageProvider;
|
|
237
285
|
wasmBinary?: Uint8Array | null;
|
|
238
286
|
}
|
|
239
287
|
```
|
|
240
288
|
|
|
241
|
-
- `storage`: provide custom storage implementation (for example, tests)
|
|
242
289
|
- `wasmBinary`:
|
|
243
290
|
- `Uint8Array`: use provided precompiled WASM
|
|
244
291
|
- `null`: force JavaScript-only compute
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure JavaScript compute functions for vector operations.
|
|
3
|
+
* These serve as the reference implementation and fallback when WASM SIMD is unavailable.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Normalizes a vector in-place to unit length.
|
|
7
|
+
* After normalization, cosine similarity reduces to a simple dot product.
|
|
8
|
+
*/
|
|
9
|
+
export declare function normalize(vec: Float32Array): void;
|
|
10
|
+
/**
|
|
11
|
+
* Computes dot products of query against all vectors in the database.
|
|
12
|
+
* Writes scores to the output array.
|
|
13
|
+
*
|
|
14
|
+
* @param query - Normalized query vector (length = dimensions)
|
|
15
|
+
* @param db - Contiguous flat array of normalized vectors (length = dbSize * dimensions)
|
|
16
|
+
* @param scores - Output array for dot product scores (length = dbSize)
|
|
17
|
+
* @param dbSize - Number of vectors in the database
|
|
18
|
+
* @param dimensions - Dimensionality of each vector
|
|
19
|
+
*/
|
|
20
|
+
export declare function searchAll(query: Float32Array, db: Float32Array, scores: Float32Array, dbSize: number, dimensions: number): void;
|