endee 1.0.6 → 1.0.8-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +279 -153
- package/dist/crypto.d.ts.map +1 -1
- package/dist/crypto.js +15 -20
- package/dist/crypto.js.map +1 -1
- package/dist/{endee.d.ts → endee.class.d.ts} +4 -7
- package/dist/endee.class.d.ts.map +1 -0
- package/dist/{endee.js → endee.class.js} +35 -27
- package/dist/endee.class.js.map +1 -0
- package/dist/exceptions.js +15 -15
- package/dist/{indexClient.d.ts → index.class.d.ts} +8 -13
- package/dist/index.class.d.ts.map +1 -0
- package/dist/{indexClient.js → index.class.js} +96 -39
- package/dist/index.class.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/types/index.d.ts +21 -6
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js.map +1 -1
- package/dist/userClient.d.ts.map +1 -1
- package/dist/userClient.js +35 -35
- package/dist/userClient.js.map +1 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +14 -4
- package/dist/utils.js.map +1 -1
- package/package.json +27 -4
- package/dist/endee.d.ts.map +0 -1
- package/dist/endee.js.map +0 -1
- package/dist/indexClient.d.ts.map +0 -1
- package/dist/indexClient.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,250 +1,381 @@
|
|
|
1
1
|
# Endee - TypeScript Vector Database Client
|
|
2
2
|
|
|
3
|
-
Endee is a TypeScript client for a vector database designed for maximum speed and efficiency. This package provides full type safety, modern ES module support, and optimized code for rapid Approximate Nearest Neighbor (ANN) searches on vector data.
|
|
3
|
+
Endee is a TypeScript client for a local vector database designed for maximum speed and efficiency. This package provides full type safety, modern ES module support, and optimized code for rapid Approximate Nearest Neighbor (ANN) searches on vector data.
|
|
4
4
|
|
|
5
5
|
## Key Features
|
|
6
6
|
|
|
7
7
|
- **TypeScript First**: Full type safety and IntelliSense support
|
|
8
8
|
- **Fast ANN Searches**: Efficient similarity searches on vector data
|
|
9
9
|
- **Multiple Distance Metrics**: Support for cosine, L2, and inner product distance metrics
|
|
10
|
+
- **Hybrid Indexes**: Support for dense vectors, sparse vectors, and hybrid (dense + sparse) searches
|
|
10
11
|
- **Metadata Support**: Attach and search with metadata and filters
|
|
11
|
-
- **Client-Side Encryption**: Optional AES-256 encryption for your metadata
|
|
12
12
|
- **High Performance**: Optimized for speed and efficiency
|
|
13
13
|
- **Modern ES Modules**: Native ES module support with proper tree-shaking
|
|
14
14
|
|
|
15
|
+
## Requirements
|
|
16
|
+
|
|
17
|
+
- Node.js >= 18.0.0
|
|
18
|
+
- Endee Local server running (see [Quick Start](https://docs.endee.io/quick-start))
|
|
19
|
+
|
|
15
20
|
## Installation
|
|
16
21
|
|
|
17
22
|
```bash
|
|
18
23
|
npm install endee
|
|
19
24
|
```
|
|
20
25
|
|
|
21
|
-
##
|
|
26
|
+
## Quick Start
|
|
22
27
|
|
|
23
|
-
|
|
24
|
-
2. Sign up or log in to your account
|
|
25
|
-
3. Navigate to **API Keys** section
|
|
26
|
-
4. Click **Generate New Token**
|
|
27
|
-
5. Copy the generated auth token and store it securely
|
|
28
|
+
### Initialize the Client
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
The Endee client connects to your local server (defaults to `http://127.0.0.1:8080/api/v1`):
|
|
30
31
|
|
|
31
32
|
```typescript
|
|
32
|
-
import { Endee, Precision } from
|
|
33
|
+
import { Endee, Precision } from 'endee';
|
|
34
|
+
|
|
35
|
+
// Connect to local Endee server (defaults to localhost:8080)
|
|
36
|
+
const client = new Endee();
|
|
37
|
+
```
|
|
33
38
|
|
|
34
|
-
|
|
35
|
-
|
|
39
|
+
**Using Authentication?** If your server has `NDD_AUTH_TOKEN` set, pass the same token when initializing:
|
|
40
|
+
|
|
41
|
+
```typescript
|
|
42
|
+
const client = new Endee('your-auth-token');
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Setting a Custom Base URL
|
|
46
|
+
|
|
47
|
+
If your server runs on a different port, use `setBaseUrl()`:
|
|
48
|
+
|
|
49
|
+
```typescript
|
|
50
|
+
const client = new Endee();
|
|
36
51
|
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
52
|
+
// Set custom base URL for non-default port
|
|
53
|
+
client.setBaseUrl('http://0.0.0.0:8081/api/v1');
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### Create a Dense Index
|
|
57
|
+
|
|
58
|
+
```typescript
|
|
59
|
+
import { Precision } from 'endee';
|
|
60
|
+
|
|
61
|
+
await client.createIndex({
|
|
62
|
+
name: 'my_vectors',
|
|
63
|
+
dimension: 384,
|
|
64
|
+
spaceType: 'cosine',
|
|
65
|
+
precision: Precision.INT8D,
|
|
42
66
|
});
|
|
67
|
+
```
|
|
43
68
|
|
|
44
|
-
|
|
45
|
-
|
|
69
|
+
**Dense Index Parameters:**
|
|
70
|
+
|
|
71
|
+
| Parameter | Description |
|
|
72
|
+
|-----------|-------------|
|
|
73
|
+
| `name` | Unique name for your index |
|
|
74
|
+
| `dimension` | Vector dimensionality (must match your embedding model's output) |
|
|
75
|
+
| `spaceType` | Distance metric - `"cosine"`, `"l2"`, or `"ip"` (inner product) |
|
|
76
|
+
| `M` | Graph connectivity - higher values increase recall but use more memory (default: 16) |
|
|
77
|
+
| `efCon` | Construction-time parameter - higher values improve index quality (default: 128) |
|
|
78
|
+
| `precision` | Quantization precision (default: `Precision.INT8D`) |
|
|
79
|
+
|
|
80
|
+
### Create a Hybrid Index
|
|
81
|
+
|
|
82
|
+
Hybrid indexes combine dense vector search with sparse vector search. Add the `sparseDimension` parameter:
|
|
83
|
+
|
|
84
|
+
```typescript
|
|
85
|
+
await client.createIndex({
|
|
86
|
+
name: 'hybrid_index',
|
|
87
|
+
dimension: 384, // Dense vector dimension
|
|
88
|
+
sparseDimension: 30000, // Sparse vector dimension (vocabulary size)
|
|
89
|
+
spaceType: 'cosine',
|
|
90
|
+
precision: Precision.INT8D,
|
|
91
|
+
});
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### List and Access Indexes
|
|
95
|
+
|
|
96
|
+
```typescript
|
|
97
|
+
// List all indexes
|
|
98
|
+
const indexes = await client.listIndexes();
|
|
99
|
+
|
|
100
|
+
// Get reference to an existing index
|
|
101
|
+
const index = await client.getIndex('my_vectors');
|
|
102
|
+
|
|
103
|
+
// Delete an index
|
|
104
|
+
await client.deleteIndex('my_vectors');
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Upserting Vectors
|
|
108
|
+
|
|
109
|
+
The `index.upsert()` method adds or updates vectors in an existing index.
|
|
110
|
+
|
|
111
|
+
```typescript
|
|
112
|
+
const index = await client.getIndex('my_index');
|
|
46
113
|
|
|
47
|
-
// Insert vectors
|
|
48
114
|
await index.upsert([
|
|
49
115
|
{
|
|
50
|
-
id:
|
|
51
|
-
vector: [0.1, 0.2, 0.3 /* ... */],
|
|
52
|
-
meta: {
|
|
116
|
+
id: 'vec1',
|
|
117
|
+
vector: [0.1, 0.2, 0.3 /* ... */],
|
|
118
|
+
meta: { title: 'First document' },
|
|
119
|
+
filter: { category: 'tech' },
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: 'vec2',
|
|
123
|
+
vector: [0.3, 0.4, 0.5 /* ... */],
|
|
124
|
+
meta: { title: 'Second document' },
|
|
125
|
+
filter: { category: 'science' },
|
|
53
126
|
},
|
|
54
127
|
]);
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
**Vector Object Fields:**
|
|
131
|
+
|
|
132
|
+
| Field | Required | Description |
|
|
133
|
+
|-------|----------|-------------|
|
|
134
|
+
| `id` | Yes | Unique identifier for the vector |
|
|
135
|
+
| `vector` | Yes | Array of floats representing the embedding |
|
|
136
|
+
| `meta` | No | Arbitrary metadata object |
|
|
137
|
+
| `filter` | No | Key-value pairs for filtering during queries |
|
|
138
|
+
|
|
139
|
+
## Querying the Index
|
|
55
140
|
|
|
56
|
-
|
|
141
|
+
The `index.query()` method performs a similarity search.
|
|
142
|
+
|
|
143
|
+
```typescript
|
|
57
144
|
const results = await index.query({
|
|
58
|
-
vector: [0.
|
|
59
|
-
topK:
|
|
145
|
+
vector: [0.15, 0.25 /* ... */],
|
|
146
|
+
topK: 5,
|
|
147
|
+
ef: 128,
|
|
148
|
+
includeVectors: true,
|
|
60
149
|
});
|
|
61
150
|
|
|
62
|
-
// Process results
|
|
63
151
|
for (const item of results) {
|
|
64
152
|
console.log(`ID: ${item.id}, Similarity: ${item.similarity}`);
|
|
65
|
-
console.log(`Metadata:`, item.meta);
|
|
66
153
|
}
|
|
67
154
|
```
|
|
68
155
|
|
|
69
|
-
|
|
156
|
+
**Query Parameters:**
|
|
70
157
|
|
|
71
|
-
|
|
158
|
+
| Parameter | Description |
|
|
159
|
+
|-----------|-------------|
|
|
160
|
+
| `vector` | Query vector (must match index dimension) |
|
|
161
|
+
| `topK` | Number of results to return (default: 10, max: 512) |
|
|
162
|
+
| `ef` | Search quality parameter (default: 128, max: 1024) |
|
|
163
|
+
| `includeVectors` | Include vector data in results (default: false) |
|
|
72
164
|
|
|
73
|
-
|
|
74
|
-
import { Endee } from "endee";
|
|
165
|
+
## Filtered Querying
|
|
75
166
|
|
|
76
|
-
|
|
77
|
-
const endee = new Endee("<auth-token>");
|
|
167
|
+
Use the `filter` parameter to restrict results. All filters are combined with **logical AND**.
|
|
78
168
|
|
|
79
|
-
|
|
80
|
-
const
|
|
169
|
+
```typescript
|
|
170
|
+
const results = await index.query({
|
|
171
|
+
vector: [0.15, 0.25 /* ... */],
|
|
172
|
+
topK: 5,
|
|
173
|
+
filter: [
|
|
174
|
+
{ category: { $eq: 'tech' } },
|
|
175
|
+
{ score: { $range: [80, 100] } },
|
|
176
|
+
],
|
|
177
|
+
});
|
|
81
178
|
```
|
|
82
179
|
|
|
83
|
-
###
|
|
180
|
+
### Filtering Operators
|
|
84
181
|
|
|
85
|
-
|
|
86
|
-
|
|
182
|
+
| Operator | Description | Example |
|
|
183
|
+
|----------|-------------|---------|
|
|
184
|
+
| `$eq` | Exact match | `{ status: { $eq: 'published' } }` |
|
|
185
|
+
| `$in` | Match any in list | `{ tags: { $in: ['ai', 'ml'] } }` |
|
|
186
|
+
| `$range` | Numeric range (inclusive) | `{ score: { $range: [70, 95] } }` |
|
|
87
187
|
|
|
88
|
-
|
|
89
|
-
const indexes = await endee.listIndexes();
|
|
188
|
+
> **Note:** The `$range` operator supports values within **[0 - 999]**. Normalize larger values before upserting.
|
|
90
189
|
|
|
91
|
-
|
|
92
|
-
await endee.createIndex({
|
|
93
|
-
name: "my_custom_index",
|
|
94
|
-
dimension: 384,
|
|
95
|
-
spaceType: "l2", // space type (cosine, l2, ip)
|
|
96
|
-
M: 32, // M: Graph connectivity parameter
|
|
97
|
-
efCon: 256, // efCon: Construction-time parameter
|
|
98
|
-
precision: Precision.INT8D // Quantization precision (default: Precision.INT8D)
|
|
99
|
-
});
|
|
190
|
+
## Hybrid Search
|
|
100
191
|
|
|
101
|
-
|
|
102
|
-
await endee.deleteIndex("my_index");
|
|
103
|
-
```
|
|
192
|
+
### Upserting Hybrid Vectors
|
|
104
193
|
|
|
105
|
-
|
|
194
|
+
Provide both dense vectors and sparse representations:
|
|
106
195
|
|
|
107
196
|
```typescript
|
|
108
|
-
|
|
109
|
-
const index = await endee.getIndex("my_index");
|
|
197
|
+
const index = await client.getIndex('hybrid_index');
|
|
110
198
|
|
|
111
|
-
// Insert multiple vectors in a batch
|
|
112
199
|
await index.upsert([
|
|
113
200
|
{
|
|
114
|
-
id:
|
|
115
|
-
vector: [/* ... */],
|
|
116
|
-
|
|
117
|
-
|
|
201
|
+
id: 'doc1',
|
|
202
|
+
vector: [0.1, 0.2 /* ... */], // Dense vector
|
|
203
|
+
sparseIndices: [10, 50, 200], // Non-zero term positions
|
|
204
|
+
sparseValues: [0.8, 0.5, 0.3], // Weights for each position
|
|
205
|
+
meta: { title: 'Document 1' },
|
|
118
206
|
},
|
|
119
207
|
{
|
|
120
|
-
id:
|
|
121
|
-
vector: [/* ... */],
|
|
122
|
-
|
|
208
|
+
id: 'doc2',
|
|
209
|
+
vector: [0.3, 0.4 /* ... */],
|
|
210
|
+
sparseIndices: [15, 100, 500],
|
|
211
|
+
sparseValues: [0.9, 0.4, 0.6],
|
|
212
|
+
meta: { title: 'Document 2' },
|
|
123
213
|
},
|
|
124
214
|
]);
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
**Hybrid Vector Fields:**
|
|
218
|
+
|
|
219
|
+
| Field | Required | Description |
|
|
220
|
+
|-------|----------|-------------|
|
|
221
|
+
| `id` | Yes | Unique identifier |
|
|
222
|
+
| `vector` | Yes | Dense embedding vector |
|
|
223
|
+
| `sparseIndices` | Yes (hybrid) | Non-zero term positions in sparse vector |
|
|
224
|
+
| `sparseValues` | Yes (hybrid) | Weights for each sparse index |
|
|
225
|
+
| `meta` | No | Metadata dictionary |
|
|
226
|
+
| `filter` | No | Filter fields |
|
|
227
|
+
|
|
228
|
+
> **Important:** `sparseIndices` and `sparseValues` must have the same length. Values in `sparseIndices` must be within `[0, sparseDimension)`.
|
|
229
|
+
|
|
230
|
+
### Querying Hybrid Index
|
|
231
|
+
|
|
232
|
+
Provide both dense and sparse query vectors:
|
|
125
233
|
|
|
126
|
-
|
|
234
|
+
```typescript
|
|
127
235
|
const results = await index.query({
|
|
128
|
-
vector: [/* ... */],
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
includeVectors: true, // Include vector data in results
|
|
236
|
+
vector: [0.15, 0.25 /* ... */], // Dense query
|
|
237
|
+
sparseIndices: [10, 100, 300], // Sparse query positions
|
|
238
|
+
sparseValues: [0.7, 0.5, 0.4], // Sparse query weights
|
|
239
|
+
topK: 5,
|
|
133
240
|
});
|
|
134
241
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
242
|
+
for (const item of results) {
|
|
243
|
+
console.log(`ID: ${item.id}, Similarity: ${item.similarity}`);
|
|
244
|
+
}
|
|
245
|
+
```
|
|
138
246
|
|
|
139
|
-
|
|
140
|
-
|
|
247
|
+
You can also query with:
|
|
248
|
+
- **Dense only**: Provide only `vector`
|
|
249
|
+
- **Sparse only**: Provide only `sparseIndices` and `sparseValues`
|
|
250
|
+
- **Hybrid**: Provide all three for combined results
|
|
141
251
|
|
|
142
|
-
|
|
143
|
-
const description = await index.describe();
|
|
144
|
-
console.log(description);
|
|
145
|
-
```
|
|
252
|
+
## Deletion Methods
|
|
146
253
|
|
|
147
|
-
|
|
254
|
+
### Delete by ID
|
|
255
|
+
Delete vector with a specifc vector id.
|
|
256
|
+
```typescript
|
|
257
|
+
await index.deleteVector('vec1');
|
|
258
|
+
```
|
|
259
|
+
### Delete by Filter
|
|
260
|
+
Delete all vectors matching specific filters.
|
|
261
|
+
```typescript
|
|
262
|
+
await index.deleteWithFilter([{'category': {'$eq' : 'tech'}}]);
|
|
263
|
+
```
|
|
148
264
|
|
|
149
|
-
|
|
265
|
+
### Delete Index
|
|
266
|
+
Delete an entire Index.
|
|
267
|
+
```typescript
|
|
268
|
+
await client.deleteIndex('my_index');
|
|
269
|
+
```
|
|
150
270
|
|
|
151
|
-
|
|
271
|
+
> **Warning:** Deletion operations are **irreversible**.
|
|
152
272
|
|
|
153
|
-
|
|
273
|
+
## Additional Operations
|
|
154
274
|
|
|
155
|
-
###
|
|
275
|
+
### Get Vector by ID
|
|
156
276
|
|
|
157
277
|
```typescript
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const endee = new Endee("<your-auth-token>");
|
|
278
|
+
const vector = await index.getVector('vec1');
|
|
279
|
+
```
|
|
161
280
|
|
|
162
|
-
|
|
163
|
-
const key = endee.generateKey();
|
|
281
|
+
### Describe Index
|
|
164
282
|
|
|
283
|
+
```typescript
|
|
284
|
+
const info = index.describe();
|
|
285
|
+
console.log(info);
|
|
286
|
+
// { name, spaceType, dimension, sparseDimension, isHybrid, count, precision, M }
|
|
165
287
|
```
|
|
166
288
|
|
|
167
|
-
|
|
289
|
+
## Precision Options
|
|
168
290
|
|
|
169
|
-
|
|
291
|
+
Endee supports different quantization precision levels:
|
|
170
292
|
|
|
171
293
|
```typescript
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
294
|
+
import { Precision } from 'endee';
|
|
295
|
+
|
|
296
|
+
Precision.BINARY; // Binary quantization (1-bit) - smallest storage, fastest search
|
|
297
|
+
Precision.INT8D; // 8-bit integer quantization (default) - balanced performance
|
|
298
|
+
Precision.INT16D; // 16-bit integer quantization - higher precision
|
|
299
|
+
Precision.FLOAT16; // 16-bit floating point - good balance
|
|
300
|
+
Precision.FLOAT32; // 32-bit floating point - highest precision
|
|
179
301
|
```
|
|
180
302
|
|
|
181
|
-
|
|
303
|
+
**Choosing Precision:**
|
|
304
|
+
|
|
305
|
+
- `BINARY`: Best for very large datasets where speed and storage are critical
|
|
306
|
+
- `INT8D` (default): Recommended for most use cases - good balance of accuracy and performance
|
|
307
|
+
- `INT16D`: When you need better accuracy than INT8D but less storage than FLOAT32
|
|
308
|
+
- `FLOAT16`: Good compromise between precision and storage for embeddings
|
|
309
|
+
- `FLOAT32`: When you need maximum precision and storage is not a concern
|
|
310
|
+
|
|
311
|
+
## Complete Example
|
|
182
312
|
|
|
183
313
|
```typescript
|
|
184
|
-
|
|
185
|
-
|
|
314
|
+
import { Endee, Precision } from 'endee';
|
|
315
|
+
|
|
316
|
+
// Initialize client
|
|
317
|
+
const client = new Endee();
|
|
318
|
+
|
|
319
|
+
// Create a dense index
|
|
320
|
+
await client.createIndex({
|
|
321
|
+
name: 'documents',
|
|
322
|
+
dimension: 384,
|
|
323
|
+
spaceType: 'cosine',
|
|
324
|
+
precision: Precision.INT8D,
|
|
325
|
+
});
|
|
186
326
|
|
|
187
|
-
//
|
|
327
|
+
// Get the index
|
|
328
|
+
const index = await client.getIndex('documents');
|
|
329
|
+
|
|
330
|
+
// Add vectors
|
|
188
331
|
await index.upsert([
|
|
189
332
|
{
|
|
190
|
-
id:
|
|
191
|
-
vector: [0.1, 0.2
|
|
192
|
-
meta: {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
333
|
+
id: 'doc1',
|
|
334
|
+
vector: [0.1, 0.2 /* ... 384 dimensions */],
|
|
335
|
+
meta: { title: 'First Document' },
|
|
336
|
+
filter: { category: 'tech' },
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
id: 'doc2',
|
|
340
|
+
vector: [0.3, 0.4 /* ... 384 dimensions */],
|
|
341
|
+
meta: { title: 'Second Document' },
|
|
342
|
+
filter: { category: 'science' },
|
|
196
343
|
},
|
|
197
344
|
]);
|
|
198
345
|
|
|
199
|
-
// Query
|
|
346
|
+
// Query the index
|
|
200
347
|
const results = await index.query({
|
|
201
|
-
vector: [0.
|
|
202
|
-
topK:
|
|
348
|
+
vector: [0.15, 0.25 /* ... */],
|
|
349
|
+
topK: 5,
|
|
203
350
|
});
|
|
204
351
|
|
|
205
|
-
// Results contain decrypted metadata
|
|
206
352
|
for (const item of results) {
|
|
207
|
-
console.log(item.
|
|
353
|
+
console.log(`ID: ${item.id}, Similarity: ${item.similarity}`);
|
|
208
354
|
}
|
|
209
355
|
```
|
|
210
356
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
- **Encryption is optional**: Only enable it if you need to protect sensitive metadata
|
|
214
|
-
- **Key management is your responsibility**: Store keys securely (e.g., environment variables, secret managers)
|
|
215
|
-
- **Vectors are not encrypted**: Only metadata is encrypted; vector data remains searchable
|
|
216
|
-
- **Key verification**: The system verifies the key checksum when accessing an encrypted index
|
|
217
|
-
- **No key recovery**: Lost keys cannot be recovered; encrypted data becomes inaccessible
|
|
357
|
+
## API Reference
|
|
218
358
|
|
|
219
|
-
|
|
359
|
+
### Endee Class
|
|
220
360
|
|
|
221
|
-
|
|
361
|
+
| Method | Description |
|
|
362
|
+
|--------|-------------|
|
|
363
|
+
| `createIndex(options)` | Create a new index (add `sparseDimension` for hybrid) |
|
|
364
|
+
| `listIndexes()` | List all indexes |
|
|
365
|
+
| `deleteIndex(name)` | Delete an index |
|
|
366
|
+
| `getIndex(name)` | Get reference to an index |
|
|
367
|
+
| `setBaseUrl(url)` | Set a custom base URL |
|
|
222
368
|
|
|
223
|
-
|
|
224
|
-
import { Precision } from "endee";
|
|
225
|
-
|
|
226
|
-
// Available precision options:
|
|
227
|
-
Precision.BINARY // Binary quantization (1-bit) - smallest storage, fastest search
|
|
228
|
-
Precision.INT8D // 8-bit integer quantization (default) - balanced performance
|
|
229
|
-
Precision.INT16D // 16-bit integer quantization - higher precision
|
|
230
|
-
Precision.FLOAT16 // 16-bit floating point - good balance
|
|
231
|
-
Precision.FLOAT32 // 32-bit floating point - highest precision
|
|
232
|
-
|
|
233
|
-
// Example usage:
|
|
234
|
-
await endee.createIndex({
|
|
235
|
-
name: "high_precision_index",
|
|
236
|
-
dimension: 1536,
|
|
237
|
-
spaceType: "cosine",
|
|
238
|
-
precision: Precision.FLOAT32 // Use full precision
|
|
239
|
-
});
|
|
240
|
-
```
|
|
369
|
+
### Index Class
|
|
241
370
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
371
|
+
| Method | Description |
|
|
372
|
+
|--------|-------------|
|
|
373
|
+
| `upsert(vectors)` | Insert or update vectors |
|
|
374
|
+
| `query(options)` | Search for similar vectors |
|
|
375
|
+
| `deleteVector(id)` | Delete a vector by ID |
|
|
376
|
+
| `deleteWithFilter(filter)` | Delete vectors by Filter |
|
|
377
|
+
| `getVector(id)` | Get a vector by ID |
|
|
378
|
+
| `describe()` | Get index info |
|
|
248
379
|
|
|
249
380
|
## TypeScript Types
|
|
250
381
|
|
|
@@ -258,15 +389,10 @@ import type {
|
|
|
258
389
|
CreateIndexOptions,
|
|
259
390
|
IndexDescription,
|
|
260
391
|
SpaceType,
|
|
261
|
-
Precision
|
|
262
|
-
} from
|
|
392
|
+
Precision,
|
|
393
|
+
} from 'endee';
|
|
263
394
|
```
|
|
264
395
|
|
|
265
|
-
## Requirements
|
|
266
|
-
|
|
267
|
-
- Node.js >= 18.0.0
|
|
268
|
-
- TypeScript >= 5.0.0 (for development)
|
|
269
|
-
|
|
270
396
|
## License
|
|
271
397
|
|
|
272
398
|
MIT
|
package/dist/crypto.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../src/crypto.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAItD;AAED,wBAAgB,OAAO,CACrB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,EAC/C,GAAG,GAAE,MAAM,GAAG,IAAW,GACxB,MAAM,CAWR;AAED,wBAAgB,SAAS,CACvB,UAAU,
|
|
1
|
+
{"version":3,"file":"crypto.d.ts","sourceRoot":"","sources":["../src/crypto.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAItD;AAED,wBAAgB,OAAO,CACrB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,EAC/C,GAAG,GAAE,MAAM,GAAG,IAAW,GACxB,MAAM,CAWR;AAED,wBAAgB,SAAS,CACvB,UAAU,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GAAG,SAAS,EACrF,GAAG,GAAE,MAAM,GAAG,IAAW,GACxB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAgCzB;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAuB/D;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAsB/D"}
|
package/dist/crypto.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Cryptographic utilities for Endee-DB
|
|
3
3
|
*/
|
|
4
|
-
import * as zlib from
|
|
5
|
-
import crypto from
|
|
4
|
+
import * as zlib from 'zlib';
|
|
5
|
+
import crypto from 'crypto';
|
|
6
6
|
export function getChecksum(key) {
|
|
7
7
|
// Convert last two characters of key to integer (hex)
|
|
8
8
|
if (key === null)
|
|
@@ -13,7 +13,7 @@ export function jsonZip(obj, key = null) {
|
|
|
13
13
|
if (!obj || Object.keys(obj).length === 0)
|
|
14
14
|
return Buffer.alloc(0);
|
|
15
15
|
const jsonStr = JSON.stringify(obj);
|
|
16
|
-
const compressed = zlib.deflateSync(Buffer.from(jsonStr,
|
|
16
|
+
const compressed = zlib.deflateSync(Buffer.from(jsonStr, 'utf-8'));
|
|
17
17
|
// If key is provided, encrypt the compressed data
|
|
18
18
|
if (key) {
|
|
19
19
|
return aesEncrypt(compressed, key);
|
|
@@ -23,12 +23,12 @@ export function jsonZip(obj, key = null) {
|
|
|
23
23
|
export function jsonUnzip(compressed, key = null) {
|
|
24
24
|
if (!compressed || (compressed instanceof Buffer && compressed.length === 0))
|
|
25
25
|
return {};
|
|
26
|
-
if (typeof compressed ===
|
|
26
|
+
if (typeof compressed === 'object' &&
|
|
27
27
|
!Buffer.isBuffer(compressed) &&
|
|
28
28
|
!(compressed instanceof Uint8Array)) {
|
|
29
29
|
return compressed;
|
|
30
30
|
}
|
|
31
|
-
if (typeof compressed ===
|
|
31
|
+
if (typeof compressed === 'string') {
|
|
32
32
|
try {
|
|
33
33
|
return JSON.parse(compressed);
|
|
34
34
|
}
|
|
@@ -37,17 +37,15 @@ export function jsonUnzip(compressed, key = null) {
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
try {
|
|
40
|
-
let buffer = Buffer.isBuffer(compressed)
|
|
41
|
-
? compressed
|
|
42
|
-
: Buffer.from(compressed);
|
|
40
|
+
let buffer = Buffer.isBuffer(compressed) ? compressed : Buffer.from(compressed);
|
|
43
41
|
// If key is provided, decrypt the compressed data first
|
|
44
42
|
if (key) {
|
|
45
43
|
buffer = aesDecrypt(buffer, key);
|
|
46
44
|
}
|
|
47
45
|
const decompressed = zlib.inflateSync(buffer);
|
|
48
|
-
return JSON.parse(decompressed.toString(
|
|
46
|
+
return JSON.parse(decompressed.toString('utf-8'));
|
|
49
47
|
}
|
|
50
|
-
catch
|
|
48
|
+
catch {
|
|
51
49
|
return {};
|
|
52
50
|
}
|
|
53
51
|
}
|
|
@@ -60,15 +58,15 @@ export function jsonUnzip(compressed, key = null) {
|
|
|
60
58
|
*/
|
|
61
59
|
export function aesEncrypt(data, keyHex) {
|
|
62
60
|
// Convert hex key to bytes
|
|
63
|
-
const key = Buffer.from(keyHex,
|
|
61
|
+
const key = Buffer.from(keyHex, 'hex');
|
|
64
62
|
if (key.length !== 32) {
|
|
65
63
|
// 256 bits = 32 bytes
|
|
66
|
-
throw new Error(
|
|
64
|
+
throw new Error('Key must be 256 bits (64 hex characters)');
|
|
67
65
|
}
|
|
68
66
|
// Generate a random 16-byte IV
|
|
69
67
|
const iv = crypto.randomBytes(16);
|
|
70
68
|
// Create cipher
|
|
71
|
-
const cipher = crypto.createCipheriv(
|
|
69
|
+
const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
|
|
72
70
|
// Pad data to 16-byte boundary (PKCS7 padding)
|
|
73
71
|
const paddedData = pkcs7Pad(data);
|
|
74
72
|
// Encrypt
|
|
@@ -85,22 +83,19 @@ export function aesEncrypt(data, keyHex) {
|
|
|
85
83
|
*/
|
|
86
84
|
export function aesDecrypt(data, keyHex) {
|
|
87
85
|
// Convert hex key to bytes
|
|
88
|
-
const key = Buffer.from(keyHex,
|
|
86
|
+
const key = Buffer.from(keyHex, 'hex');
|
|
89
87
|
if (key.length !== 32) {
|
|
90
88
|
// 256 bits = 32 bytes
|
|
91
|
-
throw new Error(
|
|
89
|
+
throw new Error('Key must be 256 bits (64 hex characters)');
|
|
92
90
|
}
|
|
93
91
|
// Extract IV and ciphertext
|
|
94
92
|
const iv = data.subarray(0, 16);
|
|
95
93
|
const ciphertext = data.subarray(16);
|
|
96
94
|
// Create decipher
|
|
97
|
-
const decipher = crypto.createDecipheriv(
|
|
95
|
+
const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
|
|
98
96
|
decipher.setAutoPadding(false); // We handle padding manually
|
|
99
97
|
// Decrypt
|
|
100
|
-
const paddedData = Buffer.concat([
|
|
101
|
-
decipher.update(ciphertext),
|
|
102
|
-
decipher.final(),
|
|
103
|
-
]);
|
|
98
|
+
const paddedData = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
|
104
99
|
// Remove PKCS7 padding
|
|
105
100
|
return pkcs7Unpad(paddedData);
|
|
106
101
|
}
|