@orion-js/vectors 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +120 -0
- package/dist/index.cjs +259 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +205 -0
- package/dist/index.d.ts +205 -0
- package/dist/index.js +246 -0
- package/dist/index.js.map +1 -0
- package/package.json +38 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Orionjs Team
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# @orion-js/vectors
|
|
2
|
+
|
|
3
|
+
The `@orion-js/vectors` module provides a provider-agnostic interface for vector storage and similarity search, with built-in support for local file-system storage and AWS S3 Vectors.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Provider-agnostic Interface**: `VectorStorageProvider` defines a standard interface for all vector storage operations.
|
|
8
|
+
- **Local Provider**: A file-system-based `LocalVectorProvider` for development, testing, and small-scale deployments.
|
|
9
|
+
- **AWS S3 Provider**: An `S3VectorProvider` that integrates with AWS S3 Vectors, a new feature for storing and querying vector embeddings in S3.
|
|
10
|
+
- **Type Safety**: Full TypeScript support with comprehensive type definitions for vector stores, indexes, and operations.
|
|
11
|
+
- **Core Functionality**:
|
|
12
|
+
- `defineVectorStore`: Idempotently create stores (buckets) and indexes.
|
|
13
|
+
- `insertVectors`, `queryVectors`, `deleteVectors`: Full CRUD and search capabilities.
|
|
14
|
+
- `cosine` and `euclidean` distance metrics.
|
|
15
|
+
- Metadata filtering in queries.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pnpm add @orion-js/vectors
|
|
21
|
+
pnpm add @aws-sdk/client-s3vectors # If using S3VectorProvider
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Usage
|
|
25
|
+
|
|
26
|
+
### 1. Define a Vector Store
|
|
27
|
+
|
|
28
|
+
First, define your vector store configuration. This is where your vectors will be stored.
|
|
29
|
+
|
|
30
|
+
```typescript
|
|
31
|
+
import { VectorStoreConfig } from '@orion-js/vectors'
|
|
32
|
+
|
|
33
|
+
const storeConfig: VectorStoreConfig = {
|
|
34
|
+
storeName: 'media-embeddings',
|
|
35
|
+
indexes: [
|
|
36
|
+
{
|
|
37
|
+
name: 'movies',
|
|
38
|
+
dimension: 1024, // Dimension of your vectors
|
|
39
|
+
distanceMetric: 'cosine', // 'cosine' or 'euclidean'
|
|
40
|
+
nonFilterableMetadataKeys: ['source_text'] // Optional
|
|
41
|
+
}
|
|
42
|
+
]
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### 2. Choose a Provider
|
|
47
|
+
|
|
48
|
+
#### LocalVectorProvider (for development/testing)
|
|
49
|
+
|
|
50
|
+
This provider stores data on the local file system, which is great for getting started without needing cloud resources.
|
|
51
|
+
|
|
52
|
+
```typescript
|
|
53
|
+
import { LocalVectorProvider } from '@orion-js/vectors'
|
|
54
|
+
|
|
55
|
+
const provider = new LocalVectorProvider({ basePath: './vector-data' })
|
|
56
|
+
|
|
57
|
+
// This will create the necessary directories and files
|
|
58
|
+
await provider.defineVectorStore(storeConfig)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
#### S3VectorProvider (for production)
|
|
62
|
+
|
|
63
|
+
This provider uses AWS S3 Vectors.
|
|
64
|
+
|
|
65
|
+
> **Note**: S3 Vectors is currently in preview.
|
|
66
|
+
|
|
67
|
+
```typescript
|
|
68
|
+
import { S3VectorProvider } from '@orion-js/vectors'
|
|
69
|
+
|
|
70
|
+
const provider = new S3VectorProvider({
|
|
71
|
+
region: 'us-east-1',
|
|
72
|
+
// Credentials can be omitted if using IAM roles
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
// This will create an S3 Vector Bucket and Index
|
|
76
|
+
await provider.defineVectorStore(storeConfig)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### 3. Perform Vector Operations
|
|
80
|
+
|
|
81
|
+
The API is the same for all providers.
|
|
82
|
+
|
|
83
|
+
```typescript
|
|
84
|
+
// Insert vectors
|
|
85
|
+
await provider.insertVectors('media-embeddings', 'movies', [
|
|
86
|
+
{
|
|
87
|
+
key: 'Star Wars',
|
|
88
|
+
data: { float32: embedding1 },
|
|
89
|
+
metadata: { genre: 'scifi', source_text: 'A long time ago...' }
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
key: 'Jurassic Park',
|
|
93
|
+
data: { float32: embedding2 },
|
|
94
|
+
metadata: { genre: 'scifi', source_text: 'An adventure 65 million years in the making.' }
|
|
95
|
+
}
|
|
96
|
+
])
|
|
97
|
+
|
|
98
|
+
// Query for similar vectors
|
|
99
|
+
const results = await provider.queryVectors('media-embeddings', 'movies', {
|
|
100
|
+
queryVector: { float32: queryEmbedding },
|
|
101
|
+
topK: 3,
|
|
102
|
+
filter: { genre: 'scifi' }, // Filter by metadata
|
|
103
|
+
returnDistance: true,
|
|
104
|
+
returnMetadata: true
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
console.log(results)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Testing
|
|
111
|
+
|
|
112
|
+
The package includes a comprehensive test suite. The `LocalVectorProvider` is fully tested.
|
|
113
|
+
|
|
114
|
+
To run the tests:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pnpm --filter @orion-js/vectors test
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
To run integration tests for `S3VectorProvider`, you need to have AWS credentials configured in your environment.
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
3
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
5
|
+
var __export = (target, all) => {
|
|
6
|
+
for (var name in all)
|
|
7
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
8
|
+
};
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
+
for (let key of __getOwnPropNames(from))
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
+
}
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
18
|
+
|
|
19
|
+
// src/index.ts
|
|
20
|
+
var index_exports = {};
|
|
21
|
+
__export(index_exports, {
|
|
22
|
+
S3VectorProvider: () => S3VectorProvider
|
|
23
|
+
});
|
|
24
|
+
module.exports = __toCommonJS(index_exports);
|
|
25
|
+
|
|
26
|
+
// src/providers/S3VectorProvider.ts
|
|
27
|
+
var import_client_s3vectors2 = require("@aws-sdk/client-s3vectors");
|
|
28
|
+
|
|
29
|
+
// src/providers/S3VectorStore.ts
|
|
30
|
+
var import_client_s3vectors = require("@aws-sdk/client-s3vectors");
|
|
31
|
+
var S3VectorStore = class {
|
|
32
|
+
config;
|
|
33
|
+
client;
|
|
34
|
+
constructor(client, config) {
|
|
35
|
+
this.client = client;
|
|
36
|
+
this.config = config;
|
|
37
|
+
}
|
|
38
|
+
async wrapOperationWithIndexCreation(namespace, operation) {
|
|
39
|
+
try {
|
|
40
|
+
return await operation();
|
|
41
|
+
} catch (error) {
|
|
42
|
+
if (error instanceof import_client_s3vectors.NotFoundException) {
|
|
43
|
+
if (error.message === "The specified index could not be found") {
|
|
44
|
+
await this.createVectorIndex(namespace, this.config, this.config.index);
|
|
45
|
+
return this.wrapOperationWithIndexCreation(namespace, operation);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
async insertVectors(namespace, vectors) {
|
|
52
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
53
|
+
const result = await this.client.send(
|
|
54
|
+
new import_client_s3vectors.PutVectorsCommand({
|
|
55
|
+
vectorBucketName: this.config.storeName,
|
|
56
|
+
indexName: namespace,
|
|
57
|
+
vectors: vectors.map((vector) => ({
|
|
58
|
+
key: vector.key,
|
|
59
|
+
data: {
|
|
60
|
+
float32: vector.vector
|
|
61
|
+
},
|
|
62
|
+
metadata: vector.metadata
|
|
63
|
+
}))
|
|
64
|
+
})
|
|
65
|
+
);
|
|
66
|
+
return result;
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
async queryVectors(namespace, options) {
|
|
70
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
71
|
+
var _a;
|
|
72
|
+
const result = await this.client.send(
|
|
73
|
+
new import_client_s3vectors.QueryVectorsCommand({
|
|
74
|
+
...options,
|
|
75
|
+
vectorBucketName: this.config.storeName,
|
|
76
|
+
indexName: namespace,
|
|
77
|
+
queryVector: {
|
|
78
|
+
float32: options.queryVector
|
|
79
|
+
}
|
|
80
|
+
})
|
|
81
|
+
);
|
|
82
|
+
const vectors = ((_a = result.vectors) == null ? void 0 : _a.map(
|
|
83
|
+
(vector) => {
|
|
84
|
+
var _a2;
|
|
85
|
+
return {
|
|
86
|
+
key: vector.key,
|
|
87
|
+
vector: (_a2 = vector.data) == null ? void 0 : _a2.float32,
|
|
88
|
+
metadata: vector.metadata
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
)) ?? [];
|
|
92
|
+
return {
|
|
93
|
+
vectors
|
|
94
|
+
};
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
async deleteVectors(namespace, keys) {
|
|
98
|
+
return await this.client.send(
|
|
99
|
+
new import_client_s3vectors.DeleteVectorsCommand({
|
|
100
|
+
vectorBucketName: this.config.storeName,
|
|
101
|
+
indexName: namespace,
|
|
102
|
+
keys
|
|
103
|
+
})
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
async listVectors(namespace, options) {
|
|
107
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
108
|
+
var _a;
|
|
109
|
+
const result = await this.client.send(
|
|
110
|
+
new import_client_s3vectors.ListVectorsCommand({
|
|
111
|
+
vectorBucketName: this.config.storeName,
|
|
112
|
+
indexName: namespace,
|
|
113
|
+
...options
|
|
114
|
+
})
|
|
115
|
+
);
|
|
116
|
+
return ((_a = result.vectors) == null ? void 0 : _a.map((vector) => {
|
|
117
|
+
var _a2;
|
|
118
|
+
return {
|
|
119
|
+
key: vector.key,
|
|
120
|
+
vector: (_a2 = vector.data) == null ? void 0 : _a2.float32,
|
|
121
|
+
metadata: vector.metadata
|
|
122
|
+
};
|
|
123
|
+
})) ?? [];
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
async listVectorKeys(namespace, options) {
|
|
127
|
+
const result = await this.listVectors(namespace, {
|
|
128
|
+
...options,
|
|
129
|
+
returnData: false,
|
|
130
|
+
returnMetadata: false
|
|
131
|
+
});
|
|
132
|
+
return result.map((vector) => vector.key);
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Create a new vector index
|
|
136
|
+
*/
|
|
137
|
+
async createVectorIndex(namespace, config, indexConfig) {
|
|
138
|
+
await this.client.send(
|
|
139
|
+
new import_client_s3vectors.CreateIndexCommand({
|
|
140
|
+
indexName: namespace,
|
|
141
|
+
dataType: "float32",
|
|
142
|
+
dimension: indexConfig.dimension,
|
|
143
|
+
distanceMetric: indexConfig.distanceMetric,
|
|
144
|
+
vectorBucketName: config.storeName,
|
|
145
|
+
metadataConfiguration: indexConfig.metadataConfiguration
|
|
146
|
+
})
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
async deleteNamespace(namespace) {
|
|
150
|
+
return await this.client.send(
|
|
151
|
+
new import_client_s3vectors.DeleteIndexCommand({
|
|
152
|
+
vectorBucketName: this.config.storeName,
|
|
153
|
+
indexName: namespace
|
|
154
|
+
})
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
// src/providers/S3VectorProvider.ts
|
|
160
|
+
var S3VectorProvider = class {
|
|
161
|
+
config;
|
|
162
|
+
client;
|
|
163
|
+
constructor(config) {
|
|
164
|
+
this.config = config;
|
|
165
|
+
this.validateConfig();
|
|
166
|
+
this.client = new import_client_s3vectors2.S3VectorsClient(this.config.s3Options);
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Validate the provider configuration
|
|
170
|
+
*/
|
|
171
|
+
validateConfig() {
|
|
172
|
+
if (!this.config.s3Options) {
|
|
173
|
+
throw new Error("S3Options are required for S3VectorProvider");
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
createVectorStore(config) {
|
|
177
|
+
this.validateVectorStoreConfig(config);
|
|
178
|
+
this.setupVectorStore(config);
|
|
179
|
+
return new S3VectorStore(this.client, config);
|
|
180
|
+
}
|
|
181
|
+
async setupVectorStore(config) {
|
|
182
|
+
await this.ensureVectorBucketExists(config);
|
|
183
|
+
}
|
|
184
|
+
async storeExists(storeName) {
|
|
185
|
+
try {
|
|
186
|
+
await this.client.send(
|
|
187
|
+
new import_client_s3vectors2.GetVectorBucketCommand({
|
|
188
|
+
vectorBucketName: storeName
|
|
189
|
+
})
|
|
190
|
+
);
|
|
191
|
+
return true;
|
|
192
|
+
} catch (error) {
|
|
193
|
+
if (this.isNotFoundError(error)) {
|
|
194
|
+
return false;
|
|
195
|
+
}
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Ensure a vector bucket exists, create if it doesn't
|
|
201
|
+
*/
|
|
202
|
+
async ensureVectorBucketExists(config) {
|
|
203
|
+
const exists = await this.storeExists(config.storeName);
|
|
204
|
+
if (!exists) {
|
|
205
|
+
await this.createVectorBucket(config);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Create a new vector bucket
|
|
210
|
+
*/
|
|
211
|
+
async createVectorBucket(config) {
|
|
212
|
+
await this.client.send(
|
|
213
|
+
new import_client_s3vectors2.CreateVectorBucketCommand({
|
|
214
|
+
vectorBucketName: config.storeName,
|
|
215
|
+
encryptionConfiguration: config.encryptionConfiguration
|
|
216
|
+
})
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Validate vector store configuration
|
|
221
|
+
*/
|
|
222
|
+
validateVectorStoreConfig(config) {
|
|
223
|
+
if (!config.storeName || config.storeName.length < 3 || config.storeName.length > 63) {
|
|
224
|
+
throw new Error("Store name must be between 3 and 63 characters");
|
|
225
|
+
}
|
|
226
|
+
if (!/^[a-z0-9-]+$/.test(config.storeName)) {
|
|
227
|
+
throw new Error("Store name must contain only lowercase letters, numbers, and hyphens");
|
|
228
|
+
}
|
|
229
|
+
if (!config.index) {
|
|
230
|
+
throw new Error("Index must be specified");
|
|
231
|
+
}
|
|
232
|
+
this.validateIndexConfig(config.index);
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Validate vector index configuration
|
|
236
|
+
*/
|
|
237
|
+
validateIndexConfig(indexConfig) {
|
|
238
|
+
if (indexConfig.dimension < 1 || indexConfig.dimension > 4096) {
|
|
239
|
+
throw new Error("Vector dimension must be between 1 and 4096");
|
|
240
|
+
}
|
|
241
|
+
if (!["cosine", "euclidean"].includes(indexConfig.distanceMetric)) {
|
|
242
|
+
throw new Error('Distance metric must be either "cosine" or "euclidean"');
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Check if an error indicates a resource was not found
|
|
247
|
+
*/
|
|
248
|
+
isNotFoundError(error) {
|
|
249
|
+
if (error instanceof import_client_s3vectors2.NotFoundException) {
|
|
250
|
+
return true;
|
|
251
|
+
}
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
256
|
+
0 && (module.exports = {
|
|
257
|
+
S3VectorProvider
|
|
258
|
+
});
|
|
259
|
+
//# sourceMappingURL=index.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/providers/S3VectorProvider.ts","../src/providers/S3VectorStore.ts"],"sourcesContent":["/**\n * @orion-js/vectors\n *\n * A package for working with vector operations.\n * Provides interfaces and implementations for vector storage providers.\n */\n\n// Core types and interfaces\nexport type {\n DistanceMetric,\n VectorDataType,\n VectorIndexConfig,\n VectorStoreConfig,\n VectorData,\n VectorQueryParams,\n VectorQueryResult,\n} from './types'\n\n// VectorStore interface and related types\nexport type {VectorStore} from './VectorStore'\n\n// Provider interfaces\nexport type {\n VectorStorageProvider,\n VectorStorageProviderConfig,\n} from './VectorStorageProvider'\n\n// AWS S3 provider\nexport type {S3VectorProviderConfig} from './providers/S3VectorProvider'\nexport {S3VectorProvider} from './providers/S3VectorProvider'\n","import type {VectorStorageProvider, VectorStorageProviderConfig} from '../VectorStorageProvider'\nimport type {VectorStoreConfig, VectorIndexConfig} from '../types'\nimport {\n CreateVectorBucketCommand,\n GetVectorBucketCommand,\n NotFoundException,\n S3VectorsClient,\n S3VectorsClientConfig,\n} from '@aws-sdk/client-s3vectors'\nimport {S3VectorStore} from './S3VectorStore'\n\n/**\n * Configuration for AWS S3 Vector provider\n */\nexport interface S3VectorProviderConfig extends VectorStorageProviderConfig {\n s3Options: S3VectorsClientConfig\n}\n\n/**\n * AWS S3 Vector storage provider implementation\n *\n * This provider implements vector storage using AWS S3's native vector capabilities.\n * It manages vector buckets and indexes according to the S3 Vector API.\n */\nexport class S3VectorProvider implements VectorStorageProvider {\n private config: S3VectorProviderConfig\n private client: S3VectorsClient\n\n constructor(config: S3VectorProviderConfig) {\n this.config = config\n this.validateConfig()\n this.client = new S3VectorsClient(this.config.s3Options)\n }\n\n /**\n * Validate the provider configuration\n */\n private validateConfig(): void {\n if (!this.config.s3Options) {\n throw new Error('S3Options are required for S3VectorProvider')\n }\n }\n\n createVectorStore(config: VectorStoreConfig) {\n this.validateVectorStoreConfig(config)\n\n this.setupVectorStore(config)\n\n // Return a VectorStore instance\n return new S3VectorStore(this.client, config)\n }\n\n private async setupVectorStore(config: VectorStoreConfig): Promise<void> {\n // Ensure the vector bucket exists\n await this.ensureVectorBucketExists(config)\n }\n\n private async storeExists(storeName: string): Promise<boolean> {\n try {\n await this.client.send(\n new GetVectorBucketCommand({\n vectorBucketName: storeName,\n }),\n )\n\n return true\n } catch (error) {\n if (this.isNotFoundError(error)) {\n return false\n }\n throw error\n }\n }\n\n /**\n * Ensure a vector bucket exists, create if it doesn't\n */\n private async ensureVectorBucketExists(config: VectorStoreConfig): Promise<void> {\n const exists = await this.storeExists(config.storeName)\n if (!exists) {\n await this.createVectorBucket(config)\n }\n }\n\n /**\n * Create a new vector bucket\n */\n private async createVectorBucket(config: VectorStoreConfig): Promise<void> {\n await this.client.send(\n new CreateVectorBucketCommand({\n vectorBucketName: config.storeName,\n encryptionConfiguration: config.encryptionConfiguration,\n }),\n )\n }\n\n /**\n * Validate vector store configuration\n */\n private validateVectorStoreConfig(config: VectorStoreConfig): void {\n if (!config.storeName || config.storeName.length < 3 || config.storeName.length > 63) {\n throw new Error('Store name must be between 3 and 63 characters')\n }\n\n if (!/^[a-z0-9-]+$/.test(config.storeName)) {\n throw new Error('Store name must contain only lowercase letters, numbers, and hyphens')\n }\n\n if (!config.index) {\n throw new Error('Index must be specified')\n }\n\n this.validateIndexConfig(config.index)\n }\n\n /**\n * Validate vector index configuration\n */\n private validateIndexConfig(indexConfig: VectorIndexConfig): void {\n if (indexConfig.dimension < 1 || indexConfig.dimension > 4096) {\n throw new Error('Vector dimension must be between 1 and 4096')\n }\n\n if (!['cosine', 'euclidean'].includes(indexConfig.distanceMetric)) {\n throw new Error('Distance metric must be either \"cosine\" or \"euclidean\"')\n }\n }\n\n /**\n * Check if an error indicates a resource was not found\n */\n private isNotFoundError(error: unknown): boolean {\n if (error instanceof NotFoundException) {\n return true\n }\n return false\n }\n}\n","import {\n CreateIndexCommand,\n DeleteIndexCommand,\n DeleteVectorsCommand,\n ListVectorsCommand,\n ListVectorsCommandInput,\n NotFoundException,\n PutVectorsCommand,\n QueryVectorsCommand,\n QueryVectorsCommandInput,\n S3VectorsClient,\n} from '@aws-sdk/client-s3vectors'\nimport {VectorData, VectorIndexConfig, VectorStoreConfig} from '../types'\nimport {VectorStore} from '../VectorStore'\n\n/**\n * S3 Vector Store implementation that provides CRUD operations for a specific vector store\n */\nexport class S3VectorStore<TVectorData extends VectorData = VectorData>\n implements VectorStore<TVectorData>\n{\n private config: VectorStoreConfig\n private client: S3VectorsClient\n\n constructor(client: S3VectorsClient, config: VectorStoreConfig) {\n this.client = client\n this.config = config\n }\n\n private async wrapOperationWithIndexCreation<TOperation extends () => Promise<any>>(\n namespace: string,\n operation: TOperation,\n ): Promise<Awaited<ReturnType<TOperation>>> {\n try {\n return await operation()\n } catch (error) {\n if (error instanceof NotFoundException) {\n if (error.message === 'The specified index could not be found') {\n await this.createVectorIndex(namespace, this.config, this.config.index)\n return this.wrapOperationWithIndexCreation(namespace, operation)\n }\n }\n throw error\n }\n }\n\n async insertVectors(namespace: string, vectors: TVectorData[]) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new PutVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n vectors: vectors.map(vector => ({\n key: vector.key,\n data: {\n float32: vector.vector,\n },\n metadata: vector.metadata,\n })),\n }),\n )\n\n return result\n })\n }\n\n async queryVectors(\n namespace: string,\n options?: Omit<QueryVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'queryVector'> & {\n queryVector: number[]\n },\n ) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new QueryVectorsCommand({\n ...options,\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n queryVector: {\n float32: options.queryVector,\n },\n }),\n )\n\n const vectors: TVectorData[] =\n result.vectors?.map(\n vector =>\n ({\n key: vector.key,\n vector: vector.data?.float32,\n metadata: vector.metadata,\n }) as TVectorData,\n ) ?? []\n\n return {\n vectors,\n }\n })\n }\n\n async deleteVectors(namespace: string, keys: string[]) {\n return await this.client.send(\n new DeleteVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n keys,\n }),\n )\n }\n\n async listVectors(\n namespace: string,\n options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName'>,\n ) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new ListVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n ...options,\n }),\n )\n\n return (result.vectors?.map(vector => ({\n key: vector.key,\n vector: vector.data?.float32,\n metadata: vector.metadata as TVectorData['metadata'],\n })) ?? []) as Partial<TVectorData>[]\n })\n }\n\n async listVectorKeys(\n namespace: string,\n options?: Omit<\n ListVectorsCommandInput,\n 'vectorBucketName' | 'indexName' | 'returnData' | 'returnMetadata'\n >,\n ) {\n const result = await this.listVectors(namespace, {\n ...options,\n returnData: false,\n returnMetadata: false,\n })\n return result.map(vector => vector.key)\n }\n\n /**\n * Create a new vector index\n */\n private async createVectorIndex(\n namespace: string,\n config: VectorStoreConfig,\n indexConfig: VectorIndexConfig,\n ): Promise<void> {\n await this.client.send(\n new CreateIndexCommand({\n indexName: namespace,\n dataType: 'float32',\n dimension: indexConfig.dimension,\n distanceMetric: indexConfig.distanceMetric,\n vectorBucketName: config.storeName,\n metadataConfiguration: indexConfig.metadataConfiguration,\n }),\n )\n }\n\n async deleteNamespace(namespace: string) {\n return await this.client.send(\n new DeleteIndexCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n }),\n )\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACEA,IAAAA,2BAMO;;;ACRP,8BAWO;AAOA,IAAM,gBAAN,MAEP;AAAA,EACU;AAAA,EACA;AAAA,EAER,YAAY,QAAyB,QAA2B;AAC9D,SAAK,SAAS;AACd,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,+BACZ,WACA,WAC0C;AAC1C,QAAI;AACF,aAAO,MAAM,UAAU;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,2CAAmB;AACtC,YAAI,MAAM,YAAY,0CAA0C;AAC9D,gBAAM,KAAK,kBAAkB,WAAW,KAAK,QAAQ,KAAK,OAAO,KAAK;AACtE,iBAAO,KAAK,+BAA+B,WAAW,SAAS;AAAA,QACjE;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,WAAmB,SAAwB;AAC7D,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AACtE,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,0CAAkB;AAAA,UACpB,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,SAAS,QAAQ,IAAI,aAAW;AAAA,YAC9B,KAAK,OAAO;AAAA,YACZ,MAAM;AAAA,cACJ,SAAS,OAAO;AAAA,YAClB;AAAA,YACA,UAAU,OAAO;AAAA,UACnB,EAAE;AAAA,QACJ,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aACJ,WACA,SAGA;AACA,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AAxE5E;AAyEM,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,4CAAoB;AAAA,UACtB,GAAG;AAAA,UACH,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,aAAa;AAAA,YACX,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,YACJ,YAAO,YAAP,mBAAgB;AAAA,QACd,YAAO;AAtFjB,cAAAC;AAuFa;AAAA,YACC,KAAK,OAAO;AAAA,YACZ,SAAQA,MAAA,OAAO,SAAP,gBAAAA,IAAa;AAAA,YACrB,UAAU,OAAO;AAAA,UACnB;AAAA;AAAA,YACC,CAAC;AAER,aAAO;AAAA,QACL;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,cAAc,WAAmB,MAAgB;AACrD,WAAO,MAAM,KAAK,OAAO;AAAA,MACvB,IAAI,6CAAqB;AAAA,QACvB,kBAAkB,KAAK,OAAO;AAAA,QAC9B,WAAW;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YACJ,WACA,SACA;AACA,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AAlH5E;AAmHM,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,2CAAmB;AAAA,UACrB,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,GAAG;AAAA,QACL,CAAC;AAAA,MACH;AAEA,eAAQ,YAAO,YAAP,mBAAgB,IAAI,YAAO;AA3HzC,YAAAA;AA2H6C;AAAA,UACrC,KAAK,OAAO;AAAA,UACZ,SAAQA,MAAA,OAAO,SAAP,gBAAAA,IAAa;AAAA,UACrB,UAAU,OAAO;AAAA,QACnB;AAAA,aAAO,CAAC;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,eACJ,WACA,SAIA;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,WAAW;AAAA,MAC/C,GAAG;AAAA,MACH,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB,CAAC;AACD,WAAO,OAAO,IAAI,YAAU,OAAO,GAAG;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBACZ,WACA,QACA,aACe;AACf,UAAM,KAAK,OAAO;AAAA,MAChB,IAAI,2CAAmB;AAAA,QACrB,WAAW;AAAA,QACX,UAAU;AAAA,QACV,WAAW,YAAY;AAAA,QACvB,gBAAgB,YAAY;AAAA,QAC5B,kBAAkB,OAAO;AAAA,QACzB,uBAAuB,YAAY;AAAA,MACrC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB,WAAmB;AACvC,WAAO,MAAM,KAAK,OAAO;AAAA,MACvB,IAAI,2CAAmB;AAAA,QACrB,kBAAkB,KAAK,OAAO;AAAA,QAC9B,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ADtJO,IAAM,mBAAN,MAAwD;AAAA,EACrD;AAAA,EACA;AAAA,EAER,YAAY,QAAgC;AAC1C,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,SAAS,IAAI,yCAAgB,KAAK,OAAO,SAAS;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA,EAKQ,iBAAuB;AAC7B,QAAI,CAAC,KAAK,OAAO,WAAW;AAC1B,YAAM,IAAI,MAAM,6CAA6C;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,kBAAkB,QAA2B;AAC3C,SAAK,0BAA0B,MAAM;AAErC,SAAK,iBAAiB,MAAM;AAG5B,WAAO,IAAI,cAAc,KAAK,QAAQ,MAAM;AAAA,EAC9C;AAAA,EAEA,MAAc,iBAAiB,QAA0C;AAEvE,UAAM,KAAK,yBAAyB,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAc,YAAY,WAAqC;AAC7D,QAAI;AACF,YAAM,KAAK,OAAO;AAAA,QAChB,IAAI,gDAAuB;AAAA,UACzB,kBAAkB;AAAA,QACpB,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,UAAI,KAAK,gBAAgB,KAAK,GAAG;AAC/B,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,yBAAyB,QAA0C;AAC/E,UAAM,SAAS,MAAM,KAAK,YAAY,OAAO,SAAS;AACtD,QAAI,CAAC,QAAQ;AACX,YAAM,KAAK,mBAAmB,MAAM;AAAA,IACtC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,mBAAmB,QAA0C;AACzE,UAAM,KAAK,OAAO;AAAA,MAChB,IAAI,mDAA0B;AAAA,QAC5B,kBAAkB,OAAO;AAAA,QACzB,yBAAyB,OAAO;AAAA,MAClC,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,0BAA0B,QAAiC;AACjE,QAAI,CAAC,OAAO,aAAa,OAAO,UAAU,SAAS,KAAK,OAAO,UAAU,SAAS,IAAI;AACpF,YAAM,IAAI,MAAM,gDAAgD;AAAA,IAClE;AAEA,QAAI,CAAC,eAAe,KAAK,OAAO,SAAS,GAAG;AAC1C,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,QAAI,CAAC,OAAO,OAAO;AACjB,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AAEA,SAAK,oBAAoB,OAAO,KAAK;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKQ,oBAAoB,aAAsC;AAChE,QAAI,YAAY,YAAY,KAAK,YAAY,YAAY,MAAM;AAC7D,YAAM,IAAI,MAAM,6CAA6C;AAAA,IAC/D;AAEA,QAAI,CAAC,CAAC,UAAU,WAAW,EAAE,SAAS,YAAY,cAAc,GAAG;AACjE,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,OAAyB;AAC/C,QAAI,iBAAiB,4CAAmB;AACtC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AACF;","names":["import_client_s3vectors","_a"]}
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import * as _aws_sdk_client_s3vectors from '@aws-sdk/client-s3vectors';
|
|
2
|
+
import { MetadataConfiguration, EncryptionConfiguration, S3VectorsClient, QueryVectorsCommandInput, ListVectorsCommandInput, S3VectorsClientConfig } from '@aws-sdk/client-s3vectors';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Distance metrics for vector similarity calculations
|
|
6
|
+
*/
|
|
7
|
+
type DistanceMetric = 'cosine' | 'euclidean';
|
|
8
|
+
/**
|
|
9
|
+
* Vector data types supported
|
|
10
|
+
*/
|
|
11
|
+
type VectorDataType = 'float32';
|
|
12
|
+
/**
|
|
13
|
+
* Configuration for a vector index
|
|
14
|
+
*/
|
|
15
|
+
interface VectorIndexConfig {
|
|
16
|
+
/**
|
|
17
|
+
* Number of dimensions in each vector (1-4096)
|
|
18
|
+
*/
|
|
19
|
+
dimension: number;
|
|
20
|
+
/**
|
|
21
|
+
* Distance metric for similarity calculations
|
|
22
|
+
*/
|
|
23
|
+
distanceMetric: DistanceMetric;
|
|
24
|
+
/**
|
|
25
|
+
* Metadata configuration for the index
|
|
26
|
+
*/
|
|
27
|
+
metadataConfiguration?: MetadataConfiguration;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Configuration for defining a vector store
|
|
31
|
+
*/
|
|
32
|
+
interface VectorStoreConfig {
|
|
33
|
+
/**
|
|
34
|
+
* Name of the vector store/bucket (3-63 characters, lowercase letters, numbers, hyphens)
|
|
35
|
+
*/
|
|
36
|
+
storeName: string;
|
|
37
|
+
/**
|
|
38
|
+
* Array of vector indexes to create in this store
|
|
39
|
+
*/
|
|
40
|
+
index: VectorIndexConfig;
|
|
41
|
+
/**
|
|
42
|
+
* Optional encryption configuration
|
|
43
|
+
*/
|
|
44
|
+
encryptionConfiguration?: EncryptionConfiguration;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Vector data for insertion
|
|
48
|
+
*/
|
|
49
|
+
interface VectorData<TMetadata extends Record<string, string | number | boolean> = Record<string, string | number | boolean>> {
|
|
50
|
+
/** Unique identifier for the vector */
|
|
51
|
+
key: string;
|
|
52
|
+
/**
|
|
53
|
+
* The vector data of the vector.
|
|
54
|
+
* Vector dimensions must match the dimension count that's configured for the vector index.
|
|
55
|
+
* For the cosine distance metric, zero vectors (vectors containing all zeros) aren't allowed.
|
|
56
|
+
* For both cosine and euclidean distance metrics, vector data must contain only valid floating-point values. Invalid values such as NaN (Not a Number) or Infinity aren't allowed.
|
|
57
|
+
*/
|
|
58
|
+
vector: number[];
|
|
59
|
+
/** Filterable metadata attached to the vector */
|
|
60
|
+
metadata?: TMetadata;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Query parameters for vector similarity search
|
|
64
|
+
*/
|
|
65
|
+
interface VectorQueryParams {
|
|
66
|
+
/** The query vector to search for similar vectors */
|
|
67
|
+
queryVector: {
|
|
68
|
+
[K in VectorDataType]: number[];
|
|
69
|
+
};
|
|
70
|
+
/** Number of top similar vectors to return */
|
|
71
|
+
topK: number;
|
|
72
|
+
/** Optional metadata filters to narrow search results */
|
|
73
|
+
filter?: Record<string, string | number | boolean>;
|
|
74
|
+
/** Whether to return distance scores */
|
|
75
|
+
returnDistance?: boolean;
|
|
76
|
+
/** Whether to return metadata */
|
|
77
|
+
returnMetadata?: boolean;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Result from a vector query
|
|
81
|
+
*/
|
|
82
|
+
interface VectorQueryResult {
|
|
83
|
+
/** The vector key */
|
|
84
|
+
key: string;
|
|
85
|
+
/** Distance score (if requested) */
|
|
86
|
+
distance?: number;
|
|
87
|
+
/** Vector metadata (if requested) */
|
|
88
|
+
metadata?: Record<string, string | number | boolean>;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Interface for vector store operations providing CRUD functionality
|
|
93
|
+
*
|
|
94
|
+
* This interface defines the contract for vector storage implementations,
|
|
95
|
+
* providing methods for creating, reading, updating, and deleting vectors
|
|
96
|
+
* within a vector store.
|
|
97
|
+
*/
|
|
98
|
+
interface VectorStore<TVectorData extends VectorData = VectorData> {
|
|
99
|
+
insertVectors(namespace: string, vectors: TVectorData[], options?: any): Promise<any>;
|
|
100
|
+
deleteVectors(namespace: string, keys: string[], options?: any): Promise<any>;
|
|
101
|
+
listVectors(namespace: string, options?: any): Promise<Partial<TVectorData>[]>;
|
|
102
|
+
listVectorKeys(namespace: string, options?: any): Promise<string[]>;
|
|
103
|
+
deleteNamespace(namespace: string): Promise<any>;
|
|
104
|
+
queryVectors(namespace: string, options?: any): Promise<{
|
|
105
|
+
vectors: TVectorData[];
|
|
106
|
+
} & Record<string, any>>;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Abstract interface for vector storage providers
|
|
111
|
+
*
|
|
112
|
+
* This interface defines the contract that all vector storage implementations must follow.
|
|
113
|
+
* It provides methods for managing vector stores, indexes, and performing vector operations.
|
|
114
|
+
*/
|
|
115
|
+
interface VectorStorageProvider {
|
|
116
|
+
/**
|
|
117
|
+
* Create and ensure a vector store exists with the specified configuration.
|
|
118
|
+
* This method will:
|
|
119
|
+
* 1. Create the vector store/bucket if it doesn't exist
|
|
120
|
+
* 2. Create all specified indexes if they don't exist
|
|
121
|
+
* 3. Validate the configuration
|
|
122
|
+
*
|
|
123
|
+
* @param config - Configuration for the vector store and its indexes
|
|
124
|
+
* @returns Promise that resolves to a VectorStore instance when setup is complete
|
|
125
|
+
*/
|
|
126
|
+
createVectorStore(config: VectorStoreConfig): VectorStore;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Base configuration for vector storage providers
|
|
130
|
+
*/
|
|
131
|
+
interface VectorStorageProviderConfig {
|
|
132
|
+
/** Provider-specific configuration */
|
|
133
|
+
[key: string]: unknown;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* S3 Vector Store implementation that provides CRUD operations for a specific vector store
|
|
138
|
+
*/
|
|
139
|
+
declare class S3VectorStore<TVectorData extends VectorData = VectorData> implements VectorStore<TVectorData> {
|
|
140
|
+
private config;
|
|
141
|
+
private client;
|
|
142
|
+
constructor(client: S3VectorsClient, config: VectorStoreConfig);
|
|
143
|
+
private wrapOperationWithIndexCreation;
|
|
144
|
+
insertVectors(namespace: string, vectors: TVectorData[]): Promise<_aws_sdk_client_s3vectors.PutVectorsCommandOutput>;
|
|
145
|
+
queryVectors(namespace: string, options?: Omit<QueryVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'queryVector'> & {
|
|
146
|
+
queryVector: number[];
|
|
147
|
+
}): Promise<{
|
|
148
|
+
vectors: TVectorData[];
|
|
149
|
+
}>;
|
|
150
|
+
deleteVectors(namespace: string, keys: string[]): Promise<_aws_sdk_client_s3vectors.DeleteVectorsCommandOutput>;
|
|
151
|
+
listVectors(namespace: string, options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName'>): Promise<Partial<TVectorData>[]>;
|
|
152
|
+
listVectorKeys(namespace: string, options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'returnData' | 'returnMetadata'>): Promise<TVectorData["key"][]>;
|
|
153
|
+
/**
|
|
154
|
+
* Create a new vector index
|
|
155
|
+
*/
|
|
156
|
+
private createVectorIndex;
|
|
157
|
+
deleteNamespace(namespace: string): Promise<_aws_sdk_client_s3vectors.DeleteIndexCommandOutput>;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Configuration for AWS S3 Vector provider
|
|
162
|
+
*/
|
|
163
|
+
interface S3VectorProviderConfig extends VectorStorageProviderConfig {
|
|
164
|
+
s3Options: S3VectorsClientConfig;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* AWS S3 Vector storage provider implementation
|
|
168
|
+
*
|
|
169
|
+
* This provider implements vector storage using AWS S3's native vector capabilities.
|
|
170
|
+
* It manages vector buckets and indexes according to the S3 Vector API.
|
|
171
|
+
*/
|
|
172
|
+
declare class S3VectorProvider implements VectorStorageProvider {
|
|
173
|
+
private config;
|
|
174
|
+
private client;
|
|
175
|
+
constructor(config: S3VectorProviderConfig);
|
|
176
|
+
/**
|
|
177
|
+
* Validate the provider configuration
|
|
178
|
+
*/
|
|
179
|
+
private validateConfig;
|
|
180
|
+
createVectorStore(config: VectorStoreConfig): S3VectorStore<VectorData<Record<string, string | number | boolean>>>;
|
|
181
|
+
private setupVectorStore;
|
|
182
|
+
private storeExists;
|
|
183
|
+
/**
|
|
184
|
+
* Ensure a vector bucket exists, create if it doesn't
|
|
185
|
+
*/
|
|
186
|
+
private ensureVectorBucketExists;
|
|
187
|
+
/**
|
|
188
|
+
* Create a new vector bucket
|
|
189
|
+
*/
|
|
190
|
+
private createVectorBucket;
|
|
191
|
+
/**
|
|
192
|
+
* Validate vector store configuration
|
|
193
|
+
*/
|
|
194
|
+
private validateVectorStoreConfig;
|
|
195
|
+
/**
|
|
196
|
+
* Validate vector index configuration
|
|
197
|
+
*/
|
|
198
|
+
private validateIndexConfig;
|
|
199
|
+
/**
|
|
200
|
+
* Check if an error indicates a resource was not found
|
|
201
|
+
*/
|
|
202
|
+
private isNotFoundError;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export { type DistanceMetric, S3VectorProvider, type S3VectorProviderConfig, type VectorData, type VectorDataType, type VectorIndexConfig, type VectorQueryParams, type VectorQueryResult, type VectorStorageProvider, type VectorStorageProviderConfig, type VectorStore, type VectorStoreConfig };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import * as _aws_sdk_client_s3vectors from '@aws-sdk/client-s3vectors';
|
|
2
|
+
import { MetadataConfiguration, EncryptionConfiguration, S3VectorsClient, QueryVectorsCommandInput, ListVectorsCommandInput, S3VectorsClientConfig } from '@aws-sdk/client-s3vectors';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Distance metrics for vector similarity calculations
|
|
6
|
+
*/
|
|
7
|
+
type DistanceMetric = 'cosine' | 'euclidean';
|
|
8
|
+
/**
|
|
9
|
+
* Vector data types supported
|
|
10
|
+
*/
|
|
11
|
+
type VectorDataType = 'float32';
|
|
12
|
+
/**
|
|
13
|
+
* Configuration for a vector index
|
|
14
|
+
*/
|
|
15
|
+
interface VectorIndexConfig {
|
|
16
|
+
/**
|
|
17
|
+
* Number of dimensions in each vector (1-4096)
|
|
18
|
+
*/
|
|
19
|
+
dimension: number;
|
|
20
|
+
/**
|
|
21
|
+
* Distance metric for similarity calculations
|
|
22
|
+
*/
|
|
23
|
+
distanceMetric: DistanceMetric;
|
|
24
|
+
/**
|
|
25
|
+
* Metadata configuration for the index
|
|
26
|
+
*/
|
|
27
|
+
metadataConfiguration?: MetadataConfiguration;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Configuration for defining a vector store
|
|
31
|
+
*/
|
|
32
|
+
interface VectorStoreConfig {
|
|
33
|
+
/**
|
|
34
|
+
* Name of the vector store/bucket (3-63 characters, lowercase letters, numbers, hyphens)
|
|
35
|
+
*/
|
|
36
|
+
storeName: string;
|
|
37
|
+
/**
|
|
38
|
+
* Array of vector indexes to create in this store
|
|
39
|
+
*/
|
|
40
|
+
index: VectorIndexConfig;
|
|
41
|
+
/**
|
|
42
|
+
* Optional encryption configuration
|
|
43
|
+
*/
|
|
44
|
+
encryptionConfiguration?: EncryptionConfiguration;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Vector data for insertion
|
|
48
|
+
*/
|
|
49
|
+
interface VectorData<TMetadata extends Record<string, string | number | boolean> = Record<string, string | number | boolean>> {
|
|
50
|
+
/** Unique identifier for the vector */
|
|
51
|
+
key: string;
|
|
52
|
+
/**
|
|
53
|
+
* The vector data of the vector.
|
|
54
|
+
* Vector dimensions must match the dimension count that's configured for the vector index.
|
|
55
|
+
* For the cosine distance metric, zero vectors (vectors containing all zeros) aren't allowed.
|
|
56
|
+
* For both cosine and euclidean distance metrics, vector data must contain only valid floating-point values. Invalid values such as NaN (Not a Number) or Infinity aren't allowed.
|
|
57
|
+
*/
|
|
58
|
+
vector: number[];
|
|
59
|
+
/** Filterable metadata attached to the vector */
|
|
60
|
+
metadata?: TMetadata;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Query parameters for vector similarity search
|
|
64
|
+
*/
|
|
65
|
+
interface VectorQueryParams {
|
|
66
|
+
/** The query vector to search for similar vectors */
|
|
67
|
+
queryVector: {
|
|
68
|
+
[K in VectorDataType]: number[];
|
|
69
|
+
};
|
|
70
|
+
/** Number of top similar vectors to return */
|
|
71
|
+
topK: number;
|
|
72
|
+
/** Optional metadata filters to narrow search results */
|
|
73
|
+
filter?: Record<string, string | number | boolean>;
|
|
74
|
+
/** Whether to return distance scores */
|
|
75
|
+
returnDistance?: boolean;
|
|
76
|
+
/** Whether to return metadata */
|
|
77
|
+
returnMetadata?: boolean;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Result from a vector query
|
|
81
|
+
*/
|
|
82
|
+
interface VectorQueryResult {
|
|
83
|
+
/** The vector key */
|
|
84
|
+
key: string;
|
|
85
|
+
/** Distance score (if requested) */
|
|
86
|
+
distance?: number;
|
|
87
|
+
/** Vector metadata (if requested) */
|
|
88
|
+
metadata?: Record<string, string | number | boolean>;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Interface for vector store operations providing CRUD functionality
|
|
93
|
+
*
|
|
94
|
+
* This interface defines the contract for vector storage implementations,
|
|
95
|
+
* providing methods for creating, reading, updating, and deleting vectors
|
|
96
|
+
* within a vector store.
|
|
97
|
+
*/
|
|
98
|
+
interface VectorStore<TVectorData extends VectorData = VectorData> {
|
|
99
|
+
insertVectors(namespace: string, vectors: TVectorData[], options?: any): Promise<any>;
|
|
100
|
+
deleteVectors(namespace: string, keys: string[], options?: any): Promise<any>;
|
|
101
|
+
listVectors(namespace: string, options?: any): Promise<Partial<TVectorData>[]>;
|
|
102
|
+
listVectorKeys(namespace: string, options?: any): Promise<string[]>;
|
|
103
|
+
deleteNamespace(namespace: string): Promise<any>;
|
|
104
|
+
queryVectors(namespace: string, options?: any): Promise<{
|
|
105
|
+
vectors: TVectorData[];
|
|
106
|
+
} & Record<string, any>>;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Abstract interface for vector storage providers
|
|
111
|
+
*
|
|
112
|
+
* This interface defines the contract that all vector storage implementations must follow.
|
|
113
|
+
* It provides methods for managing vector stores, indexes, and performing vector operations.
|
|
114
|
+
*/
|
|
115
|
+
interface VectorStorageProvider {
|
|
116
|
+
/**
|
|
117
|
+
* Create and ensure a vector store exists with the specified configuration.
|
|
118
|
+
* This method will:
|
|
119
|
+
* 1. Create the vector store/bucket if it doesn't exist
|
|
120
|
+
* 2. Create all specified indexes if they don't exist
|
|
121
|
+
* 3. Validate the configuration
|
|
122
|
+
*
|
|
123
|
+
* @param config - Configuration for the vector store and its indexes
|
|
124
|
+
* @returns Promise that resolves to a VectorStore instance when setup is complete
|
|
125
|
+
*/
|
|
126
|
+
createVectorStore(config: VectorStoreConfig): VectorStore;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Base configuration for vector storage providers
|
|
130
|
+
*/
|
|
131
|
+
interface VectorStorageProviderConfig {
|
|
132
|
+
/** Provider-specific configuration */
|
|
133
|
+
[key: string]: unknown;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* S3 Vector Store implementation that provides CRUD operations for a specific vector store
|
|
138
|
+
*/
|
|
139
|
+
declare class S3VectorStore<TVectorData extends VectorData = VectorData> implements VectorStore<TVectorData> {
|
|
140
|
+
private config;
|
|
141
|
+
private client;
|
|
142
|
+
constructor(client: S3VectorsClient, config: VectorStoreConfig);
|
|
143
|
+
private wrapOperationWithIndexCreation;
|
|
144
|
+
insertVectors(namespace: string, vectors: TVectorData[]): Promise<_aws_sdk_client_s3vectors.PutVectorsCommandOutput>;
|
|
145
|
+
queryVectors(namespace: string, options?: Omit<QueryVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'queryVector'> & {
|
|
146
|
+
queryVector: number[];
|
|
147
|
+
}): Promise<{
|
|
148
|
+
vectors: TVectorData[];
|
|
149
|
+
}>;
|
|
150
|
+
deleteVectors(namespace: string, keys: string[]): Promise<_aws_sdk_client_s3vectors.DeleteVectorsCommandOutput>;
|
|
151
|
+
listVectors(namespace: string, options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName'>): Promise<Partial<TVectorData>[]>;
|
|
152
|
+
listVectorKeys(namespace: string, options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'returnData' | 'returnMetadata'>): Promise<TVectorData["key"][]>;
|
|
153
|
+
/**
|
|
154
|
+
* Create a new vector index
|
|
155
|
+
*/
|
|
156
|
+
private createVectorIndex;
|
|
157
|
+
deleteNamespace(namespace: string): Promise<_aws_sdk_client_s3vectors.DeleteIndexCommandOutput>;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Configuration for AWS S3 Vector provider
|
|
162
|
+
*/
|
|
163
|
+
interface S3VectorProviderConfig extends VectorStorageProviderConfig {
|
|
164
|
+
s3Options: S3VectorsClientConfig;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* AWS S3 Vector storage provider implementation
|
|
168
|
+
*
|
|
169
|
+
* This provider implements vector storage using AWS S3's native vector capabilities.
|
|
170
|
+
* It manages vector buckets and indexes according to the S3 Vector API.
|
|
171
|
+
*/
|
|
172
|
+
declare class S3VectorProvider implements VectorStorageProvider {
|
|
173
|
+
private config;
|
|
174
|
+
private client;
|
|
175
|
+
constructor(config: S3VectorProviderConfig);
|
|
176
|
+
/**
|
|
177
|
+
* Validate the provider configuration
|
|
178
|
+
*/
|
|
179
|
+
private validateConfig;
|
|
180
|
+
createVectorStore(config: VectorStoreConfig): S3VectorStore<VectorData<Record<string, string | number | boolean>>>;
|
|
181
|
+
private setupVectorStore;
|
|
182
|
+
private storeExists;
|
|
183
|
+
/**
|
|
184
|
+
* Ensure a vector bucket exists, create if it doesn't
|
|
185
|
+
*/
|
|
186
|
+
private ensureVectorBucketExists;
|
|
187
|
+
/**
|
|
188
|
+
* Create a new vector bucket
|
|
189
|
+
*/
|
|
190
|
+
private createVectorBucket;
|
|
191
|
+
/**
|
|
192
|
+
* Validate vector store configuration
|
|
193
|
+
*/
|
|
194
|
+
private validateVectorStoreConfig;
|
|
195
|
+
/**
|
|
196
|
+
* Validate vector index configuration
|
|
197
|
+
*/
|
|
198
|
+
private validateIndexConfig;
|
|
199
|
+
/**
|
|
200
|
+
* Check if an error indicates a resource was not found
|
|
201
|
+
*/
|
|
202
|
+
private isNotFoundError;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export { type DistanceMetric, S3VectorProvider, type S3VectorProviderConfig, type VectorData, type VectorDataType, type VectorIndexConfig, type VectorQueryParams, type VectorQueryResult, type VectorStorageProvider, type VectorStorageProviderConfig, type VectorStore, type VectorStoreConfig };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
// src/providers/S3VectorProvider.ts
|
|
2
|
+
import {
|
|
3
|
+
CreateVectorBucketCommand,
|
|
4
|
+
GetVectorBucketCommand,
|
|
5
|
+
NotFoundException as NotFoundException2,
|
|
6
|
+
S3VectorsClient as S3VectorsClient2
|
|
7
|
+
} from "@aws-sdk/client-s3vectors";
|
|
8
|
+
|
|
9
|
+
// src/providers/S3VectorStore.ts
|
|
10
|
+
import {
|
|
11
|
+
CreateIndexCommand,
|
|
12
|
+
DeleteIndexCommand,
|
|
13
|
+
DeleteVectorsCommand,
|
|
14
|
+
ListVectorsCommand,
|
|
15
|
+
NotFoundException,
|
|
16
|
+
PutVectorsCommand,
|
|
17
|
+
QueryVectorsCommand
|
|
18
|
+
} from "@aws-sdk/client-s3vectors";
|
|
19
|
+
var S3VectorStore = class {
|
|
20
|
+
config;
|
|
21
|
+
client;
|
|
22
|
+
constructor(client, config) {
|
|
23
|
+
this.client = client;
|
|
24
|
+
this.config = config;
|
|
25
|
+
}
|
|
26
|
+
async wrapOperationWithIndexCreation(namespace, operation) {
|
|
27
|
+
try {
|
|
28
|
+
return await operation();
|
|
29
|
+
} catch (error) {
|
|
30
|
+
if (error instanceof NotFoundException) {
|
|
31
|
+
if (error.message === "The specified index could not be found") {
|
|
32
|
+
await this.createVectorIndex(namespace, this.config, this.config.index);
|
|
33
|
+
return this.wrapOperationWithIndexCreation(namespace, operation);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
async insertVectors(namespace, vectors) {
|
|
40
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
41
|
+
const result = await this.client.send(
|
|
42
|
+
new PutVectorsCommand({
|
|
43
|
+
vectorBucketName: this.config.storeName,
|
|
44
|
+
indexName: namespace,
|
|
45
|
+
vectors: vectors.map((vector) => ({
|
|
46
|
+
key: vector.key,
|
|
47
|
+
data: {
|
|
48
|
+
float32: vector.vector
|
|
49
|
+
},
|
|
50
|
+
metadata: vector.metadata
|
|
51
|
+
}))
|
|
52
|
+
})
|
|
53
|
+
);
|
|
54
|
+
return result;
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async queryVectors(namespace, options) {
|
|
58
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
59
|
+
var _a;
|
|
60
|
+
const result = await this.client.send(
|
|
61
|
+
new QueryVectorsCommand({
|
|
62
|
+
...options,
|
|
63
|
+
vectorBucketName: this.config.storeName,
|
|
64
|
+
indexName: namespace,
|
|
65
|
+
queryVector: {
|
|
66
|
+
float32: options.queryVector
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
);
|
|
70
|
+
const vectors = ((_a = result.vectors) == null ? void 0 : _a.map(
|
|
71
|
+
(vector) => {
|
|
72
|
+
var _a2;
|
|
73
|
+
return {
|
|
74
|
+
key: vector.key,
|
|
75
|
+
vector: (_a2 = vector.data) == null ? void 0 : _a2.float32,
|
|
76
|
+
metadata: vector.metadata
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
)) ?? [];
|
|
80
|
+
return {
|
|
81
|
+
vectors
|
|
82
|
+
};
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
async deleteVectors(namespace, keys) {
|
|
86
|
+
return await this.client.send(
|
|
87
|
+
new DeleteVectorsCommand({
|
|
88
|
+
vectorBucketName: this.config.storeName,
|
|
89
|
+
indexName: namespace,
|
|
90
|
+
keys
|
|
91
|
+
})
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
async listVectors(namespace, options) {
|
|
95
|
+
return await this.wrapOperationWithIndexCreation(namespace, async () => {
|
|
96
|
+
var _a;
|
|
97
|
+
const result = await this.client.send(
|
|
98
|
+
new ListVectorsCommand({
|
|
99
|
+
vectorBucketName: this.config.storeName,
|
|
100
|
+
indexName: namespace,
|
|
101
|
+
...options
|
|
102
|
+
})
|
|
103
|
+
);
|
|
104
|
+
return ((_a = result.vectors) == null ? void 0 : _a.map((vector) => {
|
|
105
|
+
var _a2;
|
|
106
|
+
return {
|
|
107
|
+
key: vector.key,
|
|
108
|
+
vector: (_a2 = vector.data) == null ? void 0 : _a2.float32,
|
|
109
|
+
metadata: vector.metadata
|
|
110
|
+
};
|
|
111
|
+
})) ?? [];
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
async listVectorKeys(namespace, options) {
|
|
115
|
+
const result = await this.listVectors(namespace, {
|
|
116
|
+
...options,
|
|
117
|
+
returnData: false,
|
|
118
|
+
returnMetadata: false
|
|
119
|
+
});
|
|
120
|
+
return result.map((vector) => vector.key);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Create a new vector index
|
|
124
|
+
*/
|
|
125
|
+
async createVectorIndex(namespace, config, indexConfig) {
|
|
126
|
+
await this.client.send(
|
|
127
|
+
new CreateIndexCommand({
|
|
128
|
+
indexName: namespace,
|
|
129
|
+
dataType: "float32",
|
|
130
|
+
dimension: indexConfig.dimension,
|
|
131
|
+
distanceMetric: indexConfig.distanceMetric,
|
|
132
|
+
vectorBucketName: config.storeName,
|
|
133
|
+
metadataConfiguration: indexConfig.metadataConfiguration
|
|
134
|
+
})
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
async deleteNamespace(namespace) {
|
|
138
|
+
return await this.client.send(
|
|
139
|
+
new DeleteIndexCommand({
|
|
140
|
+
vectorBucketName: this.config.storeName,
|
|
141
|
+
indexName: namespace
|
|
142
|
+
})
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
// src/providers/S3VectorProvider.ts
|
|
148
|
+
var S3VectorProvider = class {
|
|
149
|
+
config;
|
|
150
|
+
client;
|
|
151
|
+
constructor(config) {
|
|
152
|
+
this.config = config;
|
|
153
|
+
this.validateConfig();
|
|
154
|
+
this.client = new S3VectorsClient2(this.config.s3Options);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Validate the provider configuration
|
|
158
|
+
*/
|
|
159
|
+
validateConfig() {
|
|
160
|
+
if (!this.config.s3Options) {
|
|
161
|
+
throw new Error("S3Options are required for S3VectorProvider");
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
createVectorStore(config) {
|
|
165
|
+
this.validateVectorStoreConfig(config);
|
|
166
|
+
this.setupVectorStore(config);
|
|
167
|
+
return new S3VectorStore(this.client, config);
|
|
168
|
+
}
|
|
169
|
+
async setupVectorStore(config) {
|
|
170
|
+
await this.ensureVectorBucketExists(config);
|
|
171
|
+
}
|
|
172
|
+
async storeExists(storeName) {
|
|
173
|
+
try {
|
|
174
|
+
await this.client.send(
|
|
175
|
+
new GetVectorBucketCommand({
|
|
176
|
+
vectorBucketName: storeName
|
|
177
|
+
})
|
|
178
|
+
);
|
|
179
|
+
return true;
|
|
180
|
+
} catch (error) {
|
|
181
|
+
if (this.isNotFoundError(error)) {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
throw error;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Ensure a vector bucket exists, create if it doesn't
|
|
189
|
+
*/
|
|
190
|
+
async ensureVectorBucketExists(config) {
|
|
191
|
+
const exists = await this.storeExists(config.storeName);
|
|
192
|
+
if (!exists) {
|
|
193
|
+
await this.createVectorBucket(config);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Create a new vector bucket
|
|
198
|
+
*/
|
|
199
|
+
async createVectorBucket(config) {
|
|
200
|
+
await this.client.send(
|
|
201
|
+
new CreateVectorBucketCommand({
|
|
202
|
+
vectorBucketName: config.storeName,
|
|
203
|
+
encryptionConfiguration: config.encryptionConfiguration
|
|
204
|
+
})
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Validate vector store configuration
|
|
209
|
+
*/
|
|
210
|
+
validateVectorStoreConfig(config) {
|
|
211
|
+
if (!config.storeName || config.storeName.length < 3 || config.storeName.length > 63) {
|
|
212
|
+
throw new Error("Store name must be between 3 and 63 characters");
|
|
213
|
+
}
|
|
214
|
+
if (!/^[a-z0-9-]+$/.test(config.storeName)) {
|
|
215
|
+
throw new Error("Store name must contain only lowercase letters, numbers, and hyphens");
|
|
216
|
+
}
|
|
217
|
+
if (!config.index) {
|
|
218
|
+
throw new Error("Index must be specified");
|
|
219
|
+
}
|
|
220
|
+
this.validateIndexConfig(config.index);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Validate vector index configuration
|
|
224
|
+
*/
|
|
225
|
+
validateIndexConfig(indexConfig) {
|
|
226
|
+
if (indexConfig.dimension < 1 || indexConfig.dimension > 4096) {
|
|
227
|
+
throw new Error("Vector dimension must be between 1 and 4096");
|
|
228
|
+
}
|
|
229
|
+
if (!["cosine", "euclidean"].includes(indexConfig.distanceMetric)) {
|
|
230
|
+
throw new Error('Distance metric must be either "cosine" or "euclidean"');
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Check if an error indicates a resource was not found
|
|
235
|
+
*/
|
|
236
|
+
isNotFoundError(error) {
|
|
237
|
+
if (error instanceof NotFoundException2) {
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
export {
|
|
244
|
+
S3VectorProvider
|
|
245
|
+
};
|
|
246
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/providers/S3VectorProvider.ts","../src/providers/S3VectorStore.ts"],"sourcesContent":["import type {VectorStorageProvider, VectorStorageProviderConfig} from '../VectorStorageProvider'\nimport type {VectorStoreConfig, VectorIndexConfig} from '../types'\nimport {\n CreateVectorBucketCommand,\n GetVectorBucketCommand,\n NotFoundException,\n S3VectorsClient,\n S3VectorsClientConfig,\n} from '@aws-sdk/client-s3vectors'\nimport {S3VectorStore} from './S3VectorStore'\n\n/**\n * Configuration for AWS S3 Vector provider\n */\nexport interface S3VectorProviderConfig extends VectorStorageProviderConfig {\n s3Options: S3VectorsClientConfig\n}\n\n/**\n * AWS S3 Vector storage provider implementation\n *\n * This provider implements vector storage using AWS S3's native vector capabilities.\n * It manages vector buckets and indexes according to the S3 Vector API.\n */\nexport class S3VectorProvider implements VectorStorageProvider {\n private config: S3VectorProviderConfig\n private client: S3VectorsClient\n\n constructor(config: S3VectorProviderConfig) {\n this.config = config\n this.validateConfig()\n this.client = new S3VectorsClient(this.config.s3Options)\n }\n\n /**\n * Validate the provider configuration\n */\n private validateConfig(): void {\n if (!this.config.s3Options) {\n throw new Error('S3Options are required for S3VectorProvider')\n }\n }\n\n createVectorStore(config: VectorStoreConfig) {\n this.validateVectorStoreConfig(config)\n\n this.setupVectorStore(config)\n\n // Return a VectorStore instance\n return new S3VectorStore(this.client, config)\n }\n\n private async setupVectorStore(config: VectorStoreConfig): Promise<void> {\n // Ensure the vector bucket exists\n await this.ensureVectorBucketExists(config)\n }\n\n private async storeExists(storeName: string): Promise<boolean> {\n try {\n await this.client.send(\n new GetVectorBucketCommand({\n vectorBucketName: storeName,\n }),\n )\n\n return true\n } catch (error) {\n if (this.isNotFoundError(error)) {\n return false\n }\n throw error\n }\n }\n\n /**\n * Ensure a vector bucket exists, create if it doesn't\n */\n private async ensureVectorBucketExists(config: VectorStoreConfig): Promise<void> {\n const exists = await this.storeExists(config.storeName)\n if (!exists) {\n await this.createVectorBucket(config)\n }\n }\n\n /**\n * Create a new vector bucket\n */\n private async createVectorBucket(config: VectorStoreConfig): Promise<void> {\n await this.client.send(\n new CreateVectorBucketCommand({\n vectorBucketName: config.storeName,\n encryptionConfiguration: config.encryptionConfiguration,\n }),\n )\n }\n\n /**\n * Validate vector store configuration\n */\n private validateVectorStoreConfig(config: VectorStoreConfig): void {\n if (!config.storeName || config.storeName.length < 3 || config.storeName.length > 63) {\n throw new Error('Store name must be between 3 and 63 characters')\n }\n\n if (!/^[a-z0-9-]+$/.test(config.storeName)) {\n throw new Error('Store name must contain only lowercase letters, numbers, and hyphens')\n }\n\n if (!config.index) {\n throw new Error('Index must be specified')\n }\n\n this.validateIndexConfig(config.index)\n }\n\n /**\n * Validate vector index configuration\n */\n private validateIndexConfig(indexConfig: VectorIndexConfig): void {\n if (indexConfig.dimension < 1 || indexConfig.dimension > 4096) {\n throw new Error('Vector dimension must be between 1 and 4096')\n }\n\n if (!['cosine', 'euclidean'].includes(indexConfig.distanceMetric)) {\n throw new Error('Distance metric must be either \"cosine\" or \"euclidean\"')\n }\n }\n\n /**\n * Check if an error indicates a resource was not found\n */\n private isNotFoundError(error: unknown): boolean {\n if (error instanceof NotFoundException) {\n return true\n }\n return false\n }\n}\n","import {\n CreateIndexCommand,\n DeleteIndexCommand,\n DeleteVectorsCommand,\n ListVectorsCommand,\n ListVectorsCommandInput,\n NotFoundException,\n PutVectorsCommand,\n QueryVectorsCommand,\n QueryVectorsCommandInput,\n S3VectorsClient,\n} from '@aws-sdk/client-s3vectors'\nimport {VectorData, VectorIndexConfig, VectorStoreConfig} from '../types'\nimport {VectorStore} from '../VectorStore'\n\n/**\n * S3 Vector Store implementation that provides CRUD operations for a specific vector store\n */\nexport class S3VectorStore<TVectorData extends VectorData = VectorData>\n implements VectorStore<TVectorData>\n{\n private config: VectorStoreConfig\n private client: S3VectorsClient\n\n constructor(client: S3VectorsClient, config: VectorStoreConfig) {\n this.client = client\n this.config = config\n }\n\n private async wrapOperationWithIndexCreation<TOperation extends () => Promise<any>>(\n namespace: string,\n operation: TOperation,\n ): Promise<Awaited<ReturnType<TOperation>>> {\n try {\n return await operation()\n } catch (error) {\n if (error instanceof NotFoundException) {\n if (error.message === 'The specified index could not be found') {\n await this.createVectorIndex(namespace, this.config, this.config.index)\n return this.wrapOperationWithIndexCreation(namespace, operation)\n }\n }\n throw error\n }\n }\n\n async insertVectors(namespace: string, vectors: TVectorData[]) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new PutVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n vectors: vectors.map(vector => ({\n key: vector.key,\n data: {\n float32: vector.vector,\n },\n metadata: vector.metadata,\n })),\n }),\n )\n\n return result\n })\n }\n\n async queryVectors(\n namespace: string,\n options?: Omit<QueryVectorsCommandInput, 'vectorBucketName' | 'indexName' | 'queryVector'> & {\n queryVector: number[]\n },\n ) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new QueryVectorsCommand({\n ...options,\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n queryVector: {\n float32: options.queryVector,\n },\n }),\n )\n\n const vectors: TVectorData[] =\n result.vectors?.map(\n vector =>\n ({\n key: vector.key,\n vector: vector.data?.float32,\n metadata: vector.metadata,\n }) as TVectorData,\n ) ?? []\n\n return {\n vectors,\n }\n })\n }\n\n async deleteVectors(namespace: string, keys: string[]) {\n return await this.client.send(\n new DeleteVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n keys,\n }),\n )\n }\n\n async listVectors(\n namespace: string,\n options?: Omit<ListVectorsCommandInput, 'vectorBucketName' | 'indexName'>,\n ) {\n return await this.wrapOperationWithIndexCreation(namespace, async () => {\n const result = await this.client.send(\n new ListVectorsCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n ...options,\n }),\n )\n\n return (result.vectors?.map(vector => ({\n key: vector.key,\n vector: vector.data?.float32,\n metadata: vector.metadata as TVectorData['metadata'],\n })) ?? []) as Partial<TVectorData>[]\n })\n }\n\n async listVectorKeys(\n namespace: string,\n options?: Omit<\n ListVectorsCommandInput,\n 'vectorBucketName' | 'indexName' | 'returnData' | 'returnMetadata'\n >,\n ) {\n const result = await this.listVectors(namespace, {\n ...options,\n returnData: false,\n returnMetadata: false,\n })\n return result.map(vector => vector.key)\n }\n\n /**\n * Create a new vector index\n */\n private async createVectorIndex(\n namespace: string,\n config: VectorStoreConfig,\n indexConfig: VectorIndexConfig,\n ): Promise<void> {\n await this.client.send(\n new CreateIndexCommand({\n indexName: namespace,\n dataType: 'float32',\n dimension: indexConfig.dimension,\n distanceMetric: indexConfig.distanceMetric,\n vectorBucketName: config.storeName,\n metadataConfiguration: indexConfig.metadataConfiguration,\n }),\n )\n }\n\n async deleteNamespace(namespace: string) {\n return await this.client.send(\n new DeleteIndexCommand({\n vectorBucketName: this.config.storeName,\n indexName: namespace,\n }),\n )\n }\n}\n"],"mappings":";AAEA;AAAA,EACE;AAAA,EACA;AAAA,EACA,qBAAAA;AAAA,EACA,mBAAAC;AAAA,OAEK;;;ACRP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AAOA,IAAM,gBAAN,MAEP;AAAA,EACU;AAAA,EACA;AAAA,EAER,YAAY,QAAyB,QAA2B;AAC9D,SAAK,SAAS;AACd,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAc,+BACZ,WACA,WAC0C;AAC1C,QAAI;AACF,aAAO,MAAM,UAAU;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,mBAAmB;AACtC,YAAI,MAAM,YAAY,0CAA0C;AAC9D,gBAAM,KAAK,kBAAkB,WAAW,KAAK,QAAQ,KAAK,OAAO,KAAK;AACtE,iBAAO,KAAK,+BAA+B,WAAW,SAAS;AAAA,QACjE;AAAA,MACF;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,WAAmB,SAAwB;AAC7D,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AACtE,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,kBAAkB;AAAA,UACpB,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,SAAS,QAAQ,IAAI,aAAW;AAAA,YAC9B,KAAK,OAAO;AAAA,YACZ,MAAM;AAAA,cACJ,SAAS,OAAO;AAAA,YAClB;AAAA,YACA,UAAU,OAAO;AAAA,UACnB,EAAE;AAAA,QACJ,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aACJ,WACA,SAGA;AACA,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AAxE5E;AAyEM,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,oBAAoB;AAAA,UACtB,GAAG;AAAA,UACH,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,aAAa;AAAA,YACX,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,CAAC;AAAA,MACH;AAEA,YAAM,YACJ,YAAO,YAAP,mBAAgB;AAAA,QACd,YAAO;AAtFjB,cAAAC;AAuFa;AAAA,YACC,KAAK,OAAO;AAAA,YACZ,SAAQA,MAAA,OAAO,SAAP,gBAAAA,IAAa;AAAA,YACrB,UAAU,OAAO;AAAA,UACnB;AAAA;AAAA,YACC,CAAC;AAER,aAAO;AAAA,QACL;AAAA,MACF;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,cAAc,WAAmB,MAAgB;AACrD,WAAO,MAAM,KAAK,OAAO;AAAA,MACvB,IAAI,qBAAqB;AAAA,QACvB,kBAAkB,KAAK,OAAO;AAAA,QAC9B,WAAW;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,YACJ,WACA,SACA;AACA,WAAO,MAAM,KAAK,+BAA+B,WAAW,YAAY;AAlH5E;AAmHM,YAAM,SAAS,MAAM,KAAK,OAAO;AAAA,QAC/B,IAAI,mBAAmB;AAAA,UACrB,kBAAkB,KAAK,OAAO;AAAA,UAC9B,WAAW;AAAA,UACX,GAAG;AAAA,QACL,CAAC;AAAA,MACH;AAEA,eAAQ,YAAO,YAAP,mBAAgB,IAAI,YAAO;AA3HzC,YAAAA;AA2H6C;AAAA,UACrC,KAAK,OAAO;AAAA,UACZ,SAAQA,MAAA,OAAO,SAAP,gBAAAA,IAAa;AAAA,UACrB,UAAU,OAAO;AAAA,QACnB;AAAA,aAAO,CAAC;AAAA,IACV,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,eACJ,WACA,SAIA;AACA,UAAM,SAAS,MAAM,KAAK,YAAY,WAAW;AAAA,MAC/C,GAAG;AAAA,MACH,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB,CAAC;AACD,WAAO,OAAO,IAAI,YAAU,OAAO,GAAG;AAAA,EACxC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,kBACZ,WACA,QACA,aACe;AACf,UAAM,KAAK,OAAO;AAAA,MAChB,IAAI,mBAAmB;AAAA,QACrB,WAAW;AAAA,QACX,UAAU;AAAA,QACV,WAAW,YAAY;AAAA,QACvB,gBAAgB,YAAY;AAAA,QAC5B,kBAAkB,OAAO;AAAA,QACzB,uBAAuB,YAAY;AAAA,MACrC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,gBAAgB,WAAmB;AACvC,WAAO,MAAM,KAAK,OAAO;AAAA,MACvB,IAAI,mBAAmB;AAAA,QACrB,kBAAkB,KAAK,OAAO;AAAA,QAC9B,WAAW;AAAA,MACb,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ADtJO,IAAM,mBAAN,MAAwD;AAAA,EACrD;AAAA,EACA;AAAA,EAER,YAAY,QAAgC;AAC1C,SAAK,SAAS;AACd,SAAK,eAAe;AACpB,SAAK,SAAS,IAAIC,iBAAgB,KAAK,OAAO,SAAS;AAAA,EACzD;AAAA;AAAA;AAAA;AAAA,EAKQ,iBAAuB;AAC7B,QAAI,CAAC,KAAK,OAAO,WAAW;AAC1B,YAAM,IAAI,MAAM,6CAA6C;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,kBAAkB,QAA2B;AAC3C,SAAK,0BAA0B,MAAM;AAErC,SAAK,iBAAiB,MAAM;AAG5B,WAAO,IAAI,cAAc,KAAK,QAAQ,MAAM;AAAA,EAC9C;AAAA,EAEA,MAAc,iBAAiB,QAA0C;AAEvE,UAAM,KAAK,yBAAyB,MAAM;AAAA,EAC5C;AAAA,EAEA,MAAc,YAAY,WAAqC;AAC7D,QAAI;AACF,YAAM,KAAK,OAAO;AAAA,QAChB,IAAI,uBAAuB;AAAA,UACzB,kBAAkB;AAAA,QACpB,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,UAAI,KAAK,gBAAgB,KAAK,GAAG;AAC/B,eAAO;AAAA,MACT;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,yBAAyB,QAA0C;AAC/E,UAAM,SAAS,MAAM,KAAK,YAAY,OAAO,SAAS;AACtD,QAAI,CAAC,QAAQ;AACX,YAAM,KAAK,mBAAmB,MAAM;AAAA,IACtC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,mBAAmB,QAA0C;AACzE,UAAM,KAAK,OAAO;AAAA,MAChB,IAAI,0BAA0B;AAAA,QAC5B,kBAAkB,OAAO;AAAA,QACzB,yBAAyB,OAAO;AAAA,MAClC,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,0BAA0B,QAAiC;AACjE,QAAI,CAAC,OAAO,aAAa,OAAO,UAAU,SAAS,KAAK,OAAO,UAAU,SAAS,IAAI;AACpF,YAAM,IAAI,MAAM,gDAAgD;AAAA,IAClE;AAEA,QAAI,CAAC,eAAe,KAAK,OAAO,SAAS,GAAG;AAC1C,YAAM,IAAI,MAAM,sEAAsE;AAAA,IACxF;AAEA,QAAI,CAAC,OAAO,OAAO;AACjB,YAAM,IAAI,MAAM,yBAAyB;AAAA,IAC3C;AAEA,SAAK,oBAAoB,OAAO,KAAK;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKQ,oBAAoB,aAAsC;AAChE,QAAI,YAAY,YAAY,KAAK,YAAY,YAAY,MAAM;AAC7D,YAAM,IAAI,MAAM,6CAA6C;AAAA,IAC/D;AAEA,QAAI,CAAC,CAAC,UAAU,WAAW,EAAE,SAAS,YAAY,cAAc,GAAG;AACjE,YAAM,IAAI,MAAM,wDAAwD;AAAA,IAC1E;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,OAAyB;AAC/C,QAAI,iBAAiBC,oBAAmB;AACtC,aAAO;AAAA,IACT;AACA,WAAO;AAAA,EACT;AACF;","names":["NotFoundException","S3VectorsClient","_a","S3VectorsClient","NotFoundException"]}
|
package/package.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@orion-js/vectors",
|
|
3
|
+
"version": "4.0.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "./dist/index.cjs",
|
|
6
|
+
"module": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"import": "./dist/index.js",
|
|
11
|
+
"require": "./dist/index.cjs"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist"
|
|
15
|
+
],
|
|
16
|
+
"author": "nicolaslopezj",
|
|
17
|
+
"license": "MIT",
|
|
18
|
+
"devDependencies": {
|
|
19
|
+
"@aws-sdk/client-s3vectors": "^3.848.0",
|
|
20
|
+
"@types/node": "^18.19.86",
|
|
21
|
+
"tsup": "^8.0.1",
|
|
22
|
+
"typescript": "^5.4.5",
|
|
23
|
+
"vitest": "^3.1.1"
|
|
24
|
+
},
|
|
25
|
+
"peerDependencies": {
|
|
26
|
+
"@aws-sdk/client-s3vectors": "*"
|
|
27
|
+
},
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
30
|
+
},
|
|
31
|
+
"scripts": {
|
|
32
|
+
"test": "vitest --config vitest.config.ts run",
|
|
33
|
+
"clean": "rm -rf ./dist",
|
|
34
|
+
"build": "tsup",
|
|
35
|
+
"upgrade-interactive": "pnpm upgrade --interactive",
|
|
36
|
+
"dev": "tsup --watch"
|
|
37
|
+
}
|
|
38
|
+
}
|