@mastra/pinecone 0.2.4 → 0.2.5-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +34 -0
- package/dist/_tsup-dts-rollup.d.cts +27 -5
- package/dist/_tsup-dts-rollup.d.ts +27 -5
- package/dist/index.cjs +27 -15
- package/dist/index.js +27 -15
- package/package.json +2 -2
- package/src/vector/index.test.ts +418 -36
- package/src/vector/index.ts +57 -16
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
|
|
2
|
-
> @mastra/pinecone@0.2.
|
|
2
|
+
> @mastra/pinecone@0.2.5-alpha.2 build /home/runner/work/mastra/mastra/stores/pinecone
|
|
3
3
|
> tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
6
6
|
[34mCLI[39m Using tsconfig: tsconfig.json
|
|
7
7
|
[34mCLI[39m tsup v8.4.0
|
|
8
8
|
[34mTSC[39m Build start
|
|
9
|
-
[32mTSC[39m ⚡️ Build success in
|
|
9
|
+
[32mTSC[39m ⚡️ Build success in 6790ms
|
|
10
10
|
[34mDTS[39m Build start
|
|
11
11
|
[34mCLI[39m Target: es2022
|
|
12
|
-
[34mCLI[39m Cleaning output folder
|
|
13
|
-
[34mESM[39m Build start
|
|
14
|
-
[34mCJS[39m Build start
|
|
15
12
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
16
13
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/pinecone/dist/_tsup-dts-rollup.d.ts[39m
|
|
17
14
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
18
15
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/pinecone/dist/_tsup-dts-rollup.d.cts[39m
|
|
19
|
-
[32mDTS[39m ⚡️ Build success in
|
|
20
|
-
[
|
|
21
|
-
[
|
|
22
|
-
[
|
|
23
|
-
[
|
|
16
|
+
[32mDTS[39m ⚡️ Build success in 9167ms
|
|
17
|
+
[34mCLI[39m Cleaning output folder
|
|
18
|
+
[34mESM[39m Build start
|
|
19
|
+
[34mCJS[39m Build start
|
|
20
|
+
[32mCJS[39m [1mdist/index.cjs [22m[32m7.13 KB[39m
|
|
21
|
+
[32mCJS[39m ⚡️ Build success in 640ms
|
|
22
|
+
[32mESM[39m [1mdist/index.js [22m[32m7.09 KB[39m
|
|
23
|
+
[32mESM[39m ⚡️ Build success in 640ms
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# @mastra/pinecone
|
|
2
2
|
|
|
3
|
+
## 0.2.5-alpha.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [56c31b7]
|
|
8
|
+
- Updated dependencies [dbbbf80]
|
|
9
|
+
- Updated dependencies [99d43b9]
|
|
10
|
+
- @mastra/core@0.8.0-alpha.2
|
|
11
|
+
|
|
12
|
+
## 0.2.5-alpha.1
|
|
13
|
+
|
|
14
|
+
### Patch Changes
|
|
15
|
+
|
|
16
|
+
- 7071597: Update pinecone to include namespace and hybrid search
|
|
17
|
+
- Updated dependencies [619c39d]
|
|
18
|
+
- Updated dependencies [fe56be0]
|
|
19
|
+
- Updated dependencies [a0967a0]
|
|
20
|
+
- Updated dependencies [fca3b21]
|
|
21
|
+
- Updated dependencies [0118361]
|
|
22
|
+
- Updated dependencies [619c39d]
|
|
23
|
+
- @mastra/core@0.8.0-alpha.1
|
|
24
|
+
|
|
25
|
+
## 0.2.5-alpha.0
|
|
26
|
+
|
|
27
|
+
### Patch Changes
|
|
28
|
+
|
|
29
|
+
- Updated dependencies [107bcfe]
|
|
30
|
+
- Updated dependencies [5b4e19f]
|
|
31
|
+
- Updated dependencies [7599d77]
|
|
32
|
+
- Updated dependencies [cafae83]
|
|
33
|
+
- Updated dependencies [8076ecf]
|
|
34
|
+
- Updated dependencies [304397c]
|
|
35
|
+
- @mastra/core@0.7.1-alpha.0
|
|
36
|
+
|
|
3
37
|
## 0.2.4
|
|
4
38
|
|
|
5
39
|
### Patch Changes
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
2
2
|
import type { CreateIndexParams } from '@mastra/core/vector';
|
|
3
3
|
import type { IndexStats } from '@mastra/core/vector';
|
|
4
|
+
import type { IndexStatsDescription } from '@pinecone-database/pinecone';
|
|
4
5
|
import { MastraVector } from '@mastra/core/vector';
|
|
5
6
|
import type { OperatorSupport } from '@mastra/core/vector/filter';
|
|
6
7
|
import type { ParamsToArgs } from '@mastra/core/vector';
|
|
7
8
|
import type { QueryResult } from '@mastra/core/vector';
|
|
9
|
+
import type { QueryVectorArgs } from '@mastra/core/vector';
|
|
8
10
|
import type { QueryVectorParams } from '@mastra/core/vector';
|
|
11
|
+
import type { RecordSparseValues } from '@pinecone-database/pinecone';
|
|
12
|
+
import type { UpsertVectorArgs } from '@mastra/core/vector';
|
|
9
13
|
import type { UpsertVectorParams } from '@mastra/core/vector';
|
|
10
14
|
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
11
15
|
|
|
@@ -16,21 +20,39 @@ export declare class PineconeFilterTranslator extends BaseFilterTranslator {
|
|
|
16
20
|
private translateOperator;
|
|
17
21
|
}
|
|
18
22
|
|
|
23
|
+
declare interface PineconeIndexStats extends IndexStats {
|
|
24
|
+
namespaces?: IndexStatsDescription['namespaces'];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
declare type PineconeQueryVectorArgs = [...QueryVectorArgs, string?, RecordSparseValues?];
|
|
28
|
+
|
|
29
|
+
declare interface PineconeQueryVectorParams extends QueryVectorParams {
|
|
30
|
+
namespace?: string;
|
|
31
|
+
sparseVector?: RecordSparseValues;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
declare type PineconeUpsertVectorArgs = [...UpsertVectorArgs, string?, RecordSparseValues[]?];
|
|
35
|
+
|
|
36
|
+
declare interface PineconeUpsertVectorParams extends UpsertVectorParams {
|
|
37
|
+
namespace?: string;
|
|
38
|
+
sparseVectors?: RecordSparseValues[];
|
|
39
|
+
}
|
|
40
|
+
|
|
19
41
|
declare class PineconeVector extends MastraVector {
|
|
20
42
|
private client;
|
|
21
43
|
constructor(apiKey: string, environment?: string);
|
|
22
44
|
createIndex(...args: ParamsToArgs<CreateIndexParams>): Promise<void>;
|
|
23
|
-
upsert(...args: ParamsToArgs<
|
|
45
|
+
upsert(...args: ParamsToArgs<PineconeUpsertVectorParams> | PineconeUpsertVectorArgs): Promise<string[]>;
|
|
24
46
|
transformFilter(filter?: VectorFilter): VectorFilter;
|
|
25
|
-
query(...args: ParamsToArgs<
|
|
47
|
+
query(...args: ParamsToArgs<PineconeQueryVectorParams> | PineconeQueryVectorArgs): Promise<QueryResult[]>;
|
|
26
48
|
listIndexes(): Promise<string[]>;
|
|
27
|
-
describeIndex(indexName: string): Promise<
|
|
49
|
+
describeIndex(indexName: string): Promise<PineconeIndexStats>;
|
|
28
50
|
deleteIndex(indexName: string): Promise<void>;
|
|
29
51
|
updateIndexById(indexName: string, id: string, update: {
|
|
30
52
|
vector?: number[];
|
|
31
53
|
metadata?: Record<string, any>;
|
|
32
|
-
}): Promise<void>;
|
|
33
|
-
deleteIndexById(indexName: string, id: string): Promise<void>;
|
|
54
|
+
}, namespace?: string): Promise<void>;
|
|
55
|
+
deleteIndexById(indexName: string, id: string, namespace?: string): Promise<void>;
|
|
34
56
|
}
|
|
35
57
|
export { PineconeVector }
|
|
36
58
|
export { PineconeVector as PineconeVector_alias_1 }
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
2
2
|
import type { CreateIndexParams } from '@mastra/core/vector';
|
|
3
3
|
import type { IndexStats } from '@mastra/core/vector';
|
|
4
|
+
import type { IndexStatsDescription } from '@pinecone-database/pinecone';
|
|
4
5
|
import { MastraVector } from '@mastra/core/vector';
|
|
5
6
|
import type { OperatorSupport } from '@mastra/core/vector/filter';
|
|
6
7
|
import type { ParamsToArgs } from '@mastra/core/vector';
|
|
7
8
|
import type { QueryResult } from '@mastra/core/vector';
|
|
9
|
+
import type { QueryVectorArgs } from '@mastra/core/vector';
|
|
8
10
|
import type { QueryVectorParams } from '@mastra/core/vector';
|
|
11
|
+
import type { RecordSparseValues } from '@pinecone-database/pinecone';
|
|
12
|
+
import type { UpsertVectorArgs } from '@mastra/core/vector';
|
|
9
13
|
import type { UpsertVectorParams } from '@mastra/core/vector';
|
|
10
14
|
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
11
15
|
|
|
@@ -16,21 +20,39 @@ export declare class PineconeFilterTranslator extends BaseFilterTranslator {
|
|
|
16
20
|
private translateOperator;
|
|
17
21
|
}
|
|
18
22
|
|
|
23
|
+
declare interface PineconeIndexStats extends IndexStats {
|
|
24
|
+
namespaces?: IndexStatsDescription['namespaces'];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
declare type PineconeQueryVectorArgs = [...QueryVectorArgs, string?, RecordSparseValues?];
|
|
28
|
+
|
|
29
|
+
declare interface PineconeQueryVectorParams extends QueryVectorParams {
|
|
30
|
+
namespace?: string;
|
|
31
|
+
sparseVector?: RecordSparseValues;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
declare type PineconeUpsertVectorArgs = [...UpsertVectorArgs, string?, RecordSparseValues[]?];
|
|
35
|
+
|
|
36
|
+
declare interface PineconeUpsertVectorParams extends UpsertVectorParams {
|
|
37
|
+
namespace?: string;
|
|
38
|
+
sparseVectors?: RecordSparseValues[];
|
|
39
|
+
}
|
|
40
|
+
|
|
19
41
|
declare class PineconeVector extends MastraVector {
|
|
20
42
|
private client;
|
|
21
43
|
constructor(apiKey: string, environment?: string);
|
|
22
44
|
createIndex(...args: ParamsToArgs<CreateIndexParams>): Promise<void>;
|
|
23
|
-
upsert(...args: ParamsToArgs<
|
|
45
|
+
upsert(...args: ParamsToArgs<PineconeUpsertVectorParams> | PineconeUpsertVectorArgs): Promise<string[]>;
|
|
24
46
|
transformFilter(filter?: VectorFilter): VectorFilter;
|
|
25
|
-
query(...args: ParamsToArgs<
|
|
47
|
+
query(...args: ParamsToArgs<PineconeQueryVectorParams> | PineconeQueryVectorArgs): Promise<QueryResult[]>;
|
|
26
48
|
listIndexes(): Promise<string[]>;
|
|
27
|
-
describeIndex(indexName: string): Promise<
|
|
49
|
+
describeIndex(indexName: string): Promise<PineconeIndexStats>;
|
|
28
50
|
deleteIndex(indexName: string): Promise<void>;
|
|
29
51
|
updateIndexById(indexName: string, id: string, update: {
|
|
30
52
|
vector?: number[];
|
|
31
53
|
metadata?: Record<string, any>;
|
|
32
|
-
}): Promise<void>;
|
|
33
|
-
deleteIndexById(indexName: string, id: string): Promise<void>;
|
|
54
|
+
}, namespace?: string): Promise<void>;
|
|
55
|
+
deleteIndexById(indexName: string, id: string, namespace?: string): Promise<void>;
|
|
34
56
|
}
|
|
35
57
|
export { PineconeVector }
|
|
36
58
|
export { PineconeVector as PineconeVector_alias_1 }
|
package/dist/index.cjs
CHANGED
|
@@ -119,13 +119,17 @@ var PineconeVector = class extends vector.MastraVector {
|
|
|
119
119
|
});
|
|
120
120
|
}
|
|
121
121
|
async upsert(...args) {
|
|
122
|
-
const params = this.normalizeArgs("upsert", args
|
|
123
|
-
|
|
124
|
-
|
|
122
|
+
const params = this.normalizeArgs("upsert", args, [
|
|
123
|
+
"namespace",
|
|
124
|
+
"sparseVectors"
|
|
125
|
+
]);
|
|
126
|
+
const { indexName, vectors, metadata, ids, namespace, sparseVectors } = params;
|
|
127
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
125
128
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
126
129
|
const records = vectors.map((vector, i) => ({
|
|
127
130
|
id: vectorIds[i],
|
|
128
131
|
values: vector,
|
|
132
|
+
...sparseVectors?.[i] && { sparseValues: sparseVectors?.[i] },
|
|
129
133
|
metadata: metadata?.[i] || {}
|
|
130
134
|
}));
|
|
131
135
|
const batchSize = 100;
|
|
@@ -140,17 +144,24 @@ var PineconeVector = class extends vector.MastraVector {
|
|
|
140
144
|
return translator.translate(filter);
|
|
141
145
|
}
|
|
142
146
|
async query(...args) {
|
|
143
|
-
const params = this.normalizeArgs("query", args
|
|
144
|
-
|
|
145
|
-
|
|
147
|
+
const params = this.normalizeArgs("query", args, [
|
|
148
|
+
"namespace",
|
|
149
|
+
"sparseVector"
|
|
150
|
+
]);
|
|
151
|
+
const { indexName, queryVector, topK = 10, filter, includeVector = false, namespace, sparseVector } = params;
|
|
152
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
146
153
|
const translatedFilter = this.transformFilter(filter) ?? void 0;
|
|
147
|
-
const
|
|
154
|
+
const queryParams = {
|
|
148
155
|
vector: queryVector,
|
|
149
156
|
topK,
|
|
150
|
-
filter: translatedFilter,
|
|
151
157
|
includeMetadata: true,
|
|
152
|
-
includeValues: includeVector
|
|
153
|
-
|
|
158
|
+
includeValues: includeVector,
|
|
159
|
+
filter: translatedFilter
|
|
160
|
+
};
|
|
161
|
+
if (sparseVector) {
|
|
162
|
+
queryParams.sparseVector = sparseVector;
|
|
163
|
+
}
|
|
164
|
+
const results = await index.query(queryParams);
|
|
154
165
|
return results.matches.map((match) => ({
|
|
155
166
|
id: match.id,
|
|
156
167
|
score: match.score || 0,
|
|
@@ -169,7 +180,8 @@ var PineconeVector = class extends vector.MastraVector {
|
|
|
169
180
|
return {
|
|
170
181
|
dimension: description.dimension,
|
|
171
182
|
count: stats.totalRecordCount || 0,
|
|
172
|
-
metric: description.metric
|
|
183
|
+
metric: description.metric,
|
|
184
|
+
namespaces: stats.namespaces
|
|
173
185
|
};
|
|
174
186
|
}
|
|
175
187
|
async deleteIndex(indexName) {
|
|
@@ -179,11 +191,11 @@ var PineconeVector = class extends vector.MastraVector {
|
|
|
179
191
|
throw new Error(`Failed to delete Pinecone index: ${error.message}`);
|
|
180
192
|
}
|
|
181
193
|
}
|
|
182
|
-
async updateIndexById(indexName, id, update) {
|
|
194
|
+
async updateIndexById(indexName, id, update, namespace) {
|
|
183
195
|
if (!update.vector && !update.metadata) {
|
|
184
196
|
throw new Error("No updates provided");
|
|
185
197
|
}
|
|
186
|
-
const index = this.client.Index(indexName);
|
|
198
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
187
199
|
const updateObj = { id };
|
|
188
200
|
if (update.vector) {
|
|
189
201
|
updateObj.values = update.vector;
|
|
@@ -193,8 +205,8 @@ var PineconeVector = class extends vector.MastraVector {
|
|
|
193
205
|
}
|
|
194
206
|
await index.update(updateObj);
|
|
195
207
|
}
|
|
196
|
-
async deleteIndexById(indexName, id) {
|
|
197
|
-
const index = this.client.Index(indexName);
|
|
208
|
+
async deleteIndexById(indexName, id, namespace) {
|
|
209
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
198
210
|
await index.deleteOne(id);
|
|
199
211
|
}
|
|
200
212
|
};
|
package/dist/index.js
CHANGED
|
@@ -117,13 +117,17 @@ var PineconeVector = class extends MastraVector {
|
|
|
117
117
|
});
|
|
118
118
|
}
|
|
119
119
|
async upsert(...args) {
|
|
120
|
-
const params = this.normalizeArgs("upsert", args
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
const params = this.normalizeArgs("upsert", args, [
|
|
121
|
+
"namespace",
|
|
122
|
+
"sparseVectors"
|
|
123
|
+
]);
|
|
124
|
+
const { indexName, vectors, metadata, ids, namespace, sparseVectors } = params;
|
|
125
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
123
126
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
124
127
|
const records = vectors.map((vector, i) => ({
|
|
125
128
|
id: vectorIds[i],
|
|
126
129
|
values: vector,
|
|
130
|
+
...sparseVectors?.[i] && { sparseValues: sparseVectors?.[i] },
|
|
127
131
|
metadata: metadata?.[i] || {}
|
|
128
132
|
}));
|
|
129
133
|
const batchSize = 100;
|
|
@@ -138,17 +142,24 @@ var PineconeVector = class extends MastraVector {
|
|
|
138
142
|
return translator.translate(filter);
|
|
139
143
|
}
|
|
140
144
|
async query(...args) {
|
|
141
|
-
const params = this.normalizeArgs("query", args
|
|
142
|
-
|
|
143
|
-
|
|
145
|
+
const params = this.normalizeArgs("query", args, [
|
|
146
|
+
"namespace",
|
|
147
|
+
"sparseVector"
|
|
148
|
+
]);
|
|
149
|
+
const { indexName, queryVector, topK = 10, filter, includeVector = false, namespace, sparseVector } = params;
|
|
150
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
144
151
|
const translatedFilter = this.transformFilter(filter) ?? void 0;
|
|
145
|
-
const
|
|
152
|
+
const queryParams = {
|
|
146
153
|
vector: queryVector,
|
|
147
154
|
topK,
|
|
148
|
-
filter: translatedFilter,
|
|
149
155
|
includeMetadata: true,
|
|
150
|
-
includeValues: includeVector
|
|
151
|
-
|
|
156
|
+
includeValues: includeVector,
|
|
157
|
+
filter: translatedFilter
|
|
158
|
+
};
|
|
159
|
+
if (sparseVector) {
|
|
160
|
+
queryParams.sparseVector = sparseVector;
|
|
161
|
+
}
|
|
162
|
+
const results = await index.query(queryParams);
|
|
152
163
|
return results.matches.map((match) => ({
|
|
153
164
|
id: match.id,
|
|
154
165
|
score: match.score || 0,
|
|
@@ -167,7 +178,8 @@ var PineconeVector = class extends MastraVector {
|
|
|
167
178
|
return {
|
|
168
179
|
dimension: description.dimension,
|
|
169
180
|
count: stats.totalRecordCount || 0,
|
|
170
|
-
metric: description.metric
|
|
181
|
+
metric: description.metric,
|
|
182
|
+
namespaces: stats.namespaces
|
|
171
183
|
};
|
|
172
184
|
}
|
|
173
185
|
async deleteIndex(indexName) {
|
|
@@ -177,11 +189,11 @@ var PineconeVector = class extends MastraVector {
|
|
|
177
189
|
throw new Error(`Failed to delete Pinecone index: ${error.message}`);
|
|
178
190
|
}
|
|
179
191
|
}
|
|
180
|
-
async updateIndexById(indexName, id, update) {
|
|
192
|
+
async updateIndexById(indexName, id, update, namespace) {
|
|
181
193
|
if (!update.vector && !update.metadata) {
|
|
182
194
|
throw new Error("No updates provided");
|
|
183
195
|
}
|
|
184
|
-
const index = this.client.Index(indexName);
|
|
196
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
185
197
|
const updateObj = { id };
|
|
186
198
|
if (update.vector) {
|
|
187
199
|
updateObj.values = update.vector;
|
|
@@ -191,8 +203,8 @@ var PineconeVector = class extends MastraVector {
|
|
|
191
203
|
}
|
|
192
204
|
await index.update(updateObj);
|
|
193
205
|
}
|
|
194
|
-
async deleteIndexById(indexName, id) {
|
|
195
|
-
const index = this.client.Index(indexName);
|
|
206
|
+
async deleteIndexById(indexName, id, namespace) {
|
|
207
|
+
const index = this.client.Index(indexName).namespace(namespace || "");
|
|
196
208
|
await index.deleteOne(id);
|
|
197
209
|
}
|
|
198
210
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/pinecone",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5-alpha.2",
|
|
4
4
|
"description": "Pinecone vector store provider for Mastra",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
22
|
"@pinecone-database/pinecone": "^3.0.3",
|
|
23
|
-
"@mastra/core": "^0.
|
|
23
|
+
"@mastra/core": "^0.8.0-alpha.2"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@microsoft/api-extractor": "^7.52.1",
|
package/src/vector/index.test.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import type { QueryResult } from '@mastra/core/vector';
|
|
1
2
|
import dotenv from 'dotenv';
|
|
2
3
|
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi, afterEach } from 'vitest';
|
|
3
4
|
|
|
4
5
|
import { PineconeVector } from './';
|
|
5
|
-
import type { QueryResult } from '@mastra/core/vector';
|
|
6
6
|
|
|
7
7
|
dotenv.config();
|
|
8
8
|
|
|
@@ -13,6 +13,25 @@ const PINECONE_API_KEY = process.env.PINECONE_API_KEY!;
|
|
|
13
13
|
// }
|
|
14
14
|
// TODO: skip until we the secrets on Github
|
|
15
15
|
|
|
16
|
+
vi.setConfig({ testTimeout: 80_000, hookTimeout: 80_000 });
|
|
17
|
+
|
|
18
|
+
// Helper function to create sparse vectors for testing
|
|
19
|
+
function createSparseVector(text: string) {
|
|
20
|
+
const words = text.toLowerCase().split(/\W+/).filter(Boolean);
|
|
21
|
+
const uniqueWords = Array.from(new Set(words));
|
|
22
|
+
const indices: number[] = [];
|
|
23
|
+
const values: number[] = [];
|
|
24
|
+
|
|
25
|
+
// Create a simple term frequency vector
|
|
26
|
+
uniqueWords.forEach((word, i) => {
|
|
27
|
+
const frequency = words.filter(w => w === word).length;
|
|
28
|
+
indices.push(i);
|
|
29
|
+
values.push(frequency);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
return { indices, values };
|
|
33
|
+
}
|
|
34
|
+
|
|
16
35
|
function waitUntilReady(vectorDB: PineconeVector, indexName: string) {
|
|
17
36
|
return new Promise(resolve => {
|
|
18
37
|
const interval = setInterval(async () => {
|
|
@@ -25,22 +44,63 @@ function waitUntilReady(vectorDB: PineconeVector, indexName: string) {
|
|
|
25
44
|
} catch (error) {
|
|
26
45
|
console.log(error);
|
|
27
46
|
}
|
|
28
|
-
},
|
|
47
|
+
}, 5000);
|
|
29
48
|
});
|
|
30
49
|
}
|
|
31
50
|
|
|
32
|
-
function
|
|
51
|
+
function waitUntilIndexDeleted(vectorDB: PineconeVector, indexName: string) {
|
|
33
52
|
return new Promise((resolve, reject) => {
|
|
34
|
-
const maxAttempts =
|
|
53
|
+
const maxAttempts = 60;
|
|
35
54
|
let attempts = 0;
|
|
55
|
+
|
|
36
56
|
const interval = setInterval(async () => {
|
|
37
57
|
try {
|
|
38
|
-
const
|
|
39
|
-
if (
|
|
58
|
+
const indexes = await vectorDB.listIndexes();
|
|
59
|
+
if (!indexes.includes(indexName)) {
|
|
40
60
|
clearInterval(interval);
|
|
41
61
|
resolve(true);
|
|
42
62
|
}
|
|
43
63
|
attempts++;
|
|
64
|
+
if (attempts >= maxAttempts) {
|
|
65
|
+
clearInterval(interval);
|
|
66
|
+
reject(new Error('Timeout waiting for index to be deleted'));
|
|
67
|
+
}
|
|
68
|
+
} catch (error) {
|
|
69
|
+
console.log(error);
|
|
70
|
+
}
|
|
71
|
+
}, 5000);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function waitUntilVectorsIndexed(
|
|
76
|
+
vectorDB: PineconeVector,
|
|
77
|
+
indexName: string,
|
|
78
|
+
expectedCount: number,
|
|
79
|
+
exactCount = false,
|
|
80
|
+
) {
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
const maxAttempts = 60;
|
|
83
|
+
let attempts = 0;
|
|
84
|
+
let lastCount = 0;
|
|
85
|
+
let stableCount = 0;
|
|
86
|
+
|
|
87
|
+
const interval = setInterval(async () => {
|
|
88
|
+
try {
|
|
89
|
+
const stats = await vectorDB.describeIndex(indexName);
|
|
90
|
+
const check = exactCount ? stats?.count === expectedCount : stats?.count >= expectedCount;
|
|
91
|
+
if (stats && check) {
|
|
92
|
+
if (stats.count === lastCount) {
|
|
93
|
+
stableCount++;
|
|
94
|
+
if (stableCount >= 2) {
|
|
95
|
+
clearInterval(interval);
|
|
96
|
+
resolve(true);
|
|
97
|
+
}
|
|
98
|
+
} else {
|
|
99
|
+
stableCount = 1;
|
|
100
|
+
}
|
|
101
|
+
lastCount = stats.count;
|
|
102
|
+
}
|
|
103
|
+
attempts++;
|
|
44
104
|
if (attempts >= maxAttempts) {
|
|
45
105
|
clearInterval(interval);
|
|
46
106
|
reject(new Error('Timeout waiting for vectors to be indexed'));
|
|
@@ -48,17 +108,52 @@ function waitUntilVectorsIndexed(vectorDB: PineconeVector, indexName: string, ex
|
|
|
48
108
|
} catch (error) {
|
|
49
109
|
console.log(error);
|
|
50
110
|
}
|
|
51
|
-
},
|
|
111
|
+
}, 10000);
|
|
52
112
|
});
|
|
53
113
|
}
|
|
54
114
|
// TODO: our pinecone account is over the limit, tests don't work in CI
|
|
55
115
|
describe.skip('PineconeVector Integration Tests', () => {
|
|
56
116
|
let vectorDB: PineconeVector;
|
|
57
|
-
const testIndexName = 'test-index
|
|
117
|
+
const testIndexName = 'test-index'; // Unique index name for each test run
|
|
118
|
+
const indexNameUpdate = 'test-index-update';
|
|
119
|
+
const indexNameDelete = 'test-index-delete';
|
|
120
|
+
const indexNameNamespace = 'test-index-namespace';
|
|
121
|
+
const indexNameHybrid = 'test-index-hybrid';
|
|
58
122
|
const dimension = 3;
|
|
59
123
|
|
|
60
124
|
beforeAll(async () => {
|
|
61
125
|
vectorDB = new PineconeVector(PINECONE_API_KEY);
|
|
126
|
+
// Delete test index
|
|
127
|
+
try {
|
|
128
|
+
await vectorDB.deleteIndex(testIndexName);
|
|
129
|
+
await waitUntilIndexDeleted(vectorDB, testIndexName);
|
|
130
|
+
} catch {
|
|
131
|
+
// Ignore errors if index doesn't exist
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
await vectorDB.deleteIndex(indexNameUpdate);
|
|
135
|
+
await waitUntilIndexDeleted(vectorDB, indexNameUpdate);
|
|
136
|
+
} catch {
|
|
137
|
+
// Ignore errors if index doesn't exist
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
await vectorDB.deleteIndex(indexNameDelete);
|
|
141
|
+
await waitUntilIndexDeleted(vectorDB, indexNameDelete);
|
|
142
|
+
} catch {
|
|
143
|
+
// Ignore errors if index doesn't exist
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
await vectorDB.deleteIndex(indexNameNamespace);
|
|
147
|
+
await waitUntilIndexDeleted(vectorDB, indexNameNamespace);
|
|
148
|
+
} catch {
|
|
149
|
+
// Ignore errors if index doesn't exist
|
|
150
|
+
}
|
|
151
|
+
try {
|
|
152
|
+
await vectorDB.deleteIndex(indexNameHybrid);
|
|
153
|
+
await waitUntilIndexDeleted(vectorDB, indexNameHybrid);
|
|
154
|
+
} catch {
|
|
155
|
+
// Ignore errors if index doesn't exist
|
|
156
|
+
}
|
|
62
157
|
// Create test index
|
|
63
158
|
await vectorDB.createIndex({ indexName: testIndexName, dimension });
|
|
64
159
|
await waitUntilReady(vectorDB, testIndexName);
|
|
@@ -66,7 +161,31 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
66
161
|
|
|
67
162
|
afterAll(async () => {
|
|
68
163
|
// Cleanup: delete test index
|
|
69
|
-
|
|
164
|
+
try {
|
|
165
|
+
await vectorDB.deleteIndex(testIndexName);
|
|
166
|
+
} catch {
|
|
167
|
+
// Ignore errors if index doesn't exist
|
|
168
|
+
}
|
|
169
|
+
try {
|
|
170
|
+
await vectorDB.deleteIndex(indexNameUpdate);
|
|
171
|
+
} catch {
|
|
172
|
+
// Ignore errors if index doesn't exist
|
|
173
|
+
}
|
|
174
|
+
try {
|
|
175
|
+
await vectorDB.deleteIndex(indexNameDelete);
|
|
176
|
+
} catch {
|
|
177
|
+
// Ignore errors if index doesn't exist
|
|
178
|
+
}
|
|
179
|
+
try {
|
|
180
|
+
await vectorDB.deleteIndex(indexNameNamespace);
|
|
181
|
+
} catch {
|
|
182
|
+
// Ignore errors if index doesn't exist
|
|
183
|
+
}
|
|
184
|
+
try {
|
|
185
|
+
await vectorDB.deleteIndex(indexNameHybrid);
|
|
186
|
+
} catch {
|
|
187
|
+
// Ignore errors if index doesn't exist
|
|
188
|
+
}
|
|
70
189
|
}, 500000);
|
|
71
190
|
|
|
72
191
|
describe('Index Operations', () => {
|
|
@@ -134,8 +253,22 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
134
253
|
[7, 8, 9],
|
|
135
254
|
];
|
|
136
255
|
|
|
256
|
+
beforeEach(async () => {
|
|
257
|
+
await vectorDB.createIndex({ indexName: indexNameUpdate, dimension, metric: 'cosine' });
|
|
258
|
+
await waitUntilReady(vectorDB, indexNameUpdate);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
afterEach(async () => {
|
|
262
|
+
try {
|
|
263
|
+
await vectorDB.deleteIndex(indexNameUpdate);
|
|
264
|
+
await waitUntilIndexDeleted(vectorDB, indexNameUpdate);
|
|
265
|
+
} catch {
|
|
266
|
+
// Ignore errors if index doesn't exist
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
|
|
137
270
|
it('should update the vector by id', async () => {
|
|
138
|
-
const ids = await vectorDB.upsert({ indexName:
|
|
271
|
+
const ids = await vectorDB.upsert({ indexName: indexNameUpdate, vectors: testVectors });
|
|
139
272
|
expect(ids).toHaveLength(3);
|
|
140
273
|
|
|
141
274
|
const idToBeUpdated = ids[0];
|
|
@@ -149,12 +282,12 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
149
282
|
metadata: newMetaData,
|
|
150
283
|
};
|
|
151
284
|
|
|
152
|
-
await vectorDB.updateIndexById(
|
|
285
|
+
await vectorDB.updateIndexById(indexNameUpdate, idToBeUpdated, update);
|
|
153
286
|
|
|
154
|
-
await
|
|
287
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameUpdate, 3);
|
|
155
288
|
|
|
156
289
|
const results: QueryResult[] = await vectorDB.query({
|
|
157
|
-
indexName:
|
|
290
|
+
indexName: indexNameUpdate,
|
|
158
291
|
queryVector: newVector,
|
|
159
292
|
topK: 10,
|
|
160
293
|
includeVector: true,
|
|
@@ -166,7 +299,7 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
166
299
|
}, 500000);
|
|
167
300
|
|
|
168
301
|
it('should only update the metadata by id', async () => {
|
|
169
|
-
const ids = await vectorDB.upsert({ indexName:
|
|
302
|
+
const ids = await vectorDB.upsert({ indexName: indexNameUpdate, vectors: testVectors });
|
|
170
303
|
expect(ids).toHaveLength(3);
|
|
171
304
|
|
|
172
305
|
const idToBeUpdated = ids[0];
|
|
@@ -178,12 +311,12 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
178
311
|
metadata: newMetaData,
|
|
179
312
|
};
|
|
180
313
|
|
|
181
|
-
await vectorDB.updateIndexById(
|
|
314
|
+
await vectorDB.updateIndexById(indexNameUpdate, idToBeUpdated, update);
|
|
182
315
|
|
|
183
|
-
await
|
|
316
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameUpdate, 3);
|
|
184
317
|
|
|
185
318
|
const results: QueryResult[] = await vectorDB.query({
|
|
186
|
-
indexName:
|
|
319
|
+
indexName: indexNameUpdate,
|
|
187
320
|
queryVector: testVectors[0],
|
|
188
321
|
topK: 2,
|
|
189
322
|
includeVector: true,
|
|
@@ -195,38 +328,34 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
195
328
|
}, 500000);
|
|
196
329
|
|
|
197
330
|
it('should only update vector embeddings by id', async () => {
|
|
198
|
-
const ids = await vectorDB.upsert({ indexName:
|
|
199
|
-
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
200
|
-
|
|
331
|
+
const ids = await vectorDB.upsert({ indexName: indexNameUpdate, vectors: testVectors });
|
|
201
332
|
expect(ids).toHaveLength(3);
|
|
202
333
|
|
|
203
334
|
const idToBeUpdated = ids[0];
|
|
204
|
-
const newVector = [
|
|
335
|
+
const newVector = [4, 4, 4];
|
|
205
336
|
|
|
206
337
|
const update = {
|
|
207
338
|
vector: newVector,
|
|
208
339
|
};
|
|
209
340
|
|
|
210
|
-
await vectorDB.updateIndexById(
|
|
341
|
+
await vectorDB.updateIndexById(indexNameUpdate, idToBeUpdated, update);
|
|
211
342
|
|
|
212
|
-
await
|
|
343
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameUpdate, 3);
|
|
213
344
|
|
|
214
345
|
const results: QueryResult[] = await vectorDB.query({
|
|
215
|
-
indexName:
|
|
346
|
+
indexName: indexNameUpdate,
|
|
216
347
|
queryVector: newVector,
|
|
217
348
|
topK: 10,
|
|
218
349
|
includeVector: true,
|
|
219
|
-
filter: { ids: [idToBeUpdated] },
|
|
220
350
|
});
|
|
221
351
|
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
expect(
|
|
225
|
-
expect(resultVectors).toContain(newVector);
|
|
352
|
+
const updatedResult = results.find(r => r.id === idToBeUpdated);
|
|
353
|
+
expect(updatedResult).toBeDefined();
|
|
354
|
+
expect(updatedResult?.vector).toEqual(newVector);
|
|
226
355
|
}, 500000);
|
|
227
356
|
|
|
228
|
-
it('should throw exception when no updates are given', () => {
|
|
229
|
-
expect(vectorDB.updateIndexById(
|
|
357
|
+
it('should throw exception when no updates are given', async () => {
|
|
358
|
+
await expect(vectorDB.updateIndexById(indexNameUpdate, 'id', {})).rejects.toThrow('No updates provided');
|
|
230
359
|
});
|
|
231
360
|
|
|
232
361
|
it('should throw error for non-existent index', async () => {
|
|
@@ -236,13 +365,13 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
236
365
|
|
|
237
366
|
it('should throw error for invalid vector dimension', async () => {
|
|
238
367
|
const [id] = await vectorDB.upsert({
|
|
239
|
-
indexName:
|
|
368
|
+
indexName: indexNameUpdate,
|
|
240
369
|
vectors: [[1, 2, 3]],
|
|
241
370
|
metadata: [{ test: 'initial' }],
|
|
242
371
|
});
|
|
243
372
|
|
|
244
373
|
await expect(
|
|
245
|
-
vectorDB.updateIndexById(
|
|
374
|
+
vectorDB.updateIndexById(indexNameUpdate, id, { vector: [1, 2] }), // Wrong dimension
|
|
246
375
|
).rejects.toThrow();
|
|
247
376
|
}, 500000);
|
|
248
377
|
});
|
|
@@ -254,16 +383,31 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
254
383
|
[7, 8, 9],
|
|
255
384
|
];
|
|
256
385
|
|
|
386
|
+
beforeEach(async () => {
|
|
387
|
+
await vectorDB.createIndex({ indexName: indexNameDelete, dimension, metric: 'cosine' });
|
|
388
|
+
await waitUntilReady(vectorDB, indexNameDelete);
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
afterEach(async () => {
|
|
392
|
+
try {
|
|
393
|
+
await vectorDB.deleteIndex(indexNameDelete);
|
|
394
|
+
await waitUntilIndexDeleted(vectorDB, indexNameDelete);
|
|
395
|
+
} catch {
|
|
396
|
+
// Ignore errors if index doesn't exist
|
|
397
|
+
}
|
|
398
|
+
});
|
|
399
|
+
|
|
257
400
|
it('should delete the vector by id', async () => {
|
|
258
|
-
const ids = await vectorDB.upsert({ indexName:
|
|
401
|
+
const ids = await vectorDB.upsert({ indexName: indexNameDelete, vectors: testVectors });
|
|
259
402
|
expect(ids).toHaveLength(3);
|
|
260
403
|
const idToBeDeleted = ids[0];
|
|
261
404
|
|
|
262
|
-
await vectorDB.deleteIndexById(
|
|
405
|
+
await vectorDB.deleteIndexById(indexNameDelete, idToBeDeleted);
|
|
406
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameDelete, 2, true);
|
|
263
407
|
|
|
264
408
|
// Query all vectors similar to the deleted one
|
|
265
409
|
const results: QueryResult[] = await vectorDB.query({
|
|
266
|
-
indexName:
|
|
410
|
+
indexName: indexNameDelete,
|
|
267
411
|
queryVector: testVectors[0],
|
|
268
412
|
topK: 3,
|
|
269
413
|
includeVector: true,
|
|
@@ -275,6 +419,141 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
275
419
|
});
|
|
276
420
|
});
|
|
277
421
|
|
|
422
|
+
describe('Namespace Operations', () => {
|
|
423
|
+
const namespace1 = 'test-namespace-1';
|
|
424
|
+
const namespace2 = 'test-namespace-2';
|
|
425
|
+
const testVector = [1.0, 0.0, 0.0];
|
|
426
|
+
const testMetadata = { label: 'test' };
|
|
427
|
+
|
|
428
|
+
beforeEach(async () => {
|
|
429
|
+
await vectorDB.createIndex({ indexName: indexNameNamespace, dimension, metric: 'cosine' });
|
|
430
|
+
await waitUntilReady(vectorDB, indexNameNamespace);
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
afterEach(async () => {
|
|
434
|
+
try {
|
|
435
|
+
await vectorDB.deleteIndex(indexNameNamespace);
|
|
436
|
+
await waitUntilIndexDeleted(vectorDB, indexNameNamespace);
|
|
437
|
+
} catch {
|
|
438
|
+
// Ignore errors if index doesn't exist
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
it('should isolate vectors in different namespaces', async () => {
|
|
443
|
+
// Insert same vector in two namespaces
|
|
444
|
+
const [id1] = await vectorDB.upsert({
|
|
445
|
+
indexName: indexNameNamespace,
|
|
446
|
+
vectors: [testVector],
|
|
447
|
+
metadata: [testMetadata],
|
|
448
|
+
namespace: namespace1,
|
|
449
|
+
});
|
|
450
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 1);
|
|
451
|
+
|
|
452
|
+
const [id2] = await vectorDB.upsert({
|
|
453
|
+
indexName: indexNameNamespace,
|
|
454
|
+
vectors: [testVector],
|
|
455
|
+
metadata: [{ ...testMetadata, label: 'test2' }],
|
|
456
|
+
namespace: namespace2,
|
|
457
|
+
});
|
|
458
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 2);
|
|
459
|
+
|
|
460
|
+
// Query namespace1
|
|
461
|
+
const results1 = await vectorDB.query({
|
|
462
|
+
indexName: indexNameNamespace,
|
|
463
|
+
queryVector: testVector,
|
|
464
|
+
namespace: namespace1,
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
// Query namespace2
|
|
468
|
+
const results2 = await vectorDB.query({
|
|
469
|
+
indexName: indexNameNamespace,
|
|
470
|
+
queryVector: testVector,
|
|
471
|
+
namespace: namespace2,
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
// Verify isolation
|
|
475
|
+
expect(results1).toHaveLength(1);
|
|
476
|
+
expect(results2).toHaveLength(1);
|
|
477
|
+
expect(results1[0]?.id).toBe(id1);
|
|
478
|
+
expect(results1[0]?.metadata?.label).toBe('test');
|
|
479
|
+
expect(results2[0]?.id).toBe(id2);
|
|
480
|
+
expect(results2[0]?.metadata?.label).toBe('test2');
|
|
481
|
+
}, 500000);
|
|
482
|
+
|
|
483
|
+
it('should update vectors within specific namespace', async () => {
|
|
484
|
+
const [id] = await vectorDB.upsert({
|
|
485
|
+
indexName: indexNameNamespace,
|
|
486
|
+
vectors: [testVector],
|
|
487
|
+
metadata: [testMetadata],
|
|
488
|
+
namespace: namespace1,
|
|
489
|
+
});
|
|
490
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 1);
|
|
491
|
+
|
|
492
|
+
// Update in namespace1
|
|
493
|
+
await vectorDB.updateIndexById(indexNameNamespace, id, { metadata: { label: 'updated' } }, namespace1);
|
|
494
|
+
|
|
495
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 1);
|
|
496
|
+
|
|
497
|
+
// Query to verify update
|
|
498
|
+
const results = await vectorDB.query({
|
|
499
|
+
indexName: indexNameNamespace,
|
|
500
|
+
queryVector: testVector,
|
|
501
|
+
namespace: namespace1,
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
expect(results).toHaveLength(1);
|
|
505
|
+
expect(results[0]?.metadata?.label).toBe('updated');
|
|
506
|
+
}, 500000);
|
|
507
|
+
|
|
508
|
+
it('should delete vectors from specific namespace', async () => {
|
|
509
|
+
const [id] = await vectorDB.upsert({
|
|
510
|
+
indexName: indexNameNamespace,
|
|
511
|
+
vectors: [testVector],
|
|
512
|
+
metadata: [testMetadata],
|
|
513
|
+
namespace: namespace1,
|
|
514
|
+
});
|
|
515
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 1);
|
|
516
|
+
|
|
517
|
+
// Delete from namespace1
|
|
518
|
+
await vectorDB.deleteIndexById(indexNameNamespace, id, namespace1);
|
|
519
|
+
|
|
520
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 0, true);
|
|
521
|
+
|
|
522
|
+
// Query to verify deletion
|
|
523
|
+
const results = await vectorDB.query({
|
|
524
|
+
indexName: indexNameNamespace,
|
|
525
|
+
queryVector: testVector,
|
|
526
|
+
namespace: namespace1,
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
expect(results.length).toBe(0);
|
|
530
|
+
}, 500000);
|
|
531
|
+
|
|
532
|
+
it('should show namespace stats in describeIndex', async () => {
|
|
533
|
+
await vectorDB.upsert({
|
|
534
|
+
indexName: indexNameNamespace,
|
|
535
|
+
vectors: [testVector],
|
|
536
|
+
metadata: [testMetadata],
|
|
537
|
+
namespace: namespace1,
|
|
538
|
+
});
|
|
539
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 1);
|
|
540
|
+
await vectorDB.upsert({
|
|
541
|
+
indexName: indexNameNamespace,
|
|
542
|
+
vectors: [testVector],
|
|
543
|
+
metadata: [{ ...testMetadata, label: 'test2' }],
|
|
544
|
+
namespace: namespace2,
|
|
545
|
+
});
|
|
546
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameNamespace, 2);
|
|
547
|
+
|
|
548
|
+
const stats = await vectorDB.describeIndex(indexNameNamespace);
|
|
549
|
+
expect(stats.namespaces).toBeDefined();
|
|
550
|
+
expect(stats.namespaces?.[namespace1]).toBeDefined();
|
|
551
|
+
expect(stats.namespaces?.[namespace2]).toBeDefined();
|
|
552
|
+
expect(stats.namespaces?.[namespace1].recordCount).toBe(1);
|
|
553
|
+
expect(stats.namespaces?.[namespace2].recordCount).toBe(1);
|
|
554
|
+
}, 500000);
|
|
555
|
+
});
|
|
556
|
+
|
|
278
557
|
describe('Error Handling', () => {
|
|
279
558
|
it('should handle non-existent index query gracefully', async () => {
|
|
280
559
|
const nonExistentIndex = 'non-existent-index';
|
|
@@ -1024,6 +1303,109 @@ describe.skip('PineconeVector Integration Tests', () => {
|
|
|
1024
1303
|
expect(results.length).toBeGreaterThan(0);
|
|
1025
1304
|
});
|
|
1026
1305
|
});
|
|
1306
|
+
|
|
1307
|
+
describe('Hybrid Search Operations', () => {
|
|
1308
|
+
const testVectors = [
|
|
1309
|
+
[0.9, 0.1, 0.0], // cats (very distinct)
|
|
1310
|
+
[0.1, 0.9, 0.0], // dogs (very distinct)
|
|
1311
|
+
[0.0, 0.0, 0.9], // birds (completely different)
|
|
1312
|
+
];
|
|
1313
|
+
|
|
1314
|
+
const testMetadata = [
|
|
1315
|
+
{ text: 'cats purr and meow', animal: 'cat' },
|
|
1316
|
+
{ text: 'dogs bark and fetch', animal: 'dog' },
|
|
1317
|
+
{ text: 'birds fly and nest', animal: 'bird' },
|
|
1318
|
+
];
|
|
1319
|
+
|
|
1320
|
+
// Create sparse vectors with fixed vocabulary indices
|
|
1321
|
+
const testSparseVectors = [
|
|
1322
|
+
{ indices: [0], values: [1.0] }, // cat terms only
|
|
1323
|
+
{ indices: [1], values: [1.0] }, // dog terms only
|
|
1324
|
+
{ indices: [2], values: [1.0] }, // bird terms only
|
|
1325
|
+
];
|
|
1326
|
+
|
|
1327
|
+
beforeEach(async () => {
|
|
1328
|
+
await vectorDB.createIndex({ indexName: indexNameHybrid, dimension: 3, metric: 'dotproduct' });
|
|
1329
|
+
await waitUntilReady(vectorDB, indexNameHybrid);
|
|
1330
|
+
|
|
1331
|
+
// Upsert with both dense and sparse vectors
|
|
1332
|
+
await vectorDB.upsert({
|
|
1333
|
+
indexName: indexNameHybrid,
|
|
1334
|
+
vectors: testVectors,
|
|
1335
|
+
sparseVectors: testSparseVectors,
|
|
1336
|
+
metadata: testMetadata,
|
|
1337
|
+
});
|
|
1338
|
+
await waitUntilVectorsIndexed(vectorDB, indexNameHybrid, 3);
|
|
1339
|
+
});
|
|
1340
|
+
|
|
1341
|
+
afterEach(async () => {
|
|
1342
|
+
try {
|
|
1343
|
+
await vectorDB.deleteIndex(indexNameHybrid);
|
|
1344
|
+
await waitUntilIndexDeleted(vectorDB, indexNameHybrid);
|
|
1345
|
+
} catch {
|
|
1346
|
+
// Ignore errors if index doesn't exist
|
|
1347
|
+
}
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1350
|
+
it('should combine dense and sparse signals in hybrid search', async () => {
|
|
1351
|
+
// Query vector strongly favors cats
|
|
1352
|
+
const queryVector = [1.0, 0.0, 0.0];
|
|
1353
|
+
// But sparse vector strongly favors dogs
|
|
1354
|
+
const sparseVector = {
|
|
1355
|
+
indices: [1], // Index 1 corresponds to dog-related terms
|
|
1356
|
+
values: [1.0], // Maximum weight for dog terms
|
|
1357
|
+
};
|
|
1358
|
+
|
|
1359
|
+
const results = await vectorDB.query({
|
|
1360
|
+
indexName: indexNameHybrid,
|
|
1361
|
+
queryVector,
|
|
1362
|
+
sparseVector,
|
|
1363
|
+
topK: 2,
|
|
1364
|
+
});
|
|
1365
|
+
|
|
1366
|
+
expect(results).toHaveLength(2);
|
|
1367
|
+
|
|
1368
|
+
// Get results with just vector similarity
|
|
1369
|
+
const vectorResults = await vectorDB.query({
|
|
1370
|
+
indexName: indexNameHybrid,
|
|
1371
|
+
queryVector,
|
|
1372
|
+
topK: 2,
|
|
1373
|
+
});
|
|
1374
|
+
|
|
1375
|
+
// Results should be different when using hybrid search vs just vector
|
|
1376
|
+
expect(results[0].id).not.toBe(vectorResults[0].id);
|
|
1377
|
+
|
|
1378
|
+
// First result should be dog due to sparse vector influence
|
|
1379
|
+
expect(results[0].metadata?.animal).toBe('dog');
|
|
1380
|
+
});
|
|
1381
|
+
|
|
1382
|
+
it('should support sparse vectors as optional parameters', async () => {
|
|
1383
|
+
// Should work with just dense vectors in upsert
|
|
1384
|
+
await vectorDB.upsert({
|
|
1385
|
+
indexName: indexNameHybrid,
|
|
1386
|
+
vectors: [[0.1, 0.2, 0.3]],
|
|
1387
|
+
metadata: [{ test: 'dense only' }],
|
|
1388
|
+
});
|
|
1389
|
+
|
|
1390
|
+
// Should work with just dense vector in query
|
|
1391
|
+
const denseOnlyResults = await vectorDB.query({
|
|
1392
|
+
indexName: indexNameHybrid,
|
|
1393
|
+
queryVector: [0.1, 0.2, 0.3],
|
|
1394
|
+
topK: 1,
|
|
1395
|
+
});
|
|
1396
|
+
expect(denseOnlyResults).toHaveLength(1);
|
|
1397
|
+
|
|
1398
|
+
// Should work with both dense and sparse in query
|
|
1399
|
+
const hybridResults = await vectorDB.query({
|
|
1400
|
+
indexName: indexNameHybrid,
|
|
1401
|
+
queryVector: [0.1, 0.2, 0.3],
|
|
1402
|
+
sparseVector: createSparseVector('test query'),
|
|
1403
|
+
topK: 1,
|
|
1404
|
+
});
|
|
1405
|
+
expect(hybridResults).toHaveLength(1);
|
|
1406
|
+
});
|
|
1407
|
+
});
|
|
1408
|
+
|
|
1027
1409
|
describe('Deprecation Warnings', () => {
|
|
1028
1410
|
const indexName = 'testdeprecationwarnings';
|
|
1029
1411
|
|
package/src/vector/index.ts
CHANGED
|
@@ -6,13 +6,38 @@ import type {
|
|
|
6
6
|
UpsertVectorParams,
|
|
7
7
|
QueryVectorParams,
|
|
8
8
|
ParamsToArgs,
|
|
9
|
+
QueryVectorArgs,
|
|
10
|
+
UpsertVectorArgs,
|
|
9
11
|
} from '@mastra/core/vector';
|
|
10
12
|
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
11
13
|
import { Pinecone } from '@pinecone-database/pinecone';
|
|
12
|
-
import type {
|
|
14
|
+
import type {
|
|
15
|
+
IndexStatsDescription,
|
|
16
|
+
QueryOptions,
|
|
17
|
+
RecordSparseValues,
|
|
18
|
+
UpdateOptions,
|
|
19
|
+
} from '@pinecone-database/pinecone';
|
|
13
20
|
|
|
14
21
|
import { PineconeFilterTranslator } from './filter';
|
|
15
22
|
|
|
23
|
+
interface PineconeIndexStats extends IndexStats {
|
|
24
|
+
namespaces?: IndexStatsDescription['namespaces'];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface PineconeQueryVectorParams extends QueryVectorParams {
|
|
28
|
+
namespace?: string;
|
|
29
|
+
sparseVector?: RecordSparseValues;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
type PineconeQueryVectorArgs = [...QueryVectorArgs, string?, RecordSparseValues?];
|
|
33
|
+
|
|
34
|
+
interface PineconeUpsertVectorParams extends UpsertVectorParams {
|
|
35
|
+
namespace?: string;
|
|
36
|
+
sparseVectors?: RecordSparseValues[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
type PineconeUpsertVectorArgs = [...UpsertVectorArgs, string?, RecordSparseValues[]?];
|
|
40
|
+
|
|
16
41
|
export class PineconeVector extends MastraVector {
|
|
17
42
|
private client: Pinecone;
|
|
18
43
|
|
|
@@ -57,12 +82,15 @@ export class PineconeVector extends MastraVector {
|
|
|
57
82
|
});
|
|
58
83
|
}
|
|
59
84
|
|
|
60
|
-
async upsert(...args: ParamsToArgs<
|
|
61
|
-
const params = this.normalizeArgs<
|
|
85
|
+
async upsert(...args: ParamsToArgs<PineconeUpsertVectorParams> | PineconeUpsertVectorArgs): Promise<string[]> {
|
|
86
|
+
const params = this.normalizeArgs<PineconeUpsertVectorParams, PineconeUpsertVectorArgs>('upsert', args, [
|
|
87
|
+
'namespace',
|
|
88
|
+
'sparseVectors',
|
|
89
|
+
]);
|
|
62
90
|
|
|
63
|
-
const { indexName, vectors, metadata, ids } = params;
|
|
91
|
+
const { indexName, vectors, metadata, ids, namespace, sparseVectors } = params;
|
|
64
92
|
|
|
65
|
-
const index = this.client.Index(indexName);
|
|
93
|
+
const index = this.client.Index(indexName).namespace(namespace || '');
|
|
66
94
|
|
|
67
95
|
// Generate IDs if not provided
|
|
68
96
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
@@ -70,6 +98,7 @@ export class PineconeVector extends MastraVector {
|
|
|
70
98
|
const records = vectors.map((vector, i) => ({
|
|
71
99
|
id: vectorIds[i]!,
|
|
72
100
|
values: vector,
|
|
101
|
+
...(sparseVectors?.[i] && { sparseValues: sparseVectors?.[i] }),
|
|
73
102
|
metadata: metadata?.[i] || {},
|
|
74
103
|
}));
|
|
75
104
|
|
|
@@ -88,22 +117,32 @@ export class PineconeVector extends MastraVector {
|
|
|
88
117
|
return translator.translate(filter);
|
|
89
118
|
}
|
|
90
119
|
|
|
91
|
-
async query(...args: ParamsToArgs<
|
|
92
|
-
const params = this.normalizeArgs<
|
|
120
|
+
async query(...args: ParamsToArgs<PineconeQueryVectorParams> | PineconeQueryVectorArgs): Promise<QueryResult[]> {
|
|
121
|
+
const params = this.normalizeArgs<PineconeQueryVectorParams, PineconeQueryVectorArgs>('query', args, [
|
|
122
|
+
'namespace',
|
|
123
|
+
'sparseVector',
|
|
124
|
+
]);
|
|
93
125
|
|
|
94
|
-
const { indexName, queryVector, topK = 10, filter, includeVector = false } = params;
|
|
126
|
+
const { indexName, queryVector, topK = 10, filter, includeVector = false, namespace, sparseVector } = params;
|
|
95
127
|
|
|
96
|
-
const index = this.client.Index(indexName);
|
|
128
|
+
const index = this.client.Index(indexName).namespace(namespace || '');
|
|
97
129
|
|
|
98
130
|
const translatedFilter = this.transformFilter(filter) ?? undefined;
|
|
99
131
|
|
|
100
|
-
const
|
|
132
|
+
const queryParams: QueryOptions = {
|
|
101
133
|
vector: queryVector,
|
|
102
134
|
topK,
|
|
103
|
-
filter: translatedFilter,
|
|
104
135
|
includeMetadata: true,
|
|
105
136
|
includeValues: includeVector,
|
|
106
|
-
|
|
137
|
+
filter: translatedFilter,
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// If sparse vector is provided, use hybrid search
|
|
141
|
+
if (sparseVector) {
|
|
142
|
+
queryParams.sparseVector = sparseVector;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const results = await index.query(queryParams);
|
|
107
146
|
|
|
108
147
|
return results.matches.map(match => ({
|
|
109
148
|
id: match.id,
|
|
@@ -118,7 +157,7 @@ export class PineconeVector extends MastraVector {
|
|
|
118
157
|
return indexesResult?.indexes?.map(index => index.name) || [];
|
|
119
158
|
}
|
|
120
159
|
|
|
121
|
-
async describeIndex(indexName: string): Promise<
|
|
160
|
+
async describeIndex(indexName: string): Promise<PineconeIndexStats> {
|
|
122
161
|
const index = this.client.Index(indexName);
|
|
123
162
|
const stats = await index.describeIndexStats();
|
|
124
163
|
const description = await this.client.describeIndex(indexName);
|
|
@@ -127,6 +166,7 @@ export class PineconeVector extends MastraVector {
|
|
|
127
166
|
dimension: description.dimension,
|
|
128
167
|
count: stats.totalRecordCount || 0,
|
|
129
168
|
metric: description.metric as 'cosine' | 'euclidean' | 'dotproduct',
|
|
169
|
+
namespaces: stats.namespaces,
|
|
130
170
|
};
|
|
131
171
|
}
|
|
132
172
|
|
|
@@ -145,12 +185,13 @@ export class PineconeVector extends MastraVector {
|
|
|
145
185
|
vector?: number[];
|
|
146
186
|
metadata?: Record<string, any>;
|
|
147
187
|
},
|
|
188
|
+
namespace?: string,
|
|
148
189
|
): Promise<void> {
|
|
149
190
|
if (!update.vector && !update.metadata) {
|
|
150
191
|
throw new Error('No updates provided');
|
|
151
192
|
}
|
|
152
193
|
|
|
153
|
-
const index = this.client.Index(indexName);
|
|
194
|
+
const index = this.client.Index(indexName).namespace(namespace || '');
|
|
154
195
|
|
|
155
196
|
const updateObj: UpdateOptions = { id };
|
|
156
197
|
|
|
@@ -165,8 +206,8 @@ export class PineconeVector extends MastraVector {
|
|
|
165
206
|
await index.update(updateObj);
|
|
166
207
|
}
|
|
167
208
|
|
|
168
|
-
async deleteIndexById(indexName: string, id: string): Promise<void> {
|
|
169
|
-
const index = this.client.Index(indexName);
|
|
209
|
+
async deleteIndexById(indexName: string, id: string, namespace?: string): Promise<void> {
|
|
210
|
+
const index = this.client.Index(indexName).namespace(namespace || '');
|
|
170
211
|
await index.deleteOne(id);
|
|
171
212
|
}
|
|
172
213
|
}
|