@mastra/rag 1.2.3-alpha.0 → 1.2.3-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/package.json +18 -5
- package/.turbo/turbo-build.log +0 -4
- package/docker-compose.yaml +0 -22
- package/eslint.config.js +0 -6
- package/src/document/document.test.ts +0 -2975
- package/src/document/document.ts +0 -335
- package/src/document/extractors/base.ts +0 -30
- package/src/document/extractors/index.ts +0 -5
- package/src/document/extractors/keywords.test.ts +0 -125
- package/src/document/extractors/keywords.ts +0 -126
- package/src/document/extractors/questions.test.ts +0 -120
- package/src/document/extractors/questions.ts +0 -111
- package/src/document/extractors/summary.test.ts +0 -107
- package/src/document/extractors/summary.ts +0 -122
- package/src/document/extractors/title.test.ts +0 -121
- package/src/document/extractors/title.ts +0 -185
- package/src/document/extractors/types.ts +0 -40
- package/src/document/index.ts +0 -2
- package/src/document/prompts/base.ts +0 -77
- package/src/document/prompts/format.ts +0 -9
- package/src/document/prompts/index.ts +0 -15
- package/src/document/prompts/prompt.ts +0 -60
- package/src/document/prompts/types.ts +0 -29
- package/src/document/schema/index.ts +0 -3
- package/src/document/schema/node.ts +0 -187
- package/src/document/schema/types.ts +0 -40
- package/src/document/transformers/character.ts +0 -267
- package/src/document/transformers/html.ts +0 -346
- package/src/document/transformers/json.ts +0 -536
- package/src/document/transformers/latex.ts +0 -11
- package/src/document/transformers/markdown.ts +0 -239
- package/src/document/transformers/semantic-markdown.ts +0 -227
- package/src/document/transformers/sentence.ts +0 -314
- package/src/document/transformers/text.ts +0 -158
- package/src/document/transformers/token.ts +0 -137
- package/src/document/transformers/transformer.ts +0 -5
- package/src/document/types.ts +0 -145
- package/src/document/validation.ts +0 -158
- package/src/graph-rag/index.test.ts +0 -235
- package/src/graph-rag/index.ts +0 -306
- package/src/index.ts +0 -8
- package/src/rerank/index.test.ts +0 -150
- package/src/rerank/index.ts +0 -198
- package/src/rerank/relevance/cohere/index.ts +0 -56
- package/src/rerank/relevance/index.ts +0 -3
- package/src/rerank/relevance/mastra-agent/index.ts +0 -32
- package/src/rerank/relevance/zeroentropy/index.ts +0 -26
- package/src/tools/README.md +0 -153
- package/src/tools/document-chunker.ts +0 -34
- package/src/tools/graph-rag.test.ts +0 -115
- package/src/tools/graph-rag.ts +0 -157
- package/src/tools/index.ts +0 -3
- package/src/tools/types.ts +0 -126
- package/src/tools/vector-query-database-config.test.ts +0 -190
- package/src/tools/vector-query.test.ts +0 -477
- package/src/tools/vector-query.ts +0 -171
- package/src/utils/convert-sources.ts +0 -43
- package/src/utils/default-settings.ts +0 -38
- package/src/utils/index.ts +0 -3
- package/src/utils/tool-schemas.ts +0 -38
- package/src/utils/vector-prompts.ts +0 -832
- package/src/utils/vector-search.ts +0 -130
- package/tsconfig.build.json +0 -9
- package/tsconfig.json +0 -5
- package/tsup.config.ts +0 -17
- package/vitest.config.ts +0 -8
package/src/tools/types.ts
DELETED
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
import type { MastraVector, MastraEmbeddingModel } from '@mastra/core/vector';
|
|
2
|
-
|
|
3
|
-
import type { RerankConfig } from '../rerank';
|
|
4
|
-
|
|
5
|
-
export interface PineconeConfig {
|
|
6
|
-
namespace?: string;
|
|
7
|
-
sparseVector?: {
|
|
8
|
-
indices: number[];
|
|
9
|
-
values: number[];
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export interface PgVectorConfig {
|
|
14
|
-
minScore?: number;
|
|
15
|
-
ef?: number; // HNSW search parameter
|
|
16
|
-
probes?: number; // IVFFlat probe parameter
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// Chroma types
|
|
20
|
-
type LiteralValue = string | number | boolean;
|
|
21
|
-
type ListLiteralValue = LiteralValue[];
|
|
22
|
-
type LiteralNumber = number;
|
|
23
|
-
type LogicalOperator = '$and' | '$or';
|
|
24
|
-
type InclusionOperator = '$in' | '$nin';
|
|
25
|
-
type WhereOperator = '$gt' | '$gte' | '$lt' | '$lte' | '$ne' | '$eq';
|
|
26
|
-
type OperatorExpression = {
|
|
27
|
-
[key in WhereOperator | InclusionOperator | LogicalOperator]?: LiteralValue | ListLiteralValue;
|
|
28
|
-
};
|
|
29
|
-
type BaseWhere = {
|
|
30
|
-
[key: string]: LiteralValue | OperatorExpression;
|
|
31
|
-
};
|
|
32
|
-
type LogicalWhere = {
|
|
33
|
-
[key in LogicalOperator]?: Where[];
|
|
34
|
-
};
|
|
35
|
-
type Where = BaseWhere | LogicalWhere;
|
|
36
|
-
type WhereDocumentOperator = '$contains' | '$not_contains' | LogicalOperator;
|
|
37
|
-
type WhereDocument = {
|
|
38
|
-
[key in WhereDocumentOperator]?: LiteralValue | LiteralNumber | WhereDocument[];
|
|
39
|
-
};
|
|
40
|
-
|
|
41
|
-
export interface ChromaConfig {
|
|
42
|
-
// Add Chroma-specific configs here if needed
|
|
43
|
-
where?: Where;
|
|
44
|
-
whereDocument?: WhereDocument;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// Union type for all database-specific configs
|
|
48
|
-
export type DatabaseConfig = {
|
|
49
|
-
pinecone?: PineconeConfig;
|
|
50
|
-
pgvector?: PgVectorConfig;
|
|
51
|
-
chroma?: ChromaConfig;
|
|
52
|
-
// Add other database configs as needed
|
|
53
|
-
[key: string]: any; // Allow for future database extensions
|
|
54
|
-
};
|
|
55
|
-
|
|
56
|
-
export type VectorQueryToolOptions = {
|
|
57
|
-
id?: string;
|
|
58
|
-
description?: string;
|
|
59
|
-
indexName: string;
|
|
60
|
-
model: MastraEmbeddingModel<string>;
|
|
61
|
-
enableFilter?: boolean;
|
|
62
|
-
includeVectors?: boolean;
|
|
63
|
-
includeSources?: boolean;
|
|
64
|
-
reranker?: RerankConfig;
|
|
65
|
-
/** Database-specific configuration options */
|
|
66
|
-
databaseConfig?: DatabaseConfig;
|
|
67
|
-
} & ProviderOptions &
|
|
68
|
-
(
|
|
69
|
-
| {
|
|
70
|
-
vectorStoreName: string;
|
|
71
|
-
}
|
|
72
|
-
| {
|
|
73
|
-
vectorStoreName?: string;
|
|
74
|
-
vectorStore: MastraVector;
|
|
75
|
-
}
|
|
76
|
-
);
|
|
77
|
-
|
|
78
|
-
export type GraphRagToolOptions = {
|
|
79
|
-
id?: string;
|
|
80
|
-
description?: string;
|
|
81
|
-
indexName: string;
|
|
82
|
-
vectorStoreName: string;
|
|
83
|
-
model: MastraEmbeddingModel<string>;
|
|
84
|
-
enableFilter?: boolean;
|
|
85
|
-
includeSources?: boolean;
|
|
86
|
-
graphOptions?: {
|
|
87
|
-
dimension?: number;
|
|
88
|
-
randomWalkSteps?: number;
|
|
89
|
-
restartProb?: number;
|
|
90
|
-
threshold?: number;
|
|
91
|
-
};
|
|
92
|
-
} & ProviderOptions;
|
|
93
|
-
|
|
94
|
-
export type ProviderOptions = {
|
|
95
|
-
/**
|
|
96
|
-
* Provider-specific options for the embedding model (e.g., outputDimensionality).
|
|
97
|
-
*
|
|
98
|
-
* ⚠️ **IMPORTANT**: `providerOptions` only work with AI SDK v2 models.
|
|
99
|
-
*
|
|
100
|
-
* **For v1 models**: Configure options when creating the model:
|
|
101
|
-
* ✅ const model = openai.embedding('text-embedding-3-small', { dimensions: 512 });
|
|
102
|
-
*
|
|
103
|
-
* **For v2 models**: Use providerOptions:
|
|
104
|
-
* ✅ providerOptions: { openai: { dimensions: 512 } }
|
|
105
|
-
*/
|
|
106
|
-
providerOptions?: Record<string, Record<string, any>>;
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Default options for GraphRAG
|
|
111
|
-
* @default
|
|
112
|
-
* ```json
|
|
113
|
-
* {
|
|
114
|
-
* "dimension": 1536,
|
|
115
|
-
* "randomWalkSteps": 100,
|
|
116
|
-
* "restartProb": 0.15,
|
|
117
|
-
* "threshold": 0.7
|
|
118
|
-
* }
|
|
119
|
-
* ```
|
|
120
|
-
*/
|
|
121
|
-
export const defaultGraphOptions = {
|
|
122
|
-
dimension: 1536,
|
|
123
|
-
randomWalkSteps: 100,
|
|
124
|
-
restartProb: 0.15,
|
|
125
|
-
threshold: 0.7,
|
|
126
|
-
};
|
|
@@ -1,190 +0,0 @@
|
|
|
1
|
-
import { RuntimeContext } from '@mastra/core/runtime-context';
|
|
2
|
-
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
-
import { vectorQuerySearch } from '../utils';
|
|
4
|
-
import type { DatabaseConfig } from './types';
|
|
5
|
-
import { createVectorQueryTool } from './vector-query';
|
|
6
|
-
|
|
7
|
-
vi.mock('../utils', async importOriginal => {
|
|
8
|
-
const actual: any = await importOriginal();
|
|
9
|
-
return {
|
|
10
|
-
...actual,
|
|
11
|
-
vectorQuerySearch: vi.fn().mockResolvedValue({
|
|
12
|
-
results: [{ metadata: { text: 'test result' }, vector: [1, 2, 3] }],
|
|
13
|
-
}),
|
|
14
|
-
};
|
|
15
|
-
});
|
|
16
|
-
|
|
17
|
-
describe('createVectorQueryTool with database-specific configurations', () => {
|
|
18
|
-
const mockModel = { name: 'test-model' } as any;
|
|
19
|
-
const mockMastra = {
|
|
20
|
-
getVector: vi.fn(() => ({})),
|
|
21
|
-
getLogger: vi.fn(() => ({
|
|
22
|
-
debug: vi.fn(),
|
|
23
|
-
warn: vi.fn(),
|
|
24
|
-
error: vi.fn(),
|
|
25
|
-
})),
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
beforeEach(() => {
|
|
29
|
-
vi.clearAllMocks();
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it('should pass Pinecone configuration to vectorQuerySearch', async () => {
|
|
33
|
-
const databaseConfig: DatabaseConfig = {
|
|
34
|
-
pinecone: {
|
|
35
|
-
namespace: 'test-namespace',
|
|
36
|
-
sparseVector: {
|
|
37
|
-
indices: [0, 1, 2],
|
|
38
|
-
values: [0.1, 0.2, 0.3],
|
|
39
|
-
},
|
|
40
|
-
},
|
|
41
|
-
};
|
|
42
|
-
|
|
43
|
-
const tool = createVectorQueryTool({
|
|
44
|
-
vectorStoreName: 'pinecone',
|
|
45
|
-
indexName: 'testIndex',
|
|
46
|
-
model: mockModel,
|
|
47
|
-
databaseConfig,
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
const runtimeContext = new RuntimeContext();
|
|
51
|
-
|
|
52
|
-
await tool.execute({
|
|
53
|
-
context: { queryText: 'test query', topK: 5 },
|
|
54
|
-
mastra: mockMastra as any,
|
|
55
|
-
runtimeContext,
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
expect(vectorQuerySearch).toHaveBeenCalledWith(
|
|
59
|
-
expect.objectContaining({
|
|
60
|
-
databaseConfig,
|
|
61
|
-
}),
|
|
62
|
-
);
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
it('should pass pgVector configuration to vectorQuerySearch', async () => {
|
|
66
|
-
const databaseConfig: DatabaseConfig = {
|
|
67
|
-
pgvector: {
|
|
68
|
-
minScore: 0.7,
|
|
69
|
-
ef: 200,
|
|
70
|
-
probes: 10,
|
|
71
|
-
},
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
const tool = createVectorQueryTool({
|
|
75
|
-
vectorStoreName: 'postgres',
|
|
76
|
-
indexName: 'testIndex',
|
|
77
|
-
model: mockModel,
|
|
78
|
-
databaseConfig,
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
const runtimeContext = new RuntimeContext();
|
|
82
|
-
|
|
83
|
-
await tool.execute({
|
|
84
|
-
context: { queryText: 'test query', topK: 5 },
|
|
85
|
-
mastra: mockMastra as any,
|
|
86
|
-
runtimeContext,
|
|
87
|
-
});
|
|
88
|
-
|
|
89
|
-
expect(vectorQuerySearch).toHaveBeenCalledWith(
|
|
90
|
-
expect.objectContaining({
|
|
91
|
-
databaseConfig,
|
|
92
|
-
}),
|
|
93
|
-
);
|
|
94
|
-
});
|
|
95
|
-
|
|
96
|
-
it('should allow runtime context to override database configuration', async () => {
|
|
97
|
-
const initialConfig: DatabaseConfig = {
|
|
98
|
-
pinecone: {
|
|
99
|
-
namespace: 'initial-namespace',
|
|
100
|
-
},
|
|
101
|
-
};
|
|
102
|
-
|
|
103
|
-
const runtimeConfig: DatabaseConfig = {
|
|
104
|
-
pinecone: {
|
|
105
|
-
namespace: 'runtime-namespace',
|
|
106
|
-
},
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
const tool = createVectorQueryTool({
|
|
110
|
-
vectorStoreName: 'pinecone',
|
|
111
|
-
indexName: 'testIndex',
|
|
112
|
-
model: mockModel,
|
|
113
|
-
databaseConfig: initialConfig,
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
const runtimeContext = new RuntimeContext();
|
|
117
|
-
runtimeContext.set('databaseConfig', runtimeConfig);
|
|
118
|
-
|
|
119
|
-
await tool.execute({
|
|
120
|
-
context: { queryText: 'test query', topK: 5 },
|
|
121
|
-
mastra: mockMastra as any,
|
|
122
|
-
runtimeContext,
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
expect(vectorQuerySearch).toHaveBeenCalledWith(
|
|
126
|
-
expect.objectContaining({
|
|
127
|
-
databaseConfig: runtimeConfig, // Should use runtime config, not initial
|
|
128
|
-
}),
|
|
129
|
-
);
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
it('should work without database configuration (backward compatibility)', async () => {
|
|
133
|
-
const tool = createVectorQueryTool({
|
|
134
|
-
vectorStoreName: 'testStore',
|
|
135
|
-
indexName: 'testIndex',
|
|
136
|
-
model: mockModel,
|
|
137
|
-
// No databaseConfig provided
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
const runtimeContext = new RuntimeContext();
|
|
141
|
-
|
|
142
|
-
await tool.execute({
|
|
143
|
-
context: { queryText: 'test query', topK: 5 },
|
|
144
|
-
mastra: mockMastra as any,
|
|
145
|
-
runtimeContext,
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
expect(vectorQuerySearch).toHaveBeenCalledWith(
|
|
149
|
-
expect.objectContaining({
|
|
150
|
-
databaseConfig: undefined,
|
|
151
|
-
}),
|
|
152
|
-
);
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
it('should handle multiple database configurations', async () => {
|
|
156
|
-
const databaseConfig: DatabaseConfig = {
|
|
157
|
-
pinecone: {
|
|
158
|
-
namespace: 'test-namespace',
|
|
159
|
-
},
|
|
160
|
-
pgvector: {
|
|
161
|
-
minScore: 0.8,
|
|
162
|
-
ef: 100,
|
|
163
|
-
},
|
|
164
|
-
chroma: {
|
|
165
|
-
where: { category: 'documents' },
|
|
166
|
-
},
|
|
167
|
-
};
|
|
168
|
-
|
|
169
|
-
const tool = createVectorQueryTool({
|
|
170
|
-
vectorStoreName: 'multidb',
|
|
171
|
-
indexName: 'testIndex',
|
|
172
|
-
model: mockModel,
|
|
173
|
-
databaseConfig,
|
|
174
|
-
});
|
|
175
|
-
|
|
176
|
-
const runtimeContext = new RuntimeContext();
|
|
177
|
-
|
|
178
|
-
await tool.execute({
|
|
179
|
-
context: { queryText: 'test query', topK: 5 },
|
|
180
|
-
mastra: mockMastra as any,
|
|
181
|
-
runtimeContext,
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
expect(vectorQuerySearch).toHaveBeenCalledWith(
|
|
185
|
-
expect.objectContaining({
|
|
186
|
-
databaseConfig,
|
|
187
|
-
}),
|
|
188
|
-
);
|
|
189
|
-
});
|
|
190
|
-
});
|