@mastra/rag 1.2.2 → 1.2.3-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/index.cjs +25 -9
  3. package/dist/index.cjs.map +1 -1
  4. package/dist/index.js +25 -9
  5. package/dist/index.js.map +1 -1
  6. package/dist/tools/graph-rag.d.ts.map +1 -1
  7. package/dist/tools/types.d.ts +18 -5
  8. package/dist/tools/types.d.ts.map +1 -1
  9. package/dist/tools/vector-query.d.ts.map +1 -1
  10. package/dist/utils/vector-search.d.ts +6 -7
  11. package/dist/utils/vector-search.d.ts.map +1 -1
  12. package/package.json +19 -6
  13. package/.turbo/turbo-build.log +0 -4
  14. package/docker-compose.yaml +0 -22
  15. package/eslint.config.js +0 -6
  16. package/src/document/document.test.ts +0 -2975
  17. package/src/document/document.ts +0 -335
  18. package/src/document/extractors/base.ts +0 -30
  19. package/src/document/extractors/index.ts +0 -5
  20. package/src/document/extractors/keywords.test.ts +0 -125
  21. package/src/document/extractors/keywords.ts +0 -126
  22. package/src/document/extractors/questions.test.ts +0 -120
  23. package/src/document/extractors/questions.ts +0 -111
  24. package/src/document/extractors/summary.test.ts +0 -107
  25. package/src/document/extractors/summary.ts +0 -122
  26. package/src/document/extractors/title.test.ts +0 -121
  27. package/src/document/extractors/title.ts +0 -185
  28. package/src/document/extractors/types.ts +0 -40
  29. package/src/document/index.ts +0 -2
  30. package/src/document/prompts/base.ts +0 -77
  31. package/src/document/prompts/format.ts +0 -9
  32. package/src/document/prompts/index.ts +0 -15
  33. package/src/document/prompts/prompt.ts +0 -60
  34. package/src/document/prompts/types.ts +0 -29
  35. package/src/document/schema/index.ts +0 -3
  36. package/src/document/schema/node.ts +0 -187
  37. package/src/document/schema/types.ts +0 -40
  38. package/src/document/transformers/character.ts +0 -267
  39. package/src/document/transformers/html.ts +0 -346
  40. package/src/document/transformers/json.ts +0 -536
  41. package/src/document/transformers/latex.ts +0 -11
  42. package/src/document/transformers/markdown.ts +0 -239
  43. package/src/document/transformers/semantic-markdown.ts +0 -227
  44. package/src/document/transformers/sentence.ts +0 -314
  45. package/src/document/transformers/text.ts +0 -158
  46. package/src/document/transformers/token.ts +0 -137
  47. package/src/document/transformers/transformer.ts +0 -5
  48. package/src/document/types.ts +0 -145
  49. package/src/document/validation.ts +0 -158
  50. package/src/graph-rag/index.test.ts +0 -235
  51. package/src/graph-rag/index.ts +0 -306
  52. package/src/index.ts +0 -8
  53. package/src/rerank/index.test.ts +0 -150
  54. package/src/rerank/index.ts +0 -198
  55. package/src/rerank/relevance/cohere/index.ts +0 -56
  56. package/src/rerank/relevance/index.ts +0 -3
  57. package/src/rerank/relevance/mastra-agent/index.ts +0 -32
  58. package/src/rerank/relevance/zeroentropy/index.ts +0 -26
  59. package/src/tools/README.md +0 -153
  60. package/src/tools/document-chunker.ts +0 -34
  61. package/src/tools/graph-rag.test.ts +0 -115
  62. package/src/tools/graph-rag.ts +0 -154
  63. package/src/tools/index.ts +0 -3
  64. package/src/tools/types.ts +0 -110
  65. package/src/tools/vector-query-database-config.test.ts +0 -190
  66. package/src/tools/vector-query.test.ts +0 -418
  67. package/src/tools/vector-query.ts +0 -169
  68. package/src/utils/convert-sources.ts +0 -43
  69. package/src/utils/default-settings.ts +0 -38
  70. package/src/utils/index.ts +0 -3
  71. package/src/utils/tool-schemas.ts +0 -38
  72. package/src/utils/vector-prompts.ts +0 -832
  73. package/src/utils/vector-search.ts +0 -117
  74. package/tsconfig.build.json +0 -9
  75. package/tsconfig.json +0 -5
  76. package/tsup.config.ts +0 -17
  77. package/vitest.config.ts +0 -8
@@ -1,190 +0,0 @@
1
- import { RuntimeContext } from '@mastra/core/runtime-context';
2
- import { describe, it, expect, vi, beforeEach } from 'vitest';
3
- import { vectorQuerySearch } from '../utils';
4
- import type { DatabaseConfig } from './types';
5
- import { createVectorQueryTool } from './vector-query';
6
-
7
- vi.mock('../utils', async importOriginal => {
8
- const actual: any = await importOriginal();
9
- return {
10
- ...actual,
11
- vectorQuerySearch: vi.fn().mockResolvedValue({
12
- results: [{ metadata: { text: 'test result' }, vector: [1, 2, 3] }],
13
- }),
14
- };
15
- });
16
-
17
- describe('createVectorQueryTool with database-specific configurations', () => {
18
- const mockModel = { name: 'test-model' } as any;
19
- const mockMastra = {
20
- getVector: vi.fn(() => ({})),
21
- getLogger: vi.fn(() => ({
22
- debug: vi.fn(),
23
- warn: vi.fn(),
24
- error: vi.fn(),
25
- })),
26
- };
27
-
28
- beforeEach(() => {
29
- vi.clearAllMocks();
30
- });
31
-
32
- it('should pass Pinecone configuration to vectorQuerySearch', async () => {
33
- const databaseConfig: DatabaseConfig = {
34
- pinecone: {
35
- namespace: 'test-namespace',
36
- sparseVector: {
37
- indices: [0, 1, 2],
38
- values: [0.1, 0.2, 0.3],
39
- },
40
- },
41
- };
42
-
43
- const tool = createVectorQueryTool({
44
- vectorStoreName: 'pinecone',
45
- indexName: 'testIndex',
46
- model: mockModel,
47
- databaseConfig,
48
- });
49
-
50
- const runtimeContext = new RuntimeContext();
51
-
52
- await tool.execute({
53
- context: { queryText: 'test query', topK: 5 },
54
- mastra: mockMastra as any,
55
- runtimeContext,
56
- });
57
-
58
- expect(vectorQuerySearch).toHaveBeenCalledWith(
59
- expect.objectContaining({
60
- databaseConfig,
61
- }),
62
- );
63
- });
64
-
65
- it('should pass pgVector configuration to vectorQuerySearch', async () => {
66
- const databaseConfig: DatabaseConfig = {
67
- pgvector: {
68
- minScore: 0.7,
69
- ef: 200,
70
- probes: 10,
71
- },
72
- };
73
-
74
- const tool = createVectorQueryTool({
75
- vectorStoreName: 'postgres',
76
- indexName: 'testIndex',
77
- model: mockModel,
78
- databaseConfig,
79
- });
80
-
81
- const runtimeContext = new RuntimeContext();
82
-
83
- await tool.execute({
84
- context: { queryText: 'test query', topK: 5 },
85
- mastra: mockMastra as any,
86
- runtimeContext,
87
- });
88
-
89
- expect(vectorQuerySearch).toHaveBeenCalledWith(
90
- expect.objectContaining({
91
- databaseConfig,
92
- }),
93
- );
94
- });
95
-
96
- it('should allow runtime context to override database configuration', async () => {
97
- const initialConfig: DatabaseConfig = {
98
- pinecone: {
99
- namespace: 'initial-namespace',
100
- },
101
- };
102
-
103
- const runtimeConfig: DatabaseConfig = {
104
- pinecone: {
105
- namespace: 'runtime-namespace',
106
- },
107
- };
108
-
109
- const tool = createVectorQueryTool({
110
- vectorStoreName: 'pinecone',
111
- indexName: 'testIndex',
112
- model: mockModel,
113
- databaseConfig: initialConfig,
114
- });
115
-
116
- const runtimeContext = new RuntimeContext();
117
- runtimeContext.set('databaseConfig', runtimeConfig);
118
-
119
- await tool.execute({
120
- context: { queryText: 'test query', topK: 5 },
121
- mastra: mockMastra as any,
122
- runtimeContext,
123
- });
124
-
125
- expect(vectorQuerySearch).toHaveBeenCalledWith(
126
- expect.objectContaining({
127
- databaseConfig: runtimeConfig, // Should use runtime config, not initial
128
- }),
129
- );
130
- });
131
-
132
- it('should work without database configuration (backward compatibility)', async () => {
133
- const tool = createVectorQueryTool({
134
- vectorStoreName: 'testStore',
135
- indexName: 'testIndex',
136
- model: mockModel,
137
- // No databaseConfig provided
138
- });
139
-
140
- const runtimeContext = new RuntimeContext();
141
-
142
- await tool.execute({
143
- context: { queryText: 'test query', topK: 5 },
144
- mastra: mockMastra as any,
145
- runtimeContext,
146
- });
147
-
148
- expect(vectorQuerySearch).toHaveBeenCalledWith(
149
- expect.objectContaining({
150
- databaseConfig: undefined,
151
- }),
152
- );
153
- });
154
-
155
- it('should handle multiple database configurations', async () => {
156
- const databaseConfig: DatabaseConfig = {
157
- pinecone: {
158
- namespace: 'test-namespace',
159
- },
160
- pgvector: {
161
- minScore: 0.8,
162
- ef: 100,
163
- },
164
- chroma: {
165
- where: { category: 'documents' },
166
- },
167
- };
168
-
169
- const tool = createVectorQueryTool({
170
- vectorStoreName: 'multidb',
171
- indexName: 'testIndex',
172
- model: mockModel,
173
- databaseConfig,
174
- });
175
-
176
- const runtimeContext = new RuntimeContext();
177
-
178
- await tool.execute({
179
- context: { queryText: 'test query', topK: 5 },
180
- mastra: mockMastra as any,
181
- runtimeContext,
182
- });
183
-
184
- expect(vectorQuerySearch).toHaveBeenCalledWith(
185
- expect.objectContaining({
186
- databaseConfig,
187
- }),
188
- );
189
- });
190
- });
@@ -1,418 +0,0 @@
1
- import { RuntimeContext } from '@mastra/core/runtime-context';
2
- import { describe, it, expect, vi, beforeEach } from 'vitest';
3
- import { rerank } from '../rerank';
4
- import { vectorQuerySearch } from '../utils';
5
- import { createVectorQueryTool } from './vector-query';
6
-
7
- vi.mock('../utils', async importOriginal => {
8
- const actual: any = await importOriginal();
9
- return {
10
- ...actual,
11
- vectorQuerySearch: vi.fn().mockResolvedValue({ results: [{ metadata: { text: 'foo' }, vector: [1, 2, 3] }] }),
12
- };
13
- });
14
-
15
- vi.mock('../rerank', async importOriginal => {
16
- const actual: any = await importOriginal();
17
- return {
18
- ...actual,
19
- rerank: vi
20
- .fn()
21
- .mockResolvedValue([
22
- { result: { id: '1', metadata: { text: 'bar' }, score: 1, details: { semantic: 1, vector: 1, position: 1 } } },
23
- ]),
24
- };
25
- });
26
-
27
- describe('createVectorQueryTool', () => {
28
- const mockModel = { name: 'test-model' } as any;
29
- const mockMastra = {
30
- vectors: {
31
- testStore: {
32
- // Mock vector store methods
33
- },
34
- anotherStore: {
35
- // Mock vector store methods
36
- },
37
- },
38
- getVector: vi.fn(storeName => ({
39
- [storeName]: {
40
- // Mock vector store methods
41
- },
42
- })),
43
- logger: {
44
- debug: vi.fn(),
45
- warn: vi.fn(),
46
- info: vi.fn(),
47
- error: vi.fn(),
48
- },
49
- getLogger: vi.fn(() => ({
50
- debug: vi.fn(),
51
- warn: vi.fn(),
52
- info: vi.fn(),
53
- error: vi.fn(),
54
- })),
55
- };
56
-
57
- beforeEach(() => {
58
- vi.clearAllMocks();
59
- });
60
-
61
- describe('input schema validation', () => {
62
- it('should handle filter permissively when enableFilter is false', () => {
63
- // Create tool with enableFilter set to false
64
- const tool = createVectorQueryTool({
65
- vectorStoreName: 'testStore',
66
- indexName: 'testIndex',
67
- model: mockModel,
68
- enableFilter: false,
69
- });
70
-
71
- // Get the Zod schema
72
- const schema = tool.inputSchema;
73
-
74
- // Test with no filter (should be valid)
75
- const validInput = {
76
- queryText: 'test query',
77
- topK: 5,
78
- };
79
- expect(() => schema?.parse(validInput)).not.toThrow();
80
-
81
- // Test with filter (should throw - unexpected property)
82
- const inputWithFilter = {
83
- ...validInput,
84
- filter: '{"field": "value"}',
85
- };
86
- expect(() => schema?.parse(inputWithFilter)).not.toThrow();
87
- });
88
-
89
- it('should handle filter when enableFilter is true', () => {
90
- const tool = createVectorQueryTool({
91
- vectorStoreName: 'testStore',
92
- indexName: 'testIndex',
93
- model: mockModel,
94
- enableFilter: true,
95
- });
96
-
97
- // Get the Zod schema
98
- const schema = tool.inputSchema;
99
-
100
- // Test various filter inputs that should coerce to string
101
- const testCases = [
102
- // String inputs
103
- { filter: '{"field": "value"}' },
104
- { filter: '{}' },
105
- { filter: 'simple-string' },
106
- // Empty
107
- { filter: '' },
108
- { filter: { field: 'value' } },
109
- { filter: {} },
110
- { filter: 123 },
111
- { filter: null },
112
- { filter: undefined },
113
- ];
114
-
115
- testCases.forEach(({ filter }) => {
116
- expect(() =>
117
- schema?.parse({
118
- queryText: 'test query',
119
- topK: 5,
120
- filter,
121
- }),
122
- ).not.toThrow();
123
- });
124
-
125
- // Verify that all parsed values are strings
126
- testCases.forEach(({ filter }) => {
127
- const result = schema?.parse({
128
- queryText: 'test query',
129
- topK: 5,
130
- filter,
131
- });
132
- expect(typeof result?.filter).toBe('string');
133
- });
134
- });
135
-
136
- it('should not reject unexpected properties in both modes', () => {
137
- // Test with enableFilter false
138
- const toolWithoutFilter = createVectorQueryTool({
139
- vectorStoreName: 'testStore',
140
- indexName: 'testIndex',
141
- model: mockModel,
142
- enableFilter: false,
143
- });
144
-
145
- // Should reject unexpected property
146
- expect(() =>
147
- toolWithoutFilter.inputSchema?.parse({
148
- queryText: 'test query',
149
- topK: 5,
150
- unexpectedProp: 'value',
151
- }),
152
- ).not.toThrow();
153
-
154
- // Test with enableFilter true
155
- const toolWithFilter = createVectorQueryTool({
156
- vectorStoreName: 'testStore',
157
- indexName: 'testIndex',
158
- model: mockModel,
159
- enableFilter: true,
160
- });
161
-
162
- // Should reject unexpected property even with valid filter
163
- expect(() =>
164
- toolWithFilter.inputSchema?.parse({
165
- queryText: 'test query',
166
- topK: 5,
167
- filter: '{}',
168
- unexpectedProp: 'value',
169
- }),
170
- ).not.toThrow();
171
- });
172
- });
173
-
174
- describe('execute function', () => {
175
- it('should not process filter when enableFilter is false', async () => {
176
- const runtimeContext = new RuntimeContext();
177
-
178
- // Create tool with enableFilter set to false
179
- const tool = createVectorQueryTool({
180
- vectorStoreName: 'testStore',
181
- indexName: 'testIndex',
182
- model: mockModel,
183
- enableFilter: false,
184
- });
185
-
186
- // Execute with no filter
187
- await tool.execute?.({
188
- context: {
189
- queryText: 'test query',
190
- topK: 5,
191
- },
192
- mastra: mockMastra as any,
193
- runtimeContext,
194
- });
195
-
196
- // Check that vectorQuerySearch was called with undefined queryFilter
197
- expect(vectorQuerySearch).toHaveBeenCalledWith(
198
- expect.objectContaining({
199
- queryFilter: undefined,
200
- }),
201
- );
202
- });
203
-
204
- it('should process filter when enableFilter is true and filter is provided', async () => {
205
- const runtimeContext = new RuntimeContext();
206
- // Create tool with enableFilter set to true
207
- const tool = createVectorQueryTool({
208
- vectorStoreName: 'testStore',
209
- indexName: 'testIndex',
210
- model: mockModel,
211
- enableFilter: true,
212
- });
213
-
214
- const filterJson = '{"field": "value"}';
215
-
216
- // Execute with filter
217
- await tool.execute?.({
218
- context: {
219
- queryText: 'test query',
220
- topK: 5,
221
- filter: filterJson,
222
- },
223
- mastra: mockMastra as any,
224
- runtimeContext,
225
- });
226
-
227
- // Check that vectorQuerySearch was called with the parsed filter
228
- expect(vectorQuerySearch).toHaveBeenCalledWith(
229
- expect.objectContaining({
230
- queryFilter: { field: 'value' },
231
- }),
232
- );
233
- });
234
-
235
- it('should handle string filters correctly', async () => {
236
- const runtimeContext = new RuntimeContext();
237
- // Create tool with enableFilter set to true
238
- const tool = createVectorQueryTool({
239
- vectorStoreName: 'testStore',
240
- indexName: 'testIndex',
241
- model: mockModel,
242
- enableFilter: true,
243
- });
244
-
245
- const stringFilter = 'string-filter';
246
-
247
- // Execute with string filter
248
- await tool.execute?.({
249
- context: {
250
- queryText: 'test query',
251
- topK: 5,
252
- filter: stringFilter,
253
- },
254
- mastra: mockMastra as any,
255
- runtimeContext,
256
- });
257
-
258
- // Since this is not a valid filter, it should be ignored
259
- expect(vectorQuerySearch).toHaveBeenCalledWith(
260
- expect.objectContaining({
261
- queryFilter: undefined,
262
- }),
263
- );
264
- });
265
-
266
- it('Returns early when no Mastra server or vector store is provided', async () => {
267
- const tool = createVectorQueryTool({
268
- id: 'test',
269
- model: mockModel,
270
- indexName: 'testIndex',
271
- vectorStoreName: 'testStore',
272
- });
273
-
274
- const runtimeContext = new RuntimeContext();
275
- const result = await tool.execute({
276
- context: { queryText: 'foo', topK: 1 },
277
- runtimeContext,
278
- });
279
-
280
- expect(result).toEqual({ relevantContext: [], sources: [] });
281
- expect(vectorQuerySearch).not.toHaveBeenCalled();
282
- });
283
-
284
- it('works without a mastra server if a vector store is passed as an argument', async () => {
285
- const testStore = {
286
- testStore: {},
287
- };
288
- const tool = createVectorQueryTool({
289
- id: 'test',
290
- model: mockModel,
291
- indexName: 'testIndex',
292
- vectorStoreName: 'testStore',
293
- vectorStore: testStore as any,
294
- });
295
-
296
- const runtimeContext = new RuntimeContext();
297
- const result = await tool.execute({
298
- context: { queryText: 'foo', topK: 1 },
299
- runtimeContext,
300
- });
301
-
302
- expect(result.relevantContext[0]).toEqual({ text: 'foo' });
303
- expect(vectorQuerySearch).toHaveBeenCalledWith(
304
- expect.objectContaining({
305
- databaseConfig: undefined,
306
- indexName: 'testIndex',
307
- vectorStore: {
308
- testStore: {},
309
- },
310
- queryText: 'foo',
311
- model: mockModel,
312
- queryFilter: undefined,
313
- topK: 1,
314
- }),
315
- );
316
- });
317
-
318
- it('prefers the passed vector store over one from a passed Mastra server', async () => {
319
- const thirdStore = {
320
- thirdStore: {},
321
- };
322
- const tool = createVectorQueryTool({
323
- id: 'test',
324
- model: mockModel,
325
- indexName: 'testIndex',
326
- vectorStoreName: 'thirdStore',
327
- vectorStore: thirdStore as any,
328
- });
329
-
330
- const runtimeContext = new RuntimeContext();
331
- const result = await tool.execute({
332
- context: { queryText: 'foo', topK: 1 },
333
- mastra: mockMastra as any,
334
- runtimeContext,
335
- });
336
-
337
- expect(result.relevantContext[0]).toEqual({ text: 'foo' });
338
- expect(vectorQuerySearch).toHaveBeenCalledWith(
339
- expect.objectContaining({
340
- databaseConfig: undefined,
341
- indexName: 'testIndex',
342
- vectorStore: {
343
- thirdStore: {},
344
- },
345
- queryText: 'foo',
346
- model: mockModel,
347
- queryFilter: undefined,
348
- topK: 1,
349
- }),
350
- );
351
- });
352
- });
353
-
354
- describe('runtimeContext', () => {
355
- it('calls vectorQuerySearch with runtimeContext params', async () => {
356
- const tool = createVectorQueryTool({
357
- id: 'test',
358
- model: mockModel,
359
- indexName: 'testIndex',
360
- vectorStoreName: 'testStore',
361
- });
362
- const runtimeContext = new RuntimeContext();
363
- runtimeContext.set('indexName', 'anotherIndex');
364
- runtimeContext.set('vectorStoreName', 'anotherStore');
365
- runtimeContext.set('topK', 3);
366
- runtimeContext.set('filter', { foo: 'bar' });
367
- runtimeContext.set('includeVectors', true);
368
- runtimeContext.set('includeSources', false);
369
- const result = await tool.execute({
370
- context: { queryText: 'foo', topK: 6 },
371
- mastra: mockMastra as any,
372
- runtimeContext,
373
- });
374
- expect(result.relevantContext.length).toBeGreaterThan(0);
375
- expect(result.sources).toEqual([]); // includeSources false
376
- expect(vectorQuerySearch).toHaveBeenCalledWith(
377
- expect.objectContaining({
378
- indexName: 'anotherIndex',
379
- vectorStore: {
380
- anotherStore: {},
381
- },
382
- queryText: 'foo',
383
- model: mockModel,
384
- queryFilter: { foo: 'bar' },
385
- topK: 3,
386
- includeVectors: true,
387
- }),
388
- );
389
- });
390
-
391
- it('handles reranker from runtimeContext', async () => {
392
- const tool = createVectorQueryTool({
393
- id: 'test',
394
- model: mockModel,
395
- indexName: 'testIndex',
396
- vectorStoreName: 'testStore',
397
- });
398
- const runtimeContext = new RuntimeContext();
399
- runtimeContext.set('indexName', 'testIndex');
400
- runtimeContext.set('vectorStoreName', 'testStore');
401
- runtimeContext.set('reranker', { model: 'reranker-model', options: { topK: 1 } });
402
- // Mock rerank
403
- vi.mocked(rerank).mockResolvedValue([
404
- {
405
- result: { id: '1', metadata: { text: 'bar' }, score: 1 },
406
- score: 1,
407
- details: { semantic: 1, vector: 1, position: 1 },
408
- },
409
- ]);
410
- const result = await tool.execute({
411
- context: { queryText: 'foo', topK: 1 },
412
- mastra: mockMastra as any,
413
- runtimeContext,
414
- });
415
- expect(result.relevantContext[0]).toEqual({ text: 'bar' });
416
- });
417
- });
418
- });