@mastra/rag 1.2.3-alpha.0 → 1.2.3-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/package.json +18 -5
  3. package/.turbo/turbo-build.log +0 -4
  4. package/docker-compose.yaml +0 -22
  5. package/eslint.config.js +0 -6
  6. package/src/document/document.test.ts +0 -2975
  7. package/src/document/document.ts +0 -335
  8. package/src/document/extractors/base.ts +0 -30
  9. package/src/document/extractors/index.ts +0 -5
  10. package/src/document/extractors/keywords.test.ts +0 -125
  11. package/src/document/extractors/keywords.ts +0 -126
  12. package/src/document/extractors/questions.test.ts +0 -120
  13. package/src/document/extractors/questions.ts +0 -111
  14. package/src/document/extractors/summary.test.ts +0 -107
  15. package/src/document/extractors/summary.ts +0 -122
  16. package/src/document/extractors/title.test.ts +0 -121
  17. package/src/document/extractors/title.ts +0 -185
  18. package/src/document/extractors/types.ts +0 -40
  19. package/src/document/index.ts +0 -2
  20. package/src/document/prompts/base.ts +0 -77
  21. package/src/document/prompts/format.ts +0 -9
  22. package/src/document/prompts/index.ts +0 -15
  23. package/src/document/prompts/prompt.ts +0 -60
  24. package/src/document/prompts/types.ts +0 -29
  25. package/src/document/schema/index.ts +0 -3
  26. package/src/document/schema/node.ts +0 -187
  27. package/src/document/schema/types.ts +0 -40
  28. package/src/document/transformers/character.ts +0 -267
  29. package/src/document/transformers/html.ts +0 -346
  30. package/src/document/transformers/json.ts +0 -536
  31. package/src/document/transformers/latex.ts +0 -11
  32. package/src/document/transformers/markdown.ts +0 -239
  33. package/src/document/transformers/semantic-markdown.ts +0 -227
  34. package/src/document/transformers/sentence.ts +0 -314
  35. package/src/document/transformers/text.ts +0 -158
  36. package/src/document/transformers/token.ts +0 -137
  37. package/src/document/transformers/transformer.ts +0 -5
  38. package/src/document/types.ts +0 -145
  39. package/src/document/validation.ts +0 -158
  40. package/src/graph-rag/index.test.ts +0 -235
  41. package/src/graph-rag/index.ts +0 -306
  42. package/src/index.ts +0 -8
  43. package/src/rerank/index.test.ts +0 -150
  44. package/src/rerank/index.ts +0 -198
  45. package/src/rerank/relevance/cohere/index.ts +0 -56
  46. package/src/rerank/relevance/index.ts +0 -3
  47. package/src/rerank/relevance/mastra-agent/index.ts +0 -32
  48. package/src/rerank/relevance/zeroentropy/index.ts +0 -26
  49. package/src/tools/README.md +0 -153
  50. package/src/tools/document-chunker.ts +0 -34
  51. package/src/tools/graph-rag.test.ts +0 -115
  52. package/src/tools/graph-rag.ts +0 -157
  53. package/src/tools/index.ts +0 -3
  54. package/src/tools/types.ts +0 -126
  55. package/src/tools/vector-query-database-config.test.ts +0 -190
  56. package/src/tools/vector-query.test.ts +0 -477
  57. package/src/tools/vector-query.ts +0 -171
  58. package/src/utils/convert-sources.ts +0 -43
  59. package/src/utils/default-settings.ts +0 -38
  60. package/src/utils/index.ts +0 -3
  61. package/src/utils/tool-schemas.ts +0 -38
  62. package/src/utils/vector-prompts.ts +0 -832
  63. package/src/utils/vector-search.ts +0 -130
  64. package/tsconfig.build.json +0 -9
  65. package/tsconfig.json +0 -5
  66. package/tsup.config.ts +0 -17
  67. package/vitest.config.ts +0 -8
@@ -1,126 +0,0 @@
1
- import type { MastraVector, MastraEmbeddingModel } from '@mastra/core/vector';
2
-
3
- import type { RerankConfig } from '../rerank';
4
-
5
- export interface PineconeConfig {
6
- namespace?: string;
7
- sparseVector?: {
8
- indices: number[];
9
- values: number[];
10
- };
11
- }
12
-
13
- export interface PgVectorConfig {
14
- minScore?: number;
15
- ef?: number; // HNSW search parameter
16
- probes?: number; // IVFFlat probe parameter
17
- }
18
-
19
- // Chroma types
20
- type LiteralValue = string | number | boolean;
21
- type ListLiteralValue = LiteralValue[];
22
- type LiteralNumber = number;
23
- type LogicalOperator = '$and' | '$or';
24
- type InclusionOperator = '$in' | '$nin';
25
- type WhereOperator = '$gt' | '$gte' | '$lt' | '$lte' | '$ne' | '$eq';
26
- type OperatorExpression = {
27
- [key in WhereOperator | InclusionOperator | LogicalOperator]?: LiteralValue | ListLiteralValue;
28
- };
29
- type BaseWhere = {
30
- [key: string]: LiteralValue | OperatorExpression;
31
- };
32
- type LogicalWhere = {
33
- [key in LogicalOperator]?: Where[];
34
- };
35
- type Where = BaseWhere | LogicalWhere;
36
- type WhereDocumentOperator = '$contains' | '$not_contains' | LogicalOperator;
37
- type WhereDocument = {
38
- [key in WhereDocumentOperator]?: LiteralValue | LiteralNumber | WhereDocument[];
39
- };
40
-
41
- export interface ChromaConfig {
42
- // Add Chroma-specific configs here if needed
43
- where?: Where;
44
- whereDocument?: WhereDocument;
45
- }
46
-
47
- // Union type for all database-specific configs
48
- export type DatabaseConfig = {
49
- pinecone?: PineconeConfig;
50
- pgvector?: PgVectorConfig;
51
- chroma?: ChromaConfig;
52
- // Add other database configs as needed
53
- [key: string]: any; // Allow for future database extensions
54
- };
55
-
56
- export type VectorQueryToolOptions = {
57
- id?: string;
58
- description?: string;
59
- indexName: string;
60
- model: MastraEmbeddingModel<string>;
61
- enableFilter?: boolean;
62
- includeVectors?: boolean;
63
- includeSources?: boolean;
64
- reranker?: RerankConfig;
65
- /** Database-specific configuration options */
66
- databaseConfig?: DatabaseConfig;
67
- } & ProviderOptions &
68
- (
69
- | {
70
- vectorStoreName: string;
71
- }
72
- | {
73
- vectorStoreName?: string;
74
- vectorStore: MastraVector;
75
- }
76
- );
77
-
78
- export type GraphRagToolOptions = {
79
- id?: string;
80
- description?: string;
81
- indexName: string;
82
- vectorStoreName: string;
83
- model: MastraEmbeddingModel<string>;
84
- enableFilter?: boolean;
85
- includeSources?: boolean;
86
- graphOptions?: {
87
- dimension?: number;
88
- randomWalkSteps?: number;
89
- restartProb?: number;
90
- threshold?: number;
91
- };
92
- } & ProviderOptions;
93
-
94
- export type ProviderOptions = {
95
- /**
96
- * Provider-specific options for the embedding model (e.g., outputDimensionality).
97
- *
98
- * ⚠️ **IMPORTANT**: `providerOptions` only work with AI SDK v2 models.
99
- *
100
- * **For v1 models**: Configure options when creating the model:
101
- * ✅ const model = openai.embedding('text-embedding-3-small', { dimensions: 512 });
102
- *
103
- * **For v2 models**: Use providerOptions:
104
- * ✅ providerOptions: { openai: { dimensions: 512 } }
105
- */
106
- providerOptions?: Record<string, Record<string, any>>;
107
- };
108
-
109
- /**
110
- * Default options for GraphRAG
111
- * @default
112
- * ```json
113
- * {
114
- * "dimension": 1536,
115
- * "randomWalkSteps": 100,
116
- * "restartProb": 0.15,
117
- * "threshold": 0.7
118
- * }
119
- * ```
120
- */
121
- export const defaultGraphOptions = {
122
- dimension: 1536,
123
- randomWalkSteps: 100,
124
- restartProb: 0.15,
125
- threshold: 0.7,
126
- };
@@ -1,190 +0,0 @@
1
- import { RuntimeContext } from '@mastra/core/runtime-context';
2
- import { describe, it, expect, vi, beforeEach } from 'vitest';
3
- import { vectorQuerySearch } from '../utils';
4
- import type { DatabaseConfig } from './types';
5
- import { createVectorQueryTool } from './vector-query';
6
-
7
- vi.mock('../utils', async importOriginal => {
8
- const actual: any = await importOriginal();
9
- return {
10
- ...actual,
11
- vectorQuerySearch: vi.fn().mockResolvedValue({
12
- results: [{ metadata: { text: 'test result' }, vector: [1, 2, 3] }],
13
- }),
14
- };
15
- });
16
-
17
- describe('createVectorQueryTool with database-specific configurations', () => {
18
- const mockModel = { name: 'test-model' } as any;
19
- const mockMastra = {
20
- getVector: vi.fn(() => ({})),
21
- getLogger: vi.fn(() => ({
22
- debug: vi.fn(),
23
- warn: vi.fn(),
24
- error: vi.fn(),
25
- })),
26
- };
27
-
28
- beforeEach(() => {
29
- vi.clearAllMocks();
30
- });
31
-
32
- it('should pass Pinecone configuration to vectorQuerySearch', async () => {
33
- const databaseConfig: DatabaseConfig = {
34
- pinecone: {
35
- namespace: 'test-namespace',
36
- sparseVector: {
37
- indices: [0, 1, 2],
38
- values: [0.1, 0.2, 0.3],
39
- },
40
- },
41
- };
42
-
43
- const tool = createVectorQueryTool({
44
- vectorStoreName: 'pinecone',
45
- indexName: 'testIndex',
46
- model: mockModel,
47
- databaseConfig,
48
- });
49
-
50
- const runtimeContext = new RuntimeContext();
51
-
52
- await tool.execute({
53
- context: { queryText: 'test query', topK: 5 },
54
- mastra: mockMastra as any,
55
- runtimeContext,
56
- });
57
-
58
- expect(vectorQuerySearch).toHaveBeenCalledWith(
59
- expect.objectContaining({
60
- databaseConfig,
61
- }),
62
- );
63
- });
64
-
65
- it('should pass pgVector configuration to vectorQuerySearch', async () => {
66
- const databaseConfig: DatabaseConfig = {
67
- pgvector: {
68
- minScore: 0.7,
69
- ef: 200,
70
- probes: 10,
71
- },
72
- };
73
-
74
- const tool = createVectorQueryTool({
75
- vectorStoreName: 'postgres',
76
- indexName: 'testIndex',
77
- model: mockModel,
78
- databaseConfig,
79
- });
80
-
81
- const runtimeContext = new RuntimeContext();
82
-
83
- await tool.execute({
84
- context: { queryText: 'test query', topK: 5 },
85
- mastra: mockMastra as any,
86
- runtimeContext,
87
- });
88
-
89
- expect(vectorQuerySearch).toHaveBeenCalledWith(
90
- expect.objectContaining({
91
- databaseConfig,
92
- }),
93
- );
94
- });
95
-
96
- it('should allow runtime context to override database configuration', async () => {
97
- const initialConfig: DatabaseConfig = {
98
- pinecone: {
99
- namespace: 'initial-namespace',
100
- },
101
- };
102
-
103
- const runtimeConfig: DatabaseConfig = {
104
- pinecone: {
105
- namespace: 'runtime-namespace',
106
- },
107
- };
108
-
109
- const tool = createVectorQueryTool({
110
- vectorStoreName: 'pinecone',
111
- indexName: 'testIndex',
112
- model: mockModel,
113
- databaseConfig: initialConfig,
114
- });
115
-
116
- const runtimeContext = new RuntimeContext();
117
- runtimeContext.set('databaseConfig', runtimeConfig);
118
-
119
- await tool.execute({
120
- context: { queryText: 'test query', topK: 5 },
121
- mastra: mockMastra as any,
122
- runtimeContext,
123
- });
124
-
125
- expect(vectorQuerySearch).toHaveBeenCalledWith(
126
- expect.objectContaining({
127
- databaseConfig: runtimeConfig, // Should use runtime config, not initial
128
- }),
129
- );
130
- });
131
-
132
- it('should work without database configuration (backward compatibility)', async () => {
133
- const tool = createVectorQueryTool({
134
- vectorStoreName: 'testStore',
135
- indexName: 'testIndex',
136
- model: mockModel,
137
- // No databaseConfig provided
138
- });
139
-
140
- const runtimeContext = new RuntimeContext();
141
-
142
- await tool.execute({
143
- context: { queryText: 'test query', topK: 5 },
144
- mastra: mockMastra as any,
145
- runtimeContext,
146
- });
147
-
148
- expect(vectorQuerySearch).toHaveBeenCalledWith(
149
- expect.objectContaining({
150
- databaseConfig: undefined,
151
- }),
152
- );
153
- });
154
-
155
- it('should handle multiple database configurations', async () => {
156
- const databaseConfig: DatabaseConfig = {
157
- pinecone: {
158
- namespace: 'test-namespace',
159
- },
160
- pgvector: {
161
- minScore: 0.8,
162
- ef: 100,
163
- },
164
- chroma: {
165
- where: { category: 'documents' },
166
- },
167
- };
168
-
169
- const tool = createVectorQueryTool({
170
- vectorStoreName: 'multidb',
171
- indexName: 'testIndex',
172
- model: mockModel,
173
- databaseConfig,
174
- });
175
-
176
- const runtimeContext = new RuntimeContext();
177
-
178
- await tool.execute({
179
- context: { queryText: 'test query', topK: 5 },
180
- mastra: mockMastra as any,
181
- runtimeContext,
182
- });
183
-
184
- expect(vectorQuerySearch).toHaveBeenCalledWith(
185
- expect.objectContaining({
186
- databaseConfig,
187
- }),
188
- );
189
- });
190
- });