@mastra/rag 0.0.2-alpha.35 → 0.0.2-alpha.39

package/CHANGELOG.md

@@ -1,5 +1,39 @@
 # @mastra/rag
 
+## 0.0.2-alpha.39
+
+### Patch Changes
+
+- 592e3cf: Add custom rag tools, add vector retrieval, and update docs
+- 837a288: MAJOR Revamp of tools, workflows, syncs.
+- 0b74006: Workflow updates
+- Updated dependencies [592e3cf]
+- Updated dependencies [837a288]
+- Updated dependencies [0b74006]
+  - @mastra/core@0.1.27-alpha.55
+
+## 0.0.2-alpha.38
+
+### Patch Changes
+
+- eb45d76: Updated PG Vector filter and added rag examples in docs
+- Updated dependencies [d2cd535]
+  - @mastra/core@0.1.27-alpha.54
+
+## 0.0.2-alpha.37
+
+### Patch Changes
+
+- Updated dependencies [8e7814f]
+  - @mastra/core@0.1.27-alpha.53
+
+## 0.0.2-alpha.36
+
+### Patch Changes
+
+- Updated dependencies [eedb829]
+  - @mastra/core@0.1.27-alpha.52
+
 ## 0.0.2-alpha.35
 
 ### Patch Changes

package/dist/index.d.ts

@@ -5,4 +5,5 @@ export * from './astra-db';
 export * from './qdrant';
 export * from './upstash';
 export * from './embeddings';
+export * from './utils';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,MAAM,CAAC;AACrB,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,MAAM,CAAC;AACrB,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,SAAS,CAAC"}

package/dist/pg/filter.d.ts

@@ -0,0 +1,18 @@
+export type OperatorType = 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte' | 'like' | 'ilike' | 'in' | 'contains' | 'exists';
+export declare function isValidOperator(operator: string): operator is OperatorType;
+type FilterOperator = {
+    sql: string;
+    needsValue: boolean;
+    transformValue?: (value: any) => any;
+};
+type FilterOperatorMap = {
+    [K in OperatorType]: (key: string, paramIndex: number) => FilterOperator;
+};
+export declare const FILTER_OPERATORS: FilterOperatorMap;
+type FilterCondition = {
+    operator: OperatorType;
+    value?: any;
+};
+export type Filter = Record<string, FilterCondition | any>;
+export {};
+//# sourceMappingURL=filter.d.ts.map

package/dist/pg/filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../src/pg/filter.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,UAAU,GAAG,QAAQ,CAAC;AAGxH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,IAAI,YAAY,CAE1E;AAED,KAAK,cAAc,GAAG;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC;CACtC,CAAC;AAEF,KAAK,iBAAiB,GAAG;KACtB,CAAC,IAAI,YAAY,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,cAAc;CACzE,CAAC;AAkBF,eAAO,MAAM,gBAAgB,EAAE,iBAoC9B,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,CAAC,EAAE,GAAG,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,GAAG,CAAC,CAAC"}

package/dist/pg/index.d.ts

@@ -1,8 +1,9 @@
 import { IndexStats, QueryResult, MastraVector } from '@mastra/core';
+import { Filter } from './filter';
 export declare class PgVector extends MastraVector {
     private pool;
     constructor(connectionString: string);
-    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>, minScore?: number): Promise<QueryResult[]>;
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Filter, minScore?: number): Promise<QueryResult[]>;
     upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
     createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
     listIndexes(): Promise<string[]>;

package/dist/pg/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pg/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAGrE,qBAAa,QAAS,SAAQ,YAAY;IACxC,OAAO,CAAC,IAAI,CAAU;gBAEV,gBAAgB,EAAE,MAAM;IAoB9B,KAAK,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,GAAE,MAAW,EACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC5B,QAAQ,GAAE,MAAU,GACnB,OAAO,CAAC,WAAW,EAAE,CAAC;IA4CnB,MAAM,CACV,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAChC,GAAG,CAAC,EAAE,MAAM,EAAE,GACb,OAAO,CAAC,MAAM,EAAE,CAAC;IA8Bd,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,QAAQ,GAAG,WAAW,GAAG,YAAuB,GACvD,OAAO,CAAC,IAAI,CAAC;IAqDV,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAiBhC,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA6DrD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa7C,aAAa,CAAC,SAAS,EAAE,MAAM;IAY/B,UAAU;CAGjB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pg/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAGrE,OAAO,EAAE,MAAM,EAAqC,MAAM,UAAU,CAAC;AAErE,qBAAa,QAAS,SAAQ,YAAY;IACxC,OAAO,CAAC,IAAI,CAAU;gBAEV,gBAAgB,EAAE,MAAM;IAoB9B,KAAK,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,GAAE,MAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,GAAE,MAAU,GACnB,OAAO,CAAC,WAAW,EAAE,CAAC;IAkEnB,MAAM,CACV,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAChC,GAAG,CAAC,EAAE,MAAM,EAAE,GACb,OAAO,CAAC,MAAM,EAAE,CAAC;IA8Bd,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,QAAQ,GAAG,WAAW,GAAG,YAAuB,GACvD,OAAO,CAAC,IAAI,CAAC;IAqDV,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAiBhC,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA6DrD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa7C,aAAa,CAAC,SAAS,EAAE,MAAM;IAY/B,UAAU;CAGjB"}

package/dist/rag.cjs.development.js

@@ -11,6 +11,7 @@ var pinecone = require('@pinecone-database/pinecone');
 var astraDbTs = require('@datastax/astra-db-ts');
 var jsClientRest = require('@qdrant/js-client-rest');
 var vector = require('@upstash/vector');
+var zod = require('zod');
 
 function _arrayLikeToArray(r, a) {
   (null == a || a > r.length) && (a = r.length);
@@ -2013,6 +2014,72 @@ var MDocument = /*#__PURE__*/function () {
   return MDocument;
 }();
 
+// Type guard to check if an operator is valid
+function isValidOperator(operator) {
+  return operator in FILTER_OPERATORS;
+}
+// Helper functions to create operators
+var createBasicOperator = function createBasicOperator(symbol) {
+  return function (key, paramIndex) {
+    return {
+      sql: "metadata->>'" + key + "' " + symbol + " $" + paramIndex,
+      needsValue: true
+    };
+  };
+};
+var createNumericOperator = function createNumericOperator(symbol) {
+  return function (key, paramIndex) {
+    return {
+      sql: "(metadata->>'" + key + "')::numeric " + symbol + " $" + paramIndex,
+      needsValue: true
+    };
+  };
+};
+// Define all filter operators
+var FILTER_OPERATORS = {
+  // Equal
+  eq: /*#__PURE__*/createBasicOperator('='),
+  // Not equal
+  neq: /*#__PURE__*/createBasicOperator('!='),
+  // Greater than
+  gt: /*#__PURE__*/createNumericOperator('>'),
+  // Greater than or equal
+  gte: /*#__PURE__*/createNumericOperator('>='),
+  // Less than
+  lt: /*#__PURE__*/createNumericOperator('<'),
+  // Less than or equal
+  lte: /*#__PURE__*/createNumericOperator('<='),
+  // Pattern matching (LIKE)
+  like: /*#__PURE__*/createBasicOperator('LIKE'),
+  // Case-insensitive pattern matching (ILIKE)
+  ilike: /*#__PURE__*/createBasicOperator('ILIKE'),
+  // IN array of values
+  "in": function _in(key, paramIndex) {
+    return {
+      sql: "metadata->>'" + key + "' = ANY($" + paramIndex + ")",
+      needsValue: true
+    };
+  },
+  // JSONB contains
+  contains: function contains(key, paramIndex) {
+    return {
+      sql: "metadata @> $" + paramIndex + "::jsonb",
+      needsValue: true,
+      transformValue: function transformValue(value) {
+        var _JSON$stringify;
+        return JSON.stringify((_JSON$stringify = {}, _JSON$stringify[key] = value, _JSON$stringify));
+      }
+    };
+  },
+  // Key exists
+  exists: function exists(key) {
+    return {
+      sql: "metadata ? '" + key + "'",
+      needsValue: false
+    };
+  }
+};
+
 var PgVector = /*#__PURE__*/function (_MastraVector) {
   function PgVector(connectionString) {
     var _telemetry$traceClass;
@@ -2059,11 +2126,30 @@ var PgVector = /*#__PURE__*/function (_MastraVector) {
             filterValues = [minScore];
             vectorStr = "[" + queryVector.join(',') + "]";
             if (filter) {
-              conditions = Object.entries(filter).map(function (_ref, index) {
+              conditions = Object.entries(filter).map(function (_ref) {
                 var key = _ref[0],
-                  value = _ref[1];
-                filterValues.push(value);
-                return "metadata->>'" + key + "' = $" + (index + 2); // +2 because $1 is minScore
+                  condition = _ref[1];
+                // If condition is not a FilterCondition object, assume it's an equality check
+                if (!condition || typeof condition !== 'object' || !('operator' in condition)) {
+                  filterValues.push(condition);
+                  return "metadata->>'" + key + "' = $" + filterValues.length;
+                }
+                var operator = condition.operator,
+                  value = condition.value;
+                if (!isValidOperator(operator)) {
+                  throw new Error("Unsupported operator: " + operator);
+                }
+                // Get operation function
+                var operatorFn = FILTER_OPERATORS[operator];
+                var operatorResult = operatorFn(key, filterValues.length + 1);
+                // Handle operator cases and check if value is needed
+                if (operatorResult.needsValue) {
+                  // Transform value if needed
+                  var transformedValue = operatorResult.transformValue ? operatorResult.transformValue(value) : value;
+                  filterValues.push(transformedValue);
+                }
+                // return sql condition
+                return operatorResult.sql;
               });
               if (conditions.length > 0) {
                 filterQuery = 'AND ' + conditions.join(' AND ');
@@ -3304,11 +3390,125 @@ var embed = function embed(chunk, options) {
   return core.embed(value, options);
 };
 
+var createVectorQueryTool = function createVectorQueryTool(_ref) {
+  var vectorStoreName = _ref.vectorStoreName,
+    indexName = _ref.indexName,
+    _ref$topK = _ref.topK,
+    topK = _ref$topK === void 0 ? 10 : _ref$topK,
+    options = _ref.options,
+    _ref$useFilter = _ref.useFilter,
+    useFilter = _ref$useFilter === void 0 ? false : _ref$useFilter;
+  return core.createTool({
+    id: "VectorQuery " + vectorStoreName + " " + indexName + " Tool",
+    inputSchema: zod.z.object({
+      queryText: zod.z.string(),
+      filter: zod.z.object({
+        keyword: zod.z.string().min(1),
+        operator: zod.z.string().min(1),
+        value: zod.z.string().min(1)
+      }).optional()
+    }),
+    outputSchema: zod.z.object({
+      context: zod.z.string()
+    }),
+    description: "Fetches and combines the top " + topK + " relevant chunks from the " + vectorStoreName + " vector store using the " + indexName + " index",
+    execute: function () {
+      var _execute = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(_ref2) {
+        var _mastra$vectors;
+        var _ref2$context, queryText, filter, mastra, context, vectorStore, _yield$embed, embedding, _filter$keyword, _ref3, queryFilter, results, relevantChunks;
+        return _regeneratorRuntime().wrap(function _callee$(_context) {
+          while (1) switch (_context.prev = _context.next) {
+            case 0:
+              _ref2$context = _ref2.context, queryText = _ref2$context.queryText, filter = _ref2$context.filter, mastra = _ref2.mastra;
+              context = '';
+              vectorStore = mastra == null || (_mastra$vectors = mastra.vectors) == null ? void 0 : _mastra$vectors[vectorStoreName];
+              console.log({
+                vectorStore: vectorStore,
+                vectorStoreName: vectorStoreName
+              });
+              _context.next = 6;
+              return embed(queryText, options);
+            case 6:
+              _yield$embed = _context.sent;
+              embedding = _yield$embed.embedding;
+              if (!vectorStore) {
+                _context.next = 15;
+                break;
+              }
+              queryFilter = useFilter && filter ? (_ref3 = {}, _ref3[filter.keyword] = (_filter$keyword = {}, _filter$keyword[filter.operator] = filter.value, _filter$keyword), _ref3) : {};
+              _context.next = 12;
+              return vectorStore.query(indexName, embedding, topK, queryFilter);
+            case 12:
+              results = _context.sent;
+              relevantChunks = results.map(function (result) {
+                var _result$metadata;
+                return result == null || (_result$metadata = result.metadata) == null ? void 0 : _result$metadata.text;
+              }); // Combine the chunks into a context string
+              context = relevantChunks.join('\n\n');
+            case 15:
+              return _context.abrupt("return", {
+                context: context
+              });
+            case 16:
+            case "end":
+              return _context.stop();
+          }
+        }, _callee);
+      }));
+      function execute(_x) {
+        return _execute.apply(this, arguments);
+      }
+      return execute;
+    }()
+  });
+};
+var createDocumentChunker = function createDocumentChunker(_ref4) {
+  var doc = _ref4.doc,
+    _ref4$params = _ref4.params,
+    params = _ref4$params === void 0 ? {
+      strategy: 'recursive',
+      size: 512,
+      overlap: 50,
+      separator: '\n'
+    } : _ref4$params;
+  return core.createTool({
+    id: "Document Chunker " + params.strategy + " " + params.size,
+    inputSchema: zod.z.object({}),
+    description: "Chunks document using " + params.strategy + " strategy with size " + params.size + " and " + params.overlap + " overlap",
+    execute: function () {
+      var _execute2 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2() {
+        var chunks;
+        return _regeneratorRuntime().wrap(function _callee2$(_context2) {
+          while (1) switch (_context2.prev = _context2.next) {
+            case 0:
+              _context2.next = 2;
+              return doc.chunk(params);
+            case 2:
+              chunks = _context2.sent;
+              return _context2.abrupt("return", {
+                chunks: chunks
+              });
+            case 4:
+            case "end":
+              return _context2.stop();
+          }
+        }, _callee2);
+      }));
+      function execute() {
+        return _execute2.apply(this, arguments);
+      }
+      return execute;
+    }()
+  });
+};
+
 exports.AstraVector = AstraVector;
 exports.MDocument = MDocument;
 exports.PgVector = PgVector;
 exports.PineconeVector = PineconeVector;
 exports.QdrantVector = QdrantVector;
 exports.UpstashVector = UpstashVector;
+exports.createDocumentChunker = createDocumentChunker;
+exports.createVectorQueryTool = createVectorQueryTool;
 exports.embed = embed;
 //# sourceMappingURL=rag.cjs.development.js.map