@mastra/rag 2.0.0-beta.1 → 2.0.0-beta.2

package/CHANGELOG.md

@@ -1,5 +1,57 @@
 # @mastra/rag
 
+## 2.0.0-beta.2
+
+### Patch Changes
+
+- Add table support to markdown transformer ([#10487](https://github.com/mastra-ai/mastra/pull/10487))
+
+  Added support for markdown tables in the `MarkdownHeaderTransformer` to prevent tables from being split in the middle during document chunking. Tables are now treated as semantic units similar to code blocks.
+
+  **Changes:**
+  - Updated `MarkdownHeaderTransformer` to detect and preserve markdown tables during chunking
+  - Tables are identified by lines containing pipe characters (`|`)
+  - Tables are kept together as a single block, preventing splits that would break table structure
+  - Added comprehensive test coverage for table handling in various scenarios
+  - Works with both simple and complex tables, including multi-row tables and tables with different formatting
+
+  **Usage:**
+
+  ```typescript
+  import { MDocument } from '@mastra/rag';
+
+  const doc = MDocument.fromMarkdown(`
+  # Data Report
+  
+  ## Results
+  
+  | Name | Score | Status |
+  |------|-------|--------|
+  | Alice | 95   | Pass   |
+  | Bob   | 87   | Pass   |
+  | Carol | 78   | Pass   |
+  
+  ## Summary
+  
+  The results show...
+  `);
+
+  const chunks = await doc.chunk({
+    strategy: 'markdown',
+    headers: [
+      ['#', 'title'],
+      ['##', 'section'],
+    ],
+  });
+
+  // Tables will now be preserved intact within chunks
+  ```
+
+- Fix invalid filter handling in vector queries and graph-rag searches. Invalid filter inputs now throw explicit errors instead of silently falling back to empty filters, preventing unintended unfiltered results. ([#10375](https://github.com/mastra-ai/mastra/pull/10375))
+
+- Updated dependencies [[`21a15de`](https://github.com/mastra-ai/mastra/commit/21a15de369fe82aac26bb642ed7be73505475e8b), [`feb7ee4`](https://github.com/mastra-ai/mastra/commit/feb7ee4d09a75edb46c6669a3beaceec78811747), [`b0e2ea5`](https://github.com/mastra-ai/mastra/commit/b0e2ea5b52c40fae438b9e2f7baee6f0f89c5442), [`c456e01`](https://github.com/mastra-ai/mastra/commit/c456e0149e3c176afcefdbd9bb1d2c5917723725), [`ab035c2`](https://github.com/mastra-ai/mastra/commit/ab035c2ef6d8cc7bb25f06f1a38508bd9e6f126b), [`1a46a56`](https://github.com/mastra-ai/mastra/commit/1a46a566f45a3fcbadc1cf36bf86d351f264bfa3), [`3cf540b`](https://github.com/mastra-ai/mastra/commit/3cf540b9fbfea8f4fc8d3a2319a4e6c0b0cbfd52), [`1c6ce51`](https://github.com/mastra-ai/mastra/commit/1c6ce51f875915ab57fd36873623013699a2a65d), [`898a972`](https://github.com/mastra-ai/mastra/commit/898a9727d286c2510d6b702dfd367e6aaf5c6b0f), [`a97003a`](https://github.com/mastra-ai/mastra/commit/a97003aa1cf2f4022a41912324a1e77263b326b8), [`ccc141e`](https://github.com/mastra-ai/mastra/commit/ccc141ed27da0abc3a3fc28e9e5128152e8e37f4), [`fe3b897`](https://github.com/mastra-ai/mastra/commit/fe3b897c2ccbcd2b10e81b099438c7337feddf89), [`00123ba`](https://github.com/mastra-ai/mastra/commit/00123ba96dc9e5cd0b110420ebdba56d8f237b25), [`29c4309`](https://github.com/mastra-ai/mastra/commit/29c4309f818b24304c041bcb4a8f19b5f13f6b62), [`16785ce`](https://github.com/mastra-ai/mastra/commit/16785ced928f6f22638f4488cf8a125d99211799), [`de8239b`](https://github.com/mastra-ai/mastra/commit/de8239bdcb1d8c0cfa06da21f1569912a66bbc8a), [`b5e6cd7`](https://github.com/mastra-ai/mastra/commit/b5e6cd77fc8c8e64e0494c1d06cee3d84e795d1e), [`3759cb0`](https://github.com/mastra-ai/mastra/commit/3759cb064935b5f74c65ac2f52a1145f7352899d), [`651e772`](https://github.com/mastra-ai/mastra/commit/651e772eb1475fb13e126d3fcc01751297a88214), [`b61b93f`](https://github.com/mastra-ai/mastra/commit/b61b93f9e058b11dd2eec169853175d31dbdd567), [`bae33d9`](https://github.com/mastra-ai/mastra/commit/bae33d91a63fbb64d1e80519e1fc1acaed1e9013), [`c0b731f`](https://github.com/mastra-ai/mastra/commit/c0b731fb27d712dc8582e846df5c0332a6a0c5ba), [`43ca8f2`](https://github.com/mastra-ai/mastra/commit/43ca8f2c7334851cc7b4d3d2f037d8784bfbdd5f), [`2ca67cc`](https://github.com/mastra-ai/mastra/commit/2ca67cc3bb1f6a617353fdcab197d9efebe60d6f), [`9e67002`](https://github.com/mastra-ai/mastra/commit/9e67002b52c9be19936c420a489dbee9c5fd6a78), [`35edc49`](https://github.com/mastra-ai/mastra/commit/35edc49ac0556db609189641d6341e76771b81fc)]:
+  - @mastra/core@1.0.0-beta.5
+
 ## 2.0.0-beta.1
 
 ### Patch Changes

package/dist/document/transformers/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBACxD,OAAO,GAAE,gBAAqB;CAI3C;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAqHjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBACxD,OAAO,GAAE,gBAAqB;CAI3C;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAuD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAsIjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/graph-rag/index.d.ts

@@ -6,11 +6,12 @@
  *  - Improve graph traversal and querying using types
  */
 type SupportedEdgeType = 'semantic';
+type GraphMetadata = Record<string, any>;
 export interface GraphNode {
     id: string;
     content: string;
     embedding?: number[];
-    metadata?: Record<string, any>;
+    metadata?: GraphMetadata;
 }
 export interface RankedNode extends GraphNode {
     score: number;
@@ -46,11 +47,21 @@ export declare class GraphRAG {
     createGraph(chunks: GraphChunk[], embeddings: GraphEmbedding[]): void;
     private selectWeightedNeighbor;
     private randomWalkWithRestart;
-    query({ query, topK, randomWalkSteps, restartProb, }: {
+    /**
+     * Query the graph with a dense embedding and optional metadata filter.
+     *
+     * @param query - The embedding vector to query.
+     * @param topK - Number of top results to return.
+     * @param randomWalkSteps - Steps for random walk reranking.
+     * @param restartProb - Restart probability for random walk.
+     * @param filter - Optional strict metadata filter. All key-value pairs must match exactly.
+     */
+    query({ query, topK, randomWalkSteps, restartProb, filter, }: {
         query: number[];
         topK?: number;
         randomWalkSteps?: number;
         restartProb?: number;
+        filter?: Partial<GraphMetadata>;
     }): RankedNode[];
 }
 export {};

package/dist/graph-rag/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph-rag/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,KAAK,iBAAiB,GAAG,UAAU,CAAC;AAGpC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,UAAW,SAAQ,SAAS;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,iBAAiB,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAa,EAAE,SAAS,GAAE,MAAY;IAQ7D,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAW9B,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAe9B,QAAQ,IAAI,SAAS,EAAE;IAKvB,QAAQ,IAAI,SAAS,EAAE;IAIvB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IAIzC,KAAK,IAAI,IAAI;IAKb,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IASvD,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,gBAAgB;IAgCxB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE;IAuC9D,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,qBAAqB;IAoC7B,KAAK,CAAC,EACJ,KAAK,EACL,IAAS,EACT,eAAqB,EACrB,WAAkB,GACnB,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,UAAU,EAAE;CAoDjB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graph-rag/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,KAAK,iBAAiB,GAAG,UAAU,CAAC;AACpC,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAGzC,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,UAAW,SAAQ,SAAS;IAC3C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,iBAAiB,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,GAAE,MAAa,EAAE,SAAS,GAAE,MAAY;IAQ7D,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAW9B,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAe9B,QAAQ,IAAI,SAAS,EAAE;IAKvB,QAAQ,IAAI,SAAS,EAAE;IAIvB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IAIzC,KAAK,IAAI,IAAI;IAKb,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IASvD,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,gBAAgB;IAgCxB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,cAAc,EAAE;IAuC9D,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,qBAAqB;IA2C7B;;;;;;;;OAQG;IAEH,KAAK,CAAC,EACJ,KAAK,EACL,IAAS,EACT,eAAqB,EACrB,WAAkB,EAClB,MAAM,GACP,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KACjC,GAAG,UAAU,EAAE;CA+DjB"}

package/dist/index.cjs

@@ -5567,6 +5567,11 @@ var MarkdownHeaderTransformer = class {
         currentContent.push(line);
         continue;
       }
+      const isTableLine = strippedLine.includes("|") && strippedLine.length > 0;
+      if (isTableLine) {
+        currentContent.push(line);
+        continue;
+      }
       let headerMatched = false;
       for (const [sep, name14] of this.headersToSplitOn) {
         if (strippedLine.startsWith(sep) && (strippedLine.length === sep.length || strippedLine[sep.length] === " ")) {
@@ -6803,7 +6808,7 @@ var GraphRAG = class {
     return neighbors[neighbors.length - 1]?.id;
   }
   // Perform random walk with restart
-  randomWalkWithRestart(startNodeId, steps, restartProb) {
+  randomWalkWithRestart(startNodeId, steps, restartProb, allowedNodeIds) {
     const visits = /* @__PURE__ */ new Map();
     let currentNodeId = startNodeId;
     for (let step = 0; step < steps; step++) {
@@ -6812,7 +6817,10 @@ var GraphRAG = class {
         currentNodeId = startNodeId;
         continue;
       }
-      const neighbors = this.getNeighbors(currentNodeId);
+      let neighbors = this.getNeighbors(currentNodeId);
+      if (allowedNodeIds) {
+        neighbors = neighbors.filter((n) => allowedNodeIds.has(n.id));
+      }
       if (neighbors.length === 0) {
         currentNodeId = startNodeId;
         continue;
@@ -6826,12 +6834,22 @@ var GraphRAG = class {
     }
     return normalizedVisits;
   }
+  /**
+   * Query the graph with a dense embedding and optional metadata filter.
+   *
+   * @param query - The embedding vector to query.
+   * @param topK - Number of top results to return.
+   * @param randomWalkSteps - Steps for random walk reranking.
+   * @param restartProb - Restart probability for random walk.
+   * @param filter - Optional strict metadata filter. All key-value pairs must match exactly.
+   */
   // Retrieve relevant nodes using hybrid approach
   query({
     query,
     topK = 10,
     randomWalkSteps = 100,
-    restartProb = 0.15
+    restartProb = 0.15,
+    filter
   }) {
     if (!query || query.length !== this.dimension) {
       throw new Error(`Query embedding must have dimension ${this.dimension}`);
@@ -6845,15 +6863,20 @@ var GraphRAG = class {
     if (restartProb <= 0 || restartProb >= 1) {
       throw new Error("Restart probability must be between 0 and 1");
     }
-    const similarities = Array.from(this.nodes.values()).map((node) => ({
+    const filterEntries = Object.entries(filter ?? {});
+    const matchesFilter = (node) => filterEntries.length === 0 ? true : filterEntries.every(([key, value]) => node.metadata?.[key] === value);
+    const nodesToSearch = Array.from(this.nodes.values()).filter(matchesFilter);
+    const similarities = nodesToSearch.map((node) => ({
       node,
       similarity: this.cosineSimilarity(query, node.embedding)
     }));
     similarities.sort((a, b) => b.similarity - a.similarity);
     const topNodes = similarities.slice(0, topK);
+    const useFilter = filterEntries.length > 0;
+    const allowedNodeIds = useFilter ? new Set(nodesToSearch.map((n) => n.id)) : void 0;
     const rerankedNodes = /* @__PURE__ */ new Map();
     for (const { node, similarity } of topNodes) {
-      const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb);
+      const walkScores = this.randomWalkWithRestart(node.id, randomWalkSteps, restartProb, allowedNodeIds);
       for (const [nodeId, walkScore] of walkScores) {
         const node2 = this.nodes.get(nodeId);
         const existingScore = rerankedNodes.get(nodeId)?.score || 0;
@@ -7136,9 +7159,9 @@ var createGraphRAGTool = (options) => {
               return typeof filter === "string" ? JSON.parse(filter) : filter;
             } catch (error) {
               if (logger) {
-                logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
+                logger.error("Invalid filter", { filter, error });
               }
-              return {};
+              throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
             }
           })();
         }
@@ -7263,9 +7286,9 @@ var createVectorQueryTool = (options) => {
               return typeof filter === "string" ? JSON.parse(filter) : filter;
             } catch (error) {
               if (logger) {
-                logger.warn("Failed to parse filter as JSON, using empty filter", { filter, error });
+                logger.error("Invalid filter", { filter, error });
               }
-              return {};
+              throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
             }
           })();
         }