@mastra/rag 0.1.8-alpha.3 → 0.1.8-alpha.5

package/.turbo/turbo-build.log

@@ -1,23 +1,23 @@
 
-> @mastra/rag@0.1.8-alpha.3 build /home/runner/work/mastra/mastra/packages/rag
+> @mastra/rag@0.1.8-alpha.5 build /home/runner/work/mastra/mastra/packages/rag
 > tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake
 
 CLI Building entry: src/index.ts
 CLI Using tsconfig: tsconfig.json
 CLI tsup v8.3.6
 TSC Build start
-TSC ⚡️ Build success in 27174ms
+TSC ⚡️ Build success in 23355ms
 DTS Build start
 CLI Target: es2022
 Analysis will use the bundled TypeScript version 5.7.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.ts
 Analysis will use the bundled TypeScript version 5.7.3
 Writing package typings: /home/runner/work/mastra/mastra/packages/rag/dist/_tsup-dts-rollup.d.cts
-DTS ⚡️ Build success in 37763ms
+DTS ⚡️ Build success in 23003ms
 CLI Cleaning output folder
 ESM Build start
 CJS Build start
-CJS dist/index.cjs 83.86 KB
-CJS ⚡️ Build success in 2533ms
-ESM dist/index.js 83.18 KB
-ESM ⚡️ Build success in 2536ms
+ESM dist/index.js 89.94 KB
+ESM ⚡️ Build success in 684ms
+CJS dist/index.cjs 90.62 KB
+CJS ⚡️ Build success in 685ms

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @mastra/rag
 
+## 0.1.8-alpha.5
+
+### Patch Changes
+
+- 9e81f35: Fix query filter for vector search and rerank
+- 9e0f2c9: Update JSON Chunking to fix recursive issue
+- Updated dependencies [22643eb]
+- Updated dependencies [6feb23f]
+- Updated dependencies [f2d6727]
+- Updated dependencies [301e4ee]
+- Updated dependencies [dfbe4e9]
+- Updated dependencies [9e81f35]
+- Updated dependencies [caefaa2]
+- Updated dependencies [c151ae6]
+- Updated dependencies [52e0418]
+- Updated dependencies [03236ec]
+- Updated dependencies [3764e71]
+- Updated dependencies [df982db]
+- Updated dependencies [0461849]
+- Updated dependencies [2259379]
+- Updated dependencies [358f069]
+  - @mastra/core@0.5.0-alpha.5
+
+## 0.1.8-alpha.4
+
+### Patch Changes
+
+- Updated dependencies [d79aedf]
+  - @mastra/core@0.5.0-alpha.4
+
 ## 0.1.8-alpha.3
 
 ### Patch Changes

package/dist/_tsup-dts-rollup.d.cts

@@ -12,6 +12,7 @@ import type { TiktokenEncoding } from 'js-tiktoken';
 import type { TiktokenModel } from 'js-tiktoken';
 import type { TitleCombinePrompt } from 'llamaindex';
 import type { TitleExtractorPrompt } from 'llamaindex';
+import type { VectorFilter } from '@mastra/core/vector/filter';
 
 /**
  * Vector store specific prompts that detail supported operators and examples.
@@ -404,7 +405,37 @@ export declare class RecursiveJsonTransformer {
      */
     private listToDictPreprocessing;
     /**
-     * Split json into maximum size dictionaries while preserving structure
+     * Handles primitive values (strings, numbers, etc) by either adding them to the current chunk
+     * or creating new chunks if they don't fit
+     */
+    private handlePrimitiveValue;
+    /**
+     * Creates a nested dictionary chunk from a value and path
+     * e.g., path ['a', 'b'], value 'c' becomes { a: { b: 'c' } }
+     */
+    private createChunk;
+    /**
+     * Checks if value is within size limits
+     */
+    private isWithinSizeLimit;
+    /**
+     * Splits arrays into chunks based on size limits
+     * Handles nested objects by recursing into handleNestedObject
+     */
+    private handleArray;
+    /**
+     * Splits objects into chunks based on size limits
+     * Handles nested arrays and objects by recursing into handleArray and handleNestedObject
+     */
+    private handleNestedObject;
+    /**
+     * Splits long strings into smaller chunks at word boundaries
+     * Ensures each chunk is within maxSize limit
+     */
+    private splitLongString;
+    /**
+     * Core chunking logic that processes JSON data recursively
+     * Handles arrays, objects, and primitive values while maintaining structure
      */
     private jsonSplit;
     /**
@@ -414,6 +445,10 @@ export declare class RecursiveJsonTransformer {
         jsonData: Record<string, any>;
         convertLists?: boolean;
     }): Record<string, any>[];
+    /**
+     * Converts Unicode characters to their escaped ASCII representation
+     * e.g., 'café' becomes 'caf\u00e9'
+     */
     private escapeNonAscii;
     /**
      * Splits JSON into a list of JSON formatted strings
@@ -600,7 +635,7 @@ declare interface VectorQuerySearchParams {
     vectorStore: MastraVector;
     queryText: string;
     model: EmbeddingModel<string>;
-    queryFilter?: any;
+    queryFilter?: VectorFilter;
     topK: number;
     includeVectors?: boolean;
     maxRetries?: number;

package/dist/_tsup-dts-rollup.d.ts

@@ -12,6 +12,7 @@ import type { TiktokenEncoding } from 'js-tiktoken';
 import type { TiktokenModel } from 'js-tiktoken';
 import type { TitleCombinePrompt } from 'llamaindex';
 import type { TitleExtractorPrompt } from 'llamaindex';
+import type { VectorFilter } from '@mastra/core/vector/filter';
 
 /**
  * Vector store specific prompts that detail supported operators and examples.
@@ -404,7 +405,37 @@ export declare class RecursiveJsonTransformer {
      */
     private listToDictPreprocessing;
     /**
-     * Split json into maximum size dictionaries while preserving structure
+     * Handles primitive values (strings, numbers, etc) by either adding them to the current chunk
+     * or creating new chunks if they don't fit
+     */
+    private handlePrimitiveValue;
+    /**
+     * Creates a nested dictionary chunk from a value and path
+     * e.g., path ['a', 'b'], value 'c' becomes { a: { b: 'c' } }
+     */
+    private createChunk;
+    /**
+     * Checks if value is within size limits
+     */
+    private isWithinSizeLimit;
+    /**
+     * Splits arrays into chunks based on size limits
+     * Handles nested objects by recursing into handleNestedObject
+     */
+    private handleArray;
+    /**
+     * Splits objects into chunks based on size limits
+     * Handles nested arrays and objects by recursing into handleArray and handleNestedObject
+     */
+    private handleNestedObject;
+    /**
+     * Splits long strings into smaller chunks at word boundaries
+     * Ensures each chunk is within maxSize limit
+     */
+    private splitLongString;
+    /**
+     * Core chunking logic that processes JSON data recursively
+     * Handles arrays, objects, and primitive values while maintaining structure
      */
     private jsonSplit;
     /**
@@ -414,6 +445,10 @@ export declare class RecursiveJsonTransformer {
         jsonData: Record<string, any>;
         convertLists?: boolean;
     }): Record<string, any>[];
+    /**
+     * Converts Unicode characters to their escaped ASCII representation
+     * e.g., 'café' becomes 'caf\u00e9'
+     */
     private escapeNonAscii;
     /**
      * Splits JSON into a list of JSON formatted strings
@@ -600,7 +635,7 @@ declare interface VectorQuerySearchParams {
     vectorStore: MastraVector;
     queryText: string;
     model: EmbeddingModel<string>;
-    queryFilter?: any;
+    queryFilter?: VectorFilter;
     topK: number;
     includeVectors?: boolean;
     maxRetries?: number;

package/dist/index.cjs

@@ -638,7 +638,8 @@ var RecursiveJsonTransformer = class _RecursiveJsonTransformer {
       return safeObj;
     }
     const stringifiable = getStringifiableData(data);
-    return JSON.stringify(stringifiable).length;
+    const jsonString = JSON.stringify(stringifiable);
+    return jsonString.length;
   }
   /**
    * Transform JSON data while handling circular references
@@ -694,36 +695,216 @@ var RecursiveJsonTransformer = class _RecursiveJsonTransformer {
     return data;
   }
   /**
-   * Split json into maximum size dictionaries while preserving structure
+   * Handles primitive values (strings, numbers, etc) by either adding them to the current chunk
+   * or creating new chunks if they don't fit
+   */
+  handlePrimitiveValue(value, key, currentChunk, chunks, fullPath) {
+    const testValue = { [key]: value };
+    if (_RecursiveJsonTransformer.jsonSize(testValue) <= this.maxSize) {
+      if (_RecursiveJsonTransformer.jsonSize({ ...currentChunk, ...testValue }) <= this.maxSize) {
+        return {
+          currentChunk: { ...currentChunk, ...testValue },
+          chunks
+        };
+      } else {
+        return {
+          currentChunk: testValue,
+          chunks: [...chunks, currentChunk]
+        };
+      }
+    } else if (typeof value === "string") {
+      const stringChunks = this.splitLongString(value);
+      const newChunks = stringChunks.map((chunk) => {
+        return this.createChunk(chunk, fullPath);
+      }).filter((chunk) => _RecursiveJsonTransformer.jsonSize(chunk) <= this.maxSize);
+      return {
+        currentChunk,
+        chunks: [...chunks, ...newChunks]
+      };
+    }
+    const newChunk = this.createChunk(value, fullPath);
+    return {
+      currentChunk,
+      chunks: _RecursiveJsonTransformer.jsonSize(newChunk) <= this.maxSize ? [...chunks, newChunk] : chunks
+    };
+  }
+  /**
+   * Creates a nested dictionary chunk from a value and path
+   * e.g., path ['a', 'b'], value 'c' becomes { a: { b: 'c' } }
+   */
+  createChunk(value, path) {
+    const chunk = {};
+    _RecursiveJsonTransformer.setNestedDict(chunk, path, value);
+    return chunk.root ? chunk.root : chunk;
+  }
+  /**
+   * Checks if value is within size limits
+   */
+  isWithinSizeLimit(value, currentSize = 0) {
+    const size = _RecursiveJsonTransformer.jsonSize(value);
+    return currentSize === 0 ? size <= this.maxSize : size + currentSize <= this.maxSize || currentSize < this.minSize;
+  }
+  /**
+   * Splits arrays into chunks based on size limits
+   * Handles nested objects by recursing into handleNestedObject
+   */
+  handleArray(value, key, currentPath, depth, maxDepth) {
+    const path = currentPath.length ? [...currentPath, key] : ["root", key];
+    const chunk = this.createChunk(value, path);
+    if (this.isWithinSizeLimit(chunk)) {
+      return [chunk];
+    }
+    const chunks = [];
+    let currentGroup = [];
+    const saveCurrentGroup = () => {
+      if (currentGroup.length > 0) {
+        const groupChunk = this.createChunk(currentGroup, path);
+        if (_RecursiveJsonTransformer.jsonSize(groupChunk) >= this.minSize) {
+          chunks.push(groupChunk);
+          currentGroup = [];
+        }
+      }
+    };
+    for (const item of value) {
+      const testGroup = [...currentGroup, item];
+      const testChunk = this.createChunk(testGroup, path);
+      if (this.isWithinSizeLimit(testChunk)) {
+        currentGroup = testGroup;
+        continue;
+      }
+      saveCurrentGroup();
+      if (typeof item === "object" && item !== null) {
+        const singleItemArray = [item];
+        const singleItemChunk = this.createChunk(singleItemArray, path);
+        if (this.isWithinSizeLimit(singleItemChunk)) {
+          currentGroup = singleItemArray;
+        } else {
+          const itemPath = [...path, String(chunks.length)];
+          const nestedChunks = this.handleNestedObject(item, itemPath, depth + 1, maxDepth);
+          chunks.push(...nestedChunks);
+        }
+      } else {
+        currentGroup = [item];
+      }
+    }
+    saveCurrentGroup();
+    return chunks;
+  }
+  /**
+   * Splits objects into chunks based on size limits
+   * Handles nested arrays and objects by recursing into handleArray and handleNestedObject
+   */
+  handleNestedObject(value, fullPath, depth, maxDepth) {
+    const path = fullPath.length ? fullPath : ["root"];
+    if (depth > maxDepth) {
+      console.warn(`Maximum depth of ${maxDepth} exceeded, flattening remaining structure`);
+      return [this.createChunk(value, path)];
+    }
+    const wholeChunk = this.createChunk(value, path);
+    if (this.isWithinSizeLimit(wholeChunk)) {
+      return [wholeChunk];
+    }
+    const chunks = [];
+    let currentChunk = {};
+    const saveCurrentChunk = () => {
+      if (Object.keys(currentChunk).length > 0) {
+        const objChunk = this.createChunk(currentChunk, path);
+        if (_RecursiveJsonTransformer.jsonSize(objChunk) >= this.minSize) {
+          chunks.push(objChunk);
+          currentChunk = {};
+        }
+      }
+    };
+    for (const [key, val] of Object.entries(value)) {
+      if (val === void 0) continue;
+      if (Array.isArray(val)) {
+        saveCurrentChunk();
+        const arrayChunks = this.handleArray(val, key, path, depth, maxDepth);
+        chunks.push(...arrayChunks);
+        continue;
+      }
+      const testChunk = this.createChunk({ ...currentChunk, [key]: val }, path);
+      if (this.isWithinSizeLimit(testChunk)) {
+        currentChunk[key] = val;
+        continue;
+      }
+      saveCurrentChunk();
+      if (typeof val === "object" && val !== null) {
+        const nestedChunks = this.handleNestedObject(val, [...path, key], depth + 1, maxDepth);
+        chunks.push(...nestedChunks);
+      } else {
+        currentChunk = { [key]: val };
+      }
+    }
+    saveCurrentChunk();
+    return chunks;
+  }
+  /**
+   * Splits long strings into smaller chunks at word boundaries
+   * Ensures each chunk is within maxSize limit
+   */
+  splitLongString(value) {
+    const chunks = [];
+    let remaining = value;
+    while (remaining.length > 0) {
+      const overhead = 20;
+      const chunkSize = Math.floor(this.maxSize - overhead);
+      if (remaining.length <= chunkSize) {
+        chunks.push(remaining);
+        break;
+      }
+      const lastSpace = remaining.slice(0, chunkSize).lastIndexOf(" ");
+      const splitAt = lastSpace > 0 ? lastSpace + 1 : chunkSize;
+      chunks.push(remaining.slice(0, splitAt));
+      remaining = remaining.slice(splitAt);
+    }
+    return chunks;
+  }
+  /**
+   * Core chunking logic that processes JSON data recursively
+   * Handles arrays, objects, and primitive values while maintaining structure
    */
   jsonSplit({
     data,
     currentPath = [],
-    chunks = [{}]
+    chunks = [{}],
+    depth = 0,
+    maxDepth = 100
   }) {
-    if (data && typeof data === "object" && !Array.isArray(data)) {
-      for (const [key, value] of Object.entries(data)) {
-        const newPath = [...currentPath, key];
-        const chunkSize = _RecursiveJsonTransformer.jsonSize(chunks[chunks.length - 1] || {});
-        const size = _RecursiveJsonTransformer.jsonSize({ [key]: value });
-        const remaining = this.maxSize - chunkSize;
-        if (size < remaining) {
-          _RecursiveJsonTransformer.setNestedDict(chunks[chunks.length - 1] || {}, newPath, value);
-        } else {
-          if (chunkSize >= this.minSize) {
-            chunks.push({});
-          }
-          this.jsonSplit({
-            data: typeof value === "object" ? value : { [key]: value },
-            currentPath: newPath,
-            chunks
-          });
-        }
-      }
-    } else {
+    if (!data || typeof data !== "object") {
+      return chunks;
+    }
+    if (depth > maxDepth) {
+      console.warn(`Maximum depth of ${maxDepth} exceeded, flattening remaining structure`);
       _RecursiveJsonTransformer.setNestedDict(chunks[chunks.length - 1] || {}, currentPath, data);
+      return chunks;
+    }
+    let currentChunk = {};
+    let accumulatedChunks = chunks;
+    for (const [key, value] of Object.entries(data)) {
+      const fullPath = [...currentPath, key];
+      if (Array.isArray(value)) {
+        const arrayChunks = this.handleArray(value, key, currentPath, depth, maxDepth);
+        accumulatedChunks = [...accumulatedChunks, ...arrayChunks];
+      } else if (typeof value === "object" && value !== null) {
+        const objectChunks = this.handleNestedObject(value, fullPath, depth, maxDepth);
+        accumulatedChunks = [...accumulatedChunks, ...objectChunks];
+      } else {
+        const { currentChunk: newCurrentChunk, chunks: newChunks } = this.handlePrimitiveValue(
+          value,
+          key,
+          currentChunk,
+          accumulatedChunks,
+          fullPath
+        );
+        currentChunk = newCurrentChunk;
+        accumulatedChunks = newChunks;
+      }
     }
-    return chunks;
+    if (Object.keys(currentChunk).length > 0) {
+      accumulatedChunks = [...accumulatedChunks, currentChunk];
+    }
+    return accumulatedChunks.filter((chunk) => Object.keys(chunk).length > 0);
   }
   /**
    * Splits JSON into a list of JSON chunks
@@ -739,12 +920,19 @@ var RecursiveJsonTransformer = class _RecursiveJsonTransformer {
     }
     return chunks;
   }
+  /**
+   * Converts Unicode characters to their escaped ASCII representation
+   * e.g., 'café' becomes 'caf\u00e9'
+   */
   escapeNonAscii(obj) {
     if (typeof obj === "string") {
       return obj.replace(/[\u0080-\uffff]/g, (char) => {
         return `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`;
       });
     }
+    if (Array.isArray(obj)) {
+      return obj.map((item) => this.escapeNonAscii(item));
+    }
     if (typeof obj === "object" && obj !== null) {
       return Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, this.escapeNonAscii(value)]));
     }
@@ -763,7 +951,14 @@ var RecursiveJsonTransformer = class _RecursiveJsonTransformer {
       const escapedChunks = chunks.map((chunk) => this.escapeNonAscii(chunk));
       return escapedChunks.map((chunk) => JSON.stringify(chunk));
     }
-    return chunks.map((chunk) => JSON.stringify(chunk));
+    return chunks.map(
+      (chunk) => JSON.stringify(chunk, (key, value) => {
+        if (typeof value === "string") {
+          return value.replace(/\\u[\da-f]{4}/gi, (match) => String.fromCharCode(parseInt(match.slice(2), 16)));
+        }
+        return value;
+      })
+    );
   }
   /**
    * Create documents from a list of json objects
@@ -1336,7 +1531,10 @@ async function rerank(results, query, model, options) {
   const queryAnalysis = queryEmbedding ? analyzeQueryEmbedding(queryEmbedding) : null;
   const scoredResults = await Promise.all(
     results.map(async (result, index) => {
-      const semanticScore = await semanticProvider.getRelevanceScore(query, result?.metadata?.text);
+      let semanticScore = 0;
+      if (result?.metadata?.text) {
+        semanticScore = await semanticProvider.getRelevanceScore(query, result?.metadata?.text);
+      }
       const vectorScore = result.score;
       const positionScore = calculatePositionScore(index, resultLength);
       let finalScore = weights.semantic * semanticScore + weights.vector * vectorScore + weights.position * positionScore;
@@ -1592,7 +1790,7 @@ var vectorQuerySearch = async ({
   vectorStore,
   queryText,
   model,
-  queryFilter = {},
+  queryFilter,
   topK,
   includeVectors = false,
   maxRetries = 2
@@ -1736,10 +1934,6 @@ var createVectorQueryTool = ({
     }),
     description: toolDescription,
     execute: async ({ context: { queryText, topK, filter }, mastra }) => {
-      console.log({
-        topK,
-        filter
-      });
       const vectorStore = mastra?.vectors?.[vectorStoreName];
       if (vectorStore) {
         let queryFilter = {};