@mastra/rag 2.4.0-alpha.0 → 2.4.1-alpha.0

package/dist/index.js

@@ -4531,8 +4531,8 @@ var TextTransformer = class {
     addStartIndex = false,
     stripWhitespace = true
   }) {
-    if (overlap > maxSize) {
-      throw new Error(`Got a larger chunk overlap (${overlap}) than chunk size (${maxSize}), should be smaller.`);
+    if (overlap >= maxSize) {
+      throw new Error(`Chunk overlap (${overlap}) must be smaller than chunk size (${maxSize}).`);
     }
     this.maxSize = maxSize;
     this.overlap = overlap;
@@ -4701,17 +4701,31 @@ var CharacterTransformer = class extends TextTransformer {
   }
   __splitChunk(text) {
     const chunks = [];
-    let currentPosition = 0;
-    while (currentPosition < text.length) {
-      let chunkEnd = currentPosition;
-      while (chunkEnd < text.length && this.lengthFunction(text.slice(currentPosition, chunkEnd + 1)) <= this.maxSize) {
-        chunkEnd++;
-      }
-      const currentChunk = text.slice(currentPosition, chunkEnd);
-      const chunkLength = this.lengthFunction(currentChunk);
+    const codePointBoundaries = [0];
+    for (const codePoint of text) {
+      codePointBoundaries.push(codePointBoundaries[codePointBoundaries.length - 1] + codePoint.length);
+    }
+    const finalBoundaryIndex = codePointBoundaries.length - 1;
+    let currentBoundaryIndex = 0;
+    while (currentBoundaryIndex < finalBoundaryIndex) {
+      let chunkEndBoundaryIndex = currentBoundaryIndex;
+      while (chunkEndBoundaryIndex < finalBoundaryIndex && this.lengthFunction(
+        text.slice(codePointBoundaries[currentBoundaryIndex], codePointBoundaries[chunkEndBoundaryIndex + 1])
+      ) <= this.maxSize) {
+        chunkEndBoundaryIndex++;
+      }
+      if (chunkEndBoundaryIndex === currentBoundaryIndex) {
+        chunkEndBoundaryIndex++;
+      }
+      const chunkEnd = codePointBoundaries[chunkEndBoundaryIndex];
+      const currentChunk = text.slice(codePointBoundaries[currentBoundaryIndex], chunkEnd);
       chunks.push(currentChunk);
-      if (chunkEnd >= text.length) break;
-      currentPosition += Math.max(1, chunkLength - this.overlap);
+      if (chunkEndBoundaryIndex >= finalBoundaryIndex) break;
+      let nextBoundaryIndex = chunkEndBoundaryIndex;
+      while (this.overlap > 0 && nextBoundaryIndex > currentBoundaryIndex && this.lengthFunction(text.slice(codePointBoundaries[nextBoundaryIndex - 1], chunkEnd)) <= this.overlap) {
+        nextBoundaryIndex--;
+      }
+      currentBoundaryIndex = Math.max(currentBoundaryIndex + 1, nextBoundaryIndex);
     }
     return chunks;
   }
@@ -6970,10 +6984,13 @@ var CohereRelevanceScorer = class {
   model;
   apiKey;
   constructor(model, apiKey) {
-    this.apiKey = apiKey;
+    this.apiKey = apiKey ?? process.env.COHERE_API_KEY;
     this.model = model;
   }
   async getRelevanceScore(query, text) {
+    if (!this.apiKey) {
+      throw new Error("Cohere API key is required. Pass an apiKey or set COHERE_API_KEY.");
+    }
     const response = await fetch(`https://api.cohere.com/v2/rerank`, {
       method: "POST",
       headers: {
@@ -6992,12 +7009,20 @@ var CohereRelevanceScorer = class {
     }
     const data = await response.json();
     const relevanceScore = data.results[0]?.relevance_score;
-    if (!relevanceScore) {
+    if (typeof relevanceScore !== "number" || !Number.isFinite(relevanceScore)) {
       throw new Error("No relevance score found on Cohere response");
     }
     return relevanceScore;
   }
 };
+function parseRelevanceScore(responseText) {
+  const trimmed = responseText.trim();
+  const score = Number(trimmed);
+  if (!trimmed || !Number.isFinite(score) || score < 0 || score > 1) {
+    throw new Error(`Invalid relevance score returned by model: ${responseText}`);
+  }
+  return score;
+}
 var MastraAgentRelevanceScorer = class {
   agent;
   constructor(name14, model) {
@@ -7026,7 +7051,7 @@ Always return just the number, no explanation.`,
     } else {
       response = await this.agent.generateLegacy(prompt);
     }
-    return parseFloat(response.text);
+    return parseRelevanceScore(response.text);
   }
 };
 var ZeroEntropyRelevanceScorer = class {
@@ -7098,35 +7123,41 @@ async function executeRerank({
   }
   const resultLength = results.length;
   const queryAnalysis = queryEmbedding ? analyzeQueryEmbedding(queryEmbedding) : null;
-  const scoredResults = await Promise.all(
-    results.map(async (result, index) => {
-      let semanticScore = 0;
-      if (result?.metadata?.text) {
-        semanticScore = await scorer.getRelevanceScore(query, result?.metadata?.text);
-      }
-      const vectorScore = result.score;
-      const positionScore = calculatePositionScore(index, resultLength);
-      let finalScore = weights.semantic * semanticScore + weights.vector * vectorScore + weights.position * positionScore;
-      if (queryAnalysis) {
-        finalScore = adjustScores(finalScore, queryAnalysis);
-      }
-      return {
-        result,
-        score: finalScore,
-        details: {
-          semantic: semanticScore,
-          vector: vectorScore,
-          position: positionScore,
-          ...queryAnalysis && {
-            queryAnalysis: {
-              magnitude: queryAnalysis.magnitude,
-              dominantFeatures: queryAnalysis.dominantFeatures
+  let scoredResults;
+  try {
+    scoredResults = await Promise.all(
+      results.map(async (result, index) => {
+        let semanticScore = 0;
+        if (result?.metadata?.text) {
+          semanticScore = await scorer.getRelevanceScore(query, result?.metadata?.text);
+        }
+        const vectorScore = result.score;
+        const positionScore = calculatePositionScore(index, resultLength);
+        let finalScore = weights.semantic * semanticScore + weights.vector * vectorScore + weights.position * positionScore;
+        if (queryAnalysis) {
+          finalScore = adjustScores(finalScore, queryAnalysis);
+        }
+        return {
+          result,
+          score: finalScore,
+          details: {
+            semantic: semanticScore,
+            vector: vectorScore,
+            position: positionScore,
+            ...queryAnalysis && {
+              queryAnalysis: {
+                magnitude: queryAnalysis.magnitude,
+                dominantFeatures: queryAnalysis.dominantFeatures
+              }
             }
           }
-        }
-      };
-    })
-  );
+        };
+      })
+    );
+  } catch (err) {
+    rerankSpan?.error({ error: err, endSpan: true });
+    throw err;
+  }
   const final = scoredResults.sort((a, b) => b.score - a.score).slice(0, topK);
   rerankSpan?.end({ output: { returned: final.length } });
   return final;
@@ -7700,9 +7731,10 @@ function parseFilterValue(filter, logger) {
   if (!filter) {
     return {};
   }
+  let parsedFilter = filter;
   if (typeof filter === "string") {
     try {
-      return JSON.parse(filter);
+      parsedFilter = JSON.parse(filter);
     } catch (error) {
       if (logger) {
         logger.error("Invalid filter", { filter, error });
@@ -7710,15 +7742,14 @@ function parseFilterValue(filter, logger) {
       throw new Error(`Invalid filter format: ${error instanceof Error ? error.message : String(error)}`);
     }
   }
-  if (typeof filter !== "object" || filter === null || Array.isArray(filter)) {
+  if (typeof parsedFilter !== "object" || parsedFilter === null || Array.isArray(parsedFilter)) {
+    const receivedType = Array.isArray(parsedFilter) ? "array" : typeof parsedFilter;
     if (logger) {
       logger.error("Invalid filter", { filter, error: "Filter must be a plain object" });
     }
-    throw new Error(
-      `Invalid filter format: expected a plain object, got ${Array.isArray(filter) ? "array" : typeof filter}`
-    );
+    throw new Error(`Invalid filter format: expected a plain object, got ${receivedType}`);
   }
-  return filter;
+  return parsedFilter;
 }
 
 // src/utils/convert-sources.ts