@absolutejs/absolute 0.19.0-beta.495 → 0.19.0-beta.497

package/dist/ai/index.js

@@ -2165,6 +2165,31 @@ var collectMetadataStrings = (value) => {
 };
 var normalizeSourceForLexical = (source) => source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle");
 var toFieldText = (value) => collectMetadataStrings(value).filter(Boolean).join(" ");
+var normalizeLooseText = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+var scoreLoosePhraseMatch = (query, text) => {
+  const normalizedQuery = normalizeLooseText(query);
+  const normalizedText = normalizeLooseText(text ?? "");
+  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
+    return 0;
+  }
+  if (normalizedText.includes(normalizedQuery)) {
+    return 1;
+  }
+  const words = normalizedQuery.split(" ").filter(Boolean);
+  for (let size = Math.min(5, words.length);size >= 2; size -= 1) {
+    for (let index = 0;index <= words.length - size; index += 1) {
+      const phraseWords = words.slice(index, index + size);
+      if (phraseWords.every((word) => STOP_WORDS.has(word))) {
+        continue;
+      }
+      const phrase = phraseWords.join(" ");
+      if (normalizedText.includes(phrase)) {
+        return Math.min(1, size / 4);
+      }
+    }
+  }
+  return 0;
+};
 var scoreTokenCoverage = (queryTokens, text) => {
   const normalizedText = (text ?? "").toLowerCase();
   if (normalizedText.length === 0) {
@@ -2181,10 +2206,8 @@ var scoreTokenCoverage = (queryTokens, text) => {
 var scorePhraseMatch = (query, text) => {
   const normalizedQuery = tokenize(query).join(" ");
   const normalizedText = tokenize(text ?? "").join(" ");
-  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
-    return 0;
-  }
-  return normalizedText.includes(normalizedQuery) ? 1 : 0;
+  const tokenPhraseMatch = normalizedQuery.length > 0 && normalizedText.length > 0 ? normalizedText.includes(normalizedQuery) ? 1 : 0 : 0;
+  return Math.max(tokenPhraseMatch, scoreLoosePhraseMatch(query, text ?? ""));
 };
 var scoreWeightedField = ({
   coverageWeight,
@@ -2199,8 +2222,10 @@ var extractWeightedLexicalFields = (result) => {
   const archivePath = typeof metadata.archivePath === "string" ? metadata.archivePath : source.includes("#") ? source.split("#")[1] ?? "" : "";
   const mediaSegments = Array.isArray(metadata.mediaSegments) ? metadata.mediaSegments.map((segment) => segment && typeof segment === "object" ? toFieldText(segment) : "").filter(Boolean).join(" ") : "";
   const metadataFocus = [
+    metadata.sourceNativeKind,
     metadata.sheetName,
     metadata.sheetNames,
+    metadata.slideNumber,
     metadata.slideTitle,
     metadata.slideTitles,
     metadata.threadTopic,
@@ -2772,7 +2797,36 @@ var collectMetadataStrings2 = (value) => {
   }
   return [];
 };
-var scoreHeuristicMatch = (queryTokens, result) => {
+var normalizeLooseText2 = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+var scoreLoosePhraseMatch2 = (query, text) => {
+  const normalizedQuery = normalizeLooseText2(query);
+  const normalizedText = normalizeLooseText2(text);
+  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
+    return 0;
+  }
+  if (normalizedText.includes(normalizedQuery)) {
+    return 1;
+  }
+  const words = normalizedQuery.split(" ").filter(Boolean);
+  for (let size = Math.min(5, words.length);size >= 2; size -= 1) {
+    for (let index = 0;index <= words.length - size; index += 1) {
+      const phraseWords = words.slice(index, index + size);
+      if (phraseWords.every((word) => STOP_WORDS3.has(word))) {
+        continue;
+      }
+      const phrase = phraseWords.join(" ");
+      if (normalizedText.includes(phrase)) {
+        return Math.min(1, size / 4);
+      }
+    }
+  }
+  return 0;
+};
+var scoreHeuristicMatch = ({
+  query,
+  queryTokens,
+  result
+}) => {
   if (queryTokens.length === 0) {
     return result.score;
   }
@@ -2781,8 +2835,7 @@ var scoreHeuristicMatch = (queryTokens, result) => {
   const haystackSet = new Set(haystack);
   const overlap = queryTokens.filter((token) => haystackSet.has(token)).length;
   const overlapBoost = overlap / queryTokens.length;
-  const normalizedQuery = queryTokens.join(" ");
-  const exactPhraseBoost = normalizeText([result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")).includes(normalizedQuery) ? 1 : 0;
+  const exactPhraseBoost = Math.max(normalizeText([result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")).includes(queryTokens.join(" ")) ? 1 : 0, scoreLoosePhraseMatch2(query, [result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")));
   const sourcePathBoost = typeof result.source === "string" && queryTokens.some((token) => result.source?.toLowerCase().includes(token)) ? 0.5 : 0;
   const metadataBoost = metadataValues.length > 0 ? queryTokens.filter((token) => metadataValues.some((value) => value.toLowerCase().includes(token))).length / queryTokens.length : 0;
   return result.score + overlapBoost + exactPhraseBoost + sourcePathBoost + metadataBoost;
@@ -2810,7 +2863,11 @@ var createHeuristicRAGReranker = (options = {}) => createRAGReranker({
     return [...results].map((result, index) => ({
       index,
       result,
-      score: scoreHeuristicMatch(queryTokens, result)
+      score: scoreHeuristicMatch({
+        query,
+        queryTokens,
+        result
+      })
     })).sort((left, right) => {
       if (right.score !== left.score) {
         return right.score - left.score;
@@ -3506,12 +3563,13 @@ var createOfficeDocumentExtractor = () => ({
           ...input.metadata ?? {},
           fileKind: "office",
           ...officeMetadata,
+          sourceNativeKind: "spreadsheet_sheet",
           sheetIndex: index,
           sheetName: sheet.name
         },
         source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
-        text: `Sheet ${sheet.name}
-${sheet.text}`,
+        text: normalizeWhitespace(`Spreadsheet sheet ${sheet.name} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${sheet.text}`),
         title: input.title ? `${input.title} \xB7 ${sheet.name}` : sheet.name
       }));
     } else if (extension === ".pptx" || extension === ".odp") {
@@ -3528,12 +3586,13 @@ ${sheet.text}`,
           ...input.metadata ?? {},
           fileKind: "office",
           ...officeMetadata,
+          sourceNativeKind: "presentation_slide",
           slideIndex: slide.index,
           slideNumber: slide.index + 1
         },
         source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
-        text: `Slide ${slide.index + 1}
-${slide.text}`,
+        text: normalizeWhitespace(`Presentation slide ${slide.index + 1} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${slide.text}`),
         title: input.title ? `${input.title} \xB7 Slide ${slide.index + 1}` : `Slide ${slide.index + 1}`
       }));
     }
@@ -3600,6 +3659,7 @@ var createRAGMediaFileExtractor = (transcriber) => ({
           ...input.metadata ?? {},
           ...result.metadata ?? {},
           fileKind: "media",
+          sourceNativeKind: "media_segment",
           mediaSegmentIndex: index,
           mediaSegmentStartMs: startMs,
           mediaSegmentEndMs: endMs,
@@ -3607,8 +3667,8 @@ var createRAGMediaFileExtractor = (transcriber) => ({
           speaker: typeof segment.speaker === "string" ? segment.speaker : undefined
         },
         source,
-        text: `Transcript segment${typeof startMs === "number" ? ` ${startMs}-${endMs ?? startMs}ms` : ""}
-${text}`,
+        text: normalizeWhitespace(`Media transcript segment${typeof startMs === "number" ? ` ${startMs}-${endMs ?? startMs}ms` : ""} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${text}`),
         title: input.title ? `${input.title} \xB7 Segment ${index + 1}` : `Segment ${index + 1}`
       });
     }
@@ -8895,5 +8955,5 @@ export {
   aiChat
 };
 
-//# debugId=A1829EEFE0D80F9264756E2164756E21
+//# debugId=55FD05298CEAFBDB64756E2164756E21
 //# sourceMappingURL=index.js.map