@absolutejs/absolute 0.19.0-beta.493 → 0.19.0-beta.495

package/dist/ai/index.js

@@ -2149,6 +2149,8 @@ var STOP_WORDS = new Set([
   "why"
 ]);
 var tokenize = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => !STOP_WORDS.has(token)).map((token) => token.endsWith("ies") && token.length > 3 ? `${token.slice(0, -3)}y` : token.endsWith("ing") && token.length > 5 ? token.slice(0, -3) : token.endsWith("ed") && token.length > 4 ? token.slice(0, -2) : token.endsWith("es") && token.length > 4 ? token.slice(0, -2) : token.endsWith("s") && token.length > 3 ? token.slice(0, -1) : token).filter((token) => token.length > 1);
+var BM25_K1 = 1.2;
+var BM25_B = 0.75;
 var collectMetadataStrings = (value) => {
   if (typeof value === "string" || typeof value === "number") {
     return [String(value)];
@@ -2164,7 +2166,7 @@ var collectMetadataStrings = (value) => {
 var normalizeSourceForLexical = (source) => source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle");
 var toFieldText = (value) => collectMetadataStrings(value).filter(Boolean).join(" ");
 var scoreTokenCoverage = (queryTokens, text) => {
-  const normalizedText = text.toLowerCase();
+  const normalizedText = (text ?? "").toLowerCase();
   if (normalizedText.length === 0) {
     return 0;
   }
@@ -2178,7 +2180,7 @@ var scoreTokenCoverage = (queryTokens, text) => {
 };
 var scorePhraseMatch = (query, text) => {
   const normalizedQuery = tokenize(query).join(" ");
-  const normalizedText = tokenize(text).join(" ");
+  const normalizedText = tokenize(text ?? "").join(" ");
   if (normalizedQuery.length === 0 || normalizedText.length === 0) {
     return 0;
   }
@@ -2190,7 +2192,7 @@ var scoreWeightedField = ({
   query,
   queryTokens,
   text
-}) => scoreTokenCoverage(queryTokens, text) * coverageWeight + scorePhraseMatch(query, text) * phraseWeight;
+}) => scoreTokenCoverage(queryTokens, text ?? "") * coverageWeight + scorePhraseMatch(query, text ?? "") * phraseWeight;
 var extractWeightedLexicalFields = (result) => {
   const metadata = result.metadata ?? {};
   const source = result.source ?? "";
@@ -2209,7 +2211,7 @@ var extractWeightedLexicalFields = (result) => {
   ].flatMap((value) => collectMetadataStrings(value)).join(" ");
   return {
     archivePath,
-    chunkText: result.chunkText,
+    chunkText: result.text,
     mediaSegments,
     metadataFocus,
     metadataText: toFieldText(metadata),
@@ -2217,6 +2219,34 @@ var extractWeightedLexicalFields = (result) => {
     title: result.title ?? ""
   };
 };
+var FIELD_WEIGHTS = {
+  archivePath: 4.2,
+  chunkText: 1,
+  mediaSegments: 3.8,
+  metadataFocus: 3.2,
+  metadataText: 1.4,
+  source: 3.4,
+  title: 2
+};
+var getWeightedFieldTokens = (result) => {
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.text,
+    title: result.title
+  });
+  return {
+    archivePath: tokenize(fields.archivePath ?? ""),
+    chunkText: tokenize(fields.chunkText ?? ""),
+    mediaSegments: tokenize(fields.mediaSegments ?? ""),
+    metadataFocus: tokenize(fields.metadataFocus ?? ""),
+    metadataText: tokenize(fields.metadataText ?? ""),
+    source: tokenize(fields.source ?? ""),
+    title: tokenize(fields.title ?? "")
+  };
+};
+var countWeightedTermFrequency = (fieldTokens, token) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].filter((value) => value === token).length * FIELD_WEIGHTS[fieldName], 0);
+var computeWeightedDocumentLength = (fieldTokens) => Object.keys(FIELD_WEIGHTS).reduce((total, fieldName) => total + fieldTokens[fieldName].length * FIELD_WEIGHTS[fieldName], 0);
 var buildRAGLexicalHaystack = (result) => [
   result.title,
   result.source,
@@ -2229,7 +2259,12 @@ var scoreRAGLexicalMatch = (query, result) => {
   if (queryTokens.length === 0) {
     return 0;
   }
-  const fields = extractWeightedLexicalFields(result);
+  const fields = extractWeightedLexicalFields({
+    metadata: result.metadata,
+    source: result.source,
+    text: result.chunkText,
+    title: result.title
+  });
   const haystack = buildRAGLexicalHaystack(result).toLowerCase();
   const overallCoverage = scoreTokenCoverage(queryTokens, haystack);
   if (overallCoverage === 0) {
@@ -2291,6 +2326,65 @@ var scoreRAGLexicalMatch = (query, result) => {
   const archiveBoost = resolveArchiveBoost(queryTokens, result);
   return titleScore + sourceScore + metadataFocusScore + archivePathScore + mediaSegmentScore + metadataScore + chunkScore + coverageBoost + exactPhraseBoost + fileKindBoost + transcriptBoost + archiveBoost;
 };
+var rankRAGLexicalMatches = (query, results) => {
+  const queryTokens = tokenize(query);
+  if (queryTokens.length === 0 || results.length === 0) {
+    return [];
+  }
+  const candidates = results.map((result) => {
+    const fieldTokens = getWeightedFieldTokens(result);
+    return {
+      fieldTokens,
+      length: computeWeightedDocumentLength(fieldTokens),
+      result
+    };
+  });
+  const averageDocumentLength = candidates.reduce((total, candidate) => total + candidate.length, 0) / Math.max(1, candidates.length);
+  const uniqueQueryTokens = [...new Set(queryTokens)];
+  const documentFrequency = new Map;
+  for (const token of uniqueQueryTokens) {
+    let seen = 0;
+    for (const candidate of candidates) {
+      const tf = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (tf > 0) {
+        seen += 1;
+      }
+    }
+    documentFrequency.set(token, seen);
+  }
+  return candidates.map((candidate, index) => {
+    let bm25Score = 0;
+    for (const token of uniqueQueryTokens) {
+      const termFrequency = countWeightedTermFrequency(candidate.fieldTokens, token);
+      if (termFrequency <= 0) {
+        continue;
+      }
+      const df = documentFrequency.get(token) ?? 0;
+      const idf = Math.log(1 + (candidates.length - df + 0.5) / (df + 0.5));
+      const denominator = termFrequency + BM25_K1 * (1 - BM25_B + BM25_B * (candidate.length / Math.max(1, averageDocumentLength)));
+      bm25Score += idf * (termFrequency * (BM25_K1 + 1) / Math.max(0.000000001, denominator));
+    }
+    const heuristicScore = scoreRAGLexicalMatch(query, {
+      chunkText: candidate.result.text,
+      metadata: candidate.result.metadata,
+      source: candidate.result.source,
+      title: candidate.result.title
+    });
+    return {
+      index,
+      result: candidate.result,
+      score: bm25Score + heuristicScore * 0.35
+    };
+  }).filter((entry) => entry.score > 0).sort((left, right) => {
+    if (right.score !== left.score) {
+      return right.score - left.score;
+    }
+    return left.index - right.index;
+  }).map(({ result, score }) => ({
+    result,
+    score
+  }));
+};
 var hasAnyToken = (tokens, values) => values.some((value) => tokens.includes(value));
 var resolveFileKindBoost = (queryTokens, metadata) => {
   const fileKind = typeof metadata?.fileKind === "string" ? metadata.fileKind : "";
@@ -3138,6 +3232,30 @@ var spreadsheetText = (entries) => {
   return normalizeWhitespace(sheetValues.join(`
 `));
 };
+var spreadsheetSheetTexts = (entries) => {
+  const sharedStrings = entries.filter((entry) => entry.path === "xl/sharedStrings.xml").flatMap((entry) => [
+    ...decodeUtf8(entry.data).matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)
+  ].map((match) => decodeHtmlEntities(match[1] ?? "")));
+  const sheetNames = spreadsheetSheetNames(entries);
+  const sheetEntries = entries.filter((entry) => entry.path.startsWith("xl/worksheets/") && entry.path.endsWith(".xml")).sort((left, right) => left.path.localeCompare(right.path));
+  return sheetEntries.map((entry, index) => {
+    const values = [
+      ...decodeUtf8(entry.data).matchAll(/<v>([\s\S]*?)<\/v>/g)
+    ].map((match) => match[1] ?? "").map((value) => {
+      const sharedStringIndex = Number(value);
+      return Number.isInteger(sharedStringIndex) && sharedStrings[sharedStringIndex] ? sharedStrings[sharedStringIndex] : value;
+    });
+    const text = normalizeWhitespace(values.join(`
+`));
+    if (!text) {
+      return null;
+    }
+    return {
+      name: sheetNames[index] ?? `Sheet ${index + 1}`,
+      text
+    };
+  }).filter((entry) => Boolean(entry));
+};
 var spreadsheetSheetNames = (entries) => entries.filter((entry) => entry.path === "xl/workbook.xml").flatMap((entry) => [
   ...decodeUtf8(entry.data).matchAll(/<sheet[^>]*name="([^"]+)"/g)
 ].map((match) => match[1] ?? "")).filter(Boolean);
@@ -3147,6 +3265,10 @@ var presentationText = (entries) => {
 
 `));
 };
+var presentationSlides = (entries) => entries.filter((entry) => entry.path.startsWith("ppt/slides/") && entry.path.endsWith(".xml")).sort((left, right) => left.path.localeCompare(right.path)).map((entry, index) => ({
+  index,
+  text: normalizeWhitespace(extractXmlText(decodeUtf8(entry.data)))
+})).filter((slide) => Boolean(slide.text));
 var presentationSlideCount = (entries) => entries.filter((entry) => entry.path.startsWith("ppt/slides/") && entry.path.endsWith(".xml")).length;
 var epubText = (entries) => {
   const htmlEntries = entries.filter((entry) => /\.(xhtml|html|htm)$/i.test(entry.path));
@@ -3364,6 +3486,7 @@ var createOfficeDocumentExtractor = () => ({
     const entries = unzipEntries(input.data);
     let text = "";
     let officeMetadata = {};
+    let structuredDocuments = [];
     if (extension === ".docx" || extension === ".odt") {
       text = officeDocumentText(entries);
       officeMetadata = {
@@ -3371,19 +3494,53 @@ var createOfficeDocumentExtractor = () => ({
       };
     } else if (extension === ".xlsx" || extension === ".ods") {
       text = spreadsheetText(entries);
+      const sheets = spreadsheetSheetTexts(entries);
       officeMetadata = {
         sheetNames: spreadsheetSheetNames(entries)
       };
+      structuredDocuments = sheets.map((sheet, index) => ({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          fileKind: "office",
+          ...officeMetadata,
+          sheetIndex: index,
+          sheetName: sheet.name
+        },
+        source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
+        text: `Sheet ${sheet.name}
+${sheet.text}`,
+        title: input.title ? `${input.title} \xB7 ${sheet.name}` : sheet.name
+      }));
     } else if (extension === ".pptx" || extension === ".odp") {
       text = presentationText(entries);
+      const slides = presentationSlides(entries);
       officeMetadata = {
         slideCount: presentationSlideCount(entries)
       };
+      structuredDocuments = slides.map((slide) => ({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          fileKind: "office",
+          ...officeMetadata,
+          slideIndex: slide.index,
+          slideNumber: slide.index + 1
+        },
+        source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
+        text: `Slide ${slide.index + 1}
+${slide.text}`,
+        title: input.title ? `${input.title} \xB7 Slide ${slide.index + 1}` : `Slide ${slide.index + 1}`
+      }));
     }
     if (!text) {
       throw new Error(`AbsoluteJS could not extract readable text from ${inferNameFromInput(input)}`);
     }
-    return {
+    const summaryDocument = {
       chunking: input.chunking,
       contentType: input.contentType,
       format: "text",
@@ -3396,6 +3553,7 @@ var createOfficeDocumentExtractor = () => ({
       text,
       title: input.title
     };
+    return [summaryDocument, ...structuredDocuments];
   }
 });
 var createRAGArchiveExpander = (expander) => expander;
@@ -3425,7 +3583,36 @@ var createRAGMediaFileExtractor = (transcriber) => ({
   supports: mediaExtractorSupports,
   extract: async (input) => {
     const result = await transcriber.transcribe(input);
-    return {
+    const source = input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.media.txt`;
+    const segmentDocuments = [];
+    for (const [index, segment] of (result.segments ?? []).entries()) {
+      const text = normalizeWhitespace(segment.text ?? "");
+      if (!text) {
+        continue;
+      }
+      const startMs = typeof segment.startMs === "number" ? segment.startMs : undefined;
+      const endMs = typeof segment.endMs === "number" ? segment.endMs : undefined;
+      segmentDocuments.push({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          ...result.metadata ?? {},
+          fileKind: "media",
+          mediaSegmentIndex: index,
+          mediaSegmentStartMs: startMs,
+          mediaSegmentEndMs: endMs,
+          mediaSegments: [segment],
+          speaker: typeof segment.speaker === "string" ? segment.speaker : undefined
+        },
+        source,
+        text: `Transcript segment${typeof startMs === "number" ? ` ${startMs}-${endMs ?? startMs}ms` : ""}
+${text}`,
+        title: input.title ? `${input.title} \xB7 Segment ${index + 1}` : `Segment ${index + 1}`
+      });
+    }
+    const summaryDocument = {
       chunking: input.chunking,
       contentType: input.contentType,
       format: "text",
@@ -3435,10 +3622,11 @@ var createRAGMediaFileExtractor = (transcriber) => ({
         fileKind: "media",
         mediaSegments: result.segments
       },
-      source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.media.txt`,
+      source,
       text: result.text,
       title: result.title ?? input.title
     };
+    return [summaryDocument, ...segmentDocuments];
   }
 });
 var createRAGMediaTranscriber = (transcriber) => transcriber;
@@ -3470,7 +3658,7 @@ var expandArchiveEntry = async (entry, archiveInput, extractors) => {
     },
     name: basename(entry.path),
     source: archiveInput.source && !archiveInput.source.startsWith("http") ? `${archiveInput.source}#${entry.path}` : entry.path,
-    title: archiveInput.title
+    title: basename(entry.path)
   }, extractors);
   return documents;
 };
@@ -3599,6 +3787,7 @@ var getFirstExtractedDocument = (documents, label) => {
   }
   return document;
 };
+var loadExtractedDocuments = async (input, extractors) => extractRAGFileDocuments(input, extractors);
 var sentenceUnits = (text) => {
   const matches = text.match(/[^.!?\n]+(?:[.!?]+|$)/g);
   if (!matches) {
@@ -3821,32 +4010,55 @@ var loadRAGDocumentFromURL = async (input) => {
 };
 var loadRAGDocumentsFromUploads = async (input) => {
   const documents = await Promise.all(input.uploads.map(async (upload) => {
-    const loaded = await loadRAGDocumentUpload({
-      ...upload,
-      extractors: input.extractors
-    });
-    return {
-      ...loaded,
-      metadata: mergeMetadata(loaded.metadata, { uploadFile: upload.name }, input.baseMetadata)
-    };
+    const loaded = await loadExtractedDocuments({
+      chunking: upload.chunking,
+      contentType: upload.contentType,
+      data: decodeUploadContent(upload),
+      format: upload.format,
+      metadata: upload.metadata,
+      name: upload.name,
+      source: upload.source ?? upload.name,
+      title: upload.title
+    }, input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, { uploadFile: upload.name }, input.baseMetadata)
+    }));
   }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var loadRAGDocumentsFromURLs = async (input) => {
-  const documents = await Promise.all(input.urls.map(async (urlInput) => loadRAGDocumentFromURL({
-    ...urlInput,
-    metadata: mergeMetadata(urlInput.metadata, {
-      sourceUrl: urlInput.url
-    }, input.baseMetadata),
-    contentType: urlInput.contentType,
-    extractors: urlInput.extractors ?? input.extractors
-  })));
+  const documents = await Promise.all(input.urls.map(async (urlInput) => {
+    const url = urlInput.url.trim();
+    if (!url) {
+      throw new Error("RAG URL is required");
+    }
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
+    }
+    const data = new Uint8Array(await response.arrayBuffer());
+    const loaded = await loadExtractedDocuments({
+      chunking: urlInput.chunking,
+      contentType: urlInput.contentType ?? response.headers.get("content-type") ?? undefined,
+      data,
+      format: urlInput.format ?? inferFormatFromUrl(url),
+      metadata: urlInput.metadata,
+      name: basename(new URL(url).pathname),
+      source: urlInput.source ?? url,
+      title: urlInput.title
+    }, urlInput.extractors ?? input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, { sourceUrl: urlInput.url }, input.baseMetadata)
+    }));
+  }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var loadRAGDocumentUpload = async (input) => {
@@ -3926,21 +4138,25 @@ var loadRAGDocumentsFromDirectory = async (input) => {
   const files = await collectDirectoryFiles(root, input.recursive !== false, includeExtensions);
   const documents = await Promise.all(files.map(async (path) => {
     const source = relative(root, path).replace(/\\/g, "/");
-    const loaded = await loadRAGDocumentFile({
+    const data = await readFile(path);
+    const loaded = await loadExtractedDocuments({
+      chunking: input.defaultChunking,
+      data,
       metadata: {
-        ...input.baseMetadata ?? {},
         fileName: basename(path),
         relativePath: source
       },
       path,
-      source,
-      extractors: input.extractors
-    });
-    return loaded;
+      source
+    }, input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, undefined, input.baseMetadata)
+    }));
   }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var prepareRAGDirectoryDocuments = async (input) => prepareRAGDocuments(await loadRAGDocumentsFromDirectory(input));
@@ -6860,27 +7076,15 @@ var createInMemoryRAGStore = (options = {}) => {
     }));
   };
   const queryLexical = async (input) => {
-    const results = chunks.map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ chunk }) => matchesFilter(chunk, input.filter)).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return results.slice(0, input.topK).map((entry) => ({
-      chunkId: entry.chunk.chunkId,
-      chunkText: entry.chunk.text,
-      metadata: entry.chunk.metadata,
-      score: entry.score,
-      source: entry.chunk.source,
-      title: entry.chunk.title
+    const filtered = chunks.filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, filtered);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
+      score,
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -7511,27 +7715,15 @@ var createSQLiteRAGStore = (options = {}) => {
   };
   const queryLexical = async (input) => {
     const rawRows = toStoredRows(jsonStatements.query.all());
-    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter)).map((chunk) => ({
-      chunk,
-      score: scoreRAGLexicalMatch(input.query, {
-        chunkText: chunk.text,
-        metadata: chunk.metadata,
-        source: chunk.source,
-        title: chunk.title
-      })
-    })).filter(({ score }) => score > 0).sort((left, right) => {
-      if (right.score !== left.score) {
-        return right.score - left.score;
-      }
-      return left.chunk.chunkId.localeCompare(right.chunk.chunkId);
-    });
-    return chunks.slice(0, input.topK).map(({ chunk, score }) => ({
-      chunkId: chunk.chunkId,
-      chunkText: chunk.text,
-      metadata: chunk.metadata,
+    const chunks = mapFilterToRows(rawRows).filter((chunk) => matchesFilter(chunk, input.filter));
+    const ranked = rankRAGLexicalMatches(input.query, chunks);
+    return ranked.slice(0, input.topK).map(({ result, score }) => ({
+      chunkId: result.chunkId,
+      chunkText: result.text,
+      metadata: result.metadata,
       score,
-      source: chunk.source,
-      title: chunk.title
+      source: result.source,
+      title: result.title
     }));
   };
   const upsert = async (input) => {
@@ -8703,5 +8895,5 @@ export {
   aiChat
 };
 
-//# debugId=36F6407CE8163A4F64756E2164756E21
+//# debugId=A1829EEFE0D80F9264756E2164756E21
 //# sourceMappingURL=index.js.map