@absolutejs/absolute 0.19.0-beta.494 → 0.19.0-beta.496

package/dist/ai/index.js

@@ -2199,8 +2199,10 @@ var extractWeightedLexicalFields = (result) => {
   const archivePath = typeof metadata.archivePath === "string" ? metadata.archivePath : source.includes("#") ? source.split("#")[1] ?? "" : "";
   const mediaSegments = Array.isArray(metadata.mediaSegments) ? metadata.mediaSegments.map((segment) => segment && typeof segment === "object" ? toFieldText(segment) : "").filter(Boolean).join(" ") : "";
   const metadataFocus = [
+    metadata.sourceNativeKind,
     metadata.sheetName,
     metadata.sheetNames,
+    metadata.slideNumber,
     metadata.slideTitle,
     metadata.slideTitles,
     metadata.threadTopic,
@@ -3232,6 +3234,30 @@ var spreadsheetText = (entries) => {
   return normalizeWhitespace(sheetValues.join(`
 `));
 };
+var spreadsheetSheetTexts = (entries) => {
+  const sharedStrings = entries.filter((entry) => entry.path === "xl/sharedStrings.xml").flatMap((entry) => [
+    ...decodeUtf8(entry.data).matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)
+  ].map((match) => decodeHtmlEntities(match[1] ?? "")));
+  const sheetNames = spreadsheetSheetNames(entries);
+  const sheetEntries = entries.filter((entry) => entry.path.startsWith("xl/worksheets/") && entry.path.endsWith(".xml")).sort((left, right) => left.path.localeCompare(right.path));
+  return sheetEntries.map((entry, index) => {
+    const values = [
+      ...decodeUtf8(entry.data).matchAll(/<v>([\s\S]*?)<\/v>/g)
+    ].map((match) => match[1] ?? "").map((value) => {
+      const sharedStringIndex = Number(value);
+      return Number.isInteger(sharedStringIndex) && sharedStrings[sharedStringIndex] ? sharedStrings[sharedStringIndex] : value;
+    });
+    const text = normalizeWhitespace(values.join(`
+`));
+    if (!text) {
+      return null;
+    }
+    return {
+      name: sheetNames[index] ?? `Sheet ${index + 1}`,
+      text
+    };
+  }).filter((entry) => Boolean(entry));
+};
 var spreadsheetSheetNames = (entries) => entries.filter((entry) => entry.path === "xl/workbook.xml").flatMap((entry) => [
   ...decodeUtf8(entry.data).matchAll(/<sheet[^>]*name="([^"]+)"/g)
 ].map((match) => match[1] ?? "")).filter(Boolean);
@@ -3241,6 +3267,10 @@ var presentationText = (entries) => {
 
 `));
 };
+var presentationSlides = (entries) => entries.filter((entry) => entry.path.startsWith("ppt/slides/") && entry.path.endsWith(".xml")).sort((left, right) => left.path.localeCompare(right.path)).map((entry, index) => ({
+  index,
+  text: normalizeWhitespace(extractXmlText(decodeUtf8(entry.data)))
+})).filter((slide) => Boolean(slide.text));
 var presentationSlideCount = (entries) => entries.filter((entry) => entry.path.startsWith("ppt/slides/") && entry.path.endsWith(".xml")).length;
 var epubText = (entries) => {
   const htmlEntries = entries.filter((entry) => /\.(xhtml|html|htm)$/i.test(entry.path));
@@ -3458,6 +3488,7 @@ var createOfficeDocumentExtractor = () => ({
     const entries = unzipEntries(input.data);
     let text = "";
     let officeMetadata = {};
+    let structuredDocuments = [];
     if (extension === ".docx" || extension === ".odt") {
       text = officeDocumentText(entries);
       officeMetadata = {
@@ -3465,19 +3496,55 @@ var createOfficeDocumentExtractor = () => ({
       };
     } else if (extension === ".xlsx" || extension === ".ods") {
       text = spreadsheetText(entries);
+      const sheets = spreadsheetSheetTexts(entries);
       officeMetadata = {
         sheetNames: spreadsheetSheetNames(entries)
       };
+      structuredDocuments = sheets.map((sheet, index) => ({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          fileKind: "office",
+          ...officeMetadata,
+          sourceNativeKind: "spreadsheet_sheet",
+          sheetIndex: index,
+          sheetName: sheet.name
+        },
+        source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
+        text: normalizeWhitespace(`Spreadsheet sheet ${sheet.name} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${sheet.text}`),
+        title: input.title ? `${input.title} \xB7 ${sheet.name}` : sheet.name
+      }));
     } else if (extension === ".pptx" || extension === ".odp") {
       text = presentationText(entries);
+      const slides = presentationSlides(entries);
       officeMetadata = {
         slideCount: presentationSlideCount(entries)
       };
+      structuredDocuments = slides.map((slide) => ({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          fileKind: "office",
+          ...officeMetadata,
+          sourceNativeKind: "presentation_slide",
+          slideIndex: slide.index,
+          slideNumber: slide.index + 1
+        },
+        source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}${extension || ".office"}`,
+        text: normalizeWhitespace(`Presentation slide ${slide.index + 1} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${slide.text}`),
+        title: input.title ? `${input.title} \xB7 Slide ${slide.index + 1}` : `Slide ${slide.index + 1}`
+      }));
     }
     if (!text) {
       throw new Error(`AbsoluteJS could not extract readable text from ${inferNameFromInput(input)}`);
     }
-    return {
+    const summaryDocument = {
       chunking: input.chunking,
       contentType: input.contentType,
       format: "text",
@@ -3490,6 +3557,7 @@ var createOfficeDocumentExtractor = () => ({
       text,
       title: input.title
     };
+    return [summaryDocument, ...structuredDocuments];
   }
 });
 var createRAGArchiveExpander = (expander) => expander;
@@ -3519,7 +3587,37 @@ var createRAGMediaFileExtractor = (transcriber) => ({
   supports: mediaExtractorSupports,
   extract: async (input) => {
     const result = await transcriber.transcribe(input);
-    return {
+    const source = input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.media.txt`;
+    const segmentDocuments = [];
+    for (const [index, segment] of (result.segments ?? []).entries()) {
+      const text = normalizeWhitespace(segment.text ?? "");
+      if (!text) {
+        continue;
+      }
+      const startMs = typeof segment.startMs === "number" ? segment.startMs : undefined;
+      const endMs = typeof segment.endMs === "number" ? segment.endMs : undefined;
+      segmentDocuments.push({
+        chunking: input.chunking,
+        contentType: input.contentType,
+        format: "text",
+        metadata: {
+          ...input.metadata ?? {},
+          ...result.metadata ?? {},
+          fileKind: "media",
+          sourceNativeKind: "media_segment",
+          mediaSegmentIndex: index,
+          mediaSegmentStartMs: startMs,
+          mediaSegmentEndMs: endMs,
+          mediaSegments: [segment],
+          speaker: typeof segment.speaker === "string" ? segment.speaker : undefined
+        },
+        source,
+        text: normalizeWhitespace(`Media transcript segment${typeof startMs === "number" ? ` ${startMs}-${endMs ?? startMs}ms` : ""} from ${input.title ?? input.name ?? input.path ?? DEFAULT_BINARY_NAME}.
+${text}`),
+        title: input.title ? `${input.title} \xB7 Segment ${index + 1}` : `Segment ${index + 1}`
+      });
+    }
+    const summaryDocument = {
       chunking: input.chunking,
       contentType: input.contentType,
       format: "text",
@@ -3529,10 +3627,11 @@ var createRAGMediaFileExtractor = (transcriber) => ({
         fileKind: "media",
         mediaSegments: result.segments
       },
-      source: input.source ?? input.path ?? input.name ?? `${slugify(input.title ?? DEFAULT_BINARY_NAME)}.media.txt`,
+      source,
       text: result.text,
       title: result.title ?? input.title
     };
+    return [summaryDocument, ...segmentDocuments];
   }
 });
 var createRAGMediaTranscriber = (transcriber) => transcriber;
@@ -3564,7 +3663,7 @@ var expandArchiveEntry = async (entry, archiveInput, extractors) => {
     },
     name: basename(entry.path),
     source: archiveInput.source && !archiveInput.source.startsWith("http") ? `${archiveInput.source}#${entry.path}` : entry.path,
-    title: archiveInput.title
+    title: basename(entry.path)
   }, extractors);
   return documents;
 };
@@ -3693,6 +3792,7 @@ var getFirstExtractedDocument = (documents, label) => {
   }
   return document;
 };
+var loadExtractedDocuments = async (input, extractors) => extractRAGFileDocuments(input, extractors);
 var sentenceUnits = (text) => {
   const matches = text.match(/[^.!?\n]+(?:[.!?]+|$)/g);
   if (!matches) {
@@ -3915,32 +4015,55 @@ var loadRAGDocumentFromURL = async (input) => {
 };
 var loadRAGDocumentsFromUploads = async (input) => {
   const documents = await Promise.all(input.uploads.map(async (upload) => {
-    const loaded = await loadRAGDocumentUpload({
-      ...upload,
-      extractors: input.extractors
-    });
-    return {
-      ...loaded,
-      metadata: mergeMetadata(loaded.metadata, { uploadFile: upload.name }, input.baseMetadata)
-    };
+    const loaded = await loadExtractedDocuments({
+      chunking: upload.chunking,
+      contentType: upload.contentType,
+      data: decodeUploadContent(upload),
+      format: upload.format,
+      metadata: upload.metadata,
+      name: upload.name,
+      source: upload.source ?? upload.name,
+      title: upload.title
+    }, input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, { uploadFile: upload.name }, input.baseMetadata)
+    }));
   }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var loadRAGDocumentsFromURLs = async (input) => {
-  const documents = await Promise.all(input.urls.map(async (urlInput) => loadRAGDocumentFromURL({
-    ...urlInput,
-    metadata: mergeMetadata(urlInput.metadata, {
-      sourceUrl: urlInput.url
-    }, input.baseMetadata),
-    contentType: urlInput.contentType,
-    extractors: urlInput.extractors ?? input.extractors
-  })));
+  const documents = await Promise.all(input.urls.map(async (urlInput) => {
+    const url = urlInput.url.trim();
+    if (!url) {
+      throw new Error("RAG URL is required");
+    }
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
+    }
+    const data = new Uint8Array(await response.arrayBuffer());
+    const loaded = await loadExtractedDocuments({
+      chunking: urlInput.chunking,
+      contentType: urlInput.contentType ?? response.headers.get("content-type") ?? undefined,
+      data,
+      format: urlInput.format ?? inferFormatFromUrl(url),
+      metadata: urlInput.metadata,
+      name: basename(new URL(url).pathname),
+      source: urlInput.source ?? url,
+      title: urlInput.title
+    }, urlInput.extractors ?? input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, { sourceUrl: urlInput.url }, input.baseMetadata)
+    }));
+  }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var loadRAGDocumentUpload = async (input) => {
@@ -4020,21 +4143,25 @@ var loadRAGDocumentsFromDirectory = async (input) => {
   const files = await collectDirectoryFiles(root, input.recursive !== false, includeExtensions);
   const documents = await Promise.all(files.map(async (path) => {
     const source = relative(root, path).replace(/\\/g, "/");
-    const loaded = await loadRAGDocumentFile({
+    const data = await readFile(path);
+    const loaded = await loadExtractedDocuments({
+      chunking: input.defaultChunking,
+      data,
       metadata: {
-        ...input.baseMetadata ?? {},
         fileName: basename(path),
         relativePath: source
       },
       path,
-      source,
-      extractors: input.extractors
-    });
-    return loaded;
+      source
+    }, input.extractors);
+    return loaded.map((document) => ({
+      ...document,
+      metadata: mergeMetadata(document.metadata, undefined, input.baseMetadata)
+    }));
   }));
   return {
     defaultChunking: input.defaultChunking,
-    documents
+    documents: documents.flat()
   };
 };
 var prepareRAGDirectoryDocuments = async (input) => prepareRAGDocuments(await loadRAGDocumentsFromDirectory(input));
@@ -8773,5 +8900,5 @@ export {
   aiChat
 };
 
-//# debugId=F37A373F20F3691864756E2164756E21
+//# debugId=7CB5678D47F7B89564756E2164756E21
 //# sourceMappingURL=index.js.map