@absolutejs/rag 0.9.0 → 0.11.0

package/dist/index.js

@@ -15251,6 +15251,9 @@ var inferFormatFromName = (value) => {
 };
 var inferFormatFromContentType = (contentType) => {
   const normalizedType = (contentType || "").toLowerCase();
+  if (normalizedType.includes("html")) {
+    return "html";
+  }
   if (normalizedType.includes("xml")) {
     return "xml";
   }
@@ -19963,7 +19966,7 @@ var loadRAGDocumentFromURL = async (input) => {
     contentType: input.contentType ?? response.headers.get("content-type") ?? undefined,
     data,
     extractorRegistry: input.extractorRegistry,
-    format: input.format ?? inferFormatFromUrl(url),
+    format: input.format ?? inferFormatFromContentType(input.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
     metadata: input.metadata,
     name: basename(new URL(url).pathname),
     source: input.source ?? url,
@@ -20011,7 +20014,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
       contentType: urlInput.contentType ?? response.headers.get("content-type") ?? undefined,
       data,
       extractorRegistry: urlInput.extractorRegistry ?? input.extractorRegistry,
-      format: urlInput.format ?? inferFormatFromUrl(url),
+      format: urlInput.format ?? inferFormatFromContentType(urlInput.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
       metadata: urlInput.metadata,
       name: basename(new URL(url).pathname),
       source: urlInput.source ?? url,
@@ -36572,6 +36575,56 @@ var createRAGOriginalTextTools = (options) => {
     }
   };
 };
+// src/retrieval/quoteReferences.ts
+var createRAGQuoteReferences = () => {
+  const quotes = new Map;
+  const references = new Map;
+  const record2 = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+  const encodePassage = (passage) => {
+    if (!record2(passage) || typeof passage.text !== "string")
+      return passage;
+    const { text, ...metadata } = passage;
+    const sentences = text.split(/(?<=[.!?])\s+(?=[A-Z])|\n+/u).filter((sentence) => sentence.trim());
+    return {
+      ...metadata,
+      sentences: sentences.map((sentence) => {
+        let citation = references.get(sentence);
+        if (!citation) {
+          citation = `e${quotes.size + 1}`;
+          references.set(sentence, citation);
+          quotes.set(citation, sentence);
+        }
+        return { citation, text: sentence };
+      })
+    };
+  };
+  return {
+    encodeToolResult: (serialized) => {
+      let value;
+      try {
+        value = JSON.parse(serialized);
+      } catch {
+        return serialized;
+      }
+      if (!record2(value) || value.referenceOnly !== true)
+        return serialized;
+      if (Array.isArray(value.passages))
+        return JSON.stringify({
+          ...value,
+          passages: value.passages.map(encodePassage)
+        });
+      if (typeof value.text === "string")
+        return JSON.stringify(encodePassage(value));
+      return serialized;
+    },
+    resolve: (reference) => {
+      const quote = quotes.get(reference);
+      if (quote === undefined)
+        throw new Error("Unknown retrieved quote reference");
+      return quote;
+    }
+  };
+};
 export {
   addRAGEvaluationSuiteCase,
   addRAGEvaluationSuiteCaseHardNegative,
@@ -36732,6 +36785,7 @@ export {
   createRAGPresentationCueBenchmarkSnapshot,
   createRAGPresentationCueBenchmarkSuite,
   createRAGQueryTransform,
+  createRAGQuoteReferences,
   createRAGReranker,
   createRAGSQLiteAnswerGroundingEvaluationHistoryStore,
   createRAGSQLiteEvaluationHistoryStore,
@@ -36878,5 +36932,5 @@ export {
   xaiEmbeddings
 };
 
-//# debugId=24B2DCBF7983282F64756E2164756E21
+//# debugId=3E13A042CB51A53E64756E2164756E21
 //# sourceMappingURL=index.js.map