@absolutejs/rag 0.22.8 → 0.22.10

package/CHANGELOG.md

@@ -6,6 +6,18 @@ This file is generated by `absolute-changelog` from the entries in
 `changelog/`. Edit an entry, not this file — and add new ones under
 `changelog/unreleased/`.
 
+## 0.22.10 — 2026-10-28
+
+### Fixed
+
+- **Preserve complete newly discovered source documents as well as initially searched pages; model excerpt budgets never cap retained source storage.** (`createResearch`)
+
+## 0.22.9 — 2026-10-28
+
+### Fixed
+
+- **Retain complete extracted source text separately from bounded previews; select exact model passages across full documents and preserve originals for continued research.** (`WebReadResult`, `ResearchResult`, `readRAGWebpage`, `readRAGWebsite`, `createWebpageReaderClient`, `createResearch`)
+
 ## 0.22.8 — 2026-10-28
 
 ### Fixed

package/changelog.json

@@ -2,6 +2,37 @@
 	"contract": 1,
 	"name": "@absolutejs/rag",
 	"releases": [
+		{
+			"version": "0.22.10",
+			"date": "2026-10-28",
+			"changes": [
+				{
+					"kind": "fixed",
+					"summary": "Preserve complete newly discovered source documents as well as initially searched pages; model excerpt budgets never cap retained source storage.",
+					"symbols": [
+						"createResearch"
+					]
+				}
+			]
+		},
+		{
+			"version": "0.22.9",
+			"date": "2026-10-28",
+			"changes": [
+				{
+					"kind": "fixed",
+					"summary": "Retain complete extracted source text separately from bounded previews; select exact model passages across full documents and preserve originals for continued research.",
+					"symbols": [
+						"WebReadResult",
+						"ResearchResult",
+						"readRAGWebpage",
+						"readRAGWebsite",
+						"createWebpageReaderClient",
+						"createResearch"
+					]
+				}
+			]
+		},
 		{
 			"changes": [
 				{

package/dist/research/index.js

@@ -101400,6 +101400,28 @@ var init_pdf = __esm(() => {
   };
 });
 
+// src/research/sourceExcerpts.ts
+import { createAITextSource } from "@absolutejs/ai";
+var selectResearchExcerpts = (text, budget, query) => {
+  if (budget <= 0)
+    return [];
+  if (text.length <= budget)
+    return [text];
+  const source = createAITextSource([{ role: "user", content: text }]);
+  const hits = source.search(query);
+  const passages = hits.length ? hits : source.search("");
+  let remaining = budget;
+  const selected = [];
+  for (const passage of passages) {
+    if (!remaining)
+      break;
+    const excerpt = passage.text.slice(0, remaining);
+    selected.push({ start: passage.start, text: excerpt });
+    remaining -= excerpt.length;
+  }
+  return selected.sort((a, b) => a.start - b.start).map((passage) => passage.text);
+};
+
 // src/research/budget.ts
 var createResearchBudgetAdmission = (config) => {
   if (!config.scope.trim())
@@ -120842,7 +120864,7 @@ var readRAGWebsite = async (options) => {
     const page = await readRAGWebpage({
       ...options,
       url: next.url,
-      maxChars: Math.min(maxChars, 12000),
+      maxChars,
       signal: AbortSignal.any([signal, AbortSignal.timeout(30000)])
     });
     pages.push(page);
@@ -120898,24 +120920,27 @@ var readRAGWebsite = async (options) => {
   let remainingDocumentChars = maxChars;
   const documents = sources.flatMap((source) => {
     const page = pages.find((item) => item.finalUrl === source.url && item.text);
-    if (!page || remainingDocumentChars <= 0)
+    if (!page)
       return [];
     const imageLabels = page.evidence.images.filter((image) => image.label).map((image) => `${image.context ? `${image.context}: ` : ""}${image.label}`);
     const fullText = [
-      page.text,
+      page.fullText ?? page.text,
       ...imageLabels.length ? [
         "Image labels supplied by the website (not independently verified customer relationships):",
         ...imageLabels
       ] : []
     ].join(`
 `);
-    const text = fullText.slice(0, remainingDocumentChars);
+    const text = fullText.slice(0, Math.max(0, remainingDocumentChars));
     remainingDocumentChars -= text.length;
     return [
       {
         sourceId: source.id,
         text,
-        truncated: page.truncated || text.length < fullText.length
+        fullText,
+        originalChars: fullText.length,
+        sourceTruncated: page.status !== "ok" || page.fullText === undefined && page.truncated,
+        truncated: page.fullText === undefined && page.truncated || text.length < fullText.length
       }
     ];
   });
@@ -120934,10 +120959,11 @@ ${page.text}`).join(`
     sources,
     citationRequirements: "The final answer must contain clickable Markdown source links, not just source names or bare domain mentions. Reuse sources[].citation beside the factual paragraph, list or table row it supports. Cite the actual supporting page, not the homepage for everything. Every factual section needs its supporting links. Do not invent sources or use unread links as evidence. Before responding, check that company facts, service descriptions, customer examples and numerical claims have supporting links; retrieve or omit unsupported claims. A list of large customers does not prove smaller customers are excluded. Preserve the brands actually named in the evidence; do not append parent companies, ownership relationships, current-account status, market rankings or rebrand history from memory. Label reasoned inferences as such. A verified brand change needs at most one short, cited sentence unless the user asks for its history.",
     ...first,
+    fullText: undefined,
     status: readable.length ? incomplete ? "partial" : "ok" : "error",
     text: evidenceText.slice(0, maxChars),
     truncated: evidenceText.length > maxChars || pages.some((page) => page.truncated),
-    pages: pages.map(({ text: _text, ...page }) => page),
+    pages: pages.map(({ text: _text, fullText: _fullText, ...page }) => page),
     captionEvidence,
     coverage: {
       pagesRead: pages.length,
@@ -120948,10 +120974,10 @@ ${page.text}`).join(`
       deadlineReached: signal.aborted,
       exhaustive: false,
       stopReason: signal.aborted ? "deadline" : unvisited.length ? "page_limit" : "links_exhausted",
-      incompleteReads: pages.filter((page) => page.status !== "ok" || page.truncated).map((page) => ({
+      incompleteReads: pages.filter((page) => page.status !== "ok" || page.fullText === undefined && page.truncated).map((page) => ({
         url: page.finalUrl,
         status: page.status,
-        truncated: page.truncated,
+        truncated: page.fullText === undefined && page.truncated,
         error: page.error
       }))
     },
@@ -121066,12 +121092,15 @@ var createWebpageReaderClient = (options) => {
     if (![evidence.links, evidence.images, evidence.media].every(Array.isArray))
       throw new Error("Invalid page evidence");
     const limit = Math.max(1000, Math.min(input.maxChars ?? 24000, 1e5));
+    const fullText = typeof document2.fullText === "string" ? document2.fullText : document2.text;
     return {
       status: result.status,
       url: input.url,
       finalUrl: result.finalUrl,
       title: typeof result.title === "string" ? result.title : null,
-      text: document2.text.slice(0, limit),
+      text: fullText.slice(0, limit),
+      fullText,
+      originalChars: fullText.length,
       method: result.method,
       truncated: result.truncated === true || document2.truncated === true || document2.text.length > limit,
       fetchedAt: result.fetchedAt,
@@ -121114,6 +121143,8 @@ var readRAGWebpage = async (options) => {
     finalUrl,
     title,
     text: text.slice(0, maxChars),
+    fullText: text,
+    originalChars: text.length,
     method,
     truncated: text.length > maxChars,
     attempts,
@@ -121465,8 +121496,8 @@ ${prompt}`,
         const text = sourceText.get(source.url);
         const allowance = Math.floor(remaining * weight(source) / remainingWeight);
         remainingWeight -= weight(source);
-        source.excerpts = [text.slice(0, allowance)];
-        remaining -= source.excerpts[0].length;
+        source.excerpts = selectResearchExcerpts(text, allowance, [input.query, ...input.queries ?? []].join(" "));
+        remaining -= source.excerpts.reduce((sum, excerpt) => sum + excerpt.length, 0);
       }
       evidenceChars = limits.evidenceChars - remaining;
     };
@@ -121487,7 +121518,7 @@ ${prompt}`,
         return;
       }
       const text = source.excerpts.join(`
-`).slice(0, limits.evidenceChars);
+`);
       if (!text.trim())
         return;
       sourceText.set(source.url, text);
@@ -121550,7 +121581,7 @@ ${prompt}`,
         }
         const prior = result.sources.find((source) => source.url === url);
         if (prior) {
-          sourceText.set(prior.url, page.text.slice(0, limits.evidenceChars));
+          sourceText.set(prior.url, page.fullText ?? page.text);
           prior.contentFetchedAt = page.fetchedAt;
           prior.metadata = { ...prior.metadata, readFinalUrl: page.finalUrl };
           if (page.finalUrl !== prior.url)
@@ -121561,7 +121592,7 @@ ${prompt}`,
             id: "",
             url: page.finalUrl,
             title: page.title ?? url,
-            excerpts: [page.text],
+            excerpts: [page.fullText ?? page.text],
             retrievedAt: page.fetchedAt,
             contentFetchedAt: page.fetchedAt
           });
@@ -121663,6 +121694,7 @@ ${prompt}`,
       result.status = result.sources.length ? "partial" : "unavailable";
       return result;
     } finally {
+      result.documents = result.sources.filter((source) => source.contentFetchedAt).map((source) => ({ sourceId: source.id, url: source.url, text: sourceText.get(source.url) ?? "", originalChars: (sourceText.get(source.url) ?? "").length }));
       result.limitations = [...new Set(result.limitations)];
       input.onProgress?.({
         phase: "complete",
@@ -122511,5 +122543,5 @@ export {
   selectResearchMonitorEvents
 };
 
-//# debugId=98A81F60666A9C0F64756E2164756E21
+//# debugId=11F9F8FA19C5FC6864756E2164756E21
 //# sourceMappingURL=index.js.map