@absolutejs/rag 0.10.0 → 0.11.0

package/CHANGELOG.md

@@ -6,6 +6,16 @@ This file is generated by `absolute-changelog` from the entries in
 `changelog/`. Edit an entry, not this file — and add new ones under
 `changelog/unreleased/`.
 
+## 0.11.0 — 2026-09-18
+
+### Added
+
+- **Add a bounded public website reader with prepared evidence, per-attempt diagnostics, automatic JavaScript rendering through an optional Playwright adapter, isolated browser contexts and DNS-pinned public-only resource fetching.** (`readRAGWebpage`, `createPlaywrightWebRenderer`, `fetchPublicWebResource`)
+
+### Fixed
+
+- **Honor HTML response MIME types for extensionless URL ingestion instead of treating website markup as plain text, including XHTML and batch URL loading.** (`loadRAGDocumentFromURL`, `loadRAGDocumentsFromURLs`)
+
 ## 0.10.0 — 2026-09-16
 
 ### Added

package/README.md

@@ -125,3 +125,40 @@ it. Only encode server-owned original-text tool results. References are not
 access controls or durable source IDs: reauthorize and validate restored quotes
 against the originals before saving a result. This is opt-in; it does not change
 the original-text tools, stored originals or their existing result format.
+
+### Public websites and JavaScript rendering
+
+`loadRAGDocumentFromURL` loads a document; use `prepareRAGDocument(doc).normalizedText`
+for readable text. URL loading now honors response MIME types on extensionless URLs.
+For public websites, `@absolutejs/rag/web` provides `readRAGWebpage` with bounded
+responses, timeouts, prepared text, final URL, title, truncation and per-attempt
+retrieval diagnostics. It tries static HTML first and requests a browser for thin
+or empty application shells. A missing renderer returns `rendering_required`,
+not a claim that the website contains no information.
+
+```ts
+import { readRAGWebpage } from '@absolutejs/rag/web';
+import { createPlaywrightWebRenderer } from '@absolutejs/rag/web/playwright';
+
+const browser = createPlaywrightWebRenderer();
+try {
+  const page = await readRAGWebpage({
+    url: 'https://example.com',
+    render: browser.render,
+  });
+  // Check page.status and page.error before treating page.text as complete evidence.
+} finally {
+  await browser.close();
+}
+```
+
+The browser adapter requires the optional `playwright-core` peer and an installed
+Chromium browser (`playwright-core install --with-deps chromium`). Hosts should
+run it in a separate unprivileged process, limit concurrency, and apply memory
+limits. Contexts do not share cookies; service workers and WebSockets are blocked.
+HTTP resources and redirect hops use validated public destinations with the DNS
+answer pinned to each connection. Browser resource counts and response sizes are
+bounded. The reader does not bypass login, CAPTCHA or access restrictions, and
+reports these failures separately from incomplete rendering. A page read is not
+a crawl of every page on a domain. Host-supplied renderers/fetch implementations
+must enforce equivalent network controls.

package/changelog.json

@@ -2,6 +2,29 @@
 	"contract": 1,
 	"name": "@absolutejs/rag",
 	"releases": [
+		{
+			"changes": [
+				{
+					"kind": "fixed",
+					"summary": "Honor HTML response MIME types for extensionless URL ingestion instead of treating website markup as plain text, including XHTML and batch URL loading.",
+					"symbols": [
+						"loadRAGDocumentFromURL",
+						"loadRAGDocumentsFromURLs"
+					]
+				},
+				{
+					"kind": "added",
+					"summary": "Add a bounded public website reader with prepared evidence, per-attempt diagnostics, automatic JavaScript rendering through an optional Playwright adapter, isolated browser contexts and DNS-pinned public-only resource fetching.",
+					"symbols": [
+						"readRAGWebpage",
+						"createPlaywrightWebRenderer",
+						"fetchPublicWebResource"
+					]
+				}
+			],
+			"date": "2026-09-18",
+			"version": "0.11.0"
+		},
 		{
 			"changes": [
 				{

package/dist/adapter-kit/index.js

@@ -14971,6 +14971,9 @@ var inferFormatFromName = (value) => {
 };
 var inferFormatFromContentType = (contentType) => {
   const normalizedType = (contentType || "").toLowerCase();
+  if (normalizedType.includes("html")) {
+    return "html";
+  }
   if (normalizedType.includes("xml")) {
     return "xml";
   }
@@ -19683,7 +19686,7 @@ var loadRAGDocumentFromURL = async (input) => {
     contentType: input.contentType ?? response.headers.get("content-type") ?? undefined,
     data,
     extractorRegistry: input.extractorRegistry,
-    format: input.format ?? inferFormatFromUrl(url),
+    format: input.format ?? inferFormatFromContentType(input.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
     metadata: input.metadata,
     name: basename(new URL(url).pathname),
     source: input.source ?? url,
@@ -19731,7 +19734,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
       contentType: urlInput.contentType ?? response.headers.get("content-type") ?? undefined,
       data,
       extractorRegistry: urlInput.extractorRegistry ?? input.extractorRegistry,
-      format: urlInput.format ?? inferFormatFromUrl(url),
+      format: urlInput.format ?? inferFormatFromContentType(urlInput.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
       metadata: urlInput.metadata,
       name: basename(new URL(url).pathname),
       source: urlInput.source ?? url,
@@ -31414,5 +31417,5 @@ export {
   summarizeSQLiteCandidateCoverage
 };
 
-//# debugId=5D1CC4781115DD5864756E2164756E21
+//# debugId=9FE93518A40B2B4664756E2164756E21
 //# sourceMappingURL=index.js.map