@absolutejs/rag 0.10.0 → 0.12.0

package/CHANGELOG.md

@@ -6,6 +6,26 @@ This file is generated by `absolute-changelog` from the entries in
 `changelog/`. Edit an entry, not this file — and add new ones under
 `changelog/unreleased/`.
 
+## 0.12.0 — 2026-09-18
+
+### Added
+
+- **Add bounded multi-page website research with exact page citations, semantic links and image labels, media and caption evidence, redirect chains and explicit coverage limits.**
+
+### Changed
+
+- **Expand website result and renderer contracts with semantic evidence, redirect hops and explicit readiness/coverage metadata; existing renderers may omit the new optional renderer fields.** (`WebFetchResult`, `WebReadResult`, `WebRenderer`, `readRAGWebpage`, `createPlaywrightWebRenderer`)
+
+## 0.11.0 — 2026-09-18
+
+### Added
+
+- **Add a bounded public website reader with prepared evidence, per-attempt diagnostics, automatic JavaScript rendering through an optional Playwright adapter, isolated browser contexts and DNS-pinned public-only resource fetching.** (`readRAGWebpage`, `createPlaywrightWebRenderer`, `fetchPublicWebResource`)
+
+### Fixed
+
+- **Honor HTML response MIME types for extensionless URL ingestion instead of treating website markup as plain text, including XHTML and batch URL loading.** (`loadRAGDocumentFromURL`, `loadRAGDocumentsFromURLs`)
+
 ## 0.10.0 — 2026-09-16
 
 ### Added

package/README.md

@@ -125,3 +125,55 @@ it. Only encode server-owned original-text tool results. References are not
 access controls or durable source IDs: reauthorize and validate restored quotes
 against the originals before saving a result. This is opt-in; it does not change
 the original-text tools, stored originals or their existing result format.
+
+### Public websites and JavaScript rendering
+
+`loadRAGDocumentFromURL` loads a document; use `prepareRAGDocument(doc).normalizedText`
+for readable text. URL loading now honors response MIME types on extensionless URLs.
+For public websites, `@absolutejs/rag/web` provides `readRAGWebpage` with bounded
+responses, timeouts, prepared text, final URL, title, truncation and per-attempt
+retrieval diagnostics. It tries static HTML first and requests a browser for thin
+or empty application shells. A missing renderer returns `rendering_required`,
+not a claim that the website contains no information.
+
+```ts
+import { readRAGWebpage } from '@absolutejs/rag/web';
+import { createPlaywrightWebRenderer } from '@absolutejs/rag/web/playwright';
+
+const browser = createPlaywrightWebRenderer();
+try {
+  const page = await readRAGWebpage({
+    url: 'https://example.com',
+    render: browser.render,
+  });
+  // Check page.status and page.error before treating page.text as complete evidence.
+} finally {
+  await browser.close();
+}
+```
+
+The browser adapter requires the optional `playwright-core` peer and an installed
+Chromium browser (`playwright-core install --with-deps chromium`). Hosts should
+run it in a separate unprivileged process, limit concurrency, and apply memory
+limits. Contexts do not share cookies; service workers and WebSockets are blocked.
+HTTP resources and redirect hops use validated public destinations with the DNS
+answer pinned to each connection. Browser resource counts and response sizes are
+bounded. The reader does not bypass login, CAPTCHA or access restrictions, and
+reports these failures separately from incomplete rendering. A page read is not
+a crawl of every page on a domain. Host-supplied renderers/fetch implementations
+must enforce equivalent network controls.
+
+### Research related website pages with attributable evidence
+
+`readRAGWebsite` from `@absolutejs/rag/web` reads a supplied URL and up to three
+relevant same-origin customer, services and company pages by default. Pass the
+optional Playwright renderer as for `readRAGWebpage`. `maxPages: 1` retains a
+single-page read; `mode: "browser"` retries content missed by static extraction.
+Results attribute text to exact page URLs and retain per-page redirects,
+retrieval attempts, semantic image labels, link destinations, media URLs and
+available caption text. Coverage includes unvisited relevant links and deadline
+limits. HTTP redirects carry their actual status; client navigation is labeled
+separately. Empty image labels are not proof of an absent client list, and media
+URLs are not proof that a video was watched. Consumers must cite source URLs,
+distinguish extracted evidence from inference, and finish the requested research
+without treating a successful page fetch as complete company coverage.

package/changelog.json

@@ -2,6 +2,50 @@
 	"contract": 1,
 	"name": "@absolutejs/rag",
 	"releases": [
+		{
+			"changes": [
+				{
+					"kind": "added",
+					"summary": "Add bounded multi-page website research with exact page citations, semantic links and image labels, media and caption evidence, redirect chains and explicit coverage limits."
+				},
+				{
+					"kind": "changed",
+					"summary": "Expand website result and renderer contracts with semantic evidence, redirect hops and explicit readiness/coverage metadata; existing renderers may omit the new optional renderer fields.",
+					"symbols": [
+						"WebFetchResult",
+						"WebReadResult",
+						"WebRenderer",
+						"readRAGWebpage",
+						"createPlaywrightWebRenderer"
+					]
+				}
+			],
+			"date": "2026-09-18",
+			"version": "0.12.0"
+		},
+		{
+			"changes": [
+				{
+					"kind": "fixed",
+					"summary": "Honor HTML response MIME types for extensionless URL ingestion instead of treating website markup as plain text, including XHTML and batch URL loading.",
+					"symbols": [
+						"loadRAGDocumentFromURL",
+						"loadRAGDocumentsFromURLs"
+					]
+				},
+				{
+					"kind": "added",
+					"summary": "Add a bounded public website reader with prepared evidence, per-attempt diagnostics, automatic JavaScript rendering through an optional Playwright adapter, isolated browser contexts and DNS-pinned public-only resource fetching.",
+					"symbols": [
+						"readRAGWebpage",
+						"createPlaywrightWebRenderer",
+						"fetchPublicWebResource"
+					]
+				}
+			],
+			"date": "2026-09-18",
+			"version": "0.11.0"
+		},
 		{
 			"changes": [
 				{

package/dist/adapter-kit/index.js

@@ -1,5 +1,37 @@
 // @bun
+var __create = Object.create;
+var __getProtoOf = Object.getPrototypeOf;
 var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+function __accessProp(key) {
+  return this[key];
+}
+var __toESMCache_node;
+var __toESMCache_esm;
+var __toESM = (mod, isNodeMode, target) => {
+  var canCache = mod != null && typeof mod === "object";
+  if (canCache) {
+    var cache = isNodeMode ? __toESMCache_node ??= new WeakMap : __toESMCache_esm ??= new WeakMap;
+    var cached = cache.get(mod);
+    if (cached)
+      return cached;
+  }
+  target = mod != null ? __create(__getProtoOf(mod)) : {};
+  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
+  if (mod && typeof mod === "object" || typeof mod === "function") {
+    for (let key of __getOwnPropNames(mod))
+      if (!__hasOwnProp.call(to, key))
+        __defProp(to, key, {
+          get: __accessProp.bind(mod, key),
+          enumerable: true
+        });
+  }
+  if (canCache)
+    cache.set(mod, to);
+  return to;
+};
+var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
 var __returnValue = (v) => v;
 function __exportSetter(name, newValue) {
   this[name] = __returnValue.bind(null, newValue);
@@ -13,6 +45,7 @@ var __export = (target, all) => {
       set: __exportSetter.bind(all, name)
     });
 };
+var __require = import.meta.require;
 
 // src/quality/quality.ts
 import { mkdir, readFile } from "fs/promises";
@@ -14971,6 +15004,9 @@ var inferFormatFromName = (value) => {
 };
 var inferFormatFromContentType = (contentType) => {
   const normalizedType = (contentType || "").toLowerCase();
+  if (normalizedType.includes("html")) {
+    return "html";
+  }
   if (normalizedType.includes("xml")) {
     return "xml";
   }
@@ -19683,7 +19719,7 @@ var loadRAGDocumentFromURL = async (input) => {
     contentType: input.contentType ?? response.headers.get("content-type") ?? undefined,
     data,
     extractorRegistry: input.extractorRegistry,
-    format: input.format ?? inferFormatFromUrl(url),
+    format: input.format ?? inferFormatFromContentType(input.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
     metadata: input.metadata,
     name: basename(new URL(url).pathname),
     source: input.source ?? url,
@@ -19731,7 +19767,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
       contentType: urlInput.contentType ?? response.headers.get("content-type") ?? undefined,
       data,
       extractorRegistry: urlInput.extractorRegistry ?? input.extractorRegistry,
-      format: urlInput.format ?? inferFormatFromUrl(url),
+      format: urlInput.format ?? inferFormatFromContentType(urlInput.contentType ?? response.headers.get("content-type")) ?? inferFormatFromUrl(url),
       metadata: urlInput.metadata,
       name: basename(new URL(url).pathname),
       source: urlInput.source ?? url,
@@ -31414,5 +31450,5 @@ export {
   summarizeSQLiteCandidateCoverage
 };
 
-//# debugId=5D1CC4781115DD5864756E2164756E21
+//# debugId=77D38FB1F044066A64756E2164756E21
 //# sourceMappingURL=index.js.map