@absolutejs/rag 0.16.0 → 0.17.0

package/dist/web/index.js

@@ -111057,10 +111057,12 @@ function Document4() {
 setPrototypeOf(Document4, Document2).prototype = Document2.prototype;
 
 // src/web/transport.ts
+import {
+  isPublicNetworkAddress,
+  pinnedPublicRequest
+} from "@absolutejs/egress/transport";
 import { lookup } from "dns/promises";
 import { isIP } from "net";
-import { request as httpRequest } from "http";
-import { request as httpsRequest } from "https";
 import { brotliDecompressSync, gunzipSync, inflateSync } from "zlib";
 
 class WebReadError extends Error {
@@ -111071,14 +111073,7 @@ class WebReadError extends Error {
     this.name = "WebReadError";
   }
 }
-var isPublicWebAddress = (address) => {
-  if (isIP(address) === 4) {
-    const [a = 0, b = 0, c = 0] = address.split(".").map(Number);
-    return !(a === 0 || a === 10 || a === 127 || a >= 224 || a === 100 && b >= 64 && b <= 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && (b === 168 || b === 0 || b === 88 && c === 99) || a === 198 && (b === 18 || b === 19 || b === 51 && c === 100) || a === 203 && b === 0 && c === 113);
-  }
-  const value = address.toLowerCase();
-  return isIP(value) === 6 && /^[23]/u.test(value) && !/^2001:(?:db8|0):/u.test(value) && !value.startsWith("2002:");
-};
+var isPublicWebAddress = (address) => isPublicNetworkAddress(address);
 var validatePublicWebUrl = (raw) => {
   let url;
   try {
@@ -111108,74 +111103,38 @@ var fetchPublicWebResource = async (raw, options = {}) => {
       throw new WebReadError("blocked_url", "The destination resolved to a private or reserved network.");
     signal.throwIfAborted();
     const selected = addresses.find(({ family }) => family === 4) ?? addresses[0];
-    const response = await new Promise((resolve, reject) => {
-      const request = url.protocol === "https:" ? httpsRequest : httpRequest;
-      const req = request({
-        protocol: url.protocol,
-        hostname: selected.address,
-        family: selected.family,
-        servername: isIP(host) ? undefined : host,
-        port: url.port || undefined,
-        path: url.pathname + url.search,
-        method,
-        signal,
-        agent: false,
-        headers: {
-          "user-agent": "onSparkReader/1.0 (+https://onspark.com)",
-          accept: "*/*",
-          "accept-encoding": "identity",
-          ...options.headers,
-          host: url.host
-        }
-      }, (res) => {
-        const chunks = [];
-        let bytes = 0;
-        res.on("error", reject);
-        res.on("data", (chunk) => {
-          bytes += chunk.length;
-          if (bytes > maxBytes) {
-            const error = new WebReadError("too_large", "The response exceeded the website read limit.");
-            res.destroy(error);
-            reject(error);
-          } else
-            chunks.push(chunk);
-        });
-        res.on("end", () => {
-          try {
-            let data = Buffer.concat(chunks);
-            const encoding = res.headers["content-encoding"];
-            const limits = { maxOutputLength: maxBytes };
-            if (encoding === "gzip")
-              data = gunzipSync(data, limits);
-            else if (encoding === "br")
-              data = brotliDecompressSync(data, limits);
-            else if (encoding === "deflate")
-              data = inflateSync(data, limits);
-            const headers = Object.fromEntries(Object.entries(res.headers).filter(([key2]) => ![
-              "content-encoding",
-              "content-length",
-              "transfer-encoding",
-              "set-cookie"
-            ].includes(key2)).map(([key2, value]) => [
-              key2,
-              Array.isArray(value) ? value.join(", ") : value ?? ""
-            ]));
-            resolve({
-              url: url.href,
-              status: res.statusCode ?? 502,
-              headers,
-              body: data
-            });
-          } catch (error) {
-            reject(error);
-          }
-        });
-      });
-      req.on("error", reject);
-      if (body)
-        req.write(body);
-      req.end();
-    });
+    const incoming = await pinnedPublicRequest(new Request(url, {
+      method,
+      body,
+      signal,
+      headers: {
+        "user-agent": "AbsoluteJSReader/1.0",
+        accept: "*/*",
+        "accept-encoding": "identity",
+        ...options.headers,
+        host: url.host
+      }
+    }), { hostname: host, address: selected.address, maxResponseBytes: maxBytes });
+    let data = new Uint8Array(await incoming.arrayBuffer());
+    const encoding = incoming.headers.get("content-encoding");
+    const limits = { maxOutputLength: maxBytes };
+    if (encoding === "gzip")
+      data = gunzipSync(data, limits);
+    else if (encoding === "br")
+      data = brotliDecompressSync(data, limits);
+    else if (encoding === "deflate")
+      data = inflateSync(data, limits);
+    const response = {
+      url: url.href,
+      status: incoming.status,
+      headers: Object.fromEntries([...incoming.headers].filter(([key2]) => ![
+        "content-encoding",
+        "content-length",
+        "transfer-encoding",
+        "set-cookie"
+      ].includes(key2))),
+      body: data
+    };
     if (options.redirect === "manual")
       return response;
     if ([301, 302, 303, 307, 308].includes(response.status) && response.headers.location) {
@@ -117873,6 +117832,82 @@ ${page.text}`).join(`
     citationGuidance: "Answer the user's question first, with exact source page URLs. Distinguish retrieved facts from inference. A page limit bounds this call, not the research task: if a material question remains unanswered, read the relevant remaining links in another call, using maxPages 1 for targeted detail pages. Do not stop with a list of next steps when those reads are needed to finish the request. Conversely, unvisited links do not by themselves mean the answer is incomplete. Report only gaps that materially limit the answer, unless a retrieval audit was explicitly requested. Successful browser fallback is a completed read, not an incomplete read. Normally summarize a redirected destination in one brief sentence; omit HTTP codes, hop chains, character counts and rendering mechanics unless explicitly requested for debugging. A redirect alone does not prove a rebrand; verify that claim from a source. Image labels do not establish customer relationships and media URLs are not watched video evidence."
   };
 };
+// src/web/remote.ts
+var parseWebsiteServiceResult = (value) => {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Website reader returned an invalid response");
+  const row = value;
+  if (typeof row.text !== "string" || !["ok", "partial", "error"].includes(String(row.status)))
+    throw new Error("Website reader returned an invalid response");
+  if (row.limitations !== undefined && (!Array.isArray(row.limitations) || !row.limitations.every((item) => typeof item === "string")))
+    throw new Error("Invalid website limitations");
+  if (row.sources !== undefined && (!Array.isArray(row.sources) || !row.sources.every((source) => {
+    if (!source || typeof source !== "object" || typeof source.url !== "string")
+      return false;
+    try {
+      const url = new URL(source.url);
+      return ["http:", "https:"].includes(url.protocol) && !url.username && !url.password;
+    } catch {
+      return false;
+    }
+  })))
+    throw new Error("Invalid website sources");
+  return row;
+};
+var createWebsiteReaderClient = (options) => async (input) => {
+  const timeoutMs = options.timeoutMs ?? 85000, reserveMs = options.fallbackReserveMs ?? 15000;
+  if (!(reserveMs > 0 && timeoutMs > reserveMs))
+    throw new Error("Reader deadline must reserve fallback time");
+  const signal = AbortSignal.any([
+    ...input.signal ? [input.signal] : [],
+    AbortSignal.timeout(timeoutMs)
+  ]);
+  signal.throwIfAborted();
+  const request = {
+    url: input.url,
+    maxPages: input.maxPages ?? 8,
+    mode: input.mode ?? "auto"
+  };
+  if (!Number.isInteger(request.maxPages) || request.maxPages < 1 || request.maxPages > 12)
+    throw new Error("Reader page limit must be 1\u201312");
+  try {
+    const response = await (options.fetch ?? fetch)(options.endpoint, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(request),
+      signal: AbortSignal.any([
+        signal,
+        AbortSignal.timeout(timeoutMs - reserveMs)
+      ])
+    });
+    if (response.status === 503)
+      return {
+        status: "error",
+        text: "",
+        error: {
+          code: "reader_busy",
+          message: "Website reader is busy. Retry shortly; this is not evidence of missing information.",
+          retryAfterSeconds: 2
+        }
+      };
+    if (!response.ok)
+      throw new Error(`Website reader returned ${response.status}`);
+    const result = parseWebsiteServiceResult(await response.json());
+    signal.throwIfAborted();
+    return result;
+  } catch (error) {
+    signal.throwIfAborted();
+    const result = parseWebsiteServiceResult(await options.fallback({ ...request, signal }));
+    signal.throwIfAborted();
+    return {
+      ...result,
+      limitations: [
+        ...result.limitations ?? [],
+        "Browser service unavailable; static fallback may omit JavaScript-only evidence."
+      ]
+    };
+  }
+};
 
 // src/web/index.ts
 var messageFor = (error) => error instanceof Error ? error.message : "Website retrieval failed.";
@@ -118039,12 +118074,14 @@ var readRAGWebpage = async (options) => {
 };
 export {
   WebReadError,
+  createWebsiteReaderClient,
   fetchPublicWebResource,
   isPublicWebAddress,
+  parseWebsiteServiceResult,
   readRAGWebpage,
   readRAGWebsite,
   validatePublicWebUrl
 };
 
-//# debugId=06B1B75240CC399364756E2164756E21
+//# debugId=DCC029C3B5C261E064756E2164756E21
 //# sourceMappingURL=index.js.map