@absolutejs/rag 0.22.1 → 0.22.2

package/dist/web-index/index.js

@@ -111452,6 +111452,15 @@ var hasIncompleteAppContent = (html) => {
   const text = compact(main.textContent);
   return text.length < 100 || /^(?:loading|please wait)(?:[.\s]|$)/iu.test(text);
 };
+var hasEmptyAppRoot = (html) => {
+  const { document: document2 } = parseHTML(html);
+  return Array.from(document2.querySelectorAll("div,main,section")).some((element) => {
+    const attributes = Array.from(element.attributes);
+    const id = attributes.find((attribute2) => attribute2.name.toLowerCase() === "id")?.value.toLowerCase();
+    const classes = attributes.find((attribute2) => attribute2.name.toLowerCase() === "class")?.value.toLowerCase().split(/\s+/u) ?? [];
+    return (id !== undefined && ["root", "app", "__next"].includes(id) || classes.includes("todoapp")) && element.children.length === 0 && !element.textContent?.trim();
+  });
+};
 
 // src/web/text.ts
 var websiteHtmlText = (html) => {
@@ -118080,8 +118089,8 @@ var parseWebsiteServiceResult = (value) => {
   return row;
 };
 var createWebsiteReaderClient = (options) => async (input) => {
-  const timeoutMs = options.timeoutMs ?? 85000, reserveMs = options.fallbackReserveMs ?? 15000;
-  if (!(reserveMs > 0 && timeoutMs > reserveMs))
+  const timeoutMs = options.timeoutMs ?? 85000, reserveMs = options.fallback ? options.fallbackReserveMs ?? 15000 : 0;
+  if (!(reserveMs >= 0 && timeoutMs > reserveMs))
     throw new Error("Reader deadline must reserve fallback time");
   const signal = AbortSignal.any([
     ...input.signal ? [input.signal] : [],
@@ -118122,6 +118131,15 @@ var createWebsiteReaderClient = (options) => async (input) => {
     return result;
   } catch (error) {
     signal.throwIfAborted();
+    if (!options.fallback)
+      return {
+        status: "error",
+        text: "",
+        error: {
+          code: "reader_unavailable",
+          message: "Website reader unavailable. Retry shortly; no website evidence was retrieved."
+        }
+      };
     const result = parseWebsiteServiceResult(await options.fallback({ ...request, signal }));
     signal.throwIfAborted();
     return {
@@ -118181,7 +118199,7 @@ var titleOf = (html) => {
   const title = /<title\b[^>]*>([\s\S]*?)<\/title>/iu.exec(html)?.[1];
   return title ? prepareRAGDocument({ text: title, format: "html" }).normalizedText : null;
 };
-var isShell = (html, text) => text.length < 300 || hasIncompleteAppContent(html) || /(?:enable|requires?|turn on)\s+javascript|javascript\s+(?:is\s+)?(?:disabled|required)/iu.test(text) || /<(?:div|main|section)[^>]*(?:id=["'](?:root|app|__next)["']|class=["']todoapp["'])[^>]*>\s*<\//iu.test(html);
+var isShell = (html, text) => text.length < 300 || hasIncompleteAppContent(html) || /(?:enable|requires?|turn on)\s+javascript|javascript\s+(?:is\s+)?(?:disabled|required)/iu.test(text) || hasEmptyAppRoot(html);
 var isChallenge = (text) => /verify (?:that )?you are human|checking your browser|just a moment|enable javascript and cookies to continue|access denied/iu.test(text.slice(0, 1000));
 var readRAGWebpage = async (options) => {
   const attempts = [];
@@ -125954,5 +125972,5 @@ export {
   webIndexPostgresSchemaSql
 };
 
-//# debugId=A13572243F640B2064756E2164756E21
+//# debugId=A17D9EDD6D999CCB64756E2164756E21
 //# sourceMappingURL=index.js.map