openclaw-smart-fetch 0.2.32 → 0.2.34

package/README.md

@@ -8,6 +8,7 @@
 - 🧹 **Defuddle extraction** — clean readable content instead of noisy HTML
 - 🧠 **Useful metadata** — title, author, site, language, published date when available
 - 📦 **Downloads + large file support** — stream attachments and binaries to temp files
+- 🔁 **Client-side `<meta>` redirects** — follows sane meta refresh redirects with loop limits
 - ⚡ **Batch fetch** — fetch many URLs with bounded concurrency
 - 📝 **Multiple output formats** — `markdown`, `html`, `text`, `json`
 - 🔄 **Built-in `web_fetch` fallback** — automatically improves the core web_fetch tool

package/dist/index.js

@@ -9404,6 +9404,7 @@ var HTML_CONTENT_TYPES = [
   "text/plain",
   "text/markdown"
 ];
+var MAX_CLIENT_SIDE_REDIRECTS = 5;
 function normalizeContentType(contentType) {
   return contentType.split(";")[0]?.trim().toLowerCase() ?? "";
 }
@@ -9902,6 +9903,32 @@ function isLikelyJsonBody(body) {
 function isJsonResponse(contentType, body) {
   return isJsonContentType(contentType) || isLikelyJsonBody(body);
 }
+function decodeHtmlAttribute(value) {
+  return value.replace(/&amp;/gi, "&").replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">");
+}
+function extractClientSideRedirect(body, baseUrl) {
+  const snippet = body.slice(0, 4096);
+  const metaRefreshMatch = snippet.match(
+    /<meta\b[^>]*http-equiv=["']?refresh["']?[^>]*content=["']?([^"'>]*)["']?[^>]*>/i
+  );
+  const refreshContent = metaRefreshMatch?.[1];
+  if (!refreshContent) {
+    return null;
+  }
+  const [delayPart = "", ...rest] = decodeHtmlAttribute(refreshContent).split(";");
+  const delaySeconds = Number.parseFloat(delayPart.trim());
+  const urlMatch = rest.join(";").match(/\burl\s*=\s*(.+)$/i);
+  const rawTarget = urlMatch?.[1]?.trim().replace(/^['"]|['"]$/g, "");
+  if (!rawTarget || !Number.isFinite(delaySeconds) || delaySeconds < 0 || delaySeconds >= 30) {
+    return null;
+  }
+  try {
+    const targetUrl = new URL(rawTarget, baseUrl).toString();
+    return targetUrl === baseUrl ? null : targetUrl;
+  } catch {
+    return null;
+  }
+}
 function buildJsonResult(opts, finalUrl, rawBody, format, maxChars, browser, os) {
   const parsedJson = parseAndFormatJson(rawBody);
   if ("error" in parsedJson) {
@@ -9981,7 +10008,7 @@ function shouldStripReplies(site) {
   return site === "Hacker News" || site.startsWith("r/") || site.startsWith("GitHub - ");
 }
 function createDefuddleFetch(dependencies = runtimeDependencies) {
-  return async function defuddleFetch2(opts, hooks = {}) {
+  async function fetchWithClientRedirects(opts, hooks, clientSideRedirectCount) {
     const browser = opts.browser ?? DEFAULT_BROWSER;
     const os = opts.os ?? DEFAULT_OS;
     const format = opts.format ?? "markdown";
@@ -10109,6 +10136,27 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       }
       errorContext.phase = "loading";
       const rawBody = await response.text();
+      const clientSideRedirect = extractClientSideRedirect(rawBody, finalUrl);
+      if (clientSideRedirect) {
+        if (clientSideRedirectCount >= MAX_CLIENT_SIDE_REDIRECTS) {
+          return {
+            error: `Client-side redirect limit (${MAX_CLIENT_SIDE_REDIRECTS}) exceeded while fetching ${opts.url}.`,
+            code: "too_many_redirects",
+            phase: "loading",
+            retryable: false,
+            timeoutMs,
+            url: opts.url,
+            finalUrl,
+            mimeType: normalizeContentType(contentType) || void 0,
+            contentLength: errorContext.contentLength
+          };
+        }
+        return fetchWithClientRedirects(
+          { ...opts, url: clientSideRedirect },
+          hooks,
+          clientSideRedirectCount + 1
+        );
+      }
       const jsonResponse = isJsonResponse(contentType, rawBody);
       if (format === "json") {
         if (!jsonResponse) {
@@ -10242,7 +10290,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
           fallbackDocument,
           opts.url
         );
-        if (hasOembed404 || hasJsDisabledShell) {
+        if ((hasOembed404 || hasJsDisabledShell) && !extracted.content) {
           return {
             error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
             code: "http_error",
@@ -10314,6 +10362,9 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       emitProgress(hooks, { status: "error", progress: 1, phase: "error" });
       return fetchError;
     }
+  }
+  return function defuddleFetch2(opts, hooks = {}) {
+    return fetchWithClientRedirects(opts, hooks, 0);
   };
 }
 var defuddleFetch = createDefuddleFetch();