npm - openclaw-smart-fetch - Versions diffs - 0.2.34 → 0.2.35 - Mend

openclaw-smart-fetch 0.2.34 → 0.2.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

package/README.md CHANGED Viewed

@@ -9,6 +9,7 @@
 - 🧠 **Useful metadata** — title, author, site, language, published date when available
 - 📦 **Downloads + large file support** — stream attachments and binaries to temp files
 - 🔁 **Client-side `<meta>` redirects** — follows sane meta refresh redirects with loop limits
+- 🔗 **Alternate content fallback** — when extraction produces no/thin content, follows qualified `<link rel="alternate" type="...">` entries in `<head>` that match the requested output format
 - ⚡ **Batch fetch** — fetch many URLs with bounded concurrency
 - 📝 **Multiple output formats** — `markdown`, `html`, `text`, `json`
 - 🔄 **Built-in `web_fetch` fallback** — automatically improves the core web_fetch tool
@@ -35,6 +36,7 @@ from Defuddle's extractors and cleanup:
 Notes:
 - Defuddle is the cleanup layer: it strips common page chrome like nav, sidebars, related links, share widgets, and footers
 - It does **not** execute JavaScript or solve interactive anti-bot/login flows
+- If an HTML shell advertises alternate content in `<head>`, smart-fetch can follow matching alternates such as `text/markdown`, `text/plain`, `text/html`, or JSON media types according to the requested `format`
 ## Install

package/dist/index.js CHANGED Viewed

@@ -9405,6 +9405,8 @@ var HTML_CONTENT_TYPES = [
   "text/markdown"
 ];
 var MAX_CLIENT_SIDE_REDIRECTS = 5;
+var MAX_ALTERNATE_LINK_FALLBACKS = 3;
+var MIN_EXTRACTED_WORDS_BEFORE_ALTERNATE_FALLBACK = 30;
 function normalizeContentType(contentType) {
   return contentType.split(";")[0]?.trim().toLowerCase() ?? "";
 }
@@ -9906,6 +9908,36 @@ function isJsonResponse(contentType, body) {
 function decodeHtmlAttribute(value) {
   return value.replace(/&amp;/gi, "&").replace(/&quot;/gi, '"').replace(/&#39;|&apos;/gi, "'").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">");
 }
+function extractQualifiedAlternateLinks(document, baseUrl, format) {
+  const acceptedTypes = {
+    markdown: ["text/markdown", "text/x-markdown"],
+    text: ["text/plain", "text/markdown", "text/x-markdown"],
+    html: ["text/html", "application/xhtml+xml"],
+    json: ["application/json", "text/json"]
+  };
+  const accepted = acceptedTypes[format];
+  const head = document.head;
+  if (!head) return [];
+  const links = Array.from(head.querySelectorAll("link"));
+  const candidates = [];
+  for (const link of links) {
+    const rel = (link.getAttribute("rel") ?? "").toLowerCase().split(/\s+/);
+    if (!rel.includes("alternate")) continue;
+    const type = normalizeContentType(link.getAttribute("type") ?? "");
+    const isAccepted = accepted.some((value) => type === value) || format === "json" && type.endsWith("+json");
+    if (!isAccepted) continue;
+    const href = link.getAttribute("href");
+    if (!href) continue;
+    try {
+      const target = new URL(href, baseUrl).toString();
+      if (target !== baseUrl && !candidates.includes(target)) {
+        candidates.push(target);
+      }
+    } catch {
+    }
+  }
+  return candidates;
+}
 function extractClientSideRedirect(body, baseUrl) {
   const snippet = body.slice(0, 4096);
   const metaRefreshMatch = snippet.match(
@@ -10008,7 +10040,7 @@ function shouldStripReplies(site) {
   return site === "Hacker News" || site.startsWith("r/") || site.startsWith("GitHub - ");
 }
 function createDefuddleFetch(dependencies = runtimeDependencies) {
-  async function fetchWithClientRedirects(opts, hooks, clientSideRedirectCount) {
+  async function fetchWithClientRedirects(opts, hooks, clientSideRedirectCount, alternateLinkFallbackCount) {
     const browser = opts.browser ?? DEFAULT_BROWSER;
     const os = opts.os ?? DEFAULT_OS;
     const format = opts.format ?? "markdown";
@@ -10154,12 +10186,28 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
         return fetchWithClientRedirects(
           { ...opts, url: clientSideRedirect },
           hooks,
-          clientSideRedirectCount + 1
+          clientSideRedirectCount + 1,
+          alternateLinkFallbackCount
         );
       }
       const jsonResponse = isJsonResponse(contentType, rawBody);
       if (format === "json") {
         if (!jsonResponse) {
+          if (HTML_CONTENT_TYPES.some((value) => contentType.includes(value))) {
+            const alternateLinks2 = extractQualifiedAlternateLinks(
+              parseLinkedomHTML(rawBody, finalUrl),
+              finalUrl,
+              format
+            );
+            if (alternateLinks2.length > 0 && alternateLinkFallbackCount < MAX_ALTERNATE_LINK_FALLBACKS) {
+              return fetchWithClientRedirects(
+                { ...opts, url: alternateLinks2[0] },
+                hooks,
+                clientSideRedirectCount,
+                alternateLinkFallbackCount + 1
+              );
+            }
+          }
           return {
             error: `Not a JSON response (content-type: ${contentType})`,
             code: "unexpected_response",
@@ -10251,6 +10299,22 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       });
       const fallbackDocument = parseLinkedomHTML(rawBody, finalUrl);
       const extractionDocument = parseLinkedomHTML(rawBody, finalUrl);
+      const alternateLinks = extractQualifiedAlternateLinks(
+        fallbackDocument,
+        finalUrl,
+        format
+      );
+      const tryAlternateLinkFallback = async () => {
+        if (alternateLinks.length === 0 || alternateLinkFallbackCount >= MAX_ALTERNATE_LINK_FALLBACKS) {
+          return null;
+        }
+        return fetchWithClientRedirects(
+          { ...opts, url: alternateLinks[0] },
+          hooks,
+          clientSideRedirectCount,
+          alternateLinkFallbackCount + 1
+        );
+      };
       let extracted;
       const suppressedErrors = [];
       try {
@@ -10311,6 +10375,8 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       if (!extractedContent || wordCount === 0) {
         const fallbackText = extractDomTextFallback(fallbackDocument);
         if (!fallbackText) {
+          const alternateResult = await tryAlternateLinkFallback();
+          if (alternateResult) return alternateResult;
           return {
             error: `No content extracted from ${opts.url}. May need JS rendering or is blocked.`,
             code: "no_content",
@@ -10326,6 +10392,13 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
         extractedContent = format === "html" ? rawBody : format === "markdown" ? extractDomMarkdownFallback(fallbackDocument) || fallbackText : fallbackText;
         wordCount = estimateWordCount(fallbackText);
       }
+      const extractedTextWordCount = estimateWordCount(
+        format === "text" ? extractedContent : markdownToText(extractedContent)
+      );
+      if (Math.min(wordCount, extractedTextWordCount) < MIN_EXTRACTED_WORDS_BEFORE_ALTERNATE_FALLBACK && alternateLinks.length > 0) {
+        const alternateResult = await tryAlternateLinkFallback();
+        if (alternateResult) return alternateResult;
+      }
       if (includeReplies === false && shouldStripReplies(extracted.site ?? "")) {
         const strippedContent = stripExtractorComments(
           extractedContent,
@@ -10364,7 +10437,7 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
     }
   }
   return function defuddleFetch2(opts, hooks = {}) {
-    return fetchWithClientRedirects(opts, hooks, 0);
+    return fetchWithClientRedirects(opts, hooks, 0, 0);
   };
 }
 var defuddleFetch = createDefuddleFetch();