openclaw-smart-fetch 0.2.30 → 0.2.31

package/dist/index.js

@@ -10203,25 +10203,10 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
       });
       const fallbackDocument = parseLinkedomHTML(rawBody, finalUrl);
       const extractionDocument = parseLinkedomHTML(rawBody, finalUrl);
-      if (isTwitterJsDisabledPage(fallbackDocument, opts.url)) {
-        return {
-          error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
-          code: "http_error",
-          phase: "loading",
-          retryable: false,
-          timeoutMs,
-          url: opts.url,
-          finalUrl,
-          statusCode: 404,
-          statusText: "Not Found",
-          mimeType: normalizeContentType(contentType) || void 0,
-          contentLength: errorContext.contentLength
-        };
-      }
       let extracted;
+      const suppressedErrors = [];
       try {
         const origConsoleError = console.error;
-        const suppressedErrors = [];
         console.error = (...args) => {
           suppressedErrors.push(args);
         };
@@ -10244,6 +10229,35 @@ function createDefuddleFetch(dependencies = runtimeDependencies) {
           wordCount: 0
         };
       }
+      const isXUrl = /^https?:\/\/(www\.)?(x\.com|twitter\.com)\//i.test(
+        opts.url
+      );
+      if (isXUrl) {
+        const hasOembed404 = suppressedErrors.some(
+          (args) => args.some(
+            (arg) => typeof arg === "string" && arg.includes("oEmbed request failed: 404")
+          )
+        );
+        const hasJsDisabledShell = isTwitterJsDisabledPage(
+          fallbackDocument,
+          opts.url
+        );
+        if (hasOembed404 || hasJsDisabledShell) {
+          return {
+            error: `Server returned HTTP 404 Not Found for ${opts.url}.`,
+            code: "http_error",
+            phase: "loading",
+            retryable: false,
+            timeoutMs,
+            url: opts.url,
+            finalUrl,
+            statusCode: 404,
+            statusText: "Not Found",
+            mimeType: normalizeContentType(contentType) || void 0,
+            contentLength: errorContext.contentLength
+          };
+        }
+      }
       let extractedContent = extracted.content;
       let wordCount = extracted.wordCount;
       if (!extractedContent || wordCount === 0) {