@blockrun/franklin 3.15.7 → 3.15.8

package/dist/tools/webfetch.js

@@ -58,6 +58,22 @@ async function execute(input, ctx) {
     if (!['http:', 'https:'].includes(parsed.protocol)) {
         return { output: `Error: only http/https URLs are supported`, isError: true };
     }
+    // ── Pre-flight: known anti-bot domains ──
+    // Sites that systematically block scripted access return 403 / 429 /
+    // captcha challenges to plain GET requests no matter what UA we send.
+    // Without this guard the model burns multiple turns retrying variations
+    // (Zillow → /research/austin-tx, /homedetails/X, /sold/Y...) that all
+    // 403 the same way, padding the step counter and the user's bill.
+    // Short-circuiting here returns a single actionable error instead.
+    const blocked = isBlockedDomain(parsed.hostname);
+    if (blocked) {
+        return {
+            output: `${parsed.hostname} systematically blocks automated fetch (${blocked.reason}). ` +
+                `Switch tools: ${blocked.alternative}. Don't retry this URL with WebFetch — ` +
+                `every variant of the same hostname returns the same block.`,
+            isError: true,
+        };
+    }
     const maxLen = Math.min(max_length ?? DEFAULT_MAX_LENGTH, MAX_BODY_BYTES);
     // ── YouTube special case ──
     // Plain HTML fetch on a youtube.com URL returns the SPA bundle (a wall of
@@ -108,8 +124,19 @@ async function execute(input, ctx) {
             redirect: 'follow',
         });
         if (!response.ok) {
+            // 403 / 429 from a domain not in the static block list often still
+            // means anti-bot — many sites tier their detection (first hit OK,
+            // subsequent ones blocked) or rely on UA fingerprinting. Surface
+            // this as an actionable hint so the model switches strategy
+            // instead of retrying the same URL with a different path.
+            const isAntiBot = response.status === 403 || response.status === 429 ||
+                response.status === 503;
+            const hint = isAntiBot
+                ? ` — ${parsed.hostname} likely blocks automated fetch. Try WebSearch for the same query, ` +
+                    `or fetch a different domain that publishes the same data.`
+                : '';
             return {
-                output: `HTTP ${response.status} ${response.statusText} for ${url}`,
+                output: `HTTP ${response.status} ${response.statusText} for ${url}${hint}`,
                 isError: true,
             };
         }
@@ -176,6 +203,76 @@ async function execute(input, ctx) {
         ctx.abortSignal.removeEventListener('abort', onAbort);
     }
 }
+const BLOCKED_DOMAINS = [
+    {
+        pattern: /(^|\.)zillow\.com$/i,
+        reason: '403 to all non-browser GETs',
+        alternative: 'use WebSearch for "Austin TX home price trends" or similar',
+    },
+    {
+        pattern: /(^|\.)redfin\.com$/i,
+        reason: '403 / captcha challenge to scripted requests',
+        alternative: 'use WebSearch with the property address or zip code',
+    },
+    {
+        pattern: /(^|\.)realtor\.com$/i,
+        reason: '403 / interstitial to non-browser UAs',
+        alternative: 'use WebSearch',
+    },
+    {
+        pattern: /(^|\.)linkedin\.com$/i,
+        reason: 'auth wall on every page',
+        alternative: 'use SearchX (X is the better discovery surface for the same people) or WebSearch',
+    },
+    {
+        pattern: /(^|\.)instagram\.com$/i,
+        reason: 'auth wall + 401 to public profile fetches',
+        alternative: 'use WebSearch for the username',
+    },
+    {
+        pattern: /(^|\.)facebook\.com$/i,
+        reason: 'auth wall on most public content',
+        alternative: 'use WebSearch',
+    },
+    {
+        pattern: /(^|\.)x\.com$/i,
+        reason: 'X.com requires authenticated API',
+        alternative: 'use SearchX (the dedicated X tool) instead of WebFetch',
+    },
+    {
+        pattern: /(^|\.)twitter\.com$/i,
+        reason: 'X.com requires authenticated API',
+        alternative: 'use SearchX (the dedicated X tool) instead of WebFetch',
+    },
+    {
+        pattern: /(^|\.)tiktok\.com$/i,
+        reason: 'returns SPA shell + JS challenge',
+        alternative: 'use WebSearch with the @username',
+    },
+    {
+        pattern: /(^|\.)reuters\.com$/i,
+        reason: 'paywall + bot detection',
+        alternative: 'use WebSearch which surfaces cached headlines',
+    },
+    {
+        pattern: /(^|\.)bloomberg\.com$/i,
+        reason: 'paywall + bot detection',
+        alternative: 'use WebSearch for the same story',
+    },
+    {
+        pattern: /(^|\.)wsj\.com$/i,
+        reason: 'paywall',
+        alternative: 'use WebSearch for the same story',
+    },
+];
+function isBlockedDomain(hostname) {
+    for (const entry of BLOCKED_DOMAINS) {
+        if (entry.pattern.test(hostname)) {
+            return { reason: entry.reason, alternative: entry.alternative };
+        }
+    }
+    return null;
+}
 // ─── YouTube transcript fetcher ─────────────────────────────────────────────
 // Fetches auto-generated or uploaded captions for a YouTube video by parsing
 // the watch-page's `ytInitialPlayerResponse` JSON. Pure HTTP, no deps. Saves

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blockrun/franklin",
-  "version": "3.15.7",
+  "version": "3.15.8",
   "description": "Franklin — The AI agent with a wallet. Spends USDC autonomously to get real work done. Pay per action, no subscriptions.",
   "type": "module",
   "exports": {