@sisu-ai/tool-web-fetch 1.0.0 → 2.0.0

package/README.md

@@ -10,6 +10,7 @@ npm i @sisu-ai/tool-web-fetch
 Environment / Flags
 - `WEB_FETCH_USER_AGENT` or `HTTP_USER_AGENT` (flag: `--web-fetch-user-agent`)
 - `WEB_FETCH_MAX_BYTES` (flag: `--web-fetch-max-bytes`) — default 500kB
+- `WEB_FETCH_RESPECT_ROBOTS` (flag: `--web-fetch-respect-robots`) — `1`/`true` (default) to honor robots.txt; set `0`/`false` to disable
 
 Tool
 - Name: `webFetch`
@@ -17,6 +18,7 @@ Tool
 - Returns: `{ url, finalUrl?, status, contentType?, title?, text?, html?, json? }`
 
 Behavior
+- Respects robots.txt by default for the provided User-Agent.
 - Follows redirects and reads up to `maxBytes` to avoid huge pages.
 - If `format: 'text'` (default) and page is HTML, strips tags (removes script/style) and decodes basic entities; includes `title`.
 - If `format: 'html'`, returns raw HTML and `title`.

package/dist/index.d.ts

@@ -4,6 +4,7 @@ export interface WebFetchArgs {
     url: string;
     format?: WebFetchFormat;
     maxBytes?: number;
+    respectRobots?: boolean;
 }
 export interface WebFetchResult {
     url: string;
@@ -14,6 +15,8 @@ export interface WebFetchResult {
     text?: string;
     html?: string;
     json?: unknown;
+    robotsBlocked?: boolean;
+    robotsAgent?: string;
 }
 export declare const webFetch: Tool<WebFetchArgs>;
 export default webFetch;

package/dist/index.js

@@ -7,12 +7,43 @@ export const webFetch = {
         url: z.string().url(),
         format: z.enum(['text', 'html', 'json']).optional(),
         maxBytes: z.number().int().positive().max(5_000_000).optional(),
+        respectRobots: z.boolean().optional(),
     }),
-    handler: async ({ url, format = 'text', maxBytes }, _ctx) => {
+    handler: async ({ url, format = 'text', maxBytes, respectRobots }, ctx) => {
         const ua = firstConfigValue(['WEB_FETCH_USER_AGENT', 'HTTP_USER_AGENT'])
             || 'SisuWebFetch/0.1 (+https://github.com/finger-gun/sisu)';
         const capEnv = firstConfigValue(['WEB_FETCH_MAX_BYTES']);
         const cap = Number(maxBytes ?? (capEnv !== undefined ? Number(capEnv) : 500_000));
+        // robots.txt compliance (default on; disable with arg or env WEB_FETCH_RESPECT_ROBOTS=0)
+        const respect = (() => {
+            if (typeof respectRobots === 'boolean')
+                return respectRobots;
+            const env = firstConfigValue(['WEB_FETCH_RESPECT_ROBOTS', 'RESPECT_ROBOTS']);
+            if (env === undefined)
+                return true; // default on
+            return !(env === '0' || /^false$/i.test(env));
+        })();
+        if (respect) {
+            const decision = await robotsDecision(url, ua).catch(() => ({ allowed: true }));
+            if (!decision.allowed) {
+                ctx?.log?.info?.('[webFetch] blocked by robots.txt', {
+                    url,
+                    userAgent: ua,
+                    matchedAgent: decision.matchedAgent,
+                    ruleType: decision.ruleType,
+                    rulePattern: decision.rulePattern,
+                });
+                return {
+                    url,
+                    status: 403,
+                    contentType: 'text/plain',
+                    text: `Blocked by robots.txt (agent: ${decision.matchedAgent ?? 'unknown'}, rule: ${decision.ruleType ?? 'disallow'} ${decision.rulePattern ?? ''})`.
+                        trim(),
+                    robotsBlocked: true,
+                    robotsAgent: ua
+                };
+            }
+        }
         const res = await fetch(url, {
             redirect: 'follow',
             headers: { 'User-Agent': ua, 'Accept': '*/*' },
@@ -50,6 +81,136 @@ export const webFetch = {
     },
 };
 export default webFetch;
+const robotsCache = new Map();
+async function robotsDecision(targetUrl, userAgent) {
+    const u = new URL(targetUrl);
+    const origin = `${u.protocol}//${u.host}`;
+    const cache = robotsCache.get(origin);
+    const now = Date.now();
+    if (!cache || (now - cache.ts) > 60 * 60 * 1000) { // 1h TTL
+        const robotsUrl = `${origin}/robots.txt`;
+        try {
+            const res = await fetch(robotsUrl, { headers: { 'User-Agent': userAgent, 'Accept': 'text/plain' } });
+            const txt = await res.text();
+            const rules = res.ok ? parseRobots(txt) : null;
+            robotsCache.set(origin, { ts: now, rules });
+        }
+        catch {
+            robotsCache.set(origin, { ts: now, rules: null });
+        }
+    }
+    const rules = robotsCache.get(origin)?.rules;
+    if (!rules)
+        return { allowed: true };
+    return evaluateRobotsDetailed(rules, userAgent, u.pathname + (u.search || ''));
+}
+function parseRobots(text) {
+    const lines = text.split(/\r?\n/);
+    const groups = [];
+    let current = null;
+    for (const raw of lines) {
+        const line = raw.trim();
+        if (!line || line.startsWith('#'))
+            continue;
+        const m = line.match(/^(user-agent|allow|disallow)\s*:\s*(.*)$/i);
+        if (!m)
+            continue;
+        const key = m[1].toLowerCase();
+        const val = m[2].trim();
+        if (key === 'user-agent') {
+            // Start a new group if we already had one and it contains rules
+            if (!current || (current.allows.length + current.disallows.length) > 0) {
+                current = { agents: [], allows: [], disallows: [] };
+                groups.push(current);
+            }
+            current.agents.push(val.toLowerCase());
+        }
+        else if (key === 'allow') {
+            if (!current) {
+                current = { agents: ['*'], allows: [], disallows: [] };
+                groups.push(current);
+            }
+            current.allows.push(val);
+        }
+        else if (key === 'disallow') {
+            if (!current) {
+                current = { agents: ['*'], allows: [], disallows: [] };
+                groups.push(current);
+            }
+            current.disallows.push(val);
+        }
+    }
+    return { groups };
+}
+function evaluateRobotsDetailed(rules, userAgent, pathWithQuery) {
+    // Match exact agent token (product) ignoring case, or '*'.
+    // Example: 'SisuWebFetch/0.1 (+...)' -> baseAgent 'sisuwebfetch'
+    const baseAgent = (userAgent.split(/[\/\s]/)[0] || '').toLowerCase();
+    const agentMatches = (agent) => {
+        if (agent === '*')
+            return true;
+        return agent.toLowerCase() === baseAgent;
+    };
+    const matching = rules.groups
+        .map(g => ({ g, matchedAgent: g.agents.find(agentMatches) }))
+        .filter(x => !!x.matchedAgent);
+    const selected = matching.length
+        ? matching
+        : rules.groups.filter(g => g.agents.includes('*')).map(g => ({ g, matchedAgent: '*' }));
+    if (!selected.length)
+        return { allowed: true };
+    // longest match wins between allow and disallow
+    let bestType;
+    let bestLen = -1;
+    let bestPat;
+    let bestAgent;
+    for (const { g, matchedAgent } of selected) {
+        for (const pat of g.allows) {
+            if (!pat)
+                continue;
+            if (patternMatches(pat, pathWithQuery)) {
+                const L = pat.length;
+                if (L > bestLen) {
+                    bestLen = L;
+                    bestType = 'allow';
+                    bestPat = pat;
+                    bestAgent = matchedAgent;
+                }
+            }
+        }
+        for (const pat of g.disallows) {
+            if (!pat)
+                continue;
+            if (patternMatches(pat, pathWithQuery)) {
+                const L = pat.length;
+                if (L > bestLen) {
+                    bestLen = L;
+                    bestType = 'disallow';
+                    bestPat = pat;
+                    bestAgent = matchedAgent;
+                }
+            }
+        }
+    }
+    if (bestType === 'disallow')
+        return { allowed: false, matchedAgent: bestAgent, ruleType: 'disallow', rulePattern: bestPat };
+    return { allowed: true, matchedAgent: bestAgent, ruleType: bestType, rulePattern: bestPat };
+}
+function patternMatches(pat, path) {
+    // Support '*' wildcard and '$' end anchor; treat path as starting with '/'
+    const p = pat.trim();
+    if (p === '')
+        return false;
+    // Empty disallow means allow all; already handled by return false above
+    // Convert to regex
+    const escaped = p.replace(/[.+?^${}()|\[\]\\]/g, r => '\\' + r);
+    let reStr = '^' + escaped.replace(/\*/g, '.*');
+    if (reStr.endsWith('\$')) {
+        reStr = reStr.slice(0, -2) + '$';
+    }
+    const re = new RegExp(reStr);
+    return re.test(path);
+}
 async function readWithCap(res, cap) {
     // If body is not a stream (older fetch mocks), try res.text()
     const anyRes = res;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sisu-ai/tool-web-fetch",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -14,7 +14,7 @@
     "zod": "^3.23.8"
   },
   "peerDependencies": {
-    "@sisu-ai/core": "0.3.0"
+    "@sisu-ai/core": "1.0.0"
   },
   "repository": {
     "type": "git",