minimal-agent 0.1.9 → 0.3.0

package/src/tools/webfetch/preapproved.js

@@ -0,0 +1,267 @@
+/**
+ * ============================================================
+ *  src/tools/webfetch-preapproved.ts —— WebFetch 预批准域名白名单
+ * ------------------------------------------------------------
+ *  维护一份可信任域名列表，用于过滤潜在风险 URL
+ * ============================================================
+ */
+const PREAPPROVED_HOSTS = new Set([
+    // GitHub
+    'github.com',
+    'gist.github.com',
+    // 开发文档
+    'docs.github.com',
+    'developer.github.com',
+    'help.github.com',
+    // NPM
+    'npmjs.com',
+    'www.npmjs.com',
+    // 包注册表
+    'pypi.org',
+    'www.pypi.org',
+    'crates.io',
+    'pub.dev',
+    'packagist.org',
+    'rubygems.org',
+    // 开发平台
+    'stackoverflow.com',
+    'www.stackoverflow.com',
+    'serverfault.com',
+    'superuser.com',
+    'askubuntu.com',
+    // 文档与 wiki
+    'readthedocs.io',
+    'www.readthedocs.io',
+    'readthedocs.org',
+    'wiki.python.org',
+    'en.wikipedia.org',
+    'zh.wikipedia.org',
+    // 官方文档
+    'nodejs.org',
+    'www.nodejs.org',
+    'deno.land',
+    'www.deno.land',
+    'bun.sh',
+    'www.bun.sh',
+    // Rust
+    'doc.rust-lang.org',
+    'www.rust-lang.org',
+    'rust-lang.org',
+    // 云平台
+    'aws.amazon.com',
+    'docs.aws.amazon.com',
+    'cloud.google.com',
+    'docs.microsoft.com',
+    'azure.microsoft.com',
+    'developer.microsoft.com',
+    // AI/LLM
+    'openai.com',
+    'platform.openai.com',
+    'docs.anthropic.com',
+    'anthropic.com',
+    'claude.ai',
+    'docs.cohere.com',
+    'cohere.com',
+    // AI 模型文档
+    'ai.google.dev',
+    'ai.google.com',
+    'developers.google.com',
+    'learn.deepmind.com',
+    // AI 开发框架
+    'python.langchain.com',
+    'js.langchain.com',
+    'docs.litellm.ai',
+    'litellm.ai',
+    // 前端框架
+    'react.dev',
+    'reactjs.org',
+    'www.reactjs.org',
+    'nextjs.org',
+    'www.nextjs.org',
+    'vuejs.org',
+    'www.vuejs.org',
+    'svelte.dev',
+    'svelte.org',
+    'angular.io',
+    'www.angular.io',
+    // 构建工具
+    'vitejs.dev',
+    'vite.dev',
+    'webpack.js.org',
+    'esbuild.github.io',
+    'rollupjs.org',
+    'esbuild.github.io',
+    // CSS
+    'tailwindcss.com',
+    'www.tailwindcss.com',
+    'postcss.org',
+    // 数据库
+    'redis.io',
+    'www.redis.io',
+    'postgresql.org',
+    'www.postgresql.org',
+    'www.mysql.com',
+    'dev.mysql.com',
+    'docs.mongodb.com',
+    'www.mongodb.com',
+    'sqlite.org',
+    'www.sqlite.org',
+    // 工具类
+    'regex101.com',
+    'ihateregex.io',
+    'explainshell.com',
+    'tldr.sh',
+    // 代码分享
+    'replit.com',
+    'www.replit.com',
+    'codesandbox.io',
+    'www.codesandbox.io',
+    'codepen.io',
+    'www.codepen.io',
+    'jsfiddle.net',
+    'www.jsfiddle.net',
+    // CI/CD
+    'docs.github.com/en/actions',
+    'circleci.com',
+    'docs.circleci.com',
+    'travis-ci.org',
+    'www.travis-ci.com',
+    'jenkins.io',
+    'www.jenkins.io',
+    // 容器/云原生
+    'kubernetes.io',
+    'www.kubernetes.io',
+    'docker.com',
+    'www.docker.com',
+    'docs.docker.com',
+    // 测试
+    'jestjs.io',
+    'www.jestjs.io',
+    'vitest.dev',
+    'testing-library.com',
+    'www.testing-library.com',
+    'playwright.dev',
+    'www.playwright.dev',
+    'webdriver.io',
+    'webdriver.io',
+    // API 文档
+    'httpbin.org',
+    'restfulapi.net',
+    'swagger.io',
+    'www.swagger.io',
+    'openapi.net',
+    // 安全
+    'owasp.org',
+    'www.owasp.org',
+    'cve.mitre.org',
+    // 证书
+    'letsencrypt.org',
+    'www.letsencrypt.org',
+    'acmev2.pki.duckdns.org',
+    // 博客与技术文章
+    'medium.com',
+    'www.medium.com',
+    'dev.to',
+    'www.dev.to',
+    'hashnode.com',
+    'www.hashnode.com',
+    'devblogs.microsoft.com',
+    // 浏览器
+    'caniuse.com',
+    'developer.mozilla.org',
+    'web.dev',
+    'www.w3.org',
+    // 开源项目
+    'apache.org',
+    'www.apache.org',
+    'gnu.org',
+    'www.gnu.org',
+    'fsf.org',
+    'www.fsf.org',
+    'opensource.org',
+    'www.opensource.org',
+    // 技术社区
+    'reddit.com',
+    'www.reddit.com',
+    'news.ycombinator.com',
+    'lobste.rs',
+    // 文件格式
+    'json.org',
+    'yaml.org',
+    'www.yaml.org',
+    'toml.io',
+    'www.toml.io',
+    // 版本控制
+    'git-scm.com',
+    'www.git-scm.com',
+    'github.blog',
+    'githubstatus.com',
+    // AI 搜索
+    'tavily.com',
+    'www.tavily.com',
+    'perplexity.ai',
+    'www.perplexity.ai',
+    // AI 图片
+    'midjourney.com',
+    'www.midjourney.com',
+    'stability.ai',
+    'www.stability.ai',
+    // Embeddings / 向量
+    'qdrant.tech',
+    'www.qdrant.tech',
+    'weaviate.io',
+    'www.weaviate.io',
+    'pinecone.io',
+    'www.pinecone.io',
+    // API 平台
+    'ngrok.com',
+    'www.ngrok.com',
+    'requestly.io',
+    'www.requestly.io',
+    // MCP
+    'modelcontextprotocol.io',
+    'www.modelcontextprotocol.io',
+    'github.com/modelcontextprotocol',
+]);
+const PREAPPROVED_SUFFIXES = [
+    '.github.io',
+    '.readthedocs.io',
+    '.vercel.app',
+    '.vercel.dev',
+    '.netlify.app',
+    '.netlify.com',
+    '.cloudflare-pages.com',
+    '.pages.dev',
+    '.surge.sh',
+    '.herokuapp.com',
+    '.railway.app',
+    '.fly.dev',
+    '.repl.co',
+    '.workers.dev',
+    '.pages.plus',
+    '.coded.app',
+    '.preview.app',
+    '.staging.app',
+];
+export function isPreapprovedUrl(url) {
+    try {
+        const parsed = new URL(url);
+        return isPreapprovedHost(parsed.hostname);
+    }
+    catch {
+        return false;
+    }
+}
+export function isPreapprovedHost(hostname) {
+    const normalized = hostname.toLowerCase();
+    if (PREAPPROVED_HOSTS.has(normalized)) {
+        return true;
+    }
+    for (const suffix of PREAPPROVED_SUFFIXES) {
+        if (normalized.endsWith(suffix)) {
+            return true;
+        }
+    }
+    return false;
+}
+export const isPreapprovedDomain = isPreapprovedHost;

package/src/tools/webfetch/webfetch.js

@@ -0,0 +1,317 @@
+/**
+ * ============================================================
+ *  src/tools/webfetch.ts —— WebFetch 工具（获取 URL 内容）
+ * ------------------------------------------------------------
+ *  获取 URL → HTML 转 Markdown → 用 prompt 提取信息
+ *
+ *  输入：
+ *    url    必填，要获取内容的 URL
+ *    prompt 必填，对内容进行处理的指令
+ *
+ *  输出：
+ *    { bytes, code, codeText, result, durationMs, url }
+ *
+ *  流程：
+ *    1. URL 验证（合法格式、2000字符以内、无凭据）
+ *    2. fetch（自动 http→https，超时 60s）
+ *    3. 检测跨 host 重定向，返回提示而非自动跟随
+ *    4. HTML → Markdown（turndown）；非 HTML 原样返回
+ *    5. 输出截断到 30K chars（超长内容直接截断）
+ * ============================================================
+ */
+import { z } from 'zod';
+import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
+import { toToolParameters } from '../../utils/zodToJson.js';
+import { isPreapprovedDomain } from './preapproved.js';
+// ---------------- 1. 常量 ----------------
+const MAX_URL_LENGTH = 2000;
+const FETCH_TIMEOUT_MS = 60_000;
+const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024;
+const MAX_REDIRECTS = 10;
+const URL_CACHE = new Map();
+const CACHE_TTL_MS = 15 * 60 * 1000;
+const MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024;
+function cleanCache() {
+    const now = Date.now();
+    let totalSize = 0;
+    const entries = [];
+    for (const [url, entry] of URL_CACHE) {
+        if (now - entry.fetchedAt > CACHE_TTL_MS) {
+            URL_CACHE.delete(url);
+            continue;
+        }
+        const size = entry.bytes;
+        totalSize += size;
+        entries.push({ url, entry, size });
+    }
+    if (totalSize > MAX_CACHE_SIZE_BYTES) {
+        entries.sort((a, b) => a.entry.fetchedAt - b.entry.fetchedAt);
+        for (const { url, size } of entries) {
+            URL_CACHE.delete(url);
+            totalSize -= size;
+            if (totalSize <= MAX_CACHE_SIZE_BYTES * 0.8)
+                break;
+        }
+    }
+}
+// ---------------- 3. Zod 输入 schema ----------------
+const inputSchema = z.object({
+    url: z.string().describe('要获取内容的 URL'),
+    prompt: z.string().describe('对内容进行处理的指令，描述你想从页面提取什么信息'),
+});
+// ---------------- 4. JSON Schema（由 Zod 自动派生） ----------------
+const parameters = toToolParameters(inputSchema);
+// ---------------- 4. Description ----------------
+const description = `- Fetches content from a specified URL and processes it using an AI model.
+- Takes a URL and a prompt as input.
+- Fetches the URL content, converts HTML to markdown.
+- Processes the content with the prompt (e.g., extract summary, find specific info).
+- Returns the processed result.
+- HTTP URLs are automatically upgraded to HTTPS.
+- When a URL redirects to a different host, returns a warning with the redirect URL.
+- This tool is read-only and does not modify any files.
+- Results may be summarized if the content is very large.
+- ⚠️ IMPORTANT: This tool WILL FAIL for authenticated or private URLs.
+  Before using this tool, check if the URL points to an authenticated service
+  (e.g. Google Docs, Confluence, Jira, GitHub private repos). If so, look for
+  a specialized MCP tool that provides authenticated access.
+- 💡 For GitHub URLs (repos, issues, PRs), prefer using the \`gh\` CLI via Bash
+  instead (e.g. \`gh pr view <pr-number>\`, \`gh issue view <number>\`, \`gh api <endpoint>\`).
+- ⚡ If an MCP-provided web fetch tool is available, prefer using that tool instead,
+  as it may have fewer restrictions and better performance.
+- 🔒 Domain preapproved list includes common documentation sites (MDN, TypeScript,
+  React, Vue, Angular, Node.js, Bun, Rust, Go, Python, etc.). Other domains
+  will work but results may be less reliable.
+- 📝 This tool includes a self-cleaning 15-minute cache for faster repeated access
+  to the same URL.
+- ⚠️ For PDF files, the tool will attempt to extract readable text but results may
+  be limited. Binary images cannot be processed.`;
+// ---------------- 5. 辅助函数 ----------------
+// ---------------- 6. call() 实现 ----------------
+/** 验证 URL */
+function validateURL(url) {
+    if (url.length > MAX_URL_LENGTH) {
+        return { ok: false, error: `URL 太长（最大 ${MAX_URL_LENGTH} 字符）` };
+    }
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return { ok: false, error: `无效的 URL 格式` };
+    }
+    if (parsed.username || parsed.password) {
+        return { ok: false, error: `URL 不能包含用户名或密码` };
+    }
+    return { ok: true, parsed };
+}
+/** 获取 URL 内容（返回 HTML 字符串或重定向信息） */
+async function fetchURL(url, signal, depth = 0) {
+    const validated = validateURL(url);
+    if (!validated.ok)
+        return { type: 'error', error: validated.error };
+    let targetUrl = url;
+    const parsed = validated.parsed;
+    if (parsed.protocol === 'http:') {
+        parsed.protocol = 'https:';
+        targetUrl = parsed.toString();
+    }
+    let response;
+    try {
+        const timeoutSignal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+        const controller = new AbortController();
+        const combinedSignal = signal
+            ? AbortSignal.any([signal, timeoutSignal])
+            : timeoutSignal;
+        response = await fetch(targetUrl, {
+            signal: combinedSignal,
+            headers: {
+                Accept: 'text/markdown, text/html, */*',
+                'User-Agent': 'minimal-agent/1.0',
+            },
+        });
+    }
+    catch (e) {
+        if (signal?.aborted)
+            return { type: 'error', error: '请求被中断' };
+        return { type: 'error', error: `网络请求失败：${e.message}` };
+    }
+    const code = response.status;
+    const codeText = response.statusText;
+    const contentType = response.headers.get('content-type') ?? '';
+    const location = response.headers.get('location');
+    if (location && [301, 302, 307, 308].includes(code)) {
+        const redirectUrl = new URL(location, targetUrl).toString();
+        const originalHost = new URL(url).hostname;
+        const redirectHost = new URL(redirectUrl).hostname;
+        const stripWww = (h) => h.replace(/^www\./, '');
+        if (stripWww(originalHost) !== stripWww(redirectHost)) {
+            return {
+                type: 'redirect',
+                originalUrl: url,
+                redirectUrl,
+                statusCode: code,
+            };
+        }
+        if (depth >= MAX_REDIRECTS) {
+            return { type: 'error', error: `重定向循环超过限制（最多 ${MAX_REDIRECTS} 次）` };
+        }
+        return fetchURL(redirectUrl, signal, depth + 1);
+    }
+    // 读取内容（限制 10MB）
+    let rawBuffer;
+    try {
+        rawBuffer = await response.arrayBuffer();
+    }
+    catch (e) {
+        return { type: 'error', error: `读取响应体失败：${e.message}` };
+    }
+    if (rawBuffer.byteLength > MAX_HTTP_CONTENT_LENGTH) {
+        return { type: 'error', error: `内容太大（${rawBuffer.byteLength} bytes，超过 ${MAX_HTTP_CONTENT_LENGTH}）` };
+    }
+    // 解码
+    let content;
+    try {
+        const decoder = new TextDecoder('utf-8', { fatal: false });
+        content = decoder.decode(rawBuffer);
+    }
+    catch (e) {
+        return { type: 'error', error: `解码失败：${e.message}` };
+    }
+    return { type: 'success', content, bytes: rawBuffer.byteLength, code, codeText, contentType };
+}
+/** HTML 转 Markdown */
+async function htmlToMarkdown(html) {
+    // 动态 import turndown（~1.4MB，按需加载）
+    const TurndownService = (await import('turndown')).default;
+    const td = new TurndownService();
+    return td.turndown(html);
+}
+// ---------------- 6. call() 实现 ----------------
+async function call(input, signal) {
+    const { url } = input;
+    const start = Date.now();
+    const cacheKey = url;
+    let targetHostname;
+    try {
+        targetHostname = new URL(url).hostname;
+    }
+    catch {
+        targetHostname = '';
+    }
+    const preapproved = isPreapprovedDomain(targetHostname);
+    // 检查缓存
+    const cached = URL_CACHE.get(cacheKey);
+    if (cached && Date.now() - cached.fetchedAt <= CACHE_TTL_MS) {
+        let content = cached.content;
+        const contentType = cached.contentType;
+        if (contentType.includes('text/html')) {
+            try {
+                content = await htmlToMarkdown(content);
+            }
+            catch (e) {
+                console.warn(`turndown 失败: ${e.message}`);
+            }
+        }
+        const formattedSize = cached.bytes < 1024
+            ? `${cached.bytes} B`
+            : cached.bytes < 1024 * 1024
+                ? `${(cached.bytes / 1024).toFixed(1)} KB`
+                : `${(cached.bytes / 1024 / 1024).toFixed(1)} MB`;
+        const domainNote = preapproved
+            ? '[🔒 预批准域名，内容可信]'
+            : '[⚠️ 非预批准域名，内容可能不准确]';
+        const output = `【WebFetch 结果】
+URL: ${url}
+状态: ${cached.code} ${cached.codeText}
+大小: ${formattedSize}
+耗时: 0.00s (缓存命中)
+${domainNote}
+[📦 来自缓存（15分钟 TTL）]
+
+--- 内容 ---
+${content}`;
+        let final = output;
+        if (final.length > DEFAULT_MAX_RESULT_SIZE_CHARS) {
+            final = final.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS) + `\n\n... (输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符，已截断)`;
+        }
+        return { ok: true, content: final };
+    }
+    // 1. fetch
+    const fetched = await fetchURL(url, signal ?? new AbortController().signal);
+    const durationMs = Date.now() - start;
+    if (fetched.type === 'redirect') {
+        const statusText = fetched.statusCode === 301 ? 'Moved Permanently' :
+            fetched.statusCode === 308 ? 'Permanent Redirect' :
+                fetched.statusCode === 307 ? 'Temporary Redirect' : 'Found';
+        return {
+            ok: true,
+            content: `【重定向检测】
+
+原始 URL: ${fetched.originalUrl}
+重定向到: ${fetched.redirectUrl}
+状态: ${fetched.statusCode} ${statusText}
+
+请使用重定向后的 URL 再次调用 WebFetch 工具。`,
+        };
+    }
+    if (fetched.type === 'error') {
+        return { ok: false, error: fetched.error };
+    }
+    let { content, bytes, code, codeText, contentType } = fetched;
+    // 写入缓存
+    URL_CACHE.set(cacheKey, {
+        bytes,
+        code,
+        codeText,
+        content,
+        contentType,
+        fetchedAt: Date.now(),
+    });
+    cleanCache();
+    // 2. HTML → Markdown
+    if (contentType.includes('text/html')) {
+        try {
+            content = await htmlToMarkdown(content);
+        }
+        catch (e) {
+            console.warn(`turndown 失败: ${e.message}`);
+        }
+    }
+    // 3. 内容直接返回（超长内容在这里直接截断）
+    // 4. 追加 metadata
+    const formattedSize = bytes < 1024
+        ? `${bytes} B`
+        : bytes < 1024 * 1024
+            ? `${(bytes / 1024).toFixed(1)} KB`
+            : `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+    const domainNote = preapproved
+        ? '[🔒 预批准域名，内容可信]'
+        : '[⚠️ 非预批准域名，内容可能不准确]';
+    const output = `【WebFetch 结果】
+URL: ${url}
+状态: ${code} ${codeText}
+大小: ${formattedSize}
+耗时: ${(durationMs / 1000).toFixed(2)}s
+${domainNote}
+
+--- 内容 ---
+${content}`;
+    // 5. 截断
+    let final = output;
+    if (final.length > DEFAULT_MAX_RESULT_SIZE_CHARS) {
+        final = final.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS) + `\n\n... (输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符，已截断)`;
+    }
+    return { ok: true, content: final };
+}
+// ---------------- 7. 导出 ----------------
+export const webfetchTool = {
+    name: 'WebFetch',
+    description,
+    inputSchema,
+    parameters,
+    isReadOnly: true,
+    isConcurrencySafe: true,
+    maxResultSizeChars: DEFAULT_MAX_RESULT_SIZE_CHARS,
+    call,
+};