minimal-agent 0.10.0 → 0.12.0

package/src/tools/webfetch/webfetch.ts

@@ -0,0 +1,383 @@
+/**
+ * ============================================================
+ *  src/tools/webfetch.ts —— WebFetch 工具（获取 URL 内容）
+ * ------------------------------------------------------------
+ *  获取 URL → HTML 转 Markdown
+ *
+ *  输入：
+ *    url    必填，要获取内容的 URL
+ *    prompt 可选，旧版兼容字段，运行时忽略
+ *
+ *  输出：
+ *    { bytes, code, codeText, result, durationMs, url }
+ *
+ *  流程：
+ *    1. URL 验证（合法格式、2000字符以内、无凭据）
+ *    2. fetch（自动 http→https，超时 60s）
+ *    3. 检测跨 host 重定向，返回提示而非自动跟随
+ *    4. HTML → Markdown（turndown）；非 HTML 原样返回
+ *    5. 输出截断到 30K chars（超长内容直接截断）
+ * ============================================================
+ */
+
+import { z } from 'zod';
+
+import type { Tool, ToolResult } from '../../types.js';
+import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
+import { toToolParameters } from '../../utils/zodToJson.js';
+import { isPreapprovedDomain } from './preapproved.js';
+
+// ---------------- 1. 常量 ----------------
+const MAX_URL_LENGTH = 2000;
+const FETCH_TIMEOUT_MS = 60_000;
+const MAX_HTTP_CONTENT_LENGTH = 10 * 1024 * 1024;
+const MAX_REDIRECTS = 10;
+
+// ---------------- 2. LRU 缓存 ----------------
+type CacheEntry = {
+  bytes: number;
+  code: number;
+  codeText: string;
+  content: string;
+  contentType: string;
+  fetchedAt: number;
+};
+
+const URL_CACHE = new Map<string, CacheEntry>();
+const CACHE_TTL_MS = 15 * 60 * 1000;
+const MAX_CACHE_SIZE_BYTES = 50 * 1024 * 1024;
+
+function cleanCache(): void {
+  const now = Date.now();
+  let totalSize = 0;
+  const entries: Array<{ url: string; entry: CacheEntry; size: number }> = [];
+
+  for (const [url, entry] of URL_CACHE) {
+    if (now - entry.fetchedAt > CACHE_TTL_MS) {
+      URL_CACHE.delete(url);
+      continue;
+    }
+    const size = entry.bytes;
+    totalSize += size;
+    entries.push({ url, entry, size });
+  }
+
+  if (totalSize > MAX_CACHE_SIZE_BYTES) {
+    entries.sort((a, b) => a.entry.fetchedAt - b.entry.fetchedAt);
+    for (const { url, size } of entries) {
+      URL_CACHE.delete(url);
+      totalSize -= size;
+      if (totalSize <= MAX_CACHE_SIZE_BYTES * 0.8) break;
+    }
+  }
+}
+
+// ---------------- 3. Zod 输入 schema ----------------
+const inputSchema = z.object({
+  url: z.string().describe('要获取内容的 URL'),
+  /** @deprecated Accepted for callers compiled against the legacy contract. */
+  prompt: z.string().optional(),
+});
+type WebFetchInput = z.infer<typeof inputSchema>;
+
+// Model-visible v2 contract. WebFetch returns the page body and does not run a
+// hidden extraction model; the runtime schema still accepts legacy `prompt`.
+const parameters = toToolParameters(z.object({
+  url: z.string().describe('要读取正文的已知 URL'),
+}));
+
+// ---------------- 4. Description ----------------
+const description = `Reads the full text content of a known URL without opening a browser.
+- Takes a URL, fetches its response body, and converts HTML to Markdown.
+- Returns page content directly; it does not call another model or perform hidden summarization or extraction.
+- Use it to inspect a selected source after search, or to read a known static page.
+- It does not discover sources, authenticate, click, upload, or operate an interactive page.
+- HTTP URLs are automatically upgraded to HTTPS.
+- When a URL redirects to a different host, returns a warning with the redirect URL.
+- This tool is read-only and does not modify any files.
+- Very large results may be truncated with an explicit marker.
+- ⚠️ IMPORTANT: This tool WILL FAIL for authenticated or private URLs.
+  Do not claim access to private content when the current tool catalog does not
+  contain an authenticated integration for that service.
+- If a research source cannot be fetched, choose another source or continue searching; report the limitation when no readable source remains.
+- 💡 For GitHub URLs (repos, issues, PRs), prefer using the \`gh\` CLI via Bash
+  instead (e.g. \`gh pr view <pr-number>\`, \`gh issue view <number>\`, \`gh api <endpoint>\`).
+- The preapproved domain list controls network routing only; it is not an
+  endorsement of page accuracy. Judge source quality from the source itself.
+- 📝 This tool includes a self-cleaning 15-minute cache for faster repeated access
+  to the same URL.
+- ⚠️ For PDF files, the tool will attempt to extract readable text but results may
+  be limited. Binary images cannot be processed.`;
+
+// ---------------- 5. 辅助函数 ----------------
+
+// ---------------- 6. call() 实现 ----------------
+
+/** 验证 URL */
+function validateURL(url: string): { ok: true; parsed: URL } | { ok: false; error: string } {
+  if (url.length > MAX_URL_LENGTH) {
+    return { ok: false, error: `URL 太长（最大 ${MAX_URL_LENGTH} 字符）` };
+  }
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return { ok: false, error: `无效的 URL 格式` };
+  }
+  if (parsed.username || parsed.password) {
+    return { ok: false, error: `URL 不能包含用户名或密码` };
+  }
+  return { ok: true, parsed };
+}
+
+/** 获取 URL 内容（返回 HTML 字符串或重定向信息） */
+async function fetchURL(
+  url: string,
+  signal: AbortSignal,
+  depth = 0,
+): Promise<
+  | { type: 'success'; content: string; bytes: number; code: number; codeText: string; contentType: string }
+  | { type: 'redirect'; originalUrl: string; redirectUrl: string; statusCode: number }
+  | { type: 'error'; error: string }
+> {
+  const validated = validateURL(url);
+  if (!validated.ok) return { type: 'error', error: validated.error };
+
+  let targetUrl = url;
+  const parsed = validated.parsed;
+
+  if (parsed.protocol === 'http:') {
+    parsed.protocol = 'https:';
+    targetUrl = parsed.toString();
+  }
+
+  let response: Response;
+  try {
+    const timeoutSignal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+    const controller = new AbortController();
+    const combinedSignal = signal
+      ? AbortSignal.any([signal, timeoutSignal])
+      : timeoutSignal;
+
+    response = await fetch(targetUrl, {
+      signal: combinedSignal,
+      headers: {
+        Accept: 'text/markdown, text/html, */*',
+        'User-Agent': 'minimal-agent/1.0',
+      },
+    });
+  } catch (e) {
+    if (signal?.aborted) return { type: 'error', error: '请求被中断' };
+    return { type: 'error', error: `网络请求失败：${(e as Error).message}` };
+  }
+
+  const code = response.status;
+  const codeText = response.statusText;
+  const contentType = response.headers.get('content-type') ?? '';
+
+  const location = response.headers.get('location');
+  if (location && [301, 302, 307, 308].includes(code)) {
+    const redirectUrl = new URL(location, targetUrl).toString();
+    const originalHost = new URL(url).hostname;
+    const redirectHost = new URL(redirectUrl).hostname;
+
+    const stripWww = (h: string) => h.replace(/^www\./, '');
+    if (stripWww(originalHost) !== stripWww(redirectHost)) {
+      return {
+        type: 'redirect',
+        originalUrl: url,
+        redirectUrl,
+        statusCode: code,
+      };
+    }
+
+    if (depth >= MAX_REDIRECTS) {
+      return { type: 'error', error: `重定向循环超过限制（最多 ${MAX_REDIRECTS} 次）` };
+    }
+
+    return fetchURL(redirectUrl, signal, depth + 1);
+  }
+
+  // 读取内容（限制 10MB）
+  let rawBuffer: ArrayBuffer;
+  try {
+    rawBuffer = await response.arrayBuffer();
+  } catch (e) {
+    return { type: 'error', error: `读取响应体失败：${(e as Error).message}` };
+  }
+
+  if (rawBuffer.byteLength > MAX_HTTP_CONTENT_LENGTH) {
+    return { type: 'error', error: `内容太大（${rawBuffer.byteLength} bytes，超过 ${MAX_HTTP_CONTENT_LENGTH}）` };
+  }
+
+  // 解码
+  let content: string;
+  try {
+    const decoder = new TextDecoder('utf-8', { fatal: false });
+    content = decoder.decode(rawBuffer);
+  } catch (e) {
+    return { type: 'error', error: `解码失败：${(e as Error).message}` };
+  }
+
+  return { type: 'success', content, bytes: rawBuffer.byteLength, code, codeText, contentType };
+}
+
+/** HTML 转 Markdown */
+async function htmlToMarkdown(html: string): Promise<string> {
+  // 动态 import turndown（~1.4MB，按需加载）
+  const TurndownService = (await import('turndown')).default;
+  const td = new TurndownService();
+  return td.turndown(html);
+}
+
+// ---------------- 6. call() 实现 ----------------
+async function call(
+  input: WebFetchInput,
+  signal?: AbortSignal,
+): Promise<ToolResult> {
+  const { url } = input;
+  const start = Date.now();
+  const cacheKey = url;
+
+  let targetHostname: string;
+  try {
+    targetHostname = new URL(url).hostname;
+  } catch {
+    targetHostname = '';
+  }
+  const preapproved = isPreapprovedDomain(targetHostname);
+
+  // 检查缓存
+  const cached = URL_CACHE.get(cacheKey);
+  if (cached && Date.now() - cached.fetchedAt <= CACHE_TTL_MS) {
+    let content = cached.content;
+    const contentType = cached.contentType;
+
+    if (contentType.includes('text/html')) {
+      try {
+        content = await htmlToMarkdown(content);
+      } catch (e) {
+        console.warn(`turndown 失败: ${(e as Error).message}`);
+      }
+    }
+
+    const formattedSize = cached.bytes < 1024
+      ? `${cached.bytes} B`
+      : cached.bytes < 1024 * 1024
+        ? `${(cached.bytes / 1024).toFixed(1)} KB`
+        : `${(cached.bytes / 1024 / 1024).toFixed(1)} MB`;
+
+    const domainNote = preapproved
+      ? '[网络访问：预批准域名]'
+      : '[网络访问：普通域名；请按来源本身评估证据质量]';
+
+    const output = `【WebFetch 结果】
+URL: ${url}
+状态: ${cached.code} ${cached.codeText}
+大小: ${formattedSize}
+耗时: 0.00s (缓存命中)
+${domainNote}
+[📦 来自缓存（15分钟 TTL）]
+
+--- 内容 ---
+${content}`;
+
+    let final = output;
+    if (final.length > DEFAULT_MAX_RESULT_SIZE_CHARS) {
+      final = final.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS) + `\n\n... (输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符，已截断)`;
+    }
+
+    return { ok: true, content: final };
+  }
+
+  // 1. fetch
+  const fetched = await fetchURL(url, signal ?? new AbortController().signal);
+  const durationMs = Date.now() - start;
+
+  if (fetched.type === 'redirect') {
+    const statusText =
+      fetched.statusCode === 301 ? 'Moved Permanently' :
+      fetched.statusCode === 308 ? 'Permanent Redirect' :
+      fetched.statusCode === 307 ? 'Temporary Redirect' : 'Found';
+
+    return {
+      ok: true,
+      content: `【重定向检测】
+
+原始 URL: ${fetched.originalUrl}
+重定向到: ${fetched.redirectUrl}
+状态: ${fetched.statusCode} ${statusText}
+
+请使用重定向后的 URL 再次调用 WebFetch 工具。`,
+    };
+  }
+
+  if (fetched.type === 'error') {
+    return { ok: false, error: fetched.error };
+  }
+
+  let { content, bytes, code, codeText, contentType } = fetched;
+
+  // 写入缓存
+  URL_CACHE.set(cacheKey, {
+    bytes,
+    code,
+    codeText,
+    content,
+    contentType,
+    fetchedAt: Date.now(),
+  });
+  cleanCache();
+
+  // 2. HTML → Markdown
+  if (contentType.includes('text/html')) {
+    try {
+      content = await htmlToMarkdown(content);
+    } catch (e) {
+      console.warn(`turndown 失败: ${(e as Error).message}`);
+    }
+  }
+
+  // 3. 内容直接返回（超长内容在这里直接截断）
+  // 4. 追加 metadata
+  const formattedSize = bytes < 1024
+    ? `${bytes} B`
+    : bytes < 1024 * 1024
+      ? `${(bytes / 1024).toFixed(1)} KB`
+      : `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+
+  const domainNote = preapproved
+    ? '[网络访问：预批准域名]'
+    : '[网络访问：普通域名；请按来源本身评估证据质量]';
+
+  const output = `【WebFetch 结果】
+URL: ${url}
+状态: ${code} ${codeText}
+大小: ${formattedSize}
+耗时: ${(durationMs / 1000).toFixed(2)}s
+${domainNote}
+
+--- 内容 ---
+${content}`;
+
+  // 5. 截断
+  let final = output;
+  if (final.length > DEFAULT_MAX_RESULT_SIZE_CHARS) {
+    final = final.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS) + `\n\n... (输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符，已截断)`;
+  }
+
+  return { ok: true, content: final };
+}
+
+// ---------------- 7. 导出 ----------------
+export const webfetchTool: Tool<WebFetchInput> = {
+  name: 'WebFetch',
+  description,
+  inputSchema,
+  parameters,
+  isReadOnly: true,
+  isConcurrencySafe: true,
+  maxResultSizeChars: DEFAULT_MAX_RESULT_SIZE_CHARS,
+  interruptBehavior: () => 'cancel',
+  call,
+};

package/src/tools/websearch/websearch.js

@@ -60,11 +60,14 @@ const inputSchema = z.object({
 // ---------------- 2. JSON Schema（由 Zod 自动派生） ----------------
 const parameters = toToolParameters(inputSchema);
 // ---------------- 3. Description（给 LLM 看的工具说明） ----------------
-const description = `- Searches the public web via the Tavily Search API and returns structured results.
-- Use this when you need up-to-date information that is not in your training data, or when the user asks for recent news / docs / API references.
-- Returns the top N results, each with a title, URL, and content snippet. With \`search_depth: "advanced"\` Tavily also returns a synthesized answer at the top.
-- Prefer specific natural-language queries over keyword soup (e.g. "how does Bun handle .env files in version 1.1").
-- Requires the TAVILY_API_KEY environment variable to be set; if missing the tool returns a friendly error.`;
+const description = `Searches the public web through Tavily and returns structured source candidates.
+- Use it to discover sources, verify current information, compare multiple sources, or locate current documentation.
+- It does not open or operate a browser. Use returned URLs with WebFetch when full source text is needed.
+- Results contain title, URL and a text snippet. They are discovery evidence, not a substitute for reading a high-value source in full.
+- Keep each query focused on one sub-question. When another search is needed, change the keywords, source, or angle instead of repeating the same query.
+- Add an explicit date or year only when recency matters; stable concepts and exact names should remain exact.
+- \`search_depth: "advanced"\` is slower but can return a synthesized answer; \`topic: "news"\` favors news sources.
+- Requires a configured Tavily API key.`;
 // ---------------- 4b. 退避重试 ----------------
 //
 // 为什么需要（与单发 fetch 零重试的旧行为对比）：WebSearch 是 isConcurrencySafe 工具，
@@ -187,14 +190,12 @@ async function call(input, signal, getApiKey = async () => (await configService.
                 '请在 KakadeAI 设置 → 工具 → 联网搜索中配置，或在 env 模式提供 TAVILY_API_KEY。',
         };
     }
-    // 5b. 自动追加当前日期到 query（强制时间限制，防止搜索到过时信息）
-    const today = new Date().toISOString().slice(0, 10);
-    const queryWithDate = input.query.includes(today)
-        ? input.query
-        : `${input.query} (as of ${today})`;
     const body = {
         api_key: apiKey,
-        query: queryWithDate,
+        // Preserve the exact research query. The prompt layer adds a date only
+        // when recency matters; unconditional mutation hurts exact names,
+        // historical topics, and stable technical documentation.
+        query: input.query,
         max_results: input.max_results ?? 5,
         search_depth: input.search_depth ?? 'basic',
         topic: input.topic ?? 'general',