@raolin2025/claude-code-node 2.8.1 → 2.8.3

package/src/security/redact.js

@@ -0,0 +1,76 @@
+/**
+ * 敏感数据自动脱敏
+ *
+ * 移植自 openclaw safe-jina-fetch 设计（DESIGN.md §4.4）。
+ * 对响应内容扫描常见敏感模式，命中即替换为 [REDACTED:类型]，并在 warnings 中告警。
+ * （原 Python 版只告警不脱敏，这里按 v2.0 设计升级为自动脱敏）
+ */
+
+/** 各类敏感模式的检测规则：{ 类型, 正则, 替换值 } */
+const REDACT_RULES = [
+  // API Key / Access Token / Auth Token（key= / token= 等 8 位以上值）
+  {
+    type: 'API-Key',
+    regex: /(key|token|api[_-]?key|access[_-]?token|auth[_-]?token|secret)=(['"]?)[A-Za-z0-9_\-./+]{8,}\2/gi,
+    value: '[REDACTED:API-Key]',
+  },
+  // Bearer Token（20 位以上）
+  {
+    type: 'Bearer',
+    regex: /Bearer\s+[A-Za-z0-9_\-.]{20,}/gi,
+    value: '[REDACTED:Bearer]',
+  },
+  // AWS Access Key（AKIA 开头 16 位）
+  {
+    type: 'AWS-Key',
+    regex: /AKIA[0-9A-Z]{16}/g,
+    value: '[REDACTED:AWS-Key]',
+  },
+  // AWS Secret Access Key
+  {
+    type: 'AWS-Secret',
+    regex: /aws_secret_access_key\s*=\s*['"]?[A-Za-z0-9/+=]{20,}/gi,
+    value: '[REDACTED:AWS-Secret]',
+  },
+  // 私钥
+  {
+    type: 'Private-Key',
+    regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
+    value: '[REDACTED:Private-Key]',
+  },
+  // OpenAI 风格（sk- + 20 位以上字母数字）
+  {
+    type: 'OpenAI-Key',
+    regex: /sk-[A-Za-z0-9]{20,}/g,
+    value: '[REDACTED:OpenAI-Key]',
+  },
+  // Slack Token（xoxb / xoxa / xoxp / xoxr）
+  {
+    type: 'Slack-Token',
+    regex: /xox[baprs]-[A-Za-z0-9-]{10,}/g,
+    value: '[REDACTED:Slack-Token]',
+  },
+]
+
+/**
+ * 对文本执行敏感数据脱敏
+ * @param {string} text
+ * @returns {{text: string, redacted: string[]}} — 脱敏后的文本 + 命中的类型列表（去重）
+ */
+export function redactSensitiveData(text) {
+  if (typeof text !== 'string' || !text) return { text: text || '', redacted: [] }
+
+  let result = text
+  const hitTypes = new Set()
+
+  for (const rule of REDACT_RULES) {
+    if (rule.regex.test(result)) {
+      hitTypes.add(rule.type)
+      // 重置 lastIndex（/g 正则 test 会改变 lastIndex）
+      rule.regex.lastIndex = 0
+      result = result.replace(rule.regex, rule.value)
+    }
+  }
+
+  return { text: result, redacted: [...hitTypes] }
+}

package/src/tools/web-fetch-providers.js

@@ -0,0 +1,140 @@
+/**
+ * WebFetch 兜底 Provider — Safe Jina Reader
+ *
+ * 移植自 openclaw safe-jina-fetch 设计（DESIGN.md §5）。
+ *
+ * 作用：当 WebFetch 直连目标 URL 失败（非 ok / 网络错误 / 超时）时，
+ * 自动经 Jina Reader（https://r.jina.ai/<url>）清洗后返回 Markdown，
+ * 专门对付反爬 / 403 / 动态渲染站点。
+ *
+ * Jina 请求模式（对齐 DESIGN.md §5.3）：
+ *   - CLI / 默认: text/plain + X-Return-Format: markdown → 首行 Title: xxx 提取标题
+ *   - json:        application/json → {title, url, content} 更稳健
+ *
+ * 凭据（三选一，可选——匿名约 20 RPM）：
+ *   1. 环境变量 JINA_API_KEY / JINA_READER_KEY
+ *   2. 配置 web.fetch.jinaApiKey
+ *   3. 匿名
+ */
+
+const JINA_READER_BASE = 'https://r.jina.ai'
+
+/**
+ * 从 Jina Reader 返回的 markdown 文本中提取标题（首行 "Title: xxx"）
+ * @param {string} markdown
+ * @returns {string}
+ */
+export function extractTitleFromMarkdown(markdown) {
+  const lines = (markdown || '').split('\n')
+  for (const line of lines) {
+    const m = line.match(/^\s*Title:\s*(.+)$/i)
+    if (m) return m[1].trim()
+  }
+  return ''
+}
+
+/**
+ * 解析 Jina 返回的 markdown，提取正文（去掉 Title/URL 元信息行）
+ * @param {string} markdown
+ * @returns {string}
+ */
+export function cleanJinaMarkdown(markdown) {
+  if (typeof markdown !== 'string') return ''
+  const lines = markdown.split('\n')
+
+  // 优先找 "Markdown Content:" 标记，从其后开始
+  const mcIdx = lines.findIndex(l => /^Markdown Content:/i.test(l.trim()))
+  if (mcIdx >= 0) return lines.slice(mcIdx + 1).join('\n').trim()
+
+  // 否则去掉开头的 Title: / URL Source: 元信息行
+  const bodyStart = lines.findIndex(l => !/^(Title:|URL Source:)/i.test(l.trim()))
+  if (bodyStart > 0) return lines.slice(bodyStart).join('\n').trim()
+
+  return markdown.trim()
+}
+
+/**
+ * 调用 Jina Reader 抓取 URL
+ * @param {string} url — 目标 URL
+ * @param {object} [options]
+ * @param {string} [options.apiKey] — Jina 凭据（可选，匿名可用）
+ * @param {'markdown'|'json'} [options.format] — 返回格式
+ * @param {number} [options.timeoutMs]
+ * @returns {Promise<{ok: boolean, text: string, title: string, finalUrl: string, status?: number, error?: string}>}
+ */
+export async function safeJinaFetch(url, options = {}) {
+  const { apiKey = '', format = 'markdown', timeoutMs = 30000 } = options
+
+  // 校验目标 URL 基本合法性（协议必须是 http/https）
+  let parsed
+  try {
+    parsed = new URL(url)
+  } catch {
+    return { ok: false, text: '', title: '', finalUrl: url, error: `无效的 URL: ${url}` }
+  }
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    return { ok: false, text: '', title: '', finalUrl: url, error: `不支持的协议：${parsed.protocol}` }
+  }
+
+  const jinaUrl = `${JINA_READER_BASE}/${url}`
+  const headers = {
+    'User-Agent': 'cc-node/2.8.2',
+    'Accept': format === 'json' ? 'application/json' : 'text/plain',
+  }
+  if (format === 'markdown') headers['X-Return-Format'] = 'markdown'
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+  try {
+    const response = await fetch(jinaUrl, { headers, signal: AbortSignal.timeout(timeoutMs) })
+
+    if (!response.ok) {
+      return {
+        ok: false,
+        text: '',
+        title: '',
+        finalUrl: jinaUrl,
+        status: response.status,
+        error: `Jina Reader HTTP ${response.status}`,
+      }
+    }
+
+    if (format === 'json') {
+      const data = await response.json()
+      return {
+        ok: true,
+        text: typeof data.content === 'string' ? data.content : '',
+        title: typeof data.title === 'string' ? data.title : '',
+        finalUrl: jinaUrl,
+        status: 200,
+      }
+    }
+
+    const markdown = await response.text()
+    return {
+      ok: true,
+      text: cleanJinaMarkdown(markdown),
+      title: extractTitleFromMarkdown(markdown),
+      finalUrl: jinaUrl,
+      status: 200,
+    }
+  } catch (err) {
+    if (err.name === 'TimeoutError') {
+      return { ok: false, text: '', title: '', finalUrl: jinaUrl, error: 'Jina Reader 请求超时' }
+    }
+    return { ok: false, text: '', title: '', finalUrl: jinaUrl, error: `Jina Reader 错误: ${err.message}` }
+  }
+}
+
+/**
+ * 解析 Jina 凭据（三选一：环境变量 > 配置 > 匿名）
+ * @param {object} [config] — Config 实例（可选）
+ * @returns {string} apiKey（可能为空 = 匿名）
+ */
+export function resolveJinaApiKey(config = null) {
+  return (
+    process.env.JINA_API_KEY ||
+    process.env.JINA_READER_KEY ||
+    (config ? config.get('web.fetch.jinaApiKey') || '' : '') ||
+    ''
+  )
+}

package/src/tools/web-fetch.js

@@ -1,11 +1,31 @@
 /**
- * WebFetch 工具 — 抓取网页内容
- * 对应原版: src/tools/WebFetchTool/
+ * WebFetch 工具 — 抓取网页内容（安全管道 + Jina 兜底）
+ *
+ * 安全特性（移植自 safe-jina-fetch 设计）：
+ *  - 协议白名单：仅 http/https，file/ftp/data 拒绝
+ *  - 连接级 SSRF 防护：TCP 连接时逐地址校验（防 DNS rebinding）
+ *  - 重定向逐跳校验：最多 5 跳，每跳重新校验
+ *  - 响应大小上限（10MB）、超时（30s）、强制 SSL
+ *  - 敏感数据自动脱敏
+ *
+ * 兜底机制：
+ *  - 直连成功（2xx）→ 返回清洗后的文本/JSON
+ *  - 直连失败（非 2xx / 网络错误 / 超时）→ 自动经 Jina Reader 清洗后返回 Markdown
+ *  - 直连 200 但正文过短（< 200 字符，疑似 JS 挑战页）→ 也触发 Jina 兜底
+ *
+ * extractMode 参数：
+ *  - auto   （默认）直连优先，失败/异常走 Jina 兜底
+ *  - direct 强制直连，不兜底
+ *  - jina   强制直接走 Jina Reader（最干净，永远清洗）
  */
 import { ToolDef } from '../types/index.js'
-import { checkUrlSafety } from '../security/ssrf-guard.js'
+import { safeFetchWithRedirects, DEFAULT_FETCH_OPTIONS } from '../security/fetch-guard.js'
+import { redactSensitiveData } from '../security/redact.js'
+import { safeJinaFetch, resolveJinaApiKey } from './web-fetch-providers.js'
 
-const MAX_FETCH_CHARS = 100000
+const DEFAULT_MAX_CHARS = 100000
+// 正文过短阈值：低于此字符数视为疑似 JS 挑战页，触发 Jina 兜底（对齐 DESIGN.md §8 增强）
+const MIN_BODY_CHARS = 200
 
 /**
  * 简单的 HTML → 纯文本转换
@@ -35,7 +55,7 @@ function htmlToText(html) {
   return text
 }
 
-const VERSION = '2.0.0'
+const VERSION = '2.1.0'
 
 export const webFetchTool = new ToolDef(
   'WebFetch',
@@ -43,7 +63,9 @@ export const webFetchTool = new ToolDef(
 Usage:
 - url must be a valid HTTP/HTTPS URL
 - Returns the page content as cleaned text/markdown
-- Supports HTML pages, plain text, and JSON APIs`,
+- Supports HTML pages, plain text, and JSON APIs
+- If direct fetch fails (403/anti-crawl/network error), automatically falls back to Jina Reader for a clean Markdown version
+- extractMode: auto (direct first, fallback on failure) | direct (force direct only) | jina (always use Jina Reader)`,
   {
     type: 'object',
     properties: {
@@ -56,72 +78,104 @@ Usage:
         enum: ['text', 'json', 'raw'],
         description: 'Output format: text (cleaned HTML), json (parse as JSON), raw (raw response)',
       },
+      extractMode: {
+        type: 'string',
+        enum: ['auto', 'direct', 'jina'],
+        description: 'Extraction mode: auto (default, direct first + Jina fallback), direct (force direct), jina (always Jina Reader)',
+      },
     },
     required: ['url'],
   },
   async (input, ctx) => {
-    const { url, format = 'text' } = input
+    const { url, format = 'text', extractMode = 'auto' } = input
+
+    const config = ctx?.engine?.config?.configStore || null
+    const maxChars = config?.get?.('web.fetch.maxChars') || DEFAULT_MAX_CHARS
+    const maxBytes = config?.get?.('web.fetch.maxBytes') || DEFAULT_FETCH_OPTIONS.maxBytes
+    const timeoutMs = config?.get?.('web.fetch.timeoutMs') || DEFAULT_FETCH_OPTIONS.timeoutMs
+    const maxRedirects = config?.get?.('web.fetch.maxRedirects') || DEFAULT_FETCH_OPTIONS.maxRedirects
+    const jinaApiKey = resolveJinaApiKey(config)
 
-    // SSRF 安全检查
-    const urlSafety = await checkUrlSafety(url)
-    if (!urlSafety.allowed) {
-      return `[🚫 URL 被安全策略阻止]\n${urlSafety.reason}`
+    // extractMode=jina：强制直接走 Jina（永远清洗）
+    if (extractMode === 'jina') {
+      const jr = await safeJinaFetch(url, { apiKey: jinaApiKey, format: 'markdown', timeoutMs })
+      if (!jr.ok) return `[Error: ${jr.error}]`
+      const { text, redacted } = redactSensitiveData(jr.text)
+      const out = (jr.title ? `# ${jr.title}\n\n` : '') + text
+      const warning = redacted.length ? `\n\n[⚠️ 已脱敏: ${redacted.join(', ')}]` : ''
+      return (out + warning).slice(0, maxChars)
     }
 
-    try {
-      const response = await fetch(url, {
-        headers: {
-          'User-Agent': `ClaudeCode-Node/${VERSION}`,
-          'Accept': 'text/html,application/json,text/plain,*/*',
-        },
-        signal: AbortSignal.timeout(30000),
-      })
-
-      if (!response.ok) {
-        return `[HTTP ${response.status} ${response.statusText}]`
-      }
+    // 直连（安全管道：协议 + 连接级 SSRF + 重定向逐跳校验 + 大小/超时/SSL）
+    const result = await safeFetchWithRedirects(url, { timeoutMs, maxBytes, maxRedirects })
 
-      const contentType = response.headers.get('content-type') || ''
-      const body = await response.text()
-
-      // JSON 格式
-      if (format === 'json' || contentType.includes('application/json')) {
-        try {
-          const data = JSON.parse(body)
-          const formatted = JSON.stringify(data, null, 2)
-          return formatted.length > MAX_FETCH_CHARS
-            ? formatted.slice(0, MAX_FETCH_CHARS) + '\n[...truncated]'
-            : formatted
-        } catch {
-          return body.slice(0, MAX_FETCH_CHARS)
-        }
-      }
+    // extractMode=direct：不兜底，直接返回直连结果（无论成败）
+    if (extractMode === 'direct') {
+      return formatDirectResult(result, format, maxChars)
+    }
 
-      // 原始格式
-      if (format === 'raw') {
-        return body.length > MAX_FETCH_CHARS
-          ? body.slice(0, MAX_FETCH_CHARS) + '\n[...truncated]'
-          : body
+    // auto 模式：直连失败或内容过短 → 走 Jina 兜底
+    const failed = !result.ok || result.error
+    const tooShort = result.ok && result.body.trim().length < MIN_BODY_CHARS
+    if (failed || tooShort) {
+      if (config?.get?.('verbose')) {
+        console.error(`[web-fetch] direct ${failed ? `failed (${result.error || result.status})` : 'content too short'}, falling back to Jina Reader`)
       }
-
-      // HTML → 文本
-      if (contentType.includes('text/html')) {
-        const text = htmlToText(body)
-        return text.length > MAX_FETCH_CHARS
-          ? text.slice(0, MAX_FETCH_CHARS) + '\n[...truncated]'
-          : text
+      const jr = await safeJinaFetch(url, { apiKey: jinaApiKey, format: 'markdown', timeoutMs })
+      if (jr.ok) {
+        const { text, redacted } = redactSensitiveData(jr.text)
+        const out = (jr.title ? `# ${jr.title}\n\n` : '') + text
+        const warning = redacted.length ? `\n\n[⚠️ 已脱敏: ${redacted.join(', ')}]` : ''
+        return (out + warning).slice(0, maxChars)
       }
-
-      // 纯文本
-      return body.length > MAX_FETCH_CHARS
-        ? body.slice(0, MAX_FETCH_CHARS) + '\n[...truncated]'
-        : body
-    } catch (err) {
-      if (err.name === 'TimeoutError') {
-        return `[Error: Request timed out after 30s]`
-      }
-      return `[Error fetching URL: ${err.message}]`
+      // Jina 也失败 — 返回直连结果 + Jina 错误说明
+      return formatDirectResult(result, format, maxChars) +
+        `\n\n[Jina fallback also failed: ${jr.error}]`
     }
+
+    // 直连成功 — 返回直连结果
+    return formatDirectResult(result, format, maxChars)
   },
   'ask'
 )
+
+/**
+ * 格式化直连结果（含脱敏）
+ * @param {object} result — safeFetchWithRedirects 返回值
+ * @param {string} format
+ * @param {number} maxChars
+ */
+function formatDirectResult(result, format, maxChars) {
+  if (!result.ok) {
+    if (result.error) return `[Error: ${result.error}]`
+    return `[HTTP ${result.status} ${result.statusText || ''}]`.trim()
+  }
+
+  const contentType = result.headers['content-type'] || ''
+  const body = result.body
+
+  let out
+  if (format === 'json' || contentType.includes('application/json')) {
+    try {
+      const data = JSON.parse(body)
+      out = JSON.stringify(data, null, 2)
+    } catch {
+      out = body
+    }
+  } else if (format === 'raw') {
+    out = body
+  } else if (contentType.includes('text/html')) {
+    out = htmlToText(body)
+  } else {
+    out = body
+  }
+
+  // 敏感数据脱敏
+  const { text, redacted } = redactSensitiveData(out)
+  let final = text
+  if (result.truncated) final += '\n[...truncated]'
+  if (redacted.length) final += `\n[⚠️ 已脱敏: ${redacted.join(', ')}]`
+  if (result.redirects?.length) final += `\n[重定向: ${result.redirects.join(' → ')}]`
+
+  return final.length > maxChars ? final.slice(0, maxChars) + '\n[...truncated]' : final
+}