deepfish-ai 1.0.28 → 2.0.2

package/src/AgentRobot/BaseAgentRobot/tools/WebTools.js

@@ -1,257 +0,0 @@
-const axios = require('axios')
-const cheerio = require('cheerio')
-const puppeteer = require('puppeteer')
-
-function ok(data = null) {
-  return { success: true, data }
-}
-
-function fail(error, data = null) {
-  return { success: false, error: error?.message || String(error), data }
-}
-
-function normalizeWhitespace(text = '') {
-  return String(text || '').replace(/\s+/g, ' ').trim()
-}
-
-function absoluteUrl(baseUrl, href = '') {
-  try {
-    return new URL(href, baseUrl).toString()
-  } catch {
-    return href
-  }
-}
-
-function extractByCheerio(html = '', url = '') {
-  const $ = cheerio.load(html)
-  $('script, style, noscript').remove()
-
-  const title = normalizeWhitespace($('title').first().text())
-  const bodyText = normalizeWhitespace($('body').text())
-  const metaDescription = normalizeWhitespace(
-    $('meta[name="description"]').attr('content') ||
-      $('meta[property="og:description"]').attr('content') ||
-      '',
-  )
-
-  const links = []
-  $('a[href]').each((_, el) => {
-    const href = $(el).attr('href') || ''
-    const text = normalizeWhitespace($(el).text())
-    if (!href) return
-    links.push({
-      href: absoluteUrl(url, href),
-      text,
-    })
-  })
-
-  return {
-    title,
-    description: metaDescription,
-    content: bodyText,
-    links,
-  }
-}
-
-async function fetchStatic(url, timeout = 15000) {
-  const response = await axios.get(url, {
-    timeout,
-    headers: {
-      'User-Agent':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130 Safari/537.36 DeepFish-MCP-Web/1.0',
-      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    },
-  })
-  return response.data
-}
-
-async function fetchDynamic(url, waitUntil = 'networkidle2', timeout = 30000) {
-  const browser = await puppeteer.launch({
-    args: ['--no-sandbox', '--disable-setuid-sandbox'],
-  })
-  try {
-    const page = await browser.newPage()
-    await page.goto(url, { waitUntil, timeout })
-    return await page.content()
-  } finally {
-    await browser.close()
-  }
-}
-
-async function mcpBrowseWebpage(url, mode = 'auto', maxChars = 4000) {
-  try {
-    if (!url) {
-      return fail('url is required')
-    }
-
-    let html = ''
-    let resolvedMode = mode
-
-    if (mode === 'dynamic') {
-      html = await fetchDynamic(url)
-    } else if (mode === 'static') {
-      html = await fetchStatic(url)
-    } else {
-      try {
-        html = await fetchStatic(url)
-        resolvedMode = 'static'
-      } catch {
-        html = await fetchDynamic(url)
-        resolvedMode = 'dynamic'
-      }
-    }
-
-    const parsed = extractByCheerio(html, url)
-    const content = parsed.content.slice(0, Number(maxChars) > 0 ? Number(maxChars) : 4000)
-    const links = parsed.links.slice(0, 50)
-
-    return ok({
-      url,
-      mode: resolvedMode,
-      title: parsed.title,
-      description: parsed.description,
-      content,
-      contentLength: parsed.content.length,
-      links,
-      linkCount: parsed.links.length,
-    })
-  } catch (error) {
-    return fail(error, { url, mode, maxChars })
-  }
-}
-
-async function mcpFetchWebpageByQuery(url, query, mode = 'auto', limit = 20) {
-  try {
-    if (!url) {
-      return fail('url is required')
-    }
-    if (!query) {
-      return fail('query is required')
-    }
-
-    const browseResult = await mcpBrowseWebpage(url, mode, 200000)
-    if (!browseResult.success) {
-      return browseResult
-    }
-
-    const keyword = String(query).toLowerCase().trim()
-    const rawContent = browseResult.data.content || ''
-    const sentences = rawContent
-      .split(/[。！？.!?\n]/)
-      .map((item) => normalizeWhitespace(item))
-      .filter(Boolean)
-
-    const matchedSentences = sentences
-      .map((text) => {
-        const lower = text.toLowerCase()
-        let score = 0
-        let fromIndex = 0
-        while (true) {
-          const idx = lower.indexOf(keyword, fromIndex)
-          if (idx < 0) break
-          score += 1
-          fromIndex = idx + keyword.length
-        }
-        return { text, score }
-      })
-      .filter((item) => item.score > 0)
-      .sort((a, b) => b.score - a.score)
-      .slice(0, Number(limit) > 0 ? Number(limit) : 20)
-
-    const matchedLinks = (browseResult.data.links || [])
-      .filter((item) => {
-        const href = String(item.href || '').toLowerCase()
-        const text = String(item.text || '').toLowerCase()
-        return href.includes(keyword) || text.includes(keyword)
-      })
-      .slice(0, Number(limit) > 0 ? Number(limit) : 20)
-
-    return ok({
-      url,
-      query,
-      mode: browseResult.data.mode,
-      title: browseResult.data.title,
-      matchedSentenceCount: matchedSentences.length,
-      matchedLinkCount: matchedLinks.length,
-      matchedSentences,
-      matchedLinks,
-    })
-  } catch (error) {
-    return fail(error, { url, query, mode, limit })
-  }
-}
-
-const descriptions = [
-  {
-    type: 'function',
-    function: {
-      name: 'mcpBrowseWebpage',
-      description:
-        'MCP网页浏览: 打开并抓取任意网页的标题、摘要正文与链接列表。适合快速浏览网页内容。',
-      parameters: {
-        type: 'object',
-        properties: {
-          url: {
-            type: 'string',
-            description: '要浏览的网页地址（http或https）。',
-          },
-          mode: {
-            type: 'string',
-            description: '抓取模式：auto|static|dynamic。默认auto。',
-          },
-          maxChars: {
-            type: 'number',
-            description: '返回正文最大字符数，默认4000。',
-          },
-        },
-        required: ['url'],
-      },
-    },
-  },
-  {
-    type: 'function',
-    function: {
-      name: 'mcpFetchWebpageByQuery',
-      description:
-        'MCP网页抓取: 按关键词从网页正文和链接中提取匹配内容，返回高相关片段。',
-      parameters: {
-        type: 'object',
-        properties: {
-          url: {
-            type: 'string',
-            description: '目标网页地址（http或https）。',
-          },
-          query: {
-            type: 'string',
-            description: '要匹配的关键词。',
-          },
-          mode: {
-            type: 'string',
-            description: '抓取模式：auto|static|dynamic。默认auto。',
-          },
-          limit: {
-            type: 'number',
-            description: '返回匹配结果上限，默认20。',
-          },
-        },
-        required: ['url', 'query'],
-      },
-    },
-  },
-]
-
-const functions = {
-  mcpBrowseWebpage,
-  mcpFetchWebpageByQuery,
-}
-
-const MCPWebTool = {
-  name: 'MCPWebTool',
-  description: '提供网页浏览与内容抓取能力，支持任意网页的读取与关键词提取。',
-  platform: 'all',
-  descriptions,
-  functions,
-  isSystem: true
-}
-
-module.exports = MCPWebTool

package/src/AgentRobot/BaseAgentRobot/utils/AIRequest.js

@@ -1,297 +0,0 @@
-const { OpenAI } = require('openai')
-const {
-  refreshGithubModelsTokenIfNeeded,
-  buildDefaultHeaders,
-} = require('./copilot.js')
-
-function normalizeAiRequestConfig(aiConfig = {}) {
-  const max_tokens = (!aiConfig.maxTokens || aiConfig.maxTokens === -1)
-    ? undefined
-    : aiConfig.maxTokens * 1024 // 兼容旧配置，按 1KB ~= 1024 tokens 估算
-
-  const requestConfig = {
-    model: aiConfig.model,
-    temperature: aiConfig.temperature,
-    stream: aiConfig.stream,
-    max_tokens,
-  }
-
-  // 仅 DeepSeek 兼容端注入扩展字段，避免其它网关因未知参数报错
-  if (aiConfig.type === 'deepseek') {
-    requestConfig.extra_body = { reasoning_split: true }
-  }
-
-  return requestConfig
-}
-
-function creatClient(aiConfig) {
-  if (aiConfig.type === 'github-models') {
-    if (!aiConfig.apiKey || !String(aiConfig.apiKey).trim()) {
-      throw new Error(
-        'GitHub Models requires apiKey. Please set a GitHub token with models:read permission in your current AI config.',
-      )
-    }
-  }
-  if (aiConfig.type === 'copilot') {
-    if (!aiConfig.apiKey || !String(aiConfig.apiKey).trim()) {
-      throw new Error(
-        'GitHub Copilot requires apiKey. Please set a GitHub Copilot access token in your current AI config.',
-      )
-    }
-  }
-
-  return new OpenAI({
-    baseURL: aiConfig.baseUrl,
-    apiKey: aiConfig.apiKey || '',
-    defaultHeaders: buildDefaultHeaders(aiConfig),
-  })
-}
-
-/**
- * Ai单轮问答
- * @param {*} openAiClient OpenAI客户端
- * @param {*} aiConfig {model, temperature, maxTokens, stream}
- * @param {*} systemDescription
- * @param {*} prompt
- * @param {*} temperature
- * @returns
- */
-async function think(
-  openAiClient,
-  aiConfig,
-  messages,
-  thinkBefore = () => {},
-  thinkAfter = () => {},
-  streamThinkOutput,
-  streamContentOutput,
-  streamToolCallsOutput,
-  streamEnd,
-) {
-  try {
-    await refreshGithubModelsTokenIfNeeded(aiConfig)
-    const requestClient = creatClient(aiConfig)
-    const requestConfig = normalizeAiRequestConfig(aiConfig)
-    const opt = {
-      messages: messages,
-      ...requestConfig,
-    }
-    await thinkBefore()
-    const extraHeaders = aiConfig.type === 'copilot' ? { 'x-initiator': 'user' } : undefined
-    const response = await requestClient.chat.completions.create(opt, extraHeaders ? { headers: extraHeaders } : undefined)
-    if (aiConfig.stream) {
-      const messageRes = await _streamToNonStream(
-        response,
-        streamThinkOutput,
-        streamContentOutput,
-        streamToolCallsOutput,
-        streamEnd,
-      )
-      await thinkAfter()
-      return clearThinkTag(messageRes.choices[0].message.content)
-    }
-    await thinkAfter()
-    return clearThinkTag(response.choices[0].message.content)
-  } catch (error) {
-    throw new Error(`AI response error: ${error.message}`)
-  }
-}
-/**
- * Ai携带工具请求
- * @param {*} openAiClient
- * @param {*} aiConfig {model, temperature, maxTokens}
- * @param {*} messages
- * @param {*} functionDescriptions
- * @returns
- */
-async function thinkByTool(
-  openAiClient,
-  aiConfig,
-  messages,
-  functionDescriptions,
-  thinkBefore = () => {},
-  thinkAfter = () => {},
-  streamThinkOutput,
-  streamContentOutput,
-  streamToolCallsOutput,
-  streamEnd,
-) {
-  try {
-    await refreshGithubModelsTokenIfNeeded(aiConfig)
-    const requestClient = creatClient(aiConfig)
-    const requestConfig = normalizeAiRequestConfig(aiConfig)
-    await thinkBefore()
-    const extraHeaders = aiConfig.type === 'copilot' ? { 'x-initiator': 'agent' } : undefined
-    const response = await requestClient.chat.completions.create(
-      {
-        messages: messages,
-        tools: functionDescriptions,
-        tool_choice: 'auto',
-        ...requestConfig,
-      },
-      extraHeaders ? { headers: extraHeaders } : undefined,
-    )
-    if (aiConfig.stream) {
-      const messageRes = await _streamToNonStream(
-        response,
-        streamThinkOutput,
-        streamContentOutput,
-        streamToolCallsOutput,
-        streamEnd,
-      )
-      await thinkAfter()
-      messageRes.choices[0].message.content = clearThinkTag(messageRes.choices[0].message.content)
-      return {
-        content: messageRes.choices[0].message.content,
-        tool_calls: messageRes.choices[0].message.tool_calls,
-        message: messageRes.choices[0].message,
-      }
-    }
-    await thinkAfter()
-    response.choices[0].message.content = clearThinkTag(response.choices[0].message.content)
-    return {
-      content: response.choices[0].message.content,
-      tool_calls: response.choices[0].message.tool_calls,
-      message: response.choices[0].message,
-    }
-  } catch (error) {
-    throw new Error(`AI response error: ${error.message}`)
-  }
-}
-
-// 清除字符串中<think></think>标签以及之间的内容
-function clearThinkTag(content) {
-  return content.replace(/<think>[\s\S]*?<\/think>/g, '')
-}
-
-// 流式输出结果转非流式输出
-async function _streamToNonStream(
-  stream,
-  streamThinkOutput = () => {},
-  streamContentOutput = () => {},
-  streamToolCallsOutput = () => {},
-  streamEnd = () => {},
-) {
-  // 初始化最终响应结构（对齐 OpenAI 非流式响应格式）
-  const finalResponse = {
-    id: '',
-    object: 'chat.completion',
-    created: Math.floor(Date.now() / 1000), // 生成时间戳
-    model: '',
-    choices: [
-      {
-        index: 0,
-        message: {
-          role: 'assistant',
-          content: '',
-          reasoning_content: '',
-          tool_calls: [], // 存储完整的工具调用列表
-        },
-        finish_reason: null,
-        logprobs: null,
-      },
-    ],
-    usage: {
-      prompt_tokens: 0,
-      completion_tokens: 0,
-      total_tokens: 0,
-    },
-  }
-
-  // 工具调用缓冲区：处理多工具调用 + 分段参数拼接
-  const toolCallBuffers = new Map() // key: tool_call_id, value: toolCall object
-  const toolCallIndexMap = new Map() // key: index, value: tool_call_id
-  try {
-    // 遍历所有流式数据块
-    for await (const chunk of stream) {
-      // 1. 填充全局信息（仅首次获取）
-      if (!finalResponse.id) {
-        finalResponse.id = chunk.id || `chatcmpl-${Date.now()}`
-      }
-      if (!finalResponse.model) {
-        finalResponse.model = chunk.model || 'deepseek-reasoner'
-      }
-      const choice = chunk.choices[0]
-      if (!choice) {
-        continue
-      }
-      const delta = choice.delta
-      if (!delta) {
-        continue
-      }
-      // 2. 处理普通文本内容
-      const reasoning_content = delta.reasoning_content
-      if (reasoning_content) {
-        if (finalResponse.choices[0].message.reasoning_content.length === 0) {
-          streamThinkOutput('[Thinking...]')
-        }
-        finalResponse.choices[0].message.reasoning_content += reasoning_content
-        // 流式输出
-        streamThinkOutput(reasoning_content)
-      }
-      const content = delta.content
-      if (content) {
-        if (finalResponse.choices[0].message.content.length === 0) {
-          streamContentOutput('\r\n')
-        }
-        finalResponse.choices[0].message.content += content
-        // 流式输出
-        streamContentOutput(content)
-      }
-      // 3. 处理工具调用（核心逻辑）
-      if (delta.tool_calls && delta.tool_calls.length > 0) {
-        delta.tool_calls.forEach((toolCallChunk) => {
-          const index = toolCallChunk.index
-          if (toolCallChunk.id) {
-            const id = toolCallChunk.id
-            toolCallIndexMap.set(index, id)
-            let toolCall = toolCallBuffers.get(id)
-            if (!toolCall) {
-              toolCall = {
-                id: id,
-                type: toolCallChunk.type || 'function',
-                function: {
-                  name: toolCallChunk.function.name,
-                  arguments: '',
-                },
-              }
-              toolCallBuffers.set(id, toolCall)
-            }
-          } else {
-            const id = toolCallIndexMap.get(index)
-            const toolCall = toolCallBuffers.get(id)
-            if (toolCall && toolCallChunk.function?.arguments) {
-              if (toolCall.function.arguments.length === 0) {
-                streamToolCallsOutput(`\r\n[ToolCalls...][${toolCall.function.name}]`)
-              }
-              toolCall.function.arguments += toolCallChunk.function.arguments
-              streamToolCallsOutput(toolCallChunk.function.arguments)
-            }
-          }
-        })
-      }
-
-      // 4. 处理结束标记
-      if (choice.finish_reason) {
-        finalResponse.choices[0].finish_reason = choice.finish_reason
-
-        // 工具调用结束：将缓冲区数据写入最终响应
-        if (choice.finish_reason === 'tool_calls' && toolCallBuffers.size > 0) {
-          finalResponse.choices[0].message.content = '' // 工具调用时 content 为 null
-          finalResponse.choices[0].message.tool_calls = Array.from(
-            toolCallBuffers.values(),
-          )
-        } else {
-          finalResponse.choices[0].message.tool_calls = undefined
-        }
-      }
-    }
-    streamEnd()
-    return finalResponse
-  } catch (error) {
-    console.error('流式数据转换失败：', error.message)
-    throw error // 抛出错误让上层处理
-  }
-}
-
-
-module.exports = { creatClient, think, thinkByTool }

package/src/AgentRobot/BaseAgentRobot/utils/AIToolManager.js

@@ -1,131 +0,0 @@
-const path = require('path')
-const os = require('os')
-const fs = require('fs-extra')
-const dayjs = require('dayjs')
-const axios = require('axios')
-const echarts = require('echarts')
-const canvas = require('canvas')
-const cheerio = require('cheerio')
-const puppeteer = require('puppeteer')
-const lodash = require('lodash')
-const { AttachmentToolScanner } = require('./AttachmentToolScanner')
-
-class AIToolManager {
-  originalTools = null // 原装工具
-  attachTools = null // 附加工具, Agent后续安装的工具函数
-  functions = {} // key为函数名称，value为方法体
-  descriptions = [] // openai能识别的描述
-  tools = [] // 工具列表
-  toolCollection = null // 工具集合
-  clawSkillCollection = null // Claw技能集合
-  constructor(agentRobot) {
-    this.agentRobot = agentRobot
-    this.initTools(agentRobot.opt)
-  }
-  initTools(opt) {
-    this.originalTools = this._getOriginalTools() // 天赋技能
-    this.attachTools = opt.attachTools || [] // 附加工具, Agent后续安装的工具函数
-    const tools = [...this.originalTools, ...this.attachTools]
-    tools.forEach((tool) => {
-      this._addTool(tool)
-    })
-    Object.assign(this.functions, {
-      fs,
-      axios,
-      dayjs,
-      lodash,
-      canvas,
-      echarts,
-      cheerio,
-      puppeteer,
-    })
-
-    this.functions.agentRobot = this.agentRobot
-    this.functions.Tools = this.functions
-    // 兼容老版本
-    this.functions.aiCli = {
-      Tools: this.functions,
-    }
-    // 外部工具扫描
-    this.toolCollection = AttachmentToolScanner.getToolCollection(
-      this.agentRobot.workspace,
-      this.agentRobot.basespace,
-    ) // 加载工具集合
-    this.clawSkillCollection = AttachmentToolScanner.getClawSkillCollection(
-      this.agentRobot.basespace,
-    ) // 加载Claw技能集合
-    this.agentRobot.toolCollection = this.toolCollection
-    this.agentRobot.clawSkillCollection = this.clawSkillCollection
-  }
-
-  // 动态添加工具
-  addTool(toolName) {
-    let tool = this.tools.find((t) => t.name === toolName)
-    if (!tool) {
-      tool = this.toolCollection.find((t) => t.name === toolName)
-      if (tool) {
-        this._addTool(tool)
-      }
-    }
-  }
-
-  _addTool(tool) {
-    const platform = tool.platform || 'all'
-    if (platform === 'all' || platform === process.platform) {
-      tool.descriptions = tool.descriptions.map((item) => {
-        if (!item.type) {
-          item = {
-            type: 'function',
-            function: item,
-          }
-        }
-        return item
-      })
-      tool.isSystem = true
-      if (tool.name && !tool.isSystem) {
-        for (const funcName in tool.functions) {
-          if (!funcName.includes('_')) {
-            this.functions[`${tool.name}_${funcName}`] =
-              tool.functions[funcName]
-          } else {
-            this.functions[funcName] = tool.functions[funcName]
-          }
-        }
-        const descriptions = tool.descriptions.map((item) => {
-          if (
-            tool.name &&
-            item.function.name &&
-            !item.function.name.includes('_')
-          ) {
-            item.function.name = `${tool.name}_${item.function.name}`
-            return item
-          } else {
-            return item
-          }
-        })
-        this.descriptions.push(...descriptions)
-      } else {
-        Object.assign(this.functions, tool.functions)
-        this.descriptions.push(...tool.descriptions)
-      }
-      this.tools.push(tool)
-    }
-  }
-
-  // 获取原装工具
-  _getOriginalTools() {
-    // 自动扫描tools目录
-    const toolsPath = path.join(__dirname, '../tools')
-    const toolFiles = fs.readdirSync(toolsPath).filter((file) => {
-      return file.endsWith('.js') || file.endsWith('.cjs')
-    })
-    const tools = []
-    toolFiles.forEach((file) => {
-      const tool = require(path.join(toolsPath, file))
-      tools.push(tool)
-    })
-    return tools
-  }
-}
-
-module.exports = AIToolManager