deepspider 0.2.11 → 0.3.0

package/README.md

@@ -37,50 +37,91 @@ cp .env.example .env  # 配置环境变量
 pnpm run setup:crypto  # 安装 Python 加密库（可选）
 ```
 
-安装完成后，首次运行会提示配置环境变量（LLM API）。
+安装完成后，首次运行会提示配置 LLM API。
 
 > **注意**: 项目依赖 `isolated-vm` 原生模块，需要 C++ 编译环境：
 > - macOS: `xcode-select --install`
 > - Ubuntu: `sudo apt install build-essential`
 > - Windows: 安装 [Visual Studio Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
 
-### 环境变量配置
+### 配置
 
 DeepSpider 需要配置 LLM API 才能运行。支持任何兼容 OpenAI 格式的供应商。
 
-| 变量名 | 必填 | 说明 |
-|--------|------|------|
-| `DEEPSPIDER_API_KEY` | 是 | API 密钥 |
-| `DEEPSPIDER_BASE_URL` | 是 | API 地址 |
-| `DEEPSPIDER_MODEL` | 是 | 模型名称 |
+| 配置键 | 环境变量 | 说明 |
+|--------|----------|------|
+| `apiKey` | `DEEPSPIDER_API_KEY` | API 密钥 |
+| `baseUrl` | `DEEPSPIDER_BASE_URL` | API 地址 |
+| `model` | `DEEPSPIDER_MODEL` | 模型名称 |
 
-**常用供应商配置示例**：
+优先级：环境变量 > 配置文件 (`~/.deepspider/config/settings.json`) > 默认值
+
+**方式一：使用 CLI 命令（推荐）**
+
+```bash
+deepspider config set apiKey sk-xxx
+deepspider config set baseUrl https://api.openai.com/v1
+deepspider config set model gpt-4o
+```
+
+**方式二：环境变量**
+
+```bash
+export DEEPSPIDER_API_KEY=sk-xxx
+export DEEPSPIDER_BASE_URL=https://api.openai.com/v1
+export DEEPSPIDER_MODEL=gpt-4o
+```
+
+**常用供应商示例**：
 
 ```bash
 # OpenAI
-DEEPSPIDER_API_KEY=sk-xxx
-DEEPSPIDER_BASE_URL=https://api.openai.com/v1
-DEEPSPIDER_MODEL=gpt-4o
+deepspider config set baseUrl https://api.openai.com/v1
+deepspider config set model gpt-4o
 
 # DeepSeek
-DEEPSPIDER_API_KEY=sk-xxx
-DEEPSPIDER_BASE_URL=https://api.deepseek.com/v1
-DEEPSPIDER_MODEL=deepseek-chat
-
-# 其他 OpenAI 兼容供应商
-DEEPSPIDER_API_KEY=your-key
-DEEPSPIDER_BASE_URL=https://your-provider.com/v1
-DEEPSPIDER_MODEL=model-name
+deepspider config set baseUrl https://api.deepseek.com/v1
+deepspider config set model deepseek-chat
 ```
 
 ### 使用
 
+#### 全局安装（npm/pnpm install -g）
+
 ```bash
-# Agent 模式（推荐）- 指定目标网站
-pnpm run agent https://example.com
+# 启动 Agent - 指定目标网站
+deepspider https://example.com
 
-# Agent 模式 - 纯交互（不启动浏览器）
-pnpm run agent
+# 启动 Agent - 纯交互模式
+deepspider
+
+# 查看帮助
+deepspider --help
+
+# 管理配置
+deepspider config list            # 查看所有配置
+deepspider config set apiKey sk-xxx
+deepspider config set model gpt-4o
+
+# 检查更新
+deepspider update
+```
+
+#### 克隆仓库
+
+```bash
+# 配置（二选一）
+cp .env.example .env  # 编辑 .env 文件
+# 或使用 CLI 命令
+node bin/cli.js config set apiKey sk-xxx
+node bin/cli.js config set baseUrl https://api.openai.com/v1
+node bin/cli.js config set model gpt-4o
+
+# 安装 Python 依赖（可选，用于执行生成的 Python 代码）
+pnpm run setup:crypto
+
+# 启动 Agent
+pnpm run agent https://example.com
 
 # MCP 服务（供 Claude Code 等调用）
 pnpm run mcp
@@ -140,12 +181,19 @@ pnpm test
 
 ```
 deepspider/
+├── bin/cli.js               # CLI 入口（命令路由）
 ├── src/
 │   ├── agent/               # DeepAgent 系统
 │   │   ├── tools/           # 工具集（90+）
 │   │   ├── subagents/       # 子代理
 │   │   ├── skills/          # 领域技能
 │   │   └── prompts/         # 系统提示
+│   ├── cli/                 # CLI 命令
+│   │   ├── config.js        # 配置 re-export
+│   │   └── commands/        # 子命令（version/help/config/update）
+│   ├── config/              # 核心配置
+│   │   ├── paths.js         # 路径常量
+│   │   └── settings.js      # 配置读写（环境变量/文件/默认值）
 │   ├── browser/             # 浏览器运行时
 │   │   ├── client.js        # Patchright 客户端
 │   │   ├── cdp.js           # CDP 会话管理
@@ -156,7 +204,6 @@ deepspider/
 │   ├── env/                 # 环境补丁模块
 │   ├── store/               # 数据存储
 │   └── mcp/                 # MCP 服务
-├── bin/cli.js               # CLI 入口
 └── test/                    # 测试
 ```
 

package/bin/cli.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+/**
+ * DeepSpider CLI 入口
+ * 路由命令到对应处理模块
+ */
+
+import 'dotenv/config';
+
+const args = process.argv.slice(2);
+const first = args[0];
+
+switch (first) {
+  case '-v':
+  case '--version': {
+    const { run } = await import('../src/cli/commands/version.js');
+    run();
+    break;
+  }
+
+  case '-h':
+  case '--help': {
+    const { run } = await import('../src/cli/commands/help.js');
+    run();
+    break;
+  }
+
+  case 'config': {
+    const { run } = await import('../src/cli/commands/config.js');
+    run(args.slice(1));
+    break;
+  }
+
+  case 'update': {
+    const { run } = await import('../src/cli/commands/update.js');
+    await run();
+    break;
+  }
+
+  default: {
+    // URL 或无参数 → 启动 Agent
+    const { init } = await import('../src/agent/run.js');
+    await init();
+    break;
+  }
+}

package/package.json

@@ -1,18 +1,24 @@
 {
   "name": "deepspider",
-  "version": "0.2.11",
+  "version": "0.3.0",
   "description": "智能爬虫工程平台 - 基于 DeepAgents + Patchright 的 AI 爬虫 Agent",
   "type": "module",
   "main": "src/index.js",
+  "files": [
+    "bin/",
+    "src/",
+    "requirements-crypto.txt",
+    ".env.example"
+  ],
   "bin": {
-    "deepspider": "./src/agent/run.js"
+    "deepspider": "./bin/cli.js"
   },
   "scripts": {
     "start": "node src/index.js",
     "dev": "node --watch src/index.js",
-    "cli": "node src/agent/run.js",
+    "cli": "node bin/cli.js",
     "mcp": "node src/mcp/server.js",
-    "agent": "node src/agent/run.js",
+    "agent": "node bin/cli.js",
     "test": "node --test test/",
     "lint": "eslint src/",
     "lint:fix": "eslint src/ --fix",

package/src/agent/core/PanelBridge.js

@@ -0,0 +1,133 @@
+/**
+ * DeepSpider - 面板通信桥接
+ * 处理与浏览器面板的消息通信
+ */
+
+export class PanelBridge {
+  constructor(browserGetter, debugFn = () => {}) {
+    this.getBrowser = browserGetter;
+    this.debug = debugFn;
+    this.textBuffer = '';
+    this.hasStartedAssistantMsg = false;
+  }
+
+  /**
+   * 重置状态
+   */
+  reset() {
+    this.textBuffer = '';
+    this.hasStartedAssistantMsg = false;
+  }
+
+  /**
+   * 通过 CDP 在页面主世界执行 JavaScript
+   */
+  async evaluateInPage(code) {
+    const browser = this.getBrowser();
+    const cdp = await browser?.getCDPSession?.();
+    if (!cdp) return null;
+
+    try {
+      const result = await cdp.send('Runtime.evaluate', {
+        expression: code,
+        returnByValue: true,
+      });
+      return result.result?.value;
+    } catch (e) {
+      this.debug('evaluateInPage 失败:', e.message);
+      return null;
+    }
+  }
+
+  /**
+   * 发送消息到前端面板
+   */
+  async sendToPanel(role, content) {
+    if (!content?.trim()) return;
+
+    const browser = this.getBrowser();
+    const page = browser?.getPage?.();
+    if (!page) return;
+
+    try {
+      const escaped = JSON.stringify(content.trim());
+      const code = `window.__deepspider__?.addMessage?.('${role}', ${escaped})`;
+      await this.evaluateInPage(code);
+    } catch (e) {
+      // ignore
+    }
+  }
+
+  /**
+   * 累积文本到缓冲区（用于 LLM 流式输出）
+   */
+  async appendToPanel(text) {
+    if (!text) return;
+    this.textBuffer += text;
+
+    // 每累积一定量或遇到换行时刷新
+    if (this.textBuffer.length > 200 || text.includes('\n')) {
+      await this.flushPanelText();
+    }
+  }
+
+  /**
+   * 刷新累积的文本到面板
+   */
+  async flushPanelText() {
+    if (!this.textBuffer.trim()) return;
+
+    const browser = this.getBrowser();
+    const page = browser?.getPage?.();
+    if (!page) {
+      this.textBuffer = '';
+      return;
+    }
+
+    try {
+      const content = this.textBuffer.trim();
+      const escaped = JSON.stringify(content);
+
+      if (!this.hasStartedAssistantMsg) {
+        const code = `(function() {
+          const fn = window.__deepspider__?.addMessage;
+          if (typeof fn === 'function') {
+            fn('assistant', ${escaped});
+            return { ok: true };
+          }
+          return { ok: false };
+        })()`;
+        await this.evaluateInPage(code);
+        this.hasStartedAssistantMsg = true;
+      } else {
+        const code = `(function() {
+          const fn = window.__deepspider__?.appendToLastMessage;
+          if (typeof fn === 'function') {
+            fn('assistant', ${escaped});
+            return { ok: true };
+          }
+          return { ok: false };
+        })()`;
+        await this.evaluateInPage(code);
+      }
+    } catch (e) {
+      // ignore
+    }
+
+    this.textBuffer = '';
+  }
+
+  /**
+   * 设置忙碌状态
+   */
+  async setBusy(busy) {
+    await this.evaluateInPage(`window.__deepspider__?.setBusy?.(${busy})`);
+  }
+
+  /**
+   * 完成消息，触发渲染
+   */
+  async finalizeMessage(role) {
+    await this.evaluateInPage(`window.__deepspider__?.finalizeMessage?.("${role}")`);
+  }
+}

package/src/agent/core/RetryManager.js

@@ -0,0 +1,51 @@
+/**
+ * DeepSpider - 重试管理器
+ * 处理 API 调用的重试策略
+ */
+
+// 默认重试配置
+const DEFAULT_CONFIG = {
+  maxRetries: 3,
+  baseDelayMs: 2000,
+  maxDelayMs: 30000,
+};
+
+export class RetryManager {
+  constructor(config = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * 计算重试延迟（指数退避 + 抖动）
+   */
+  getDelay(retryCount) {
+    const delay = Math.min(
+      this.config.baseDelayMs * Math.pow(2, retryCount),
+      this.config.maxDelayMs
+    );
+    // 添加 0-25% 的随机抖动
+    const jitter = delay * Math.random() * 0.25;
+    return Math.round(delay + jitter);
+  }
+
+  /**
+   * 是否可以重试
+   */
+  canRetry(retryCount) {
+    return retryCount < this.config.maxRetries;
+  }
+
+  /**
+   * 获取最大重试次数
+   */
+  get maxRetries() {
+    return this.config.maxRetries;
+  }
+}
+
+/**
+ * 延迟函数
+ */
+export function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

package/src/agent/core/StreamHandler.js

@@ -0,0 +1,263 @@
+/**
+ * DeepSpider - 流式输出处理器
+ * 处理 Agent 的流式事件
+ */
+
+import { isApiServiceError, isToolSchemaError } from '../errors/ErrorClassifier.js';
+import { RetryManager, sleep } from './RetryManager.js';
+
+// DeepSeek 特殊标记清理
+const DSML_PATTERN = /｜DSML｜/g;
+function cleanDSML(text) {
+  return text ? text.replace(DSML_PATTERN, '') : text;
+}
+
+// 人工介入配置
+const INTERVENTION_CONFIG = {
+  idleTimeoutMs: 120000,  // 2分钟无响应触发提示
+  checkIntervalMs: 30000, // 30秒检测一次
+};
+
+export class StreamHandler {
+  constructor({ agent, config, panelBridge, riskTools = [], debug = () => {} }) {
+    this.agent = agent;
+    this.config = config;
+    this.panelBridge = panelBridge;
+    this.riskTools = riskTools;
+    this.debug = debug;
+    this.retryManager = new RetryManager();
+  }
+
+  /**
+   * 流式对话 - 显示思考过程（带重试）
+   */
+  async chatStream(input, retryCount = 0) {
+    let finalResponse = '';
+    let lastEventTime = Date.now();
+    let eventCount = 0;
+    let lastToolCall = null;
+
+    // 重置面板状态
+    this.panelBridge.reset();
+    await this.panelBridge.setBusy(true);
+
+    this.debug(`chatStream: 开始处理, 输入长度=${input.length}`);
+
+    // 心跳检测
+    let interventionNotified = false;
+    const heartbeat = this._createHeartbeat(
+      () => lastEventTime,
+      () => eventCount,
+      () => lastToolCall,
+      () => interventionNotified,
+      (v) => { interventionNotified = v; }
+    );
+
+    try {
+      this.debug('chatStream: 创建事件流');
+      const eventStream = await this.agent.streamEvents(
+        { messages: [{ role: 'user', content: input }] },
+        { ...this.config, version: 'v2' }
+      );
+
+      this.debug('chatStream: 开始遍历事件');
+      for await (const event of eventStream) {
+        lastEventTime = Date.now();
+        eventCount++;
+
+        if (event.event === 'on_tool_start') {
+          lastToolCall = event.name;
+        }
+
+        await this._handleStreamEvent(event);
+
+        if (event.event === 'on_chat_model_end' && event.name === 'ChatOpenAI') {
+          const output = event.data?.output;
+          if (output?.content) {
+            finalResponse = output.content;
+            this.debug(`chatStream: 收到最终响应, 长度=${finalResponse.length}`);
+          }
+        }
+      }
+
+      clearInterval(heartbeat);
+      console.log(`\n[完成] 共处理 ${eventCount} 个事件`);
+
+      await this.panelBridge.flushPanelText();
+      await this.panelBridge.finalizeMessage('assistant');
+      await this.panelBridge.setBusy(false);
+
+      this.debug(`chatStream: 完成, 响应长度=${finalResponse.length}`);
+      return finalResponse || '[无响应]';
+    } catch (error) {
+      clearInterval(heartbeat);
+      return this._handleError(error, input, eventCount, lastToolCall, retryCount);
+    }
+  }
+
+  /**
+   * 从检查点恢复流式对话
+   */
+  async chatStreamResume(retryCount = 0) {
+    let finalResponse = '';
+    let lastEventTime = Date.now();
+    let eventCount = 0;
+
+    await this.panelBridge.setBusy(true);
+    this.debug(`chatStreamResume: 从检查点恢复, retryCount=${retryCount}`);
+
+    const heartbeat = setInterval(() => {
+      const elapsed = Math.round((Date.now() - lastEventTime) / 1000);
+      if (elapsed > 30) {
+        console.log(`\n[心跳] 恢复中，已等待 ${elapsed}s`);
+      }
+    }, 30000);
+
+    try {
+      const eventStream = await this.agent.streamEvents(
+        { messages: [] },
+        { ...this.config, version: 'v2' }
+      );
+
+      for await (const event of eventStream) {
+        lastEventTime = Date.now();
+        eventCount++;
+        await this._handleStreamEvent(event);
+
+        if (event.event === 'on_chat_model_end' && event.name === 'ChatOpenAI') {
+          const output = event.data?.output;
+          if (output?.content) {
+            finalResponse = output.content;
+          }
+        }
+      }
+
+      clearInterval(heartbeat);
+      await this.panelBridge.flushPanelText();
+      await this.panelBridge.setBusy(false);
+      console.log(`\n[恢复完成] 共处理 ${eventCount} 个事件`);
+      return finalResponse || '[无响应]';
+    } catch (error) {
+      clearInterval(heartbeat);
+      await this.panelBridge.setBusy(false);
+      const errMsg = error.message || String(error);
+      console.error(`\n[恢复失败] ${errMsg}`);
+
+      if (isApiServiceError(errMsg) && this.retryManager.canRetry(retryCount)) {
+        const delay = this.retryManager.getDelay(retryCount);
+        console.log(`\n[重试 ${retryCount + 1}/${this.retryManager.maxRetries}] ${delay}ms 后再次恢复...`);
+        await sleep(delay);
+        return this.chatStreamResume(retryCount + 1);
+      }
+
+      return `恢复失败: ${errMsg}`;
+    }
+  }
+
+  /**
+   * 创建心跳检测定时器
+   */
+  _createHeartbeat(getLastEventTime, getEventCount, getLastToolCall, getNotified, setNotified) {
+    return setInterval(() => {
+      const elapsed = Math.round((Date.now() - getLastEventTime()) / 1000);
+      if (elapsed > 30) {
+        console.log(`\n[心跳] 已等待 ${elapsed}s, 事件数=${getEventCount()}, 最后工具=${getLastToolCall() || '无'}`);
+      }
+
+      const isRiskTool = getLastToolCall() && this.riskTools.includes(getLastToolCall());
+      if (elapsed * 1000 > INTERVENTION_CONFIG.idleTimeoutMs && !getNotified() && isRiskTool) {
+        setNotified(true);
+        const msg = '⚠️ 页面操作后长时间无响应，可能遇到验证码或风控，请检查浏览器';
+        console.log('\n[提示] ' + msg);
+        this.panelBridge.sendToPanel('system', msg).catch(() => {});
+      }
+    }, INTERVENTION_CONFIG.checkIntervalMs);
+  }
+
+  /**
+   * 处理流式事件
+   */
+  async _handleStreamEvent(event) {
+    const { event: eventType, name, data } = event;
+
+    // 过滤内部事件
+    if (name?.startsWith('ChannelWrite') ||
+        name?.startsWith('Branch') ||
+        name?.includes('Middleware') ||
+        name === 'RunnableSequence' ||
+        name === 'model_request' ||
+        name === 'tools') {
+      return;
+    }
+
+    this.debug(`handleStreamEvent: ${eventType}, name=${name}`);
+
+    switch (eventType) {
+      case 'on_chat_model_stream':
+        let chunk = data?.chunk?.content;
+        if (chunk && typeof chunk === 'string') {
+          chunk = cleanDSML(chunk);
+          process.stdout.write(chunk);
+          await this.panelBridge.appendToPanel(chunk);
+        }
+        break;
+
+      case 'on_tool_start':
+        this.debug('handleStreamEvent: 工具开始，先刷新缓冲区');
+        await this.panelBridge.flushPanelText();
+        this.panelBridge.hasStartedAssistantMsg = false;
+        const input = data?.input || {};
+        const inputStr = typeof input === 'string' ? input : JSON.stringify(input);
+        const preview = inputStr.length > 100 ? inputStr.slice(0, 100) + '...' : inputStr;
+        console.log(`\n[调用] ${name}(${preview})`);
+        await this.panelBridge.sendToPanel('system', `[调用] ${name}`);
+        break;
+
+      case 'on_tool_end':
+        const output = data?.output;
+        let result = '';
+        this.debug(`on_tool_end: name=${name}, output type=${typeof output}`);
+
+        if (typeof output === 'string') {
+          result = output.slice(0, 80);
+        } else if (output?.content) {
+          result = String(output.content).slice(0, 80);
+        }
+        if (result) {
+          console.log(`[结果] ${result}${result.length >= 80 ? '...' : ''}`);
+          await this.panelBridge.sendToPanel('system', `[结果] ${result.slice(0, 50)}${result.length > 50 ? '...' : ''}`);
+        }
+        break;
+    }
+  }
+
+  /**
+   * 处理错误和重试
+   */
+  async _handleError(error, input, eventCount, lastToolCall, retryCount) {
+    const errMsg = error.message || String(error);
+    await this.panelBridge.setBusy(false);
+    console.error(`\n[异常] 事件数=${eventCount}, 最后工具=${lastToolCall || '无'}, 错误: ${errMsg}`);
+
+    if (this.retryManager.canRetry(retryCount)) {
+      if (isApiServiceError(errMsg)) {
+        const delay = this.retryManager.getDelay(retryCount);
+        console.log(`\n[重试 ${retryCount + 1}/${this.retryManager.maxRetries}] API错误，${delay}ms 后从检查点恢复...`);
+        await this.panelBridge.sendToPanel('system',
+          `服务暂时不可用，${Math.round(delay/1000)}s 后重试 (${retryCount + 1}/${this.retryManager.maxRetries})`);
+        await sleep(delay);
+        return this.chatStreamResume(retryCount + 1);
+      }
+
+      if (isToolSchemaError(errMsg)) {
+        console.log(`\n[重试 ${retryCount + 1}/${this.retryManager.maxRetries}] 工具参数错误，发送修正请求...`);
+        await this.panelBridge.sendToPanel('system',
+          `工具调用失败，正在修正 (${retryCount + 1}/${this.retryManager.maxRetries})`);
+        const resumeInput = `工具调用失败: ${errMsg}\n请检查参数格式并重试。`;
+        return this.chatStream(resumeInput, retryCount + 1);
+      }
+    }
+
+    return `错误: ${errMsg}`;
+  }
+}

package/src/agent/core/index.js

@@ -0,0 +1,7 @@
+/**
+ * DeepSpider - Core 模块索引
+ */
+
+export { StreamHandler } from './StreamHandler.js';
+export { RetryManager, sleep } from './RetryManager.js';
+export { PanelBridge } from './PanelBridge.js';