deepspider 0.3.1 → 0.3.2

package/.env.example

@@ -5,6 +5,9 @@ DEEPSPIDER_API_KEY=your_api_key_here
 DEEPSPIDER_BASE_URL=https://api.openai.com/v1
 DEEPSPIDER_MODEL=gpt-4o
 
+# 浏览器持久化（可选，保持登录态）
+# DEEPSPIDER_PERSIST_BROWSER=true
+
 # LangSmith 追踪配置（可选）
 LANGSMITH_TRACING=true
 LANGSMITH_API_KEY=your_langsmith_api_key_here

package/README.md

@@ -53,6 +53,7 @@ DeepSpider 需要配置 LLM API 才能运行。支持任何兼容 OpenAI 格式
 | `apiKey` | `DEEPSPIDER_API_KEY` | API 密钥 |
 | `baseUrl` | `DEEPSPIDER_BASE_URL` | API 地址 |
 | `model` | `DEEPSPIDER_MODEL` | 模型名称 |
+| `persistBrowserData` | `DEEPSPIDER_PERSIST_BROWSER` | 持久化浏览器数据（保持登录态） |
 
 优先级：环境变量 > 配置文件 (`~/.deepspider/config/settings.json`) > 默认值
 
@@ -92,6 +93,9 @@ deepspider config set model deepseek-chat
 # 启动 Agent - 指定目标网站
 deepspider https://example.com
 
+# 启动 Agent - 持久化浏览器数据（一次性）
+deepspider --persist https://example.com
+
 # 启动 Agent - 纯交互模式
 deepspider
 
@@ -103,6 +107,9 @@ deepspider config list            # 查看所有配置
 deepspider config set apiKey sk-xxx
 deepspider config set model gpt-4o
 
+# 持久化浏览器数据（需要登录的网站，下次启动自动恢复登录态）
+deepspider config set persistBrowserData true
+
 # 检查更新
 deepspider update
 ```
@@ -150,19 +157,14 @@ pnpm test
        ┌───────────────┼───────────────┐
        ▼               ▼               ▼
 ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
-│static-agent │ │captcha-agent│ │anti-detect  │
-│ 静态分析    │ │ 验证码处理  │ │ 反检测      │
+│reverse-agent│ │captcha-agent│ │anti-detect  │
+│ 逆向分析    │ │ 验证码处理  │ │ 反检测      │
 └──────┬──────┘ └─────────────┘ └─────────────┘
        ▼
 ┌─────────────┐
-│dynamic-agent│
-│ 动态调试    │
-└──────┬──────┘
-       ▼
-┌─────────────┐     ┌─────────────┐
-│sandbox-agent│ ──▶ │js2python    │
-│ 沙箱验证    │     │ 代码转换    │
-└─────────────┘     └─────────────┘
+│js2python    │
+│ 代码转换    │
+└─────────────┘
 ```
 
 ### 子代理体系
@@ -170,9 +172,7 @@ pnpm test
 | 子代理 | 职责 | 核心工具 |
 |--------|------|----------|
 | crawler | 爬虫编排：整合各模块、生成完整脚本 | file, store, crawler |
-| static | 静态分析：解包、反混淆、加密定位 | webcrack, deobfuscate, analyze |
-| dynamic | 动态分析：浏览器控制、Hook、数据采集 | browser, debug, capture |
-| sandbox | 沙箱执行：环境补全、代码执行 | sandbox, env, patch |
+| reverse | 逆向分析全流程：反混淆、断点调试、Hook、沙箱验证、补环境 | tracing, deobfuscate, debug, capture, sandbox, env |
 | js2python | JS转Python：加密代码转换、验证 | python, analyzer |
 | captcha | 验证码处理：OCR、滑块、点选 | captcha_ocr, captcha_slide |
 | anti-detect | 反检测：指纹管理、代理池 | proxy, fingerprint |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepspider",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "智能爬虫工程平台 - 基于 DeepAgents + Patchright 的 AI 爬虫 Agent",
   "type": "module",
   "main": "src/index.js",
@@ -19,7 +19,7 @@
     "cli": "node bin/cli.js",
     "mcp": "node src/mcp/server.js",
     "agent": "node bin/cli.js",
-    "test": "node --test test/",
+    "test": "node --test 'test/*.test.js'",
     "lint": "eslint src/",
     "lint:fix": "eslint src/ --fix",
     "setup:crypto": "uv venv .venv --python 3.11 2>/dev/null || true && uv pip install -r requirements-crypto.txt",
@@ -54,20 +54,20 @@
     "@babel/parser": "^7.28.6",
     "@babel/traverse": "^7.28.6",
     "@babel/types": "^7.28.6",
-    "@langchain/anthropic": "^1.3.12",
-    "@langchain/core": "^1.1.17",
+    "@langchain/anthropic": "^1.3.17",
+    "@langchain/core": "^1.1.24",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/openai": "^1.2.3",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "crypto-js": "^4.2.0",
-    "deepagents": "^1.6.0",
+    "deepagents": "^1.7.6",
     "dotenv": "^17.2.3",
     "hono": "4.11.7",
     "isolated-vm": "^6.0.2",
     "js-md5": "^0.8.3",
     "js-sha256": "^0.11.1",
     "jsencrypt": "^3.5.4",
-    "langchain": "^1.2.15",
+    "langchain": "^1.2.24",
     "marked": "^17.0.1",
     "patchright": "^1.57.0",
     "sm-crypto": "^0.4.0",

package/src/agent/core/PanelBridge.js

@@ -1,22 +1,12 @@
 /**
  * DeepSpider - 面板通信桥接
- * 处理与浏览器面板的消息通信
+ * 处理与浏览器面板的结构化消息通信
  */
 
 export class PanelBridge {
   constructor(browserGetter, debugFn = () => {}) {
     this.getBrowser = browserGetter;
     this.debug = debugFn;
-    this.textBuffer = '';
-    this.hasStartedAssistantMsg = false;
-  }
-
-  /**
-   * 重置状态
-   */
-  reset() {
-    this.textBuffer = '';
-    this.hasStartedAssistantMsg = false;
   }
 
   /**
@@ -40,81 +30,31 @@ export class PanelBridge {
   }
 
   /**
-   * 发送消息到前端面板
+   * 发送结构化消息到前端面板
    */
-  async sendToPanel(role, content) {
-    if (!content?.trim()) return;
-
+  async sendMessage(type, data) {
     const browser = this.getBrowser();
     const page = browser?.getPage?.();
     if (!page) return;
 
     try {
-      const escaped = JSON.stringify(content.trim());
-      const code = `window.__deepspider__?.addMessage?.('${role}', ${escaped})`;
-      await this.evaluateInPage(code);
+      const escapedType = JSON.stringify(type);
+      const escapedData = JSON.stringify(data);
+      await this.evaluateInPage(
+        `window.__deepspider__?.addStructuredMessage?.(${escapedType}, ${escapedData})`
+      );
     } catch {
       // ignore
     }
   }
 
   /**
-   * 累积文本到缓冲区（用于 LLM 流式输出）
+   * 按 role 发送消息到前端面板
    */
-  async appendToPanel(text) {
-    if (!text) return;
-    this.textBuffer += text;
-
-    // 每累积一定量或遇到换行时刷新
-    if (this.textBuffer.length > 200 || text.includes('\n')) {
-      await this.flushPanelText();
-    }
-  }
-
-  /**
-   * 刷新累积的文本到面板
-   */
-  async flushPanelText() {
-    if (!this.textBuffer.trim()) return;
-
-    const browser = this.getBrowser();
-    const page = browser?.getPage?.();
-    if (!page) {
-      this.textBuffer = '';
-      return;
-    }
-
-    try {
-      const content = this.textBuffer.trim();
-      const escaped = JSON.stringify(content);
-
-      if (!this.hasStartedAssistantMsg) {
-        const code = `(function() {
-          const fn = window.__deepspider__?.addMessage;
-          if (typeof fn === 'function') {
-            fn('assistant', ${escaped});
-            return { ok: true };
-          }
-          return { ok: false };
-        })()`;
-        await this.evaluateInPage(code);
-        this.hasStartedAssistantMsg = true;
-      } else {
-        const code = `(function() {
-          const fn = window.__deepspider__?.appendToLastMessage;
-          if (typeof fn === 'function') {
-            fn('assistant', ${escaped});
-            return { ok: true };
-          }
-          return { ok: false };
-        })()`;
-        await this.evaluateInPage(code);
-      }
-    } catch {
-      // ignore
-    }
-
-    this.textBuffer = '';
+  async sendToPanel(role, content) {
+    if (!content?.trim()) return;
+    const type = role === 'system' ? 'system' : role === 'user' ? 'user' : 'text';
+    await this.sendMessage(type, { content: content.trim() });
   }
 
   /**
@@ -125,9 +65,21 @@ export class PanelBridge {
   }
 
   /**
-   * 完成消息，触发渲染
+   * 删除面板中最后一条 assistant 消息
+   * 用于 interrupt 场景：LLM 在调用 interrupt 工具前输出的冗余描述文字需要清除
    */
-  async finalizeMessage(role) {
-    await this.evaluateInPage(`window.__deepspider__?.finalizeMessage?.("${role}")`);
+  async removeLastAssistantMessage() {
+    await this.evaluateInPage(`
+      (function() {
+        const ds = window.__deepspider__;
+        if (!ds?.chatMessages) return;
+        for (let i = ds.chatMessages.length - 1; i >= 0; i--) {
+          if (ds.chatMessages[i].role === 'assistant') {
+            ds.chatMessages.splice(i, 1);
+            break;
+          }
+        }
+      })()
+    `);
   }
 }

package/src/agent/core/StreamHandler.js

@@ -1,8 +1,9 @@
 /**
  * DeepSpider - 流式输出处理器
- * 处理 Agent 的流式事件
+ * 处理 Agent 的流式事件，支持 interrupt HITL 交互
  */
 
+import { Command } from '@langchain/langgraph';
 import { isApiServiceError, isToolSchemaError } from '../errors/ErrorClassifier.js';
 import { RetryManager, sleep } from './RetryManager.js';
 
@@ -26,6 +27,7 @@ export class StreamHandler {
     this.riskTools = riskTools;
     this.debug = debug;
     this.retryManager = new RetryManager();
+    this.fullResponse = '';
   }
 
   /**
@@ -37,8 +39,8 @@ export class StreamHandler {
     let eventCount = 0;
     let lastToolCall = null;
 
-    // 重置面板状态
-    this.panelBridge.reset();
+    // 重置状态
+    this.fullResponse = '';
     await this.panelBridge.setBusy(true);
 
     this.debug(`chatStream: 开始处理, 输入长度=${input.length}`);
@@ -83,8 +85,12 @@ export class StreamHandler {
       clearInterval(heartbeat);
       console.log(`\n[完成] 共处理 ${eventCount} 个事件`);
 
-      await this.panelBridge.flushPanelText();
-      await this.panelBridge.finalizeMessage('assistant');
+      // 发送剩余累积文本
+      await this._flushFullResponse();
+
+      // 检测 interrupt 并渲染到面板
+      await this._checkAndRenderInterrupt();
+
       await this.panelBridge.setBusy(false);
 
       this.debug(`chatStream: 完成, 响应长度=${finalResponse.length}`);
@@ -96,13 +102,69 @@ export class StreamHandler {
   }
 
   /**
-   * 从检查点恢复流式对话
+   * 用 Command({ resume }) 恢复被 interrupt 暂停的 graph
+   */
+  async resumeInterrupt(value) {
+    let finalResponse = '';
+    let lastEventTime = Date.now();
+    let eventCount = 0;
+
+    this.fullResponse = '';
+    await this.panelBridge.setBusy(true);
+    this.debug(`resumeInterrupt: 恢复 interrupt, value=${value}`);
+
+    const heartbeat = setInterval(() => {
+      const elapsed = Math.round((Date.now() - lastEventTime) / 1000);
+      if (elapsed > 30) {
+        console.log(`\n[心跳] 恢复中，已等待 ${elapsed}s`);
+      }
+    }, 30000);
+
+    try {
+      const eventStream = await this.agent.streamEvents(
+        new Command({ resume: value }),
+        { ...this.config, version: 'v2' }
+      );
+
+      for await (const event of eventStream) {
+        lastEventTime = Date.now();
+        eventCount++;
+        await this._handleStreamEvent(event);
+
+        if (event.event === 'on_chat_model_end' && event.name === 'ChatOpenAI') {
+          const output = event.data?.output;
+          if (output?.content) {
+            finalResponse = output.content;
+          }
+        }
+      }
+
+      clearInterval(heartbeat);
+
+      await this._flushFullResponse();
+      await this._checkAndRenderInterrupt();
+      await this.panelBridge.setBusy(false);
+
+      console.log(`\n[恢复完成] 共处理 ${eventCount} 个事件`);
+      return finalResponse || '[无响应]';
+    } catch (error) {
+      clearInterval(heartbeat);
+      await this.panelBridge.setBusy(false);
+      const errMsg = error.message || String(error);
+      console.error(`\n[恢复失败] ${errMsg}`);
+      return `恢复失败: ${errMsg}`;
+    }
+  }
+
+  /**
+   * 从检查点恢复流式对话（错误重试用）
    */
   async chatStreamResume(retryCount = 0) {
     let finalResponse = '';
     let lastEventTime = Date.now();
     let eventCount = 0;
 
+    this.fullResponse = '';
     await this.panelBridge.setBusy(true);
     this.debug(`chatStreamResume: 从检查点恢复, retryCount=${retryCount}`);
 
@@ -133,8 +195,11 @@ export class StreamHandler {
       }
 
       clearInterval(heartbeat);
-      await this.panelBridge.flushPanelText();
+
+      await this._flushFullResponse();
+      await this._checkAndRenderInterrupt();
       await this.panelBridge.setBusy(false);
+
       console.log(`\n[恢复完成] 共处理 ${eventCount} 个事件`);
       return finalResponse || '[无响应]';
     } catch (error) {
@@ -154,6 +219,62 @@ export class StreamHandler {
     }
   }
 
+  /**
+   * 发送剩余累积文本到面板
+   */
+  async _flushFullResponse() {
+    if (this.fullResponse?.trim()) {
+      await this.panelBridge.sendToPanel('assistant', this.fullResponse);
+    }
+    this.fullResponse = '';
+  }
+
+  /**
+   * 检测 graph interrupt 状态，将 interrupt payload 渲染为面板结构化消息
+   *
+   * interrupt payload 协议：
+   *   { type: 'choices', question, options: [{id, label, description?}] }
+   *   { type: 'confirm', question, confirmText?, cancelText? }
+   */
+  async _checkAndRenderInterrupt() {
+    try {
+      const state = await this.agent.getState(this.config);
+      this.debug(`_checkAndRenderInterrupt: state.next=${JSON.stringify(state?.next)}, tasks=${state?.tasks?.length ?? 'undefined'}`);
+
+      if (!state?.tasks) {
+        this.debug('_checkAndRenderInterrupt: state.tasks 为空，尝试检查 next');
+        // 某些 LangGraph 版本用 next 为空数组表示 interrupt
+        // 如果 next 不为空，说明 graph 正常结束，无 interrupt
+        return false;
+      }
+
+      let found = false;
+      for (const task of state.tasks) {
+        this.debug(`_checkAndRenderInterrupt: task id=${task.id}, interrupts=${task.interrupts?.length ?? 0}`);
+        if (!task.interrupts?.length) continue;
+        for (const intr of task.interrupts) {
+          const payload = intr.value;
+          this.debug(`_checkAndRenderInterrupt: interrupt payload=${JSON.stringify(payload)?.slice(0, 200)}`);
+          if (!payload?.type) continue;
+
+          console.log(`\n[交互] 等待用户 ${payload.type === 'choices' ? '选择' : '确认'}...`);
+
+          if (payload.type === 'choices' || payload.type === 'confirm') {
+            // 删除面板中 interrupt 工具调用前 LLM 输出的冗余描述文字
+            await this.panelBridge.removeLastAssistantMessage();
+            await this.panelBridge.sendMessage(payload.type, payload);
+            found = true;
+          }
+        }
+      }
+      return found;
+    } catch (e) {
+      this.debug('_checkAndRenderInterrupt 失败:', e.message);
+      console.log(`[DEBUG] _checkAndRenderInterrupt error: ${e.message}`);
+      return false;
+    }
+  }
+
   /**
    * 创建心跳检测定时器
    */
@@ -197,15 +318,20 @@ export class StreamHandler {
         let chunk = data?.chunk?.content;
         if (chunk && typeof chunk === 'string') {
           chunk = cleanDSML(chunk);
+          // CLI 侧仍流式输出
           process.stdout.write(chunk);
-          await this.panelBridge.appendToPanel(chunk);
+          // 面板侧只累积，不推送
+          this.fullResponse = (this.fullResponse || '') + chunk;
         }
         break;
 
       case 'on_tool_start':
-        this.debug('handleStreamEvent: 工具开始，先刷新缓冲区');
-        await this.panelBridge.flushPanelText();
-        this.panelBridge.hasStartedAssistantMsg = false;
+        // 工具调用前，先把已累积的 LLM 文字发送到面板
+        if (this.fullResponse?.trim()) {
+          await this.panelBridge.sendToPanel('assistant', this.fullResponse);
+          this.fullResponse = '';
+        }
+        this.debug('handleStreamEvent: 工具开始');
         const input = data?.input || {};
         const inputStr = typeof input === 'string' ? input : JSON.stringify(input);
         const preview = inputStr.length > 100 ? inputStr.slice(0, 100) + '...' : inputStr;
@@ -250,11 +376,10 @@ export class StreamHandler {
       }
 
       if (isToolSchemaError(errMsg)) {
-        console.log(`\n[重试 ${retryCount + 1}/${this.retryManager.maxRetries}] 工具参数错误，发送修正请求...`);
+        console.log(`\n[重试 ${retryCount + 1}/${this.retryManager.maxRetries}] 工具参数错误，从检查点恢复...`);
         await this.panelBridge.sendToPanel('system',
           `工具调用失败，正在修正 (${retryCount + 1}/${this.retryManager.maxRetries})`);
-        const resumeInput = `工具调用失败: ${errMsg}\n请检查参数格式并重试。`;
-        return this.chatStream(resumeInput, retryCount + 1);
+        return this.chatStreamResume(retryCount + 1);
       }
     }
 

package/src/agent/index.js

@@ -1,10 +1,12 @@
 /**
  * DeepSpider - DeepAgent 主入口
- * 基于 DeepAgents 最佳实践重构
+ * 使用 createAgent 手动组装 middleware 栈，替换 createDeepAgent
+ * 目的：用自定义 subagent middleware 支持 context 结构化传递
  */
 
 import 'dotenv/config';
-import { createDeepAgent, StateBackend, FilesystemBackend } from 'deepagents';
+import { StateBackend, FilesystemBackend, createFilesystemMiddleware, createPatchToolCallsMiddleware } from 'deepagents';
+import { createAgent, toolRetryMiddleware, summarizationMiddleware, anthropicPromptCachingMiddleware, todoListMiddleware, humanInTheLoopMiddleware } from 'langchain';
 import { ChatOpenAI } from '@langchain/openai';
 import { MemorySaver } from '@langchain/langgraph';
 
@@ -13,6 +15,12 @@ import { allSubagents } from './subagents/index.js';
 import { systemPrompt } from './prompts/system.js';
 import { createReportMiddleware } from './middleware/report.js';
 import { createFilterToolsMiddleware } from './middleware/filterTools.js';
+import { createCustomSubAgentMiddleware } from './middleware/subagent.js';
+import { createToolGuardMiddleware } from './middleware/toolGuard.js';
+import { createValidationWorkflowMiddleware } from './middleware/validationWorkflow.js';
+
+// createDeepAgent 内部拼接的 BASE_PROMPT
+const BASE_PROMPT = 'In order to complete the objective that the user asks of you, you have access to a number of standard tools.';
 
 // 从环境变量读取配置
 const config = {
@@ -72,22 +80,53 @@ export function createDeepSpiderAgent(options = {}) {
       }
     : undefined;
 
-  // 中间件配置
-  const middleware = [
-    createFilterToolsMiddleware(),  // 过滤内置的 write_file/read_file
-    createReportMiddleware({ onReportReady }),
+  // 框架级子代理默认中间件（对照 createDeepAgent 内部的 subagentMiddleware）
+  const subagentDefaultMiddleware = [
+    todoListMiddleware(),
+    createFilesystemMiddleware({ backend }),
+    summarizationMiddleware({ model: llm, trigger: { tokens: 170000 }, keep: { messages: 6 } }),
+    anthropicPromptCachingMiddleware({ unsupportedModelBehavior: 'ignore' }),
+    createPatchToolCallsMiddleware(),
   ];
 
-  return createDeepAgent({
+  // 组装完整 middleware 栈（对照 createDeepAgent 源码 dist/index.js:3791-3826）
+  return createAgent({
     name: 'deepspider',
     model: llm,
     tools: coreTools,
-    subagents: allSubagents,
-    systemPrompt,
-    backend,
+    systemPrompt: `${systemPrompt}\n\n${BASE_PROMPT}`,
+    middleware: [
+      // === 框架内置 middleware ===
+      todoListMiddleware(),
+      createFilesystemMiddleware({ backend }),
+      createCustomSubAgentMiddleware({
+        defaultModel: llm,
+        defaultTools: coreTools,
+        subagents: allSubagents,
+        defaultMiddleware: subagentDefaultMiddleware,
+        generalPurposeAgent: false,
+        defaultInterruptOn: interruptOn,
+      }),
+      summarizationMiddleware({ model: llm, trigger: { tokens: 170000 }, keep: { messages: 6 } }),
+      anthropicPromptCachingMiddleware({ unsupportedModelBehavior: 'ignore' }),
+      createPatchToolCallsMiddleware(),
+      // === HITL（如果启用）===
+      ...(interruptOn ? [humanInTheLoopMiddleware({ interruptOn })] : []),
+      // === 自定义 middleware ===
+      toolRetryMiddleware({
+        maxRetries: 0,
+        onFailure: (err) => {
+          // GraphInterrupt / ParentCommand 等 LangGraph 内部控制流异常必须透传，不能吞掉
+          if (err?.is_bubble_up === true) throw err;
+          return `Tool call failed: ${err.message}\nPlease fix the arguments and retry.`;
+        },
+      }),
+      createToolGuardMiddleware(),
+      createFilterToolsMiddleware(),
+      createValidationWorkflowMiddleware(),
+      createReportMiddleware({ onReportReady }),
+    ],
     checkpointer,
-    interruptOn,
-    middleware,
   });
 }