deepspider 0.3.2 → 0.5.0

package/src/agent/tools/analysis.js

@@ -7,16 +7,26 @@ import { tool } from '@langchain/core/tools';
 import { getBrowserClient } from '../../browser/index.js';
 
 /**
- * 通过 CDP 执行 JS（复用 session）
+ * 通过 CDP 执行 JS（复用 session，带超时保护）
  */
-async function evaluateViaCDP(client, expression) {
+async function evaluateViaCDP(client, expression, timeout = 5000) {
   const cdp = await client.getCDPSession();
   if (!cdp) return null;
-  const result = await cdp.send('Runtime.evaluate', {
-    expression,
-    returnByValue: true,
-  });
-  return result.result?.value;
+  try {
+    const result = await Promise.race([
+      cdp.send('Runtime.evaluate', {
+        expression,
+        returnByValue: true,
+      }),
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('CDP evaluate timeout')), timeout)
+      ),
+    ]);
+    return result.result?.value;
+  } catch (e) {
+    console.error('[analysis:evaluateViaCDP] 超时或错误:', e.message);
+    return null;
+  }
 }
 
 /**

package/src/agent/tools/browser.js

@@ -9,16 +9,29 @@ import { getBrowser } from '../../browser/index.js';
 import { getScreenshotPath } from './utils.js';
 
 /**
- * 通过 CDP 执行 JS
+ * 安全获取 CDP session，断开时返回友好错误而非 TypeError
  */
-async function cdpEvaluate(browser, expression, returnByValue = true) {
+async function safeCDP(browser) {
   const cdp = await browser.getCDPSession();
-  if (!cdp) throw new Error('CDP session not available');
-  const result = await cdp.send('Runtime.evaluate', {
-    expression,
-    returnByValue,
-    awaitPromise: true,
-  });
+  if (!cdp) throw new Error('CDP session 不可用，浏览器可能已关闭或断开连接');
+  return cdp;
+}
+
+/**
+ * 通过 CDP 执行 JS（带超时保护）
+ */
+async function cdpEvaluate(browser, expression, returnByValue = true, timeout = 5000) {
+  const cdp = await safeCDP(browser);
+  const result = await Promise.race([
+    cdp.send('Runtime.evaluate', {
+      expression,
+      returnByValue,
+      awaitPromise: true,
+    }),
+    new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('cdpEvaluate timeout (page loading/paused?)')), timeout)
+    ),
+  ]);
   if (result.exceptionDetails) {
     throw new Error(result.exceptionDetails.text || 'CDP evaluate error');
   }
@@ -112,7 +125,7 @@ export const waitForSelector = tool(
 export const reloadPage = tool(
   async () => {
     const browser = await getBrowser();
-    const cdp = await browser.getCDPSession();
+    const cdp = await safeCDP(browser);
     await cdp.send('Page.reload');
     const url = await cdpEvaluate(browser, 'location.href');
     return JSON.stringify({ success: true, url });
@@ -130,7 +143,7 @@ export const reloadPage = tool(
 export const goBack = tool(
   async () => {
     const browser = await getBrowser();
-    const cdp = await browser.getCDPSession();
+    const cdp = await safeCDP(browser);
     const history = await cdp.send('Page.getNavigationHistory');
     if (history.currentIndex > 0) {
       const entry = history.entries[history.currentIndex - 1];
@@ -152,7 +165,7 @@ export const goBack = tool(
 export const goForward = tool(
   async () => {
     const browser = await getBrowser();
-    const cdp = await browser.getCDPSession();
+    const cdp = await safeCDP(browser);
     const history = await cdp.send('Page.getNavigationHistory');
     if (history.currentIndex < history.entries.length - 1) {
       const entry = history.entries[history.currentIndex + 1];
@@ -174,7 +187,7 @@ export const goForward = tool(
 export const scrollPage = tool(
   async ({ direction, distance }) => {
     const browser = await getBrowser();
-    const cdp = await browser.getCDPSession();
+    const cdp = await safeCDP(browser);
     const deltaY = direction === 'up' ? -distance : distance;
     await cdp.send('Input.dispatchMouseEvent', {
       type: 'mouseWheel', x: 100, y: 100, deltaX: 0, deltaY
@@ -350,7 +363,7 @@ export const getElementHtml = tool(
 export const getCookies = tool(
   async ({ domain, format }) => {
     const browser = await getBrowser();
-    const cdp = await browser.getCDPSession();
+    const cdp = await safeCDP(browser);
 
     // 获取当前页面 URL 用于过滤
     const currentUrl = await cdpEvaluate(browser, 'location.href');

package/src/agent/tools/crawler.js

@@ -108,7 +108,7 @@ export const e2eTest = tool(
     description: '端到端测试爬虫脚本',
     schema: z.object({
       script_path: z.string().describe('脚本路径'),
-      test_params: z.record(z.string(), z.unknown()).optional().describe('测试参数'),
+      test_params: z.object({}).passthrough().optional().describe('测试参数'),
     }),
   }
 );

package/src/agent/tools/crawlerGenerator.js

@@ -73,8 +73,8 @@ export const delegateCrawlerGeneration = tool(
             xpath: z.string(),
             type: z.string(),
           })),
-          entry: z.any().nullable(),
-          pagination: z.any().nullable(),
+          entry: z.string().optional().describe('入口 URL 或选择器'),
+          pagination: z.string().optional().describe('分页选择器或 URL 模式'),
         })),
       }).describe('爬虫配置'),
       domain: z.string().describe('目标网站域名'),

package/src/agent/tools/evolve.js

@@ -79,9 +79,10 @@ last_merged: null
 
 /**
  * evolve_skill 工具
+ * 使用结构化格式记录经验，符合 evolved.md 模板规范
  */
 export const evolveSkill = tool(
-  async ({ skill, title, scenario, insight, isCore }) => {
+  async ({ skill, title, scenario, conclusion, technicalDetails, correctExample, incorrectExample, why, extensions, isCore }) => {
     const skillInfo = getSkillPath(skill);
     if (!skillInfo) {
       return JSON.stringify({
@@ -109,11 +110,44 @@ export const evolveSkill = tool(
 
     const data = parseEvolvedMd(content);
 
-    // 生成新条目
+    // 生成新条目（结构化格式）
     const date = new Date().toISOString().split('T')[0];
-    const entry = `### [${date}] ${title}
-**场景**: ${scenario}
-**经验**: ${insight}`;
+    let entry = `### [${date}] ${title}
+
+**一句话结论**: ${conclusion}
+
+**场景**: ${scenario}`;
+
+    // 技术细节（表格形式）
+    if (technicalDetails && Object.keys(technicalDetails).length > 0) {
+      entry += '\n\n**技术细节**:\n| 项目 | 值/说明 |\n|------|---------|';
+      for (const [key, value] of Object.entries(technicalDetails)) {
+        entry += `\n| ${key} | ${value} |`;
+      }
+    }
+
+    // 正确做法
+    if (correctExample) {
+      entry += `\n\n**正确做法**:\n\`\`\`python\n${correctExample}\n\`\`\``;
+    }
+
+    // 错误陷阱
+    if (incorrectExample) {
+      entry += `\n\n**错误陷阱** ⚠️:\n\`\`\`python\n${incorrectExample}\n\`\`\``;
+    }
+
+    // 原因解释
+    if (why) {
+      entry += `\n\n**为什么**: ${why}`;
+    }
+
+    // 举一反三
+    if (extensions && extensions.length > 0) {
+      entry += '\n\n**举一反三**:';
+      for (const item of extensions) {
+        entry += `\n- ${item}`;
+      }
+    }
 
     if (isCore) {
       // 追加到核心经验
@@ -153,13 +187,18 @@ export const evolveSkill = tool(
   },
   {
     name: 'evolve_skill',
-    description: '记录分析过程中学到的经验。支持现有 skill 或 new:<name> 创建新 skill',
+    description: '记录分析过程中学到的经验。使用结构化格式（一句话结论、技术细节、正确/错误示例、陷阱标记）',
     schema: z.object({
       skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, crawler, captcha, anti-detect, report, general，或 new:<name> 创建新 skill'),
       title: z.string().describe('经验标题，简短描述'),
       scenario: z.string().describe('具体场景/案例'),
-      insight: z.string().describe('一句话总结经验'),
-      isCore: z.boolean().default(false).describe('是否为核心经验'),
+      conclusion: z.string().describe('一句话核心结论（merge时必须提取到SKILL.md最前面）'),
+      technicalDetails: z.record(z.string(), z.string()).optional().describe('技术细节表格，如 {"参数类型": "int", "默认值": "0", "取值范围": "0或1"}'),
+      correctExample: z.string().optional().describe('正确代码示例'),
+      incorrectExample: z.string().optional().describe('错误代码示例（带陷阱标记）'),
+      why: z.string().optional().describe('解释根本原因'),
+      extensions: z.array(z.string()).optional().describe('类似场景列表（举一反三）'),
+      isCore: z.boolean().default(false).describe('是否为核心经验（已验证的高价值经验）'),
     }),
   }
 );

package/src/agent/tools/index.js

@@ -38,10 +38,9 @@ export { antiDetectTools } from './anti-detect.js';
 export { crawlerTools } from './crawler.js';
 export { crawlerGeneratorTools, generateCrawlerWithConfirm, delegateCrawlerGeneration } from './crawlerGenerator.js';
 export { nodejsTools, runNodeCode } from './nodejs.js';
+export { pythonTools, executePythonCode } from './python.js';
 export { hookManagerTools, listHooks, enableHook, disableHook, injectHook, setHookConfig } from './hookManager.js';
 export { scratchpadTools, saveMemo, loadMemo, listMemo } from './scratchpad.js';
-// pythonTools 只在 js2python 子代理中使用，不导出到主工具集
-
 // 所有工具
 import { sandboxTools } from './sandbox.js';
 import { analyzerTools } from './analyzer.js';
@@ -54,7 +53,7 @@ import { profileTools } from './profile.js';
 import { runtimeTools } from './runtime.js';
 import { debugTools } from './debug.js';
 import { captureTools } from './capture.js';
-import { browserTools } from './browser.js';
+import { browserTools, clickElement, scrollPage, fillInput, getInteractiveElements, getPageInfo, hoverElement, pressKey } from './browser.js';
 import { reportTools } from './report.js';
 import { webcrackTools } from './webcrack.js';
 import { preprocessTools } from './preprocess.js';
@@ -76,6 +75,7 @@ import { antiDetectTools } from './anti-detect.js';
 import { crawlerTools } from './crawler.js';
 import { crawlerGeneratorTools } from './crawlerGenerator.js';
 import { nodejsTools } from './nodejs.js';
+import { executePythonCode } from './python.js';
 import { hookManagerTools } from './hookManager.js';
 import { scratchpadTools } from './scratchpad.js';
 
@@ -143,8 +143,12 @@ export const coreTools = [
   ...evolveTools,
   // Node.js 执行（委托前快速验证假设）- 已添加网络请求防护
   ...nodejsTools,
+  // Python 执行（用于加密验证、数据处理等任务）
+  executePythonCode,
   // 工作记忆
   ...scratchpadTools,
   // 爬虫代码生成（带 HITL 确认）
   ...crawlerGeneratorTools,
+  // 页面交互（自主数据搜寻：滚动加载、点击触发请求）
+  clickElement, scrollPage, fillInput, getInteractiveElements, getPageInfo, hoverElement, pressKey,
 ];

package/src/agent/tools/patch.js

@@ -21,7 +21,7 @@ export const generatePatch = tool(
     description: '为缺失的环境属性生成补丁代码。',
     schema: z.object({
       property: z.string().describe('缺失的属性路径，如 navigator.userAgent'),
-      context: z.record(z.string(), z.unknown()).optional().describe('上下文信息'),
+      context: z.object({}).passthrough().optional().describe('上下文信息'),
     }),
   }
 );

package/src/agent/tools/store.js

@@ -30,7 +30,7 @@ export const saveToStore = tool(
       type: z.enum(['env-module', 'crypto-pattern', 'obfuscation']).describe('类型'),
       name: z.string().describe('名称'),
       code: z.string().describe('代码'),
-      metadata: z.record(z.string(), z.unknown()).optional().describe('元数据'),
+      metadata: z.object({}).passthrough().optional().describe('元数据'),
     }),
   }
 );

package/src/browser/client.js

@@ -206,7 +206,11 @@ export class BrowserClient extends EventEmitter {
 
       try {
         // 通过简单的 Runtime.evaluate 验证 session 是否还活着
-        await this.cdpSession.send('Runtime.evaluate', { expression: '1' });
+        // 必须加超时：页面 loading/断点暂停时 Runtime.evaluate 会永远挂住
+        await Promise.race([
+          this.cdpSession.send('Runtime.evaluate', { expression: '1' }),
+          new Promise((_, reject) => setTimeout(() => reject(new Error('CDP health check timeout')), 3000)),
+        ]);
         this._cdpLastCheck = now;
         return this.cdpSession;
       } catch {

package/src/browser/ui/analysisPanel.js

@@ -657,6 +657,22 @@ export function getAnalysisPanelScript() {
       .deepspider-confirm-no {
         background: rgba(255,255,255,0.05); border: 1px solid rgba(255,255,255,0.15); color: #8b949e;
       }
+      /* 恢复 session 横幅 */
+      .deepspider-resume-banner {
+        background: rgba(99, 179, 237, 0.08); border: 1px solid rgba(99, 179, 237, 0.2);
+        border-radius: 10px; padding: 12px 14px; margin: 4px 0;
+      }
+      .deepspider-resume-btn {
+        width: 100%; padding: 8px; border-radius: 8px; border: none;
+        background: linear-gradient(135deg, #63b3ed, #4299e1); color: #fff;
+        font-size: 12px; font-weight: 500; cursor: pointer; transition: all 0.2s;
+      }
+      .deepspider-resume-btn:hover { opacity: 0.85; }
+      .deepspider-resume-dismiss {
+        width: 100%; padding: 6px; border: none; background: none;
+        color: #8b949e; font-size: 11px; cursor: pointer; margin-top: 4px;
+      }
+      .deepspider-resume-dismiss:hover { color: #c9d1d9; }
       .deepspider-msg-system {
         background: transparent;
         text-align: center;
@@ -1270,6 +1286,7 @@ export function getAnalysisPanelScript() {
         bindFilePathClicks(messagesEl);
         bindChoiceClicks(messagesEl);
         bindConfirmClicks(messagesEl);
+        bindResumeClicks(messagesEl);
       }
       messagesEl.scrollTop = messagesEl.scrollHeight;
     }
@@ -1286,6 +1303,10 @@ export function getAnalysisPanelScript() {
           return renderChoicesMessage(m);
         case 'confirm':
           return renderConfirmMessage(m);
+        case 'resume-available':
+          return renderResumeMessage(m);
+        case 'file-saved':
+          return renderFileSavedMessage(m);
         default:
           return '<div class="deepspider-msg deepspider-msg-system">' + escapeHtml(JSON.stringify(m.data)) + '</div>';
       }
@@ -1326,6 +1347,32 @@ export function getAnalysisPanelScript() {
       return html;
     }
 
+    function renderResumeMessage(m) {
+      if (m.answered) return '';
+      const d = m.data;
+      return '<div class="deepspider-resume-banner">' +
+        '<div style="margin-bottom:6px;">检测到上次未完成的分析</div>' +
+        '<div style="font-size:11px;color:#8b949e;margin-bottom:8px;">' +
+          escapeHtml(d.domain) + ' · ' + escapeHtml(d.timeAgo) + ' · ' + escapeHtml(String(d.messageCount)) + '条消息</div>' +
+        '<button class="deepspider-resume-btn" data-resume-thread="' + escapeHtml(d.threadId) + '">恢复上次分析</button>' +
+        '<button class="deepspider-resume-dismiss" data-resume-dismiss="true">忽略</button>' +
+        '</div>';
+    }
+
+    function renderFileSavedMessage(m) {
+      var d = m.data;
+      var icon = d.type === 'py' ? '🐍' : d.type === 'report' ? '📊' : '📄';
+      var label = d.type === 'py' ? 'Python 脚本' : d.type === 'report' ? '分析报告' : '文件';
+      return '<div class="deepspider-msg deepspider-msg-system" style="background:#1a2332;border-left:3px solid #388bfd;padding:8px 12px;">' +
+        '<div style="display:flex;align-items:center;gap:6px;">' +
+        '<span>' + icon + '</span>' +
+        '<span style="color:#58a6ff;">' + escapeHtml(label) + '已保存</span>' +
+        '</div>' +
+        '<div class="deepspider-file-path" style="font-size:11px;color:#8b949e;margin-top:4px;cursor:pointer;" data-file-path="' + escapeHtml(d.path) + '">' +
+        escapeHtml(d.path) +
+        '</div></div>';
+    }
+
     function bindChoiceClicks(container) {
       container.querySelectorAll('.deepspider-choice-btn:not([style*="pointer-events"])').forEach(btn => {
         btn.onclick = () => {
@@ -1367,6 +1414,31 @@ export function getAnalysisPanelScript() {
       });
     }
 
+    function bindResumeClicks(container) {
+      container.querySelectorAll('.deepspider-resume-btn').forEach(btn => {
+        btn.onclick = () => {
+          const threadId = btn.dataset.resumeThread;
+          const msgs = deepspider.chatMessages;
+          for (let i = msgs.length - 1; i >= 0; i--) {
+            if (msgs[i].type === 'resume-available') { msgs[i].answered = true; break; }
+          }
+          addMessage('system', '正在恢复上次分析...');
+          if (typeof __deepspider_send__ === 'function') {
+            __deepspider_send__(JSON.stringify({ __ds__: true, type: 'resume', threadId }));
+          }
+        };
+      });
+      container.querySelectorAll('.deepspider-resume-dismiss').forEach(btn => {
+        btn.onclick = () => {
+          const msgs = deepspider.chatMessages;
+          for (let i = msgs.length - 1; i >= 0; i--) {
+            if (msgs[i].type === 'resume-available') { msgs[i].answered = true; break; }
+          }
+          renderMessages();
+        };
+      });
+    }
+
     function escapeHtml(str) {
       return String(str).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
     }