deepspider 0.3.1 → 0.3.2

package/src/agent/tools/index.js

@@ -11,9 +11,9 @@ export { patchTools, generatePatch, matchModule } from './patch.js';
 export { envTools, listEnvModules, loadEnvModule, loadAllEnvModules } from './env.js';
 export { profileTools, listProfiles, loadProfile, generateProfileCode } from './profile.js';
 export { runtimeTools, launchBrowser, navigateTo, browserClose, addInitScript, clearCookies } from './runtime.js';
-export { debugTools, setBreakpoint, setXHRBreakpoint, getCallStack, getFrameVariables, evaluateAtBreakpoint, resumeExecution, stepOver } from './debug.js';
+export { debugTools, setBreakpoint, setXHRBreakpoint, getCallStack, getFrameVariables, evaluateAtBreakpoint, resumeExecution, stepOver, getAgentLogs } from './debug.js';
 export { captureTools, collectEnv, collectProperty, autoFixEnv, getHookLogs } from './capture.js';
-export { browserTools, clickElement, fillInput, waitForSelector } from './browser.js';
+export { browserTools, clickElement, fillInput, waitForSelector, takeScreenshot, reloadPage, goBack, goForward, scrollPage, pressKey, hoverElement, getPageInfo, getPageSource, getElementHtml, getCookies, getInteractiveElements } from './browser.js';
 export { reportTools, saveAnalysisReport } from './report.js';
 export { webcrackTools, unpackBundle, analyzeBundle } from './webcrack.js';
 export { preprocessTools, preprocessCode } from './preprocess.js';
@@ -24,17 +24,22 @@ export { asyncTools, generatePromiseHook, generateTimerHook } from './async.js';
 export { antiDebugTools, generateAntiDebugger, generateAntiConsoleDetect, generateAntiCDP, generateFullAntiDebug } from './antidebug.js';
 export { verifyTools, verifyMD5, verifySHA256, verifyHMAC, verifyAES, identifyEncryption } from './verify.js';
 export { cryptoHookTools, generateCryptoJSHook, generateRSAHook } from './cryptohook.js';
-export { correlateTools, analyzeCorrelation, locateCryptoSource, analyzeHeaderEncryption, analyzeCookieEncryption, analyzeResponseDecryption } from './correlate.js';
+// 合并工具（reverse-agent 使用）
+export { generateHookTools, generateHook } from './generateHook.js';
+export { verifyAlgorithmTools, verifyAlgorithm } from './verifyAlgorithm.js';
+export { correlateTools, analyzeCorrelation, locateCryptoSource, analyzeHeaderEncryption, analyzeCookieEncryption, analyzeResponseDecryption, analyzeRequestParams } from './correlate.js';
 export { extractorTools, listFunctions, getFunctionCode } from './extractor.js';
-export { tracingTools, getSiteList, searchInResponses, getRequestDetail, getRequestList, getScriptList, getScriptSource, searchInScripts, clearSiteData, clearAllData } from './tracing.js';
+export { tracingTools, getSiteList, searchInResponses, getRequestDetail, getRequestList, getRequestInitiator, getScriptList, getScriptSource, searchInScripts, clearSiteData, clearAllData } from './tracing.js';
 export { analysisTools, getPendingAnalysis, getPendingChat, sendPanelMessage, startSelector } from './analysis.js';
 export { fileTools, artifactSave, artifactLoad, artifactEdit, artifactGlob, artifactGrep } from './file.js';
 export { evolveTools, evolveSkill } from './evolve.js';
 export { captchaTools } from './captcha.js';
 export { antiDetectTools } from './anti-detect.js';
 export { crawlerTools } from './crawler.js';
+export { crawlerGeneratorTools, generateCrawlerWithConfirm, delegateCrawlerGeneration } from './crawlerGenerator.js';
 export { nodejsTools, runNodeCode } from './nodejs.js';
 export { hookManagerTools, listHooks, enableHook, disableHook, injectHook, setHookConfig } from './hookManager.js';
+export { scratchpadTools, saveMemo, loadMemo, listMemo } from './scratchpad.js';
 // pythonTools 只在 js2python 子代理中使用，不导出到主工具集
 
 // 所有工具
@@ -62,15 +67,17 @@ import { verifyTools } from './verify.js';
 import { cryptoHookTools } from './cryptohook.js';
 import { correlateTools } from './correlate.js';
 import { extractorTools } from './extractor.js';
-import { tracingTools } from './tracing.js';
+import { tracingTools, getSiteList, getRequestList, searchInResponses, getRequestDetail, getRequestInitiator } from './tracing.js';
 import { analysisTools } from './analysis.js';
 import { fileTools } from './file.js';
 import { evolveTools } from './evolve.js';
 import { captchaTools } from './captcha.js';
 import { antiDetectTools } from './anti-detect.js';
 import { crawlerTools } from './crawler.js';
+import { crawlerGeneratorTools } from './crawlerGenerator.js';
 import { nodejsTools } from './nodejs.js';
 import { hookManagerTools } from './hookManager.js';
+import { scratchpadTools } from './scratchpad.js';
 
 export const allTools = [
   ...sandboxTools,
@@ -104,34 +111,40 @@ export const allTools = [
   ...captchaTools,
   ...antiDetectTools,
   ...crawlerTools,
+  ...crawlerGeneratorTools,
   ...nodejsTools,
   ...hookManagerTools,
+  ...scratchpadTools,
 ];
 
 /**
  * 主 Agent 核心工具
- * 只包含必要的运行时、交互和数据查询工具
- * pythonTools 只在 js2python 子代理中使用
+ * 职责：浏览器生命周期、简单页面交互、数据查询、委托调度
+ *
+ * 以下工具有意不包含在主 agent 中，由专属子代理持有：
+ * - hookManagerTools (inject_hook 等) → reverse-agent
+ * - captureTools (collect_env, get_hook_logs 等) → reverse-agent
+ * - sandboxTools → reverse-agent
+ * - debugTools → reverse-agent
+ * - 静态分析工具 → reverse-agent
  */
 export const coreTools = [
-  // 浏览器运行时
+  // 浏览器运行时（生命周期管理）
   ...runtimeTools,
-  // 页面交互
-  ...browserTools,
-  // 浏览器分析交互
+  // 浏览器分析面板交互
   ...analysisTools,
-  // 数据溯源查询
-  ...tracingTools,
-  // 沙箱执行（验证代码）
-  ...sandboxTools,
-  // Hook 日志
-  ...captureTools,
+  // 数据查询（仅调度所需的最小集：列表、搜索、详情、initiator）
+  getSiteList, getRequestList, searchInResponses, getRequestDetail, getRequestInitiator,
+  // 报告生成
+  ...reportTools,
   // 文件操作
   ...fileTools,
   // 经验进化
   ...evolveTools,
-  // Node.js 执行（支持 require）
+  // Node.js 执行（委托前快速验证假设）- 已添加网络请求防护
   ...nodejsTools,
-  // Hook 动态管理
-  ...hookManagerTools,
+  // 工作记忆
+  ...scratchpadTools,
+  // 爬虫代码生成（带 HITL 确认）
+  ...crawlerGeneratorTools,
 ];

package/src/agent/tools/nodejs.js

@@ -17,10 +17,19 @@ const PROJECT_ROOT = join(__dirname, '../../..');
 // 输出大小限制
 const MAX_OUTPUT_SIZE = 100000;
 
+// 超时上限（防止 LLM 传入过大值）
+const MAX_TIMEOUT = 30000;
+
+// 连续超时计数器
+let consecutiveTimeouts = 0;
+const MAX_CONSECUTIVE_TIMEOUTS = 3;
+
 /**
  * 执行 Node.js 代码
  */
 async function executeNode(code, timeout = 10000) {
+  const effectiveTimeout = Math.min(timeout, MAX_TIMEOUT);
+
   return new Promise((resolve) => {
     const proc = spawn('node', ['-e', code], {
       env: { ...process.env },
@@ -30,12 +39,17 @@ async function executeNode(code, timeout = 10000) {
     let stdout = '';
     let stderr = '';
     let killed = false;
+    let killTimer = null;
 
-    // 手动实现超时
+    // SIGTERM 超时
     const timer = setTimeout(() => {
       killed = true;
       proc.kill('SIGTERM');
-    }, timeout);
+      // SIGKILL 兜底：环境检测死循环可能忽略 SIGTERM
+      killTimer = setTimeout(() => {
+        try { proc.kill('SIGKILL'); } catch { /* already dead */ }
+      }, 2000);
+    }, effectiveTimeout);
 
     proc.stdout.on('data', (data) => {
       if (stdout.length < MAX_OUTPUT_SIZE) {
@@ -51,21 +65,25 @@ async function executeNode(code, timeout = 10000) {
 
     proc.on('close', (exitCode) => {
       clearTimeout(timer);
+      if (killTimer) clearTimeout(killTimer);
       resolve({
         success: !killed && exitCode === 0,
         stdout: stdout.trim(),
-        stderr: killed ? 'Timeout: process killed' : stderr.trim(),
+        stderr: killed ? `Timeout after ${effectiveTimeout}ms: process killed` : stderr.trim(),
         exitCode: killed ? -1 : exitCode,
+        timedOut: killed,
       });
     });
 
     proc.on('error', (err) => {
       clearTimeout(timer);
+      if (killTimer) clearTimeout(killTimer);
       resolve({
         success: false,
         stdout: '',
         stderr: err.message,
         exitCode: -1,
+        timedOut: false,
       });
     });
   });
@@ -76,8 +94,25 @@ async function executeNode(code, timeout = 10000) {
  */
 export const runNodeCode = tool(
   async ({ code, timeout }) => {
-    const result = await executeNode(code, timeout || 10000);
-    return JSON.stringify(result);
+    // 连续超时保护：降级为短超时探测，而非完全拒绝
+    const isBlocked = consecutiveTimeouts >= MAX_CONSECUTIVE_TIMEOUTS;
+    const effectiveTimeout = isBlocked ? 5000 : (timeout || 10000);
+    const result = await executeNode(code, effectiveTimeout);
+
+    // 更新连续超时计数
+    if (result.timedOut) {
+      consecutiveTimeouts++;
+      if (consecutiveTimeouts >= MAX_CONSECUTIVE_TIMEOUTS) {
+        result.stderr += `\n\n⚠️ 已连续超时 ${consecutiveTimeouts} 次，后续调用将降级为 5s 短超时。代码很可能存在死循环或触发了环境检测。请停止重试相同逻辑，改用 sandbox_execute（沙箱执行）、断点调试或静态分析。`;
+      } else {
+        result.stderr += `\n\n⚠️ 连续超时 ${consecutiveTimeouts}/${MAX_CONSECUTIVE_TIMEOUTS} 次。如果代码包含从网站提取的混淆代码，可能存在环境检测导致死循环。建议：1) 检查代码是否有 while(true)/setInterval 等循环 2) 改用 sandbox_execute 在受控环境中执行`;
+      }
+    } else {
+      consecutiveTimeouts = 0; // 成功执行或非超时失败，重置计数
+    }
+
+    const { timedOut: _, ...output } = result;
+    return JSON.stringify(output);
   },
   {
     name: 'run_node_code',
@@ -93,7 +128,7 @@ export const runNodeCode = tool(
 示例：const CryptoJS = require('crypto-js'); console.log(CryptoJS.MD5('test').toString());`,
     schema: z.object({
       code: z.string().describe('要执行的 JS 代码，可使用 require 引入加密库'),
-      timeout: z.number().optional().default(10000).describe('超时时间（毫秒）'),
+      timeout: z.number().optional().default(10000).describe('超时时间（毫秒），上限 30000'),
     }),
   }
 );

package/src/agent/tools/sandbox.js

@@ -76,4 +76,24 @@ export const sandboxReset = tool(
   }
 );
 
-export const sandboxTools = [sandboxExecute, sandboxInject, sandboxReset];
+/**
+ * 自动补环境执行工具
+ */
+export const sandboxAutoFix = tool(
+  async ({ code, timeout, maxIterations }) => {
+    const sb = await getSandbox();
+    const result = await sb.executeWithAutoFix(code, { timeout, maxIterations });
+    return JSON.stringify(result, null, 2);
+  },
+  {
+    name: 'sandbox_auto_fix',
+    description: '自动补环境闭环执行：加载预置模块 → 执行代码 → 发现缺失环境 → 自动生成补丁 → 重试，直到成功或无法继续。适合快速验证混淆代码能否在沙箱中运行。',
+    schema: z.object({
+      code: z.string().describe('要执行的目标JS代码'),
+      timeout: z.number().optional().default(5000).describe('单次执行超时时间(ms)'),
+      maxIterations: z.number().optional().default(10).describe('最大迭代次数'),
+    }),
+  }
+);
+
+export const sandboxTools = [sandboxExecute, sandboxInject, sandboxReset, sandboxAutoFix];

package/src/agent/tools/scratchpad.js

@@ -0,0 +1,70 @@
+/**
+ * DeepSpider - 工作记忆工具（Scratchpad）
+ * 保存/读取关键发现，防止多步推理中丢失上下文
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { writeFileSync, readFileSync, existsSync, readdirSync } from 'fs';
+import { join } from 'path';
+import { DEEPSPIDER_HOME, ensureDir } from '../../config/paths.js';
+
+const MEMO_DIR = join(DEEPSPIDER_HOME, 'memo');
+
+/** 清理 key：只保留字母、数字、连字符、下划线，防止路径穿越 */
+function sanitizeKey(key) {
+  return key.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 100);
+}
+
+export const saveMemo = tool(
+  async ({ key, content }) => {
+    ensureDir(MEMO_DIR);
+    const safeKey = sanitizeKey(key);
+    const filePath = join(MEMO_DIR, `${safeKey}.txt`);
+    writeFileSync(filePath, content, 'utf-8');
+    return JSON.stringify({ success: true, key: safeKey, size: content.length });
+  },
+  {
+    name: 'save_memo',
+    description: '保存工作记忆。用于记录关键发现、中间结果、待验证假设等，防止多步推理中丢失上下文',
+    schema: z.object({
+      key: z.string().describe('记忆键名，如 "encryption-analysis"、"key-source"'),
+      content: z.string().describe('要保存的内容'),
+    }),
+  }
+);
+
+export const loadMemo = tool(
+  async ({ key }) => {
+    const safeKey = sanitizeKey(key);
+    const filePath = join(MEMO_DIR, `${safeKey}.txt`);
+    if (!existsSync(filePath)) {
+      return JSON.stringify({ success: false, error: `memo "${safeKey}" not found` });
+    }
+    const content = readFileSync(filePath, 'utf-8');
+    return JSON.stringify({ success: true, key: safeKey, content });
+  },
+  {
+    name: 'load_memo',
+    description: '读取之前保存的工作记忆',
+    schema: z.object({
+      key: z.string().describe('记忆键名'),
+    }),
+  }
+);
+
+export const listMemo = tool(
+  async () => {
+    ensureDir(MEMO_DIR);
+    const files = readdirSync(MEMO_DIR).filter(f => f.endsWith('.txt'));
+    const memos = files.map(f => f.replace('.txt', ''));
+    return JSON.stringify({ success: true, memos, count: memos.length });
+  },
+  {
+    name: 'list_memo',
+    description: '列出所有已保存的工作记忆',
+    schema: z.object({}),
+  }
+);
+
+export const scratchpadTools = [saveMemo, loadMemo, listMemo];

package/src/agent/tools/tracing.js

@@ -188,11 +188,37 @@ export const clearAllData = tool(
   }
 );
 
+/**
+ * 获取请求的 initiator（调用栈）
+ */
+export const getRequestInitiator = tool(
+  async ({ site, id }) => {
+    const store = getDataStore();
+    const result = await store.getResponse(site, id);
+    if (!result) {
+      return JSON.stringify({ error: '未找到该请求' });
+    }
+    if (!result.initiator) {
+      return JSON.stringify({ error: '该请求无 initiator 信息（可能是旧数据或浏览器内部请求）' });
+    }
+    return JSON.stringify(result.initiator, null, 2);
+  },
+  {
+    name: 'get_request_initiator',
+    description: '获取发起该请求的 JS 代码位置（脚本URL + 行号 + 函数名）。返回调用栈的前5帧。用于从目标请求反向定位加密函数入口，是逆向分析的第一步。',
+    schema: z.object({
+      site: z.string().describe('站点 hostname'),
+      id: z.string().describe('请求 ID'),
+    }),
+  }
+);
+
 export const tracingTools = [
   getSiteList,
   searchInResponses,
   getRequestDetail,
   getRequestList,
+  getRequestInitiator,
   getScriptList,
   getScriptSource,
   searchInScripts,

package/src/agent/tools/verifyAlgorithm.js

@@ -0,0 +1,117 @@
+/**
+ * DeepSpider - 统一算法验证工具
+ * 合并 verify_md5/sha256/hmac/aes + identify_encryption
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import crypto from 'crypto';
+
+/**
+ * 识别密文特征
+ */
+function identifyPattern(ciphertext) {
+  const features = [];
+  const len = ciphertext.length;
+
+  if (len === 32) features.push('可能是 MD5');
+  if (len === 40) features.push('可能是 SHA1');
+  if (len === 64) features.push('可能是 SHA256');
+  if (len === 128) features.push('可能是 SHA512');
+  if (/^[A-Za-z0-9+/]+=*$/.test(ciphertext)) features.push('Base64 编码');
+  if (/^[0-9a-fA-F]+$/.test(ciphertext)) features.push('Hex 编码');
+  if (/^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(ciphertext)) features.push('JWT Token');
+
+  return features;
+}
+
+export const verifyAlgorithm = tool(
+  async ({ algorithm, input, expected, key, iv, hmacHash, aesMode }) => {
+    // 识别模式：不传 algorithm，只传 expected
+    if (!algorithm) {
+      return JSON.stringify({
+        ciphertext: expected.slice(0, 50) + (expected.length > 50 ? '...' : ''),
+        length: expected.length,
+        features: identifyPattern(expected),
+      }, null, 2);
+    }
+
+    // 验证模式：需要 input + expected
+    if (!input) {
+      return JSON.stringify({ error: `验证 ${algorithm} 需要 input 参数` });
+    }
+
+    let computed;
+    let algoLabel;
+
+    switch (algorithm) {
+      case 'md5':
+      case 'sha1':
+      case 'sha256':
+      case 'sha512': {
+        computed = crypto.createHash(algorithm).update(input).digest('hex');
+        algoLabel = algorithm.toUpperCase();
+        break;
+      }
+      case 'hmac': {
+        if (!key) return JSON.stringify({ error: 'HMAC 需要 key 参数' });
+        const hash = hmacHash || 'sha256';
+        computed = crypto.createHmac(hash, key).update(input).digest('hex');
+        algoLabel = `HMAC-${hash.toUpperCase()}`;
+        break;
+      }
+      case 'aes': {
+        if (!key) return JSON.stringify({ error: 'AES 需要 key 参数' });
+        try {
+          const keyBuf = Buffer.from(key, 'utf8');
+          const ivBuf = iv ? Buffer.from(iv, 'utf8') : Buffer.alloc(16, 0);
+          const mode = aesMode || 'cbc';
+          const cipher = crypto.createCipheriv(
+            `aes-${keyBuf.length * 8}-${mode}`,
+            keyBuf,
+            mode === 'ecb' ? null : ivBuf
+          );
+          computed = cipher.update(input, 'utf8', 'base64') + cipher.final('base64');
+          algoLabel = `AES-${keyBuf.length * 8}-${mode.toUpperCase()}`;
+        } catch (e) {
+          return JSON.stringify({ error: e.message });
+        }
+        break;
+      }
+      default:
+        return JSON.stringify({ error: `未知算法: ${algorithm}` });
+    }
+
+    const match = algorithm === 'aes'
+      ? computed === expected
+      : computed.toLowerCase() === expected.toLowerCase();
+
+    return JSON.stringify({
+      algorithm: algoLabel,
+      input,
+      computed,
+      expected,
+      match,
+      conclusion: match ? `标准 ${algoLabel}` : '可能魔改或参数不同',
+    }, null, 2);
+  },
+  {
+    name: 'verify_algorithm',
+    description: `验证是否为标准加密算法，或根据密文特征识别算法类型。
+
+验证模式：传入 algorithm + input + expected，对比计算结果
+识别模式：只传 expected（密文），自动识别可能的算法类型`,
+    schema: z.object({
+      algorithm: z.enum(['md5', 'sha1', 'sha256', 'sha512', 'hmac', 'aes']).optional()
+        .describe('算法类型。不传则进入识别模式'),
+      input: z.string().optional().describe('原始输入'),
+      expected: z.string().describe('目标加密结果（验证模式）或待识别的密文（识别模式）'),
+      key: z.string().optional().describe('密钥（HMAC/AES 需要）'),
+      iv: z.string().optional().describe('IV 向量（AES 可选）'),
+      hmacHash: z.enum(['md5', 'sha1', 'sha256', 'sha512']).optional().describe('HMAC 哈希算法，默认 sha256'),
+      aesMode: z.enum(['cbc', 'ecb']).optional().describe('AES 模式，默认 cbc'),
+    }),
+  }
+);
+
+export const verifyAlgorithmTools = [verifyAlgorithm];

package/src/browser/EnvBridge.js

@@ -26,24 +26,38 @@ export class EnvBridge {
 
     for (const path of missingPaths) {
       try {
-        // 1. 从真实浏览器采集
+        // 1. 尝试从真实浏览器采集
         const collected = await this.collector.collect(path, { depth: 2 });
 
-        if (!collected.success) {
-          results.failed.push({ path, reason: 'collect_failed', error: collected.error });
-          continue;
-        }
+        if (collected.success) {
+          results.collected.push(path);
+          this.collectedData.set(path, collected);
 
-        results.collected.push(path);
-        this.collectedData.set(path, collected);
+          // 2. 生成补丁代码
+          const patch = this._generatePatch(path, collected);
+          if (patch) {
+            results.patched.push({ path, code: patch });
+            continue;
+          }
+        }
 
-        // 2. 生成补丁代码
-        const patch = this._generatePatch(path, collected);
-        if (patch) {
-          results.patched.push({ path, code: patch });
+        // 3. 采集失败时 fallback 到 PatchGenerator
+        const fallback = await this.patchGenerator.generate(path);
+        if (fallback.code) {
+          results.patched.push({ path, code: fallback.code, source: fallback.source });
+        } else {
+          results.failed.push({ path, reason: 'no_patch' });
         }
 
       } catch (e) {
+        // 浏览器不可用时也 fallback
+        try {
+          const fallback = await this.patchGenerator.generate(path);
+          if (fallback.code) {
+            results.patched.push({ path, code: fallback.code, source: fallback.source });
+            continue;
+          }
+        } catch { /* ignore */ }
         results.failed.push({ path, reason: 'error', error: e.message });
       }
     }
@@ -66,7 +80,7 @@ export class EnvBridge {
     // 根据数据类型生成不同的补丁
     switch (data.type) {
       case 'string':
-        return `${parentPath}.${propName} = "${data.value}";`;
+        return `${parentPath}.${propName} = ${JSON.stringify(data.value)};`;
 
       case 'number':
         return `${parentPath}.${propName} = ${data.value};`;
@@ -123,7 +137,7 @@ export class EnvBridge {
 
   _serializeValue(data) {
     switch (data.type) {
-      case 'string': return `"${data.value}"`;
+      case 'string': return JSON.stringify(data.value);
       case 'number': return data.value;
       case 'boolean': return data.value;
       case 'null': return 'null';