deepspider 0.3.1 → 0.3.2

package/src/agent/tools/correlate.js

@@ -5,6 +5,7 @@
 
 import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
+import { getDataStore } from '../../store/DataStore.js';
 
 /**
  * 分析请求-加密关联
@@ -74,22 +75,40 @@ export const analyzeCorrelation = tool(
 
 /**
  * 解析调用栈顶部
+ * 支持两种格式：
+ * 1. 字符串栈（来自 Error.stack）
+ * 2. callFrames 数组（来自 CDP initiator）
  */
 function parseStackTop(stack) {
   if (!stack) return null;
-  const lines = stack.split('\n').slice(2, 5);
-  return lines.map(line => {
-    const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/) ||
-                  line.match(/at\s+(.+?):(\d+):(\d+)/);
-    if (match) {
-      return {
-        func: match[1] || 'anonymous',
-        file: match[2] || match[1],
-        line: parseInt(match[3] || match[2])
-      };
-    }
-    return { raw: line.trim() };
-  });
+
+  // 处理 callFrames 数组格式（来自 CDP initiator）
+  if (Array.isArray(stack)) {
+    return stack.slice(0, 3).map(frame => ({
+      func: frame.functionName || frame.func || '(anonymous)',
+      file: frame.url || frame.file || '',
+      line: frame.lineNumber || frame.line || 0
+    }));
+  }
+
+  // 处理字符串栈格式（来自 Error.stack）
+  if (typeof stack === 'string') {
+    const lines = stack.split('\n').slice(2, 5);
+    return lines.map(line => {
+      const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/) ||
+                    line.match(/at\s+(.+?):(\d+):(\d+)/);
+      if (match) {
+        return {
+          func: match[1] || 'anonymous',
+          file: match[2] || match[1],
+          line: parseInt(match[3] || match[2])
+        };
+      }
+      return { raw: line.trim() };
+    });
+  }
+
+  return null;
 }
 
 /**
@@ -183,11 +202,12 @@ export const analyzeCookieEncryption = tool(
   async ({ logs, cookieName }) => {
     const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
 
-    // 找到设置该 cookie 的日志
+    // 找到设置该 cookie 的日志（匹配 cookie 键名）
     const cookieLogs = parsed.filter(entry => {
       if (entry._type !== 'cookie') return false;
       if (entry.action !== 'write') return false;
-      return entry.value?.includes(cookieName);
+      // cookie hook 日志的 value 格式为 "name=value"，匹配键名部分
+      return entry.value?.startsWith(cookieName + '=') || entry.name === cookieName;
     });
 
     if (cookieLogs.length === 0) {
@@ -294,10 +314,104 @@ export const analyzeResponseDecryption = tool(
   }
 );
 
+/**
+ * 识别加密模式
+ */
+function identifyPattern(value) {
+  if (/^[0-9a-fA-F]+$/.test(value)) {
+    if (value.length === 32) return 'hash-md5';
+    if (value.length === 40) return 'hash-sha1';
+    if (value.length === 64) return 'hash-sha256';
+    return 'hex';
+  }
+  if (/^[A-Za-z0-9+/]{20,}={0,2}$/.test(value)) return 'base64';
+  if (/^ey[A-Za-z0-9_-]+\./.test(value)) return 'jwt';
+  return 'unknown';
+}
+
+/**
+ * 判断值是否像加密结果
+ */
+function looksEncrypted(value) {
+  if (/^[0-9a-fA-F]{32,}$/.test(value)) return true;
+  if (/^[A-Za-z0-9+/]{20,}={0,2}$/.test(value)) return true;
+  if (/^ey[A-Za-z0-9_-]+\./.test(value)) return true;
+  return false;
+}
+
+/**
+ * 解析请求 body
+ */
+function parseBody(body) {
+  if (!body) return {};
+  try {
+    return JSON.parse(body);
+  } catch {
+    // 尝试 form-urlencoded
+    try {
+      return Object.fromEntries(new URLSearchParams(body));
+    } catch {
+      return { _raw: body.slice(0, 200) };
+    }
+  }
+}
+
+/**
+ * 分析请求参数结构（不依赖 Hook 日志）
+ */
+export const analyzeRequestParams = tool(
+  async ({ site, id }) => {
+    const store = getDataStore();
+    const detail = await store.getResponse(site, id);
+    if (!detail) return JSON.stringify({ error: '未找到该请求' });
+
+    let urlParams = {};
+    try {
+      urlParams = Object.fromEntries(new URL(detail.url).searchParams);
+    } catch { /* invalid URL */ }
+
+    const bodyParams = parseBody(detail.requestBody);
+
+    // 识别可疑参数
+    const suspiciousParams = [];
+    const allParams = { ...urlParams, ...bodyParams };
+    for (const [key, value] of Object.entries(allParams)) {
+      const str = String(value);
+      if (str.length > 20 && looksEncrypted(str)) {
+        suspiciousParams.push({
+          name: key,
+          value: str.slice(0, 50) + (str.length > 50 ? '...' : ''),
+          length: str.length,
+          pattern: identifyPattern(str),
+        });
+      }
+    }
+
+    return JSON.stringify({
+      url: detail.url,
+      method: detail.method,
+      urlParams,
+      bodyParams: typeof bodyParams === 'object' && !bodyParams._raw
+        ? bodyParams : { _raw: detail.requestBody?.slice(0, 200) },
+      suspiciousParams,
+      initiator: detail.initiator || null,
+    }, null, 2);
+  },
+  {
+    name: 'analyze_request_params',
+    description: '分析请求的参数结构，自动识别可疑的加密参数（hex/base64/hash）。不依赖 Hook 日志，可直接使用。',
+    schema: z.object({
+      site: z.string().describe('站点 hostname'),
+      id: z.string().describe('请求 ID'),
+    }),
+  }
+);
+
 export const correlateTools = [
   analyzeCorrelation,
   locateCryptoSource,
   analyzeHeaderEncryption,
   analyzeCookieEncryption,
   analyzeResponseDecryption,
+  analyzeRequestParams,
 ];

package/src/agent/tools/crawler.js

@@ -113,4 +113,5 @@ export const e2eTest = tool(
   }
 );
 
-export const crawlerTools = [siteAnalyze, complexityAssess, e2eTest];
+// e2eTest 暂未实现（stub），不导出到子代理工具列表，避免子代理浪费轮次调用空壳工具
+export const crawlerTools = [siteAnalyze, complexityAssess];

package/src/agent/tools/crawlerGenerator.js

@@ -0,0 +1,90 @@
+/**
+ * DeepSpider - 爬虫代码生成工具
+ * 通过 LangGraph interrupt 机制实现面板交互式选择
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { interrupt } from '@langchain/langgraph';
+
+/**
+ * 请求用户选择爬虫框架并生成代码
+ * interrupt payload 遵循统一协议，StreamHandler 自动渲染到面板
+ */
+export const generateCrawlerWithConfirm = tool(
+  async ({ analysisSummary, domain }) => {
+    const userChoice = interrupt({
+      type: 'choices',
+      question: '分析完成！选择爬虫框架生成完整脚本：',
+      options: [
+        { id: 'requests', label: 'requests', description: '简单易用，适合快速原型' },
+        { id: 'httpx', label: 'httpx', description: '异步高性能，适合大规模并发' },
+        { id: 'scrapy', label: 'Scrapy', description: '企业级框架，适合复杂项目' },
+        { id: 'skip', label: '不需要', description: '仅保存当前分析结果' },
+      ],
+    });
+
+    return JSON.stringify({
+      success: true,
+      framework: userChoice,
+      domain,
+      message: userChoice === '不需要'
+        ? '用户选择不生成爬虫脚本'
+        : `用户选择使用 ${userChoice} 框架生成爬虫`,
+    });
+  },
+  {
+    name: 'generate_crawler_code',
+    description: `分析完成后，向用户展示可点击的框架选项（requests/httpx/Scrapy/不需要）。
+
+用户点击后，工具返回用户选择的框架名称。根据返回值委托 crawler 子代理生成代码。`,
+    schema: z.object({
+      analysisSummary: z.string().describe('分析结果摘要'),
+      domain: z.string().describe('目标网站域名'),
+    }),
+  }
+);
+
+/**
+ * 根据用户选择的框架委托 crawler 子代理生成代码
+ */
+export const delegateCrawlerGeneration = tool(
+  async ({ framework, config, domain }) => {
+    return JSON.stringify({
+      success: true,
+      ready: true,
+      framework,
+      config,
+      domain,
+      message: `准备使用 ${framework} 框架生成爬虫，请调用 task 工具委托 crawler 子代理`,
+    });
+  },
+  {
+    name: 'delegate_crawler_generation',
+    description: '准备参数，委托 crawler 子代理生成特定框架的爬虫代码',
+    schema: z.object({
+      framework: z.enum(['requests', 'httpx', 'scrapy']).describe('用户选择的爬虫框架'),
+      config: z.object({
+        url: z.string(),
+        stages: z.array(z.object({
+          name: z.string(),
+          fields: z.array(z.object({
+            name: z.string(),
+            xpath: z.string(),
+            type: z.string(),
+          })),
+          entry: z.any().nullable(),
+          pagination: z.any().nullable(),
+        })),
+      }).describe('爬虫配置'),
+      domain: z.string().describe('目标网站域名'),
+    }),
+  }
+);
+
+export const crawlerGeneratorTools = [
+  generateCrawlerWithConfirm,
+  delegateCrawlerGeneration,
+];
+
+export default crawlerGeneratorTools;

package/src/agent/tools/debug.js

@@ -6,6 +6,7 @@ import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
 import { getBrowser } from '../../browser/index.js';
 import { CDPSession } from '../../browser/cdp.js';
+import { logStore } from '../logger.js';
 
 let cdpSession = null;
 let isPaused = false;
@@ -19,18 +20,26 @@ async function getSession() {
     const browser = await getBrowser();
     cdpSession = await CDPSession.fromBrowser(browser);
 
-    // 监听暂停事件
+    // 过滤反调试 debugger 语句的噪音：只在命中我们设的断点时打日志
+    let lastPauseIsBreakpoint = false;
+
     cdpSession.on('Debugger.paused', (params) => {
-      isPaused = true;
-      currentCallFrames = params.callFrames || [];
-      console.log('[debug] Debugger paused, callFrames:', currentCallFrames.length);
+      lastPauseIsBreakpoint = params.reason === 'breakpoint' || params.hitBreakpoints?.length > 0;
+      if (lastPauseIsBreakpoint) {
+        isPaused = true;
+        currentCallFrames = params.callFrames || [];
+        const top = currentCallFrames[0];
+        const func = top?.functionName || '(anonymous)';
+        const url = top?.url?.split('/').pop() || top?.url || '?';
+        const line = top?.location?.lineNumber ?? '?';
+        console.log(`[debug] Breakpoint hit: ${func} @ ${url}:${line}`);
+      }
     });
 
-    // 监听恢复事件
     cdpSession.on('Debugger.resumed', () => {
       isPaused = false;
       currentCallFrames = [];
-      console.log('[debug] Debugger resumed');
+      lastPauseIsBreakpoint = false;
     });
   }
   return cdpSession;
@@ -235,6 +244,33 @@ export const stepOver = tool(
   }
 );
 
+/**
+ * 查询 Agent 执行日志
+ */
+export const getAgentLogs = tool(
+  async ({ category, level, limit, toolName }) => {
+    if (category === 'stats') {
+      return JSON.stringify(logStore.getStats(), null, 2);
+    }
+    const logs = logStore.query({ category, level, limit, toolName });
+    return JSON.stringify(logs, null, 2);
+  },
+  {
+    name: 'get_agent_logs',
+    description: '获取当前 Agent 会话的执行日志，包括 LLM 调用、工具调用、错误等。用于调试和分析 Agent 执行过程。category=stats 可获取统计概览。',
+    schema: z.object({
+      category: z.enum(['LLM', 'TOOL', 'CHAIN', 'AGENT', 'stats']).optional()
+        .describe('日志类别：LLM/TOOL/CHAIN/AGENT，或 stats 获取统计'),
+      level: z.enum(['INFO', 'DEBUG', 'ERROR']).optional()
+        .describe('日志级别'),
+      limit: z.number().optional().default(50)
+        .describe('返回条数（默认50，最近的N条）'),
+      toolName: z.string().optional()
+        .describe('按工具名过滤（仅 TOOL 类别有效）'),
+    }),
+  }
+);
+
 export const debugTools = [
   setBreakpoint,
   setXHRBreakpoint,
@@ -243,4 +279,5 @@ export const debugTools = [
   evaluateAtBreakpoint,
   resumeExecution,
   stepOver,
+  getAgentLogs,
 ];

package/src/agent/tools/evolve.js

@@ -23,6 +23,9 @@ function getSkillPath(skillName) {
     'sandbox': SKILLS.sandbox,
     'env': SKILLS.env,
     'js2python': SKILLS.js2python,
+    'crawler': SKILLS.crawler,
+    'captcha': SKILLS.captcha,
+    'anti-detect': SKILLS.antiDetect,
     'report': SKILLS.report,
     'general': SKILLS.general,
   };
@@ -83,7 +86,7 @@ export const evolveSkill = tool(
     if (!skillInfo) {
       return JSON.stringify({
         success: false,
-        error: `未知的 skill: ${skill}。可用: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general。或使用 new:<name> 创建新 skill。`
+        error: `未知的 skill: ${skill}。可用: static-analysis, dynamic-analysis, sandbox, env, js2python, crawler, captcha, anti-detect, report, general。或使用 new:<name> 创建新 skill。`
       });
     }
 
@@ -152,7 +155,7 @@ export const evolveSkill = tool(
     name: 'evolve_skill',
     description: '记录分析过程中学到的经验。支持现有 skill 或 new:<name> 创建新 skill',
     schema: z.object({
-      skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general，或 new:<name> 创建新 skill'),
+      skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, crawler, captcha, anti-detect, report, general，或 new:<name> 创建新 skill'),
       title: z.string().describe('经验标题，简短描述'),
       scenario: z.string().describe('具体场景/案例'),
       insight: z.string().describe('一句话总结经验'),

package/src/agent/tools/extractor.js

@@ -34,16 +34,20 @@ export const listFunctions = tool(
  */
 export const getFunctionCode = tool(
   async ({ code, funcName }) => {
+    // buildDependencyGraph 先调用，extractSlice 内部会复用 this.ast 缓存
+    const graph = astAnalyzer.buildDependencyGraph(code);
+    const deps = graph.get(funcName) || [];
     const slice = astAnalyzer.extractSlice(code, funcName);
     return JSON.stringify({
       funcName,
       found: !!slice,
       code: slice || '未找到该函数',
+      dependencies: deps,
     }, null, 2);
   },
   {
     name: 'get_function_code',
-    description: '获取指定函数的代码片段',
+    description: '提取指定函数的完整代码（含递归依赖函数和全局变量）。返回可独立运行的代码片段 + 依赖函数列表',
     schema: z.object({
       code: z.string().describe('源代码'),
       funcName: z.string().describe('函数名'),

package/src/agent/tools/file.js

@@ -6,7 +6,7 @@
 import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
 import { writeFileSync, readFileSync, existsSync, readdirSync } from 'fs';
-import { dirname, join, isAbsolute, relative } from 'path';
+import { dirname, join, isAbsolute, relative, resolve } from 'path';
 import { PATHS, ensureDir, DEEPSPIDER_HOME } from '../../config/paths.js';
 
 const OUTPUT_DIR = PATHS.OUTPUT_DIR;
@@ -17,15 +17,24 @@ function ensureFileDir(filePath) {
 }
 
 function getSafePath(filePath) {
+  let resolved;
   if (isAbsolute(filePath)) {
     // 如果是 ~/.deepspider/ 目录下的路径，直接使用
     if (filePath.startsWith(DEEPSPIDER_HOME)) {
-      return filePath;
+      resolved = filePath;
+    } else {
+      // 其他绝对路径：放到 OUTPUT_DIR 下
+      resolved = join(OUTPUT_DIR, filePath.replace(/^\/+/, ''));
     }
-    // 其他绝对路径：放到 OUTPUT_DIR 下
-    return join(OUTPUT_DIR, filePath.replace(/^\/+/, ''));
+  } else {
+    resolved = join(OUTPUT_DIR, filePath);
   }
-  return join(OUTPUT_DIR, filePath);
+  // 防止 ../ 穿越到 DEEPSPIDER_HOME 之外
+  const normalized = resolve(resolved);
+  if (!normalized.startsWith(DEEPSPIDER_HOME)) {
+    throw new Error(`路径不允许超出 ${DEEPSPIDER_HOME}: ${filePath}`);
+  }
+  return normalized;
 }
 
 export const artifactSave = tool(

package/src/agent/tools/generateHook.js

@@ -0,0 +1,66 @@
+/**
+ * DeepSpider - 统一 Hook 代码生成工具
+ * 合并 hookTools + cryptoHookTools + asyncTools + antiDebugTools
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { NetworkHook } from '../../env/NetworkHook.js';
+import { CookieHook } from '../../env/CookieHook.js';
+import { CryptoHook } from '../../env/CryptoHook.js';
+import { AsyncHook } from '../../env/AsyncHook.js';
+import { AntiAntiDebug } from '../../env/AntiAntiDebug.js';
+
+const networkHook = new NetworkHook();
+const cookieHook = new CookieHook();
+const cryptoHook = new CryptoHook();
+const asyncHook = new AsyncHook();
+const antiDebug = new AntiAntiDebug();
+
+const HOOK_TYPES = {
+  // 网络
+  xhr: { gen: () => networkHook.generateXHRHookCode({ captureBody: true, captureResponse: true }), usage: "getLogs('xhr')" },
+  fetch: { gen: () => networkHook.generateFetchHookCode({ captureBody: true, captureResponse: true }), usage: "getLogs('fetch')" },
+  cookie: { gen: () => cookieHook.generateCookieHookCode({ trackRead: true, trackWrite: true }), usage: "getLogs('cookie')" },
+  // 加密
+  cryptojs: { gen: () => cryptoHook.generateCryptoJSHookCode(), usage: "getLogs('crypto')" },
+  sm_crypto: { gen: () => cryptoHook.generateSMCryptoHookCode(), usage: "getLogs('crypto')" },
+  rsa: { gen: () => cryptoHook.generateRSAHookCode(), usage: "getLogs('crypto')" },
+  generic_crypto: { gen: () => cryptoHook.generateGenericCryptoHookCode(), usage: "getLogs('crypto')" },
+  // 异步
+  promise: { gen: () => asyncHook.generatePromiseHookCode(), usage: "getLogs('async')" },
+  timer: { gen: () => asyncHook.generateTimerHookCode(), usage: "getLogs('timer')" },
+  // 反反调试
+  anti_debugger: { gen: () => antiDebug.generateAntiDebuggerCode(), usage: '绕过无限 debugger' },
+  anti_console: { gen: () => antiDebug.generateAntiConsoleDetectCode(), usage: '绕过控制台检测' },
+  anti_cdp: { gen: () => antiDebug.generateAntiCDPDetectCode(), usage: '绕过 CDP 检测' },
+  anti_debug_full: { gen: () => antiDebug.generateFullAntiDebugCode(), usage: '完整反反调试（包含以上所有）' },
+};
+
+const typeEnum = /** @type {[string, ...string[]]} */ (Object.keys(HOOK_TYPES));
+
+export const generateHook = tool(
+  async ({ type }) => {
+    const entry = HOOK_TYPES[type];
+    if (!entry) {
+      return JSON.stringify({ success: false, error: `未知类型: ${type}，可选: ${typeEnum.join(', ')}` });
+    }
+    const code = entry.gen();
+    return JSON.stringify({ success: true, type, code, usage: entry.usage }, null, 2);
+  },
+  {
+    name: 'generate_hook',
+    description: `生成 Hook 代码。生成后需通过 inject_hook 注入浏览器。
+
+类型：
+- 网络: xhr, fetch, cookie
+- 加密: cryptojs（CryptoJS）, sm_crypto（国密）, rsa（JSEncrypt/node-forge）, generic_crypto（通用）
+- 异步: promise, timer
+- 反反调试: anti_debugger, anti_console, anti_cdp, anti_debug_full（完整）`,
+    schema: z.object({
+      type: z.enum(typeEnum).describe('Hook 类型'),
+    }),
+  }
+);
+
+export const generateHookTools = [generateHook];

package/src/agent/tools/hookManager.js

@@ -8,16 +8,28 @@ import { tool } from '@langchain/core/tools';
 import { getBrowser } from '../../browser/index.js';
 
 /**
- * 通过 CDP 执行 JS
+ * 通过 CDP 执行 JS（带超时保护）
  */
-async function evaluateViaCDP(browser, expression) {
+async function evaluateViaCDP(browser, expression, timeout = 5000) {
   const cdp = await browser.getCDPSession();
   if (!cdp) return null;
-  const result = await cdp.send('Runtime.evaluate', {
+
+  const evaluatePromise = cdp.send('Runtime.evaluate', {
     expression,
     returnByValue: true,
   });
-  return result.result?.value;
+
+  const timeoutPromise = new Promise((_, reject) =>
+    setTimeout(() => reject(new Error('CDP evaluate timeout')), timeout)
+  );
+
+  try {
+    const result = await Promise.race([evaluatePromise, timeoutPromise]);
+    return result.result?.value;
+  } catch (e) {
+    console.error('[evaluateViaCDP] 超时或错误:', e.message);
+    return null;
+  }
 }
 
 /**
@@ -115,14 +127,12 @@ export const injectHook = tool(
       if (!browser.getPage()) {
         return JSON.stringify({ success: false, error: '浏览器未就绪' });
       }
-      const escapedCode = code
-        .replace(/\\/g, '\\\\')
-        .replace(/'/g, "\\'")
-        .replace(/\n/g, '\\n');
+      // 用 JSON.stringify 安全转义，避免手动转义遗漏特殊字符
+      const safeCode = JSON.stringify(code);
 
       const result = await evaluateViaCDP(
         browser,
-        `JSON.stringify(window.__deepspider__?.injectHook?.('${escapedCode}'))`
+        `JSON.stringify(window.__deepspider__?.injectHook?.(${safeCode}))`
       );
       return result || JSON.stringify({ success: false, error: '注入失败' });
     } catch (e) {