deepspider 0.3.0 → 0.3.2

package/src/agent/tools/browser.js

@@ -406,10 +406,169 @@ export const getCookies = tool(
   }
 );
 
+/**
+ * CSS 选择器转义（Node.js 没有 CSS.escape）
+ */
+function cssEscape(str) {
+  return str.replace(/([^\w-])/g, '\\$1');
+}
+
+/**
+ * 转义属性选择器中的值（双引号）
+ */
+function escapeAttrValue(str) {
+  return str.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
+}
+
+/**
+ * 可交互角色白名单
+ */
+const INTERACTIVE_ROLES = new Set([
+  'button', 'link', 'menuitem', 'tab',
+  'checkbox', 'radio', 'combobox', 'textbox',
+  'switch', 'option', 'menuitemcheckbox', 'menuitemradio',
+  'searchbox', 'spinbutton', 'slider',
+]);
+
+/**
+ * 从 DOM.describeNode 结果生成 CSS 选择器
+ */
+function buildSelector(nodeInfo) {
+  const { localName, attributes } = nodeInfo;
+  if (!localName) return '';
+
+  // attributes 是 [key, value, key, value, ...] 扁平数组
+  const attrs = {};
+  for (let i = 0; i < (attributes || []).length; i += 2) {
+    attrs[attributes[i]] = attributes[i + 1];
+  }
+
+  // 优先用 id
+  if (attrs.id) return `#${cssEscape(attrs.id)}`;
+
+  // 其次用 data-* 属性
+  for (const [k, v] of Object.entries(attrs)) {
+    if (k.startsWith('data-') && v) return `${localName}[${k}="${escapeAttrValue(v)}"]`;
+  }
+
+  // 用 aria-label
+  if (attrs['aria-label']) {
+    return `${localName}[aria-label="${escapeAttrValue(attrs['aria-label'])}"]`;
+  }
+
+  // 用 name 属性（input/select 等）
+  if (attrs.name) {
+    return `${localName}[name="${escapeAttrValue(attrs.name)}"]`;
+  }
+
+  // 用 class（拼接所有类名提高唯一性）
+  if (attrs.class) {
+    const classes = attrs.class.split(/\s+/).filter(Boolean);
+    if (classes.length) return `${localName}.${classes.map(c => cssEscape(c)).join('.')}`;
+  }
+
+  // 兜底用 tagName
+  return localName;
+}
+
+/**
+ * 获取页面可交互元素列表 - CDP Accessibility Tree
+ */
+export const getInteractiveElements = tool(
+  async ({ roles, limit }) => {
+    const browser = await getBrowser();
+    const cdp = await browser.getCDPSession();
+    if (!cdp) throw new Error('CDP session not available');
+
+    // 启用所需 CDP 域
+    await cdp.send('Accessibility.enable');
+    await cdp.send('DOM.enable');
+
+    // 获取完整无障碍树
+    const { nodes } = await cdp.send('Accessibility.getFullAXTree');
+
+    // 确定过滤角色集
+    const filterRoles = roles?.length
+      ? new Set(roles)
+      : INTERACTIVE_ROLES;
+
+    // 过滤可交互节点
+    const candidates = [];
+    for (const node of nodes) {
+      if (node.ignored) continue;
+      const role = node.role?.value;
+      if (!role || !filterRoles.has(role)) continue;
+      candidates.push(node);
+    }
+
+    const totalBeforeTruncate = candidates.length;
+    const truncated = candidates.length > limit;
+    const selected = candidates.slice(0, limit);
+
+    // 并发获取选择器（分批，每批 20）
+    const BATCH = 20;
+    const elements = [];
+    for (let i = 0; i < selected.length; i += BATCH) {
+      const batch = selected.slice(i, i + BATCH);
+      const results = await Promise.all(batch.map(async (node) => {
+        // 提取属性
+        const props = {};
+        for (const p of (node.properties || [])) {
+          props[p.name] = p.value?.value;
+        }
+
+        // 通过 backendDOMNodeId 获取选择器
+        let selector = '';
+        if (node.backendDOMNodeId) {
+          try {
+            const desc = await cdp.send('DOM.describeNode', {
+              backendNodeId: node.backendDOMNodeId,
+            });
+            selector = buildSelector(desc.node);
+          } catch {
+            // 节点可能已从 DOM 移除
+          }
+        }
+
+        return {
+          role: node.role?.value,
+          name: node.name?.value || '',
+          selector,
+          clickable: !props.disabled,
+          disabled: !!props.disabled,
+          focused: !!props.focused,
+          description: node.description?.value || '',
+        };
+      }));
+      elements.push(...results);
+    }
+
+    return JSON.stringify({
+      elements,
+      total: totalBeforeTruncate,
+      truncated,
+    });
+  },
+  {
+    name: 'get_interactive_elements',
+    description: `获取页面上所有可交互元素（按钮、链接、输入框等），基于 Accessibility Tree。
+
+用途：在点击/操作元素前，先调用此工具了解页面上有哪些可交互元素，避免盲目猜测选择器。
+返回的 selector 可直接传给 click_element / fill_input 使用。
+
+支持的角色：button, link, menuitem, tab, checkbox, radio, combobox, textbox, switch, option, searchbox, spinbutton, slider 等。`,
+    schema: z.object({
+      roles: z.array(z.string()).optional().describe('过滤角色列表，如 ["button", "link"]，不传则返回所有可交互角色'),
+      limit: z.number().default(100).describe('最大返回数量，默认 100，避免结果过大'),
+    }),
+  }
+);
+
 export const browserTools = [
   clickElement,
   fillInput,
   waitForSelector,
+  takeScreenshot,
   reloadPage,
   goBack,
   goForward,
@@ -420,4 +579,5 @@ export const browserTools = [
   getPageSource,
   getElementHtml,
   getCookies,
+  getInteractiveElements,
 ];

package/src/agent/tools/captcha.js

@@ -106,7 +106,7 @@ export const captchaSlideDetect = tool(
 
       // 获取背景图和滑块图
       const bgElement = await page.$(bg_selector);
-      const slideElement = slide_selector ? await page.$(slide_selector) : null;
+      const _slideElement = slide_selector ? await page.$(slide_selector) : null;
 
       if (!bgElement) {
         return JSON.stringify({ success: false, error: '未找到背景图' });

package/src/agent/tools/capture.js

@@ -33,6 +33,15 @@ export const collectProperty = tool(
     const browser = await getBrowser();
     const collector = new EnvCollector(browser.getPage());
     const data = await collector.collect(path, { depth });
+
+    // 变量未定义时追加 fallback 引导，引导 LLM 走断点路径
+    if (data?.success === false && /undefined|null/.test(data?.error || '')) {
+      return JSON.stringify({
+        ...data,
+        hint: `⚠️ 变量 ${path} 未定义。该变量可能仅在特定函数执行时存在（如请求发起时动态赋值，执行后被清理）。\n请使用 set_breakpoint 在目标函数处设断点，断点命中后用 evaluate_at_breakpoint 采集变量值。不要猜测变量值。`,
+      }, null, 2);
+    }
+
     return JSON.stringify(data, null, 2);
   },
   {
@@ -67,14 +76,26 @@ export const autoFixEnv = tool(
 /**
  * 通过 CDP 在页面执行 JS（复用 session）
  */
-async function evaluateViaCDP(browser, expression) {
+async function evaluateViaCDP(browser, expression, timeout = 5000) {
   const cdp = await browser.getCDPSession();
   if (!cdp) return null;
-  const result = await cdp.send('Runtime.evaluate', {
+
+  const evaluatePromise = cdp.send('Runtime.evaluate', {
     expression,
     returnByValue: true,
   });
-  return result.result?.value;
+
+  const timeoutPromise = new Promise((_, reject) =>
+    setTimeout(() => reject(new Error('CDP evaluate timeout')), timeout)
+  );
+
+  try {
+    const result = await Promise.race([evaluatePromise, timeoutPromise]);
+    return result.result?.value;
+  } catch (e) {
+    console.error('[evaluateViaCDP] 超时或错误:', e.message);
+    return null;
+  }
 }
 
 /**

package/src/agent/tools/correlate.js

@@ -5,6 +5,7 @@
 
 import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
+import { getDataStore } from '../../store/DataStore.js';
 
 /**
  * 分析请求-加密关联
@@ -74,22 +75,40 @@ export const analyzeCorrelation = tool(
 
 /**
  * 解析调用栈顶部
+ * 支持两种格式：
+ * 1. 字符串栈（来自 Error.stack）
+ * 2. callFrames 数组（来自 CDP initiator）
  */
 function parseStackTop(stack) {
   if (!stack) return null;
-  const lines = stack.split('\n').slice(2, 5);
-  return lines.map(line => {
-    const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/) ||
-                  line.match(/at\s+(.+?):(\d+):(\d+)/);
-    if (match) {
-      return {
-        func: match[1] || 'anonymous',
-        file: match[2] || match[1],
-        line: parseInt(match[3] || match[2])
-      };
-    }
-    return { raw: line.trim() };
-  });
+
+  // 处理 callFrames 数组格式（来自 CDP initiator）
+  if (Array.isArray(stack)) {
+    return stack.slice(0, 3).map(frame => ({
+      func: frame.functionName || frame.func || '(anonymous)',
+      file: frame.url || frame.file || '',
+      line: frame.lineNumber || frame.line || 0
+    }));
+  }
+
+  // 处理字符串栈格式（来自 Error.stack）
+  if (typeof stack === 'string') {
+    const lines = stack.split('\n').slice(2, 5);
+    return lines.map(line => {
+      const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/) ||
+                    line.match(/at\s+(.+?):(\d+):(\d+)/);
+      if (match) {
+        return {
+          func: match[1] || 'anonymous',
+          file: match[2] || match[1],
+          line: parseInt(match[3] || match[2])
+        };
+      }
+      return { raw: line.trim() };
+    });
+  }
+
+  return null;
 }
 
 /**
@@ -183,11 +202,12 @@ export const analyzeCookieEncryption = tool(
   async ({ logs, cookieName }) => {
     const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
 
-    // 找到设置该 cookie 的日志
+    // 找到设置该 cookie 的日志（匹配 cookie 键名）
     const cookieLogs = parsed.filter(entry => {
       if (entry._type !== 'cookie') return false;
       if (entry.action !== 'write') return false;
-      return entry.value?.includes(cookieName);
+      // cookie hook 日志的 value 格式为 "name=value"，匹配键名部分
+      return entry.value?.startsWith(cookieName + '=') || entry.name === cookieName;
     });
 
     if (cookieLogs.length === 0) {
@@ -294,10 +314,104 @@ export const analyzeResponseDecryption = tool(
   }
 );
 
+/**
+ * 识别加密模式
+ */
+function identifyPattern(value) {
+  if (/^[0-9a-fA-F]+$/.test(value)) {
+    if (value.length === 32) return 'hash-md5';
+    if (value.length === 40) return 'hash-sha1';
+    if (value.length === 64) return 'hash-sha256';
+    return 'hex';
+  }
+  if (/^[A-Za-z0-9+/]{20,}={0,2}$/.test(value)) return 'base64';
+  if (/^ey[A-Za-z0-9_-]+\./.test(value)) return 'jwt';
+  return 'unknown';
+}
+
+/**
+ * 判断值是否像加密结果
+ */
+function looksEncrypted(value) {
+  if (/^[0-9a-fA-F]{32,}$/.test(value)) return true;
+  if (/^[A-Za-z0-9+/]{20,}={0,2}$/.test(value)) return true;
+  if (/^ey[A-Za-z0-9_-]+\./.test(value)) return true;
+  return false;
+}
+
+/**
+ * 解析请求 body
+ */
+function parseBody(body) {
+  if (!body) return {};
+  try {
+    return JSON.parse(body);
+  } catch {
+    // 尝试 form-urlencoded
+    try {
+      return Object.fromEntries(new URLSearchParams(body));
+    } catch {
+      return { _raw: body.slice(0, 200) };
+    }
+  }
+}
+
+/**
+ * 分析请求参数结构（不依赖 Hook 日志）
+ */
+export const analyzeRequestParams = tool(
+  async ({ site, id }) => {
+    const store = getDataStore();
+    const detail = await store.getResponse(site, id);
+    if (!detail) return JSON.stringify({ error: '未找到该请求' });
+
+    let urlParams = {};
+    try {
+      urlParams = Object.fromEntries(new URL(detail.url).searchParams);
+    } catch { /* invalid URL */ }
+
+    const bodyParams = parseBody(detail.requestBody);
+
+    // 识别可疑参数
+    const suspiciousParams = [];
+    const allParams = { ...urlParams, ...bodyParams };
+    for (const [key, value] of Object.entries(allParams)) {
+      const str = String(value);
+      if (str.length > 20 && looksEncrypted(str)) {
+        suspiciousParams.push({
+          name: key,
+          value: str.slice(0, 50) + (str.length > 50 ? '...' : ''),
+          length: str.length,
+          pattern: identifyPattern(str),
+        });
+      }
+    }
+
+    return JSON.stringify({
+      url: detail.url,
+      method: detail.method,
+      urlParams,
+      bodyParams: typeof bodyParams === 'object' && !bodyParams._raw
+        ? bodyParams : { _raw: detail.requestBody?.slice(0, 200) },
+      suspiciousParams,
+      initiator: detail.initiator || null,
+    }, null, 2);
+  },
+  {
+    name: 'analyze_request_params',
+    description: '分析请求的参数结构，自动识别可疑的加密参数（hex/base64/hash）。不依赖 Hook 日志，可直接使用。',
+    schema: z.object({
+      site: z.string().describe('站点 hostname'),
+      id: z.string().describe('请求 ID'),
+    }),
+  }
+);
+
 export const correlateTools = [
   analyzeCorrelation,
   locateCryptoSource,
   analyzeHeaderEncryption,
   analyzeCookieEncryption,
   analyzeResponseDecryption,
+  analyzeRequestParams,
 ];

package/src/agent/tools/crawler.js

@@ -113,4 +113,5 @@ export const e2eTest = tool(
   }
 );
 
-export const crawlerTools = [siteAnalyze, complexityAssess, e2eTest];
+// e2eTest 暂未实现（stub），不导出到子代理工具列表，避免子代理浪费轮次调用空壳工具
+export const crawlerTools = [siteAnalyze, complexityAssess];

package/src/agent/tools/crawlerGenerator.js

@@ -0,0 +1,90 @@
+/**
+ * DeepSpider - 爬虫代码生成工具
+ * 通过 LangGraph interrupt 机制实现面板交互式选择
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { interrupt } from '@langchain/langgraph';
+
+/**
+ * 请求用户选择爬虫框架并生成代码
+ * interrupt payload 遵循统一协议，StreamHandler 自动渲染到面板
+ */
+export const generateCrawlerWithConfirm = tool(
+  async ({ analysisSummary, domain }) => {
+    const userChoice = interrupt({
+      type: 'choices',
+      question: '分析完成！选择爬虫框架生成完整脚本：',
+      options: [
+        { id: 'requests', label: 'requests', description: '简单易用，适合快速原型' },
+        { id: 'httpx', label: 'httpx', description: '异步高性能，适合大规模并发' },
+        { id: 'scrapy', label: 'Scrapy', description: '企业级框架，适合复杂项目' },
+        { id: 'skip', label: '不需要', description: '仅保存当前分析结果' },
+      ],
+    });
+
+    return JSON.stringify({
+      success: true,
+      framework: userChoice,
+      domain,
+      message: userChoice === '不需要'
+        ? '用户选择不生成爬虫脚本'
+        : `用户选择使用 ${userChoice} 框架生成爬虫`,
+    });
+  },
+  {
+    name: 'generate_crawler_code',
+    description: `分析完成后，向用户展示可点击的框架选项（requests/httpx/Scrapy/不需要）。
+
+用户点击后，工具返回用户选择的框架名称。根据返回值委托 crawler 子代理生成代码。`,
+    schema: z.object({
+      analysisSummary: z.string().describe('分析结果摘要'),
+      domain: z.string().describe('目标网站域名'),
+    }),
+  }
+);
+
+/**
+ * 根据用户选择的框架委托 crawler 子代理生成代码
+ */
+export const delegateCrawlerGeneration = tool(
+  async ({ framework, config, domain }) => {
+    return JSON.stringify({
+      success: true,
+      ready: true,
+      framework,
+      config,
+      domain,
+      message: `准备使用 ${framework} 框架生成爬虫，请调用 task 工具委托 crawler 子代理`,
+    });
+  },
+  {
+    name: 'delegate_crawler_generation',
+    description: '准备参数，委托 crawler 子代理生成特定框架的爬虫代码',
+    schema: z.object({
+      framework: z.enum(['requests', 'httpx', 'scrapy']).describe('用户选择的爬虫框架'),
+      config: z.object({
+        url: z.string(),
+        stages: z.array(z.object({
+          name: z.string(),
+          fields: z.array(z.object({
+            name: z.string(),
+            xpath: z.string(),
+            type: z.string(),
+          })),
+          entry: z.any().nullable(),
+          pagination: z.any().nullable(),
+        })),
+      }).describe('爬虫配置'),
+      domain: z.string().describe('目标网站域名'),
+    }),
+  }
+);
+
+export const crawlerGeneratorTools = [
+  generateCrawlerWithConfirm,
+  delegateCrawlerGeneration,
+];
+
+export default crawlerGeneratorTools;

package/src/agent/tools/debug.js

@@ -6,6 +6,7 @@ import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
 import { getBrowser } from '../../browser/index.js';
 import { CDPSession } from '../../browser/cdp.js';
+import { logStore } from '../logger.js';
 
 let cdpSession = null;
 let isPaused = false;
@@ -19,18 +20,26 @@ async function getSession() {
     const browser = await getBrowser();
     cdpSession = await CDPSession.fromBrowser(browser);
 
-    // 监听暂停事件
+    // 过滤反调试 debugger 语句的噪音：只在命中我们设的断点时打日志
+    let lastPauseIsBreakpoint = false;
+
     cdpSession.on('Debugger.paused', (params) => {
-      isPaused = true;
-      currentCallFrames = params.callFrames || [];
-      console.log('[debug] Debugger paused, callFrames:', currentCallFrames.length);
+      lastPauseIsBreakpoint = params.reason === 'breakpoint' || params.hitBreakpoints?.length > 0;
+      if (lastPauseIsBreakpoint) {
+        isPaused = true;
+        currentCallFrames = params.callFrames || [];
+        const top = currentCallFrames[0];
+        const func = top?.functionName || '(anonymous)';
+        const url = top?.url?.split('/').pop() || top?.url || '?';
+        const line = top?.location?.lineNumber ?? '?';
+        console.log(`[debug] Breakpoint hit: ${func} @ ${url}:${line}`);
+      }
     });
 
-    // 监听恢复事件
     cdpSession.on('Debugger.resumed', () => {
       isPaused = false;
       currentCallFrames = [];
-      console.log('[debug] Debugger resumed');
+      lastPauseIsBreakpoint = false;
     });
   }
   return cdpSession;
@@ -235,6 +244,33 @@ export const stepOver = tool(
   }
 );
 
+/**
+ * 查询 Agent 执行日志
+ */
+export const getAgentLogs = tool(
+  async ({ category, level, limit, toolName }) => {
+    if (category === 'stats') {
+      return JSON.stringify(logStore.getStats(), null, 2);
+    }
+    const logs = logStore.query({ category, level, limit, toolName });
+    return JSON.stringify(logs, null, 2);
+  },
+  {
+    name: 'get_agent_logs',
+    description: '获取当前 Agent 会话的执行日志，包括 LLM 调用、工具调用、错误等。用于调试和分析 Agent 执行过程。category=stats 可获取统计概览。',
+    schema: z.object({
+      category: z.enum(['LLM', 'TOOL', 'CHAIN', 'AGENT', 'stats']).optional()
+        .describe('日志类别：LLM/TOOL/CHAIN/AGENT，或 stats 获取统计'),
+      level: z.enum(['INFO', 'DEBUG', 'ERROR']).optional()
+        .describe('日志级别'),
+      limit: z.number().optional().default(50)
+        .describe('返回条数（默认50，最近的N条）'),
+      toolName: z.string().optional()
+        .describe('按工具名过滤（仅 TOOL 类别有效）'),
+    }),
+  }
+);
+
 export const debugTools = [
   setBreakpoint,
   setXHRBreakpoint,
@@ -243,4 +279,5 @@ export const debugTools = [
   evaluateAtBreakpoint,
   resumeExecution,
   stepOver,
+  getAgentLogs,
 ];

package/src/agent/tools/evolve.js

@@ -23,6 +23,9 @@ function getSkillPath(skillName) {
     'sandbox': SKILLS.sandbox,
     'env': SKILLS.env,
     'js2python': SKILLS.js2python,
+    'crawler': SKILLS.crawler,
+    'captcha': SKILLS.captcha,
+    'anti-detect': SKILLS.antiDetect,
     'report': SKILLS.report,
     'general': SKILLS.general,
   };
@@ -83,7 +86,7 @@ export const evolveSkill = tool(
     if (!skillInfo) {
       return JSON.stringify({
         success: false,
-        error: `未知的 skill: ${skill}。可用: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general。或使用 new:<name> 创建新 skill。`
+        error: `未知的 skill: ${skill}。可用: static-analysis, dynamic-analysis, sandbox, env, js2python, crawler, captcha, anti-detect, report, general。或使用 new:<name> 创建新 skill。`
       });
     }
 
@@ -100,7 +103,7 @@ export const evolveSkill = tool(
     let content = '';
     try {
       content = fs.readFileSync(evolvedPath, 'utf-8');
-    } catch (e) {
+    } catch {
       // 文件不存在，使用空内容
     }
 
@@ -152,7 +155,7 @@ export const evolveSkill = tool(
     name: 'evolve_skill',
     description: '记录分析过程中学到的经验。支持现有 skill 或 new:<name> 创建新 skill',
     schema: z.object({
-      skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general，或 new:<name> 创建新 skill'),
+      skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, crawler, captcha, anti-detect, report, general，或 new:<name> 创建新 skill'),
       title: z.string().describe('经验标题，简短描述'),
       scenario: z.string().describe('具体场景/案例'),
       insight: z.string().describe('一句话总结经验'),

package/src/agent/tools/extractor.js

@@ -34,16 +34,20 @@ export const listFunctions = tool(
  */
 export const getFunctionCode = tool(
   async ({ code, funcName }) => {
+    // buildDependencyGraph 先调用，extractSlice 内部会复用 this.ast 缓存
+    const graph = astAnalyzer.buildDependencyGraph(code);
+    const deps = graph.get(funcName) || [];
     const slice = astAnalyzer.extractSlice(code, funcName);
     return JSON.stringify({
       funcName,
       found: !!slice,
       code: slice || '未找到该函数',
+      dependencies: deps,
     }, null, 2);
   },
   {
     name: 'get_function_code',
-    description: '获取指定函数的代码片段',
+    description: '提取指定函数的完整代码（含递归依赖函数和全局变量）。返回可独立运行的代码片段 + 依赖函数列表',
     schema: z.object({
       code: z.string().describe('源代码'),
       funcName: z.string().describe('函数名'),