deepspider 0.1.0

package/src/agent/tools/capture.js

@@ -0,0 +1,179 @@
+/**
+ * DeepSpider - 数据采集工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { getBrowser } from '../../browser/index.js';
+import { EnvCollector } from '../../browser/collector.js';
+import { EnvBridge } from '../../browser/EnvBridge.js';
+
+/**
+ * 采集环境数据快照
+ */
+export const collectEnv = tool(
+  async () => {
+    const browser = await getBrowser();
+    const collector = new EnvCollector(browser.getPage());
+    const data = await collector.collectFullSnapshot();
+    return JSON.stringify(data, null, 2);
+  },
+  {
+    name: 'collect_env',
+    description: '采集浏览器完整环境快照（navigator、screen、canvas、webgl、fonts 等）',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 动态采集指定属性
+ */
+export const collectProperty = tool(
+  async ({ path, depth }) => {
+    const browser = await getBrowser();
+    const collector = new EnvCollector(browser.getPage());
+    const data = await collector.collect(path, { depth });
+    return JSON.stringify(data, null, 2);
+  },
+  {
+    name: 'collect_property',
+    description: '从真实浏览器采集指定属性路径的值',
+    schema: z.object({
+      path: z.string().describe('属性路径，如 navigator.connection.effectiveType'),
+      depth: z.number().optional().default(2).describe('采集深度'),
+    }),
+  }
+);
+
+/**
+ * 自动补环境
+ */
+export const autoFixEnv = tool(
+  async ({ missingPaths }) => {
+    const browser = await getBrowser();
+    const bridge = new EnvBridge(browser.getPage());
+    const result = await bridge.runFullPipeline(missingPaths);
+    return JSON.stringify(result, null, 2);
+  },
+  {
+    name: 'auto_fix_env',
+    description: '根据缺失属性列表，自动从真实浏览器采集并生成补丁代码',
+    schema: z.object({
+      missingPaths: z.array(z.string()).describe('缺失的属性路径列表'),
+    }),
+  }
+);
+
+/**
+ * 通过 CDP 在页面执行 JS（复用 session）
+ */
+async function evaluateViaCDP(browser, expression) {
+  const cdp = await browser.getCDPSession();
+  if (!cdp) return null;
+  const result = await cdp.send('Runtime.evaluate', {
+    expression,
+    returnByValue: true,
+  });
+  return result.result?.value;
+}
+
+/**
+ * 获取 Hook 日志
+ */
+export const getHookLogs = tool(
+  async ({ type, limit }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪', logs: [] });
+      }
+
+      // 通过 CDP 从浏览器获取日志
+      const expression = type
+        ? `window.__deepspider__?.getLogs?.('${type}') || '[]'`
+        : `window.__deepspider__?.getAllLogs?.() || '[]'`;
+
+      const logsJson = await evaluateViaCDP(browser, expression);
+      if (!logsJson) {
+        return JSON.stringify({ success: false, error: 'Hook 未加载', logs: [] });
+      }
+
+      let logs = JSON.parse(logsJson);
+
+      // 限制返回数量
+      if (limit && Array.isArray(logs) && logs.length > limit) {
+        logs = logs.slice(-limit);
+      }
+
+      return JSON.stringify({
+        success: true,
+        count: Array.isArray(logs) ? logs.length : Object.keys(logs).length,
+        logs
+      });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message, logs: [] });
+    }
+  },
+  {
+    name: 'get_hook_logs',
+    description: '获取 Hook 捕获的日志（XHR、Fetch、Cookie、加密调用等）',
+    schema: z.object({
+      type: z.string().optional().describe('日志类型: xhr, fetch, cookie, crypto, json, eval, storage, encoding, websocket, env, debug, dom。不填则获取全部'),
+      limit: z.number().optional().default(50).describe('返回日志数量限制，默认50条'),
+    }),
+  }
+);
+
+/**
+ * 搜索 Hook 日志
+ */
+export const searchHookLogs = tool(
+  async ({ keyword }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const expression = `window.__deepspider__?.searchLogs?.('${keyword}') || '[]'`;
+      const result = await evaluateViaCDP(browser, expression);
+      return JSON.stringify({ success: true, results: JSON.parse(result || '[]') });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'search_hook_logs',
+    description: '按关键词搜索 Hook 日志',
+    schema: z.object({
+      keyword: z.string().describe('搜索关键词'),
+    }),
+  }
+);
+
+/**
+ * 追踪值来源
+ */
+export const traceValue = tool(
+  async ({ value }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const expression = `window.__deepspider__?.traceValue?.('${value}') || '[]'`;
+      const result = await evaluateViaCDP(browser, expression);
+      return JSON.stringify({ success: true, traces: JSON.parse(result || '[]') });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'trace_value',
+    description: '追踪某个值的来源（在哪个加密函数或请求中出现）',
+    schema: z.object({
+      value: z.string().describe('要追踪的值'),
+    }),
+  }
+);
+
+export const captureTools = [collectEnv, collectProperty, autoFixEnv, getHookLogs, searchHookLogs, traceValue];

package/src/agent/tools/correlate.js

@@ -0,0 +1,303 @@
+/**
+ * DeepSpider - 关联分析工具
+ * 分析请求与加密调用的关联关系
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+
+/**
+ * 分析请求-加密关联
+ */
+export const analyzeCorrelation = tool(
+  async ({ logs }) => {
+    const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
+
+    // 按请求ID分组
+    const byRequest = new Map();
+    const orphanCrypto = [];
+
+    for (const entry of parsed) {
+      if (entry._type === 'xhr' || entry._type === 'fetch') {
+        const id = entry.requestId;
+        if (!byRequest.has(id)) {
+          byRequest.set(id, { request: null, response: null, crypto: [] });
+        }
+        if (entry.action === 'send' || entry.action === 'request') {
+          byRequest.get(id).request = entry;
+        } else if (entry.action === 'response') {
+          byRequest.get(id).response = entry;
+          byRequest.get(id).crypto = entry.linkedCrypto || [];
+        }
+      } else if (entry._type === 'crypto') {
+        if (!entry.requestId) {
+          orphanCrypto.push(entry);
+        }
+      }
+    }
+
+    // 生成分析报告
+    const correlations = [];
+    for (const [id, data] of byRequest) {
+      if (data.request) {
+        correlations.push({
+          requestId: id,
+          url: data.request.url,
+          method: data.request.method,
+          headers: data.request.requestHeaders,
+          cryptoCalls: data.crypto.map(c => ({
+            algo: c.algo,
+            hasKey: !!c.key,
+            stackTop: parseStackTop(c.stack)
+          }))
+        });
+      }
+    }
+
+    return JSON.stringify({
+      correlations,
+      orphanCrypto: orphanCrypto.length,
+      summary: {
+        totalRequests: correlations.length,
+        requestsWithCrypto: correlations.filter(c => c.cryptoCalls.length > 0).length
+      }
+    }, null, 2);
+  },
+  {
+    name: 'analyze_correlation',
+    description: '分析请求与加密调用的关联关系，找出每个请求使用了哪些加密',
+    schema: z.object({
+      logs: z.string().describe('__deepspider__.getAllLogs() 返回的日志'),
+    }),
+  }
+);
+
+/**
+ * 解析调用栈顶部
+ */
+function parseStackTop(stack) {
+  if (!stack) return null;
+  const lines = stack.split('\n').slice(2, 5);
+  return lines.map(line => {
+    const match = line.match(/at\s+(.+?)\s+\((.+?):(\d+):(\d+)\)/) ||
+                  line.match(/at\s+(.+?):(\d+):(\d+)/);
+    if (match) {
+      return {
+        func: match[1] || 'anonymous',
+        file: match[2] || match[1],
+        line: parseInt(match[3] || match[2])
+      };
+    }
+    return { raw: line.trim() };
+  });
+}
+
+/**
+ * 从调用栈定位加密函数
+ */
+export const locateCryptoSource = tool(
+  async ({ cryptoLog }) => {
+    const entry = typeof cryptoLog === 'string' ? JSON.parse(cryptoLog) : cryptoLog;
+    const stack = parseStackTop(entry.stack);
+
+    // 过滤掉 Hook 相关的栈帧
+    const filtered = stack.filter(frame => {
+      if (!frame.file) return true;
+      return !frame.file.includes('deepspider') &&
+             !frame.func?.includes('native') &&
+             !frame.func?.includes('hook');
+    });
+
+    return JSON.stringify({
+      algo: entry.algo,
+      sourceLocation: filtered[0] || null,
+      callChain: filtered.slice(0, 3),
+      suggestion: filtered[0] ?
+        `在 ${filtered[0].file}:${filtered[0].line} 设置断点分析` :
+        '无法定位源码位置'
+    }, null, 2);
+  },
+  {
+    name: 'locate_crypto_source',
+    description: '从加密日志的调用栈定位加密函数的源码位置',
+    schema: z.object({
+      cryptoLog: z.string().describe('单条加密日志'),
+    }),
+  }
+);
+
+/**
+ * 分析 Header 加密来源
+ */
+export const analyzeHeaderEncryption = tool(
+  async ({ logs, headerName }) => {
+    const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
+
+    // 找到设置该 header 的请求
+    const relevant = parsed.filter(entry => {
+      if (entry._type !== 'xhr' && entry._type !== 'fetch') return false;
+      if (!entry.requestHeaders && !entry.headers) return false;
+      const headers = entry.requestHeaders || entry.headers;
+      return headerName in headers;
+    });
+
+    if (relevant.length === 0) {
+      return JSON.stringify({ found: false, message: `未找到设置 ${headerName} 的请求` });
+    }
+
+    // 分析关联的加密调用
+    const analysis = relevant.map(req => {
+      const headerValue = (req.requestHeaders || req.headers)[headerName];
+      return {
+        url: req.url,
+        headerValue: headerValue?.slice(0, 50),
+        linkedCrypto: req.linkedCrypto || [],
+        timestamp: req.timestamp
+      };
+    });
+
+    return JSON.stringify({
+      found: true,
+      headerName,
+      occurrences: analysis.length,
+      analysis,
+      suggestion: analysis[0]?.linkedCrypto?.length > 0 ?
+        '已找到关联的加密调用，使用 locate_crypto_source 定位源码' :
+        '未找到直接关联的加密，可能在请求前已完成加密'
+    }, null, 2);
+  },
+  {
+    name: 'analyze_header_encryption',
+    description: '分析指定 Header 的加密来源',
+    schema: z.object({
+      logs: z.string().describe('__deepspider__.getAllLogs() 返回的日志'),
+      headerName: z.string().describe('要分析的 Header 名称，如 X-Sign'),
+    }),
+  }
+);
+
+/**
+ * 分析 Cookie 加密来源
+ */
+export const analyzeCookieEncryption = tool(
+  async ({ logs, cookieName }) => {
+    const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
+
+    // 找到设置该 cookie 的日志
+    const cookieLogs = parsed.filter(entry => {
+      if (entry._type !== 'cookie') return false;
+      if (entry.action !== 'write') return false;
+      return entry.value?.includes(cookieName);
+    });
+
+    if (cookieLogs.length === 0) {
+      return JSON.stringify({ found: false, message: `未找到设置 ${cookieName} 的操作` });
+    }
+
+    // 分析调用栈
+    const analysis = cookieLogs.map(log => ({
+      value: log.value?.slice(0, 100),
+      timestamp: log.timestamp,
+      stackTop: parseStackTop(log.stack),
+    }));
+
+    // 查找时间相近的加密调用
+    const cryptoLogs = parsed.filter(e => e._type === 'crypto');
+    const linkedCrypto = [];
+
+    for (const cookieLog of cookieLogs) {
+      const nearby = cryptoLogs.filter(c =>
+        Math.abs(c.timestamp - cookieLog.timestamp) < 100
+      );
+      linkedCrypto.push(...nearby);
+    }
+
+    return JSON.stringify({
+      found: true,
+      cookieName,
+      occurrences: analysis.length,
+      analysis,
+      nearbyCrypto: linkedCrypto.map(c => ({
+        algo: c.algo,
+        timestamp: c.timestamp,
+        stackTop: parseStackTop(c.stack)
+      })),
+      suggestion: linkedCrypto.length > 0 ?
+        '找到时间相近的加密调用，可能是 Cookie 值的来源' :
+        '未找到相近加密调用，Cookie 值可能来自其他计算'
+    }, null, 2);
+  },
+  {
+    name: 'analyze_cookie_encryption',
+    description: '分析指定 Cookie 的加密来源（通过时间和调用栈关联）',
+    schema: z.object({
+      logs: z.string().describe('__deepspider__.getAllLogs() 返回的日志'),
+      cookieName: z.string().describe('要分析的 Cookie 名称'),
+    }),
+  }
+);
+
+/**
+ * 分析响应解密
+ */
+export const analyzeResponseDecryption = tool(
+  async ({ logs, urlPattern }) => {
+    const parsed = typeof logs === 'string' ? JSON.parse(logs) : logs;
+    const regex = urlPattern ? new RegExp(urlPattern) : null;
+
+    // 找到响应日志
+    const responseLogs = parsed.filter(entry => {
+      if (entry._type !== 'xhr' && entry._type !== 'fetch') return false;
+      if (entry.action !== 'response') return false;
+      if (regex && !regex.test(entry.url)) return false;
+      return true;
+    });
+
+    // 分析每个响应后的解密调用
+    const analysis = [];
+    const cryptoLogs = parsed.filter(e => e._type === 'crypto');
+
+    for (const resp of responseLogs) {
+      // 查找响应后 500ms 内的解密调用
+      const decrypts = cryptoLogs.filter(c => {
+        const timeDiff = c.timestamp - resp.timestamp;
+        return timeDiff > 0 && timeDiff < 500 &&
+               c.algo?.toLowerCase().includes('decrypt');
+      });
+
+      analysis.push({
+        url: resp.url,
+        status: resp.status,
+        responsePreview: resp.response?.slice(0, 50),
+        timestamp: resp.timestamp,
+        decryptCalls: decrypts.map(d => ({
+          algo: d.algo,
+          timeDiff: d.timestamp - resp.timestamp,
+          stackTop: parseStackTop(d.stack)
+        }))
+      });
+    }
+
+    return JSON.stringify({
+      totalResponses: analysis.length,
+      withDecrypt: analysis.filter(a => a.decryptCalls.length > 0).length,
+      analysis: analysis.slice(0, 10),
+    }, null, 2);
+  },
+  {
+    name: 'analyze_response_decryption',
+    description: '分析响应后的解密调用（通过时间窗口关联）',
+    schema: z.object({
+      logs: z.string().describe('__deepspider__.getAllLogs() 返回的日志'),
+      urlPattern: z.string().optional().describe('URL 匹配模式'),
+    }),
+  }
+);
+
+export const correlateTools = [
+  analyzeCorrelation,
+  locateCryptoSource,
+  analyzeHeaderEncryption,
+  analyzeCookieEncryption,
+  analyzeResponseDecryption,
+];

package/src/agent/tools/crawler.js

@@ -0,0 +1,116 @@
+/**
+ * DeepSpider - 爬虫编排工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { getBrowser } from '../../browser/index.js';
+
+export const siteAnalyze = tool(
+  async ({ url }) => {
+    try {
+      const browser = await getBrowser();
+      const page = browser.getPage();
+
+      if (url) {
+        await page.goto(url, { waitUntil: 'networkidle' });
+      }
+
+      const analysis = await page.evaluate(() => {
+        const features = {
+          hasLogin: !!document.querySelector('input[type="password"]'),
+          hasCaptcha: !!document.querySelector('[class*="captcha"], [id*="captcha"]'),
+          hasEncryption: false,
+        };
+
+        // 检测加密库
+        const scripts = Array.from(document.scripts).map(s => s.src);
+        features.hasEncryption = scripts.some(s =>
+          s.includes('crypto') || s.includes('encrypt') || s.includes('sign')
+        );
+
+        return features;
+      });
+
+      return JSON.stringify({ success: true, ...analysis });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'site_analyze',
+    description: '分析目标网站特征',
+    schema: z.object({
+      url: z.string().optional().describe('目标URL，不填则分析当前页面'),
+    }),
+  }
+);
+
+export const complexityAssess = tool(
+  async ({ features }) => {
+    let level = 1;
+    const reasons = [];
+
+    if (features.hasEncryption) {
+      level = Math.max(level, 2);
+      reasons.push('存在加密');
+    }
+    if (features.hasCaptcha) {
+      level = Math.max(level, 2);
+      reasons.push('存在验证码');
+    }
+    if (features.hasLogin) {
+      level = Math.max(level, 2);
+      reasons.push('需要登录');
+    }
+    if (features.hasFingerprint) {
+      level = 3;
+      reasons.push('指纹检测');
+    }
+
+    return JSON.stringify({
+      success: true,
+      level,
+      reasons,
+      recommendation: level === 1 ? 'simple' : level === 2 ? 'medium' : 'complex',
+    });
+  },
+  {
+    name: 'complexity_assess',
+    description: '评估网站复杂度等级',
+    schema: z.object({
+      features: z.object({
+        hasEncryption: z.boolean().optional(),
+        hasCaptcha: z.boolean().optional(),
+        hasLogin: z.boolean().optional(),
+        hasFingerprint: z.boolean().optional(),
+      }),
+    }),
+  }
+);
+
+export const e2eTest = tool(
+  async ({ script_path, test_params }) => {
+    try {
+      // TODO: 执行 Python 脚本进行 E2E 测试
+      return JSON.stringify({
+        success: true,
+        message: '需要集成 Python 执行环境',
+        script_path,
+        test_params,
+      });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'e2e_test',
+    description: '端到端测试爬虫脚本',
+    schema: z.object({
+      script_path: z.string().describe('脚本路径'),
+      test_params: z.record(z.string(), z.unknown()).optional().describe('测试参数'),
+    }),
+  }
+);
+
+export const crawlerTools = [siteAnalyze, complexityAssess, e2eTest];

package/src/agent/tools/cryptohook.js

@@ -0,0 +1,80 @@
+/**
+ * DeepSpider - 加密函数 Hook 工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { CryptoHook } from '../../env/CryptoHook.js';
+
+const cryptoHook = new CryptoHook();
+
+export const generateCryptoJSHook = tool(
+  async () => {
+    const code = cryptoHook.generateCryptoJSHookCode();
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('crypto') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_cryptojs_hook',
+    description: '生成 CryptoJS Hook（AES/DES/MD5/SHA/HMAC）',
+    schema: z.object({}),
+  }
+);
+
+export const generateSMCryptoHook = tool(
+  async () => {
+    const code = cryptoHook.generateSMCryptoHookCode();
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('crypto') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_sm_crypto_hook',
+    description: '生成国密 Hook（SM2/SM3/SM4）',
+    schema: z.object({}),
+  }
+);
+
+export const generateRSAHook = tool(
+  async () => {
+    const code = cryptoHook.generateRSAHookCode();
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('crypto') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_rsa_hook',
+    description: '生成 RSA Hook（JSEncrypt/node-forge）',
+    schema: z.object({}),
+  }
+);
+
+export const generateGenericCryptoHook = tool(
+  async () => {
+    const code = cryptoHook.generateGenericCryptoHookCode();
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('crypto') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_generic_crypto_hook',
+    description: '生成通用加密 Hook（基于函数名关键词匹配）',
+    schema: z.object({}),
+  }
+);
+
+export const cryptoHookTools = [
+  generateCryptoJSHook,
+  generateSMCryptoHook,
+  generateRSAHook,
+  generateGenericCryptoHook,
+];