deepspider 0.1.0

package/src/agent/tools/index.js

@@ -0,0 +1,137 @@
+/**
+ * DeepSpider - 工具索引
+ */
+
+export { sandboxTools, sandboxExecute, sandboxInject, sandboxReset, getSandbox } from './sandbox.js';
+export { analyzerTools, analyzeAst, analyzeCallstack, analyzeEncryption } from './analyzer.js';
+export { deobfuscatorTools, deobfuscate, deobfuscatePipeline, detectObfuscator, decodeStrings } from './deobfuscator.js';
+export { traceTools, traceVariable, traceRequestParams, findCallPattern } from './trace.js';
+export { storeTools, saveToStore, queryStore, listStore } from './store.js';
+export { patchTools, generatePatch, matchModule } from './patch.js';
+export { envTools, listEnvModules, loadEnvModule, loadAllEnvModules } from './env.js';
+export { profileTools, listProfiles, loadProfile, generateProfileCode } from './profile.js';
+export { runtimeTools, launchBrowser, navigateTo, browserClose, addInitScript, clearCookies } from './runtime.js';
+export { debugTools, setBreakpoint, setXHRBreakpoint, getCallStack, getFrameVariables, evaluateAtBreakpoint, resumeExecution, stepOver } from './debug.js';
+export { captureTools, collectEnv, collectProperty, autoFixEnv, getHookLogs } from './capture.js';
+export { browserTools, clickElement, fillInput, waitForSelector } from './browser.js';
+export { reportTools, saveAnalysisReport } from './report.js';
+export { webcrackTools, unpackBundle, analyzeBundle } from './webcrack.js';
+export { preprocessTools, preprocessCode } from './preprocess.js';
+export { envDumpTools, generateEnvDumpCode, generateBaseEnvCode, parseEnvLogs } from './envdump.js';
+export { extractTools, generateExtractScript, generateBatchExtractScript, convertToPatchCode, classifyPatch } from './extract.js';
+export { hookTools, generateXHRHook, generateFetchHook, generateCookieHook } from './hook.js';
+export { asyncTools, generatePromiseHook, generateTimerHook } from './async.js';
+export { antiDebugTools, generateAntiDebugger, generateAntiConsoleDetect, generateAntiCDP, generateFullAntiDebug } from './antidebug.js';
+export { verifyTools, verifyMD5, verifySHA256, verifyHMAC, verifyAES, identifyEncryption } from './verify.js';
+export { cryptoHookTools, generateCryptoJSHook, generateRSAHook } from './cryptohook.js';
+export { correlateTools, analyzeCorrelation, locateCryptoSource, analyzeHeaderEncryption, analyzeCookieEncryption, analyzeResponseDecryption } from './correlate.js';
+export { extractorTools, listFunctions, getFunctionCode } from './extractor.js';
+export { tracingTools, getSiteList, searchInResponses, getRequestDetail, getRequestList, getScriptList, getScriptSource, searchInScripts, clearSiteData, clearAllData } from './tracing.js';
+export { analysisTools, getPendingAnalysis, getPendingChat, sendPanelMessage, startSelector } from './analysis.js';
+export { fileTools, artifactSave, artifactLoad, artifactEdit, artifactGlob, artifactGrep } from './file.js';
+export { evolveTools, evolveSkill } from './evolve.js';
+export { captchaTools } from './captcha.js';
+export { antiDetectTools } from './anti-detect.js';
+export { crawlerTools } from './crawler.js';
+export { nodejsTools, runNodeCode } from './nodejs.js';
+export { hookManagerTools, listHooks, enableHook, disableHook, injectHook, setHookConfig } from './hookManager.js';
+// pythonTools 只在 js2python 子代理中使用，不导出到主工具集
+
+// 所有工具
+import { sandboxTools } from './sandbox.js';
+import { analyzerTools } from './analyzer.js';
+import { deobfuscatorTools } from './deobfuscator.js';
+import { traceTools } from './trace.js';
+import { storeTools } from './store.js';
+import { patchTools } from './patch.js';
+import { envTools } from './env.js';
+import { profileTools } from './profile.js';
+import { runtimeTools } from './runtime.js';
+import { debugTools } from './debug.js';
+import { captureTools } from './capture.js';
+import { browserTools } from './browser.js';
+import { reportTools } from './report.js';
+import { webcrackTools } from './webcrack.js';
+import { preprocessTools } from './preprocess.js';
+import { envDumpTools } from './envdump.js';
+import { extractTools } from './extract.js';
+import { hookTools } from './hook.js';
+import { asyncTools } from './async.js';
+import { antiDebugTools } from './antidebug.js';
+import { verifyTools } from './verify.js';
+import { cryptoHookTools } from './cryptohook.js';
+import { correlateTools } from './correlate.js';
+import { extractorTools } from './extractor.js';
+import { tracingTools } from './tracing.js';
+import { analysisTools } from './analysis.js';
+import { fileTools } from './file.js';
+import { evolveTools } from './evolve.js';
+import { captchaTools } from './captcha.js';
+import { antiDetectTools } from './anti-detect.js';
+import { crawlerTools } from './crawler.js';
+import { nodejsTools } from './nodejs.js';
+import { hookManagerTools } from './hookManager.js';
+
+export const allTools = [
+  ...sandboxTools,
+  ...analyzerTools,
+  ...deobfuscatorTools,
+  ...traceTools,
+  ...storeTools,
+  ...patchTools,
+  ...envTools,
+  ...profileTools,
+  ...runtimeTools,
+  ...debugTools,
+  ...captureTools,
+  ...browserTools,
+  ...reportTools,
+  ...webcrackTools,
+  ...preprocessTools,
+  ...envDumpTools,
+  ...extractTools,
+  ...hookTools,
+  ...asyncTools,
+  ...antiDebugTools,
+  ...verifyTools,
+  ...cryptoHookTools,
+  ...correlateTools,
+  ...extractorTools,
+  ...tracingTools,
+  ...analysisTools,
+  ...fileTools,
+  ...evolveTools,
+  ...captchaTools,
+  ...antiDetectTools,
+  ...crawlerTools,
+  ...nodejsTools,
+  ...hookManagerTools,
+];
+
+/**
+ * 主 Agent 核心工具
+ * 只包含必要的运行时、交互和数据查询工具
+ * pythonTools 只在 js2python 子代理中使用
+ */
+export const coreTools = [
+  // 浏览器运行时
+  ...runtimeTools,
+  // 页面交互
+  ...browserTools,
+  // 浏览器分析交互
+  ...analysisTools,
+  // 数据溯源查询
+  ...tracingTools,
+  // 沙箱执行（验证代码）
+  ...sandboxTools,
+  // Hook 日志
+  ...captureTools,
+  // 文件操作
+  ...fileTools,
+  // 经验进化
+  ...evolveTools,
+  // Node.js 执行（支持 require）
+  ...nodejsTools,
+  // Hook 动态管理
+  ...hookManagerTools,
+];

package/src/agent/tools/nodejs.js

@@ -0,0 +1,101 @@
+/**
+ * DeepSpider - Node.js 执行工具
+ * 用于执行带有 require 依赖的 JS 代码
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+// 项目根目录（用于 require 加密库）
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROJECT_ROOT = join(__dirname, '../../..');
+
+// 输出大小限制
+const MAX_OUTPUT_SIZE = 100000;
+
+/**
+ * 执行 Node.js 代码
+ */
+async function executeNode(code, timeout = 10000) {
+  return new Promise((resolve) => {
+    const proc = spawn('node', ['-e', code], {
+      env: { ...process.env },
+      cwd: PROJECT_ROOT,
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let killed = false;
+
+    // 手动实现超时
+    const timer = setTimeout(() => {
+      killed = true;
+      proc.kill('SIGTERM');
+    }, timeout);
+
+    proc.stdout.on('data', (data) => {
+      if (stdout.length < MAX_OUTPUT_SIZE) {
+        stdout += data.toString();
+      }
+    });
+
+    proc.stderr.on('data', (data) => {
+      if (stderr.length < MAX_OUTPUT_SIZE) {
+        stderr += data.toString();
+      }
+    });
+
+    proc.on('close', (exitCode) => {
+      clearTimeout(timer);
+      resolve({
+        success: !killed && exitCode === 0,
+        stdout: stdout.trim(),
+        stderr: killed ? 'Timeout: process killed' : stderr.trim(),
+        exitCode: killed ? -1 : exitCode,
+      });
+    });
+
+    proc.on('error', (err) => {
+      clearTimeout(timer);
+      resolve({
+        success: false,
+        stdout: '',
+        stderr: err.message,
+        exitCode: -1,
+      });
+    });
+  });
+}
+
+/**
+ * 执行 Node.js 代码
+ */
+export const runNodeCode = tool(
+  async ({ code, timeout }) => {
+    const result = await executeNode(code, timeout || 10000);
+    return JSON.stringify(result);
+  },
+  {
+    name: 'run_node_code',
+    description: `在 Node.js 环境中执行 JS 代码，支持 require 引入已安装的加密库。
+
+可用的加密库：
+- crypto-js: AES/DES/MD5/SHA 等常用加密
+- jsencrypt: RSA 加密
+- sm-crypto: 国密 SM2/SM3/SM4
+- js-md5: MD5 哈希
+- js-sha256: SHA256 哈希
+
+示例：const CryptoJS = require('crypto-js'); console.log(CryptoJS.MD5('test').toString());`,
+    schema: z.object({
+      code: z.string().describe('要执行的 JS 代码，可使用 require 引入加密库'),
+      timeout: z.number().optional().default(10000).describe('超时时间（毫秒）'),
+    }),
+  }
+);
+
+export const nodejsTools = [runNodeCode];

package/src/agent/tools/patch.js

@@ -0,0 +1,46 @@
+/**
+ * DeepSpider - 补丁生成工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { PatchGenerator } from '../../core/PatchGenerator.js';
+
+const patchGen = new PatchGenerator();
+
+/**
+ * 生成单个补丁
+ */
+export const generatePatch = tool(
+  async ({ property, context }) => {
+    const result = await patchGen.generate(property, context);
+    return JSON.stringify(result, null, 2);
+  },
+  {
+    name: 'generate_patch',
+    description: '为缺失的环境属性生成补丁代码。',
+    schema: z.object({
+      property: z.string().describe('缺失的属性路径，如 navigator.userAgent'),
+      context: z.record(z.string(), z.unknown()).optional().describe('上下文信息'),
+    }),
+  }
+);
+
+/**
+ * 批量生成补丁
+ */
+export const matchModule = tool(
+  async ({ missingProperties }) => {
+    const result = await patchGen.generateBatch(missingProperties);
+    return JSON.stringify(result, null, 2);
+  },
+  {
+    name: 'match_module',
+    description: '批量匹配缺失属性，生成补丁集。',
+    schema: z.object({
+      missingProperties: z.array(z.string()).describe('缺失的属性路径列表'),
+    }),
+  }
+);
+
+export const patchTools = [generatePatch, matchModule];

package/src/agent/tools/preprocess.js

@@ -0,0 +1,71 @@
+/**
+ * DeepSpider - 代码预处理工具
+ * 智能处理各种打包/混淆代码
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { webcrack } from 'webcrack';
+
+/**
+ * 智能预处理代码
+ * 自动检测并处理 Webpack/Browserify 或直接反混淆
+ */
+export const preprocessCode = tool(
+  async ({ code, outputDir }) => {
+    try {
+      const result = await webcrack(code);
+
+      const output = {
+        success: true,
+        bundleDetected: !!result.bundle?.type,
+        bundleType: result.bundle?.type || null,
+        code: result.code,
+        modules: [],
+      };
+
+      // 如果检测到 bundle，提取模块信息
+      if (result.bundle?.modules) {
+        for (const [id, module] of result.bundle.modules) {
+          output.modules.push({
+            id,
+            path: module.path || '',
+            isEntry: module.isEntry || false,
+          });
+        }
+      }
+
+      output.moduleCount = output.modules.length;
+
+      // 保存到目录（如果指定）
+      if (outputDir && result.bundle) {
+        await result.save(outputDir);
+        output.outputDir = outputDir;
+      }
+
+      // 添加处理建议
+      if (output.bundleDetected) {
+        output.suggestion = '已解包，可继续用 deobfuscate 处理各模块';
+      } else {
+        output.suggestion = '未检测到 bundle 结构，已完成基础反混淆，可继续用 deobfuscate 深度处理';
+      }
+
+      return JSON.stringify(output);
+    } catch (error) {
+      return JSON.stringify({
+        success: false,
+        error: error.message,
+      });
+    }
+  },
+  {
+    name: 'preprocess_code',
+    description: '智能预处理 JS 代码：自动检测 Webpack/Browserify 并解包，或直接反混淆 Vite/Rollup 代码',
+    schema: z.object({
+      code: z.string().describe('待处理的 JS 代码'),
+      outputDir: z.string().optional().describe('输出目录（可选，仅对 bundle 有效）'),
+    }),
+  }
+);
+
+export const preprocessTools = [preprocessCode];

package/src/agent/tools/profile.js

@@ -0,0 +1,122 @@
+/**
+ * DeepSpider - 浏览器 Profile 工具
+ * 提供浏览器指纹配置加载能力
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const profilesDir = join(__dirname, '../../config/profiles');
+
+/**
+ * 列出可用的浏览器 Profile
+ */
+export const listProfiles = tool(
+  async () => {
+    return JSON.stringify({
+      profiles: ['chrome', 'firefox', 'safari'],
+      description: {
+        chrome: 'Chrome 120 Windows - 最常用的浏览器指纹',
+        firefox: 'Firefox Windows - 备选浏览器指纹',
+        safari: 'Safari macOS - Apple 设备指纹',
+      },
+    }, null, 2);
+  },
+  {
+    name: 'list_profiles',
+    description: '列出所有可用的浏览器指纹 Profile。',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 加载指定的浏览器 Profile
+ */
+export const loadProfile = tool(
+  async ({ name }) => {
+    try {
+      const path = join(profilesDir, `${name}.json`);
+      const profile = JSON.parse(readFileSync(path, 'utf-8'));
+      return JSON.stringify({ success: true, name, profile }, null, 2);
+    } catch (e) {
+      return JSON.stringify({
+        success: false,
+        error: `Profile ${name} 加载失败: ${e.message}`,
+        available: ['chrome', 'firefox', 'safari'],
+      });
+    }
+  },
+  {
+    name: 'load_profile',
+    description: '加载指定的浏览器指纹 Profile，包含 navigator、screen、window 等完整配置。',
+    schema: z.object({
+      name: z.enum(['chrome', 'firefox', 'safari']).describe('Profile 名称'),
+    }),
+  }
+);
+
+/**
+ * 生成 Profile 注入代码
+ */
+export const generateProfileCode = tool(
+  async ({ name }) => {
+    try {
+      const path = join(profilesDir, `${name}.json`);
+      const profile = JSON.parse(readFileSync(path, 'utf-8'));
+
+      const lines = [];
+
+      // Navigator 属性
+      if (profile.navigator) {
+        for (const [k, v] of Object.entries(profile.navigator)) {
+          if (typeof v === 'string') {
+            lines.push(`navigator.${k} = "${v}";`);
+          } else if (Array.isArray(v)) {
+            lines.push(`navigator.${k} = ${JSON.stringify(v)};`);
+          } else if (typeof v !== 'object') {
+            lines.push(`navigator.${k} = ${v};`);
+          }
+        }
+      }
+
+      // Screen 属性
+      if (profile.screen) {
+        for (const [k, v] of Object.entries(profile.screen)) {
+          lines.push(`screen.${k} = ${v};`);
+        }
+      }
+
+      // Window 属性
+      if (profile.window) {
+        for (const [k, v] of Object.entries(profile.window)) {
+          lines.push(`window.${k} = ${v};`);
+        }
+      }
+
+      return JSON.stringify({
+        success: true,
+        name,
+        code: lines.join('\n'),
+      }, null, 2);
+    } catch (e) {
+      return JSON.stringify({
+        success: false,
+        error: `生成 Profile 代码失败: ${e.message}`,
+      });
+    }
+  },
+  {
+    name: 'generate_profile_code',
+    description: '根据 Profile 生成可注入沙箱的 JavaScript 代码。',
+    schema: z.object({
+      name: z.enum(['chrome', 'firefox', 'safari']).describe('Profile 名称'),
+    }),
+  }
+);
+
+export const profileTools = [listProfiles, loadProfile, generateProfileCode];