deepspider 0.1.0

package/src/agent/tools/extract.js

@@ -0,0 +1,114 @@
+/**
+ * DeepSpider - 浏览器环境提取工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { EnvExtractor } from '../../env/EnvExtractor.js';
+
+const extractor = new EnvExtractor();
+
+/**
+ * 生成环境提取脚本
+ */
+export const generateExtractScript = tool(
+  async ({ objPath, includeProto }) => {
+    const script = extractor.generateExtractScript(objPath, { includeProto });
+    return JSON.stringify({
+      success: true,
+      objPath,
+      script,
+      usage: '在浏览器控制台或通过 browser_evaluate 执行此脚本',
+    }, null, 2);
+  },
+  {
+    name: 'generate_extract_script',
+    description: '生成从浏览器提取指定对象环境的脚本代码',
+    schema: z.object({
+      objPath: z.string().describe('要提取的对象路径，如 navigator, screen.__proto__'),
+      includeProto: z.boolean().default(true).describe('是否包含原型链属性'),
+    }),
+  }
+);
+
+/**
+ * 生成批量提取脚本
+ */
+export const generateBatchExtractScript = tool(
+  async ({ objPaths }) => {
+    const script = extractor.generateBatchExtractScript(objPaths);
+    return JSON.stringify({
+      success: true,
+      targets: objPaths,
+      script,
+    }, null, 2);
+  },
+  {
+    name: 'generate_batch_extract_script',
+    description: '生成批量提取多个对象环境的脚本',
+    schema: z.object({
+      objPaths: z.array(z.string()).describe('要提取的对象路径列表'),
+    }),
+  }
+);
+
+/**
+ * 将提取结果转换为补丁代码
+ */
+export const convertToPatchCode = tool(
+  async ({ extractResult, objPath }) => {
+    const patchCode = extractor.generatePatchCode(extractResult, objPath);
+    const patchType = extractor.classifyPatch(objPath);
+
+    return JSON.stringify({
+      success: true,
+      objPath,
+      patchType,
+      patchCode,
+      suggestion: patchType === 'universal'
+        ? '建议持久化到通用环境库'
+        : '建议按需使用',
+    }, null, 2);
+  },
+  {
+    name: 'convert_to_patch_code',
+    description: '将浏览器提取的环境数据转换为可注入的补丁代码',
+    schema: z.object({
+      extractResult: z.string().describe('浏览器执行提取脚本返回的 JSON'),
+      objPath: z.string().describe('对象路径'),
+    }),
+  }
+);
+
+/**
+ * 分类补丁类型
+ */
+export const classifyPatch = tool(
+  async ({ objPath }) => {
+    const patchType = extractor.classifyPatch(objPath);
+    return JSON.stringify({
+      objPath,
+      type: patchType,
+      description: {
+        'universal': '通用环境，建议持久化',
+        'browser-chrome': 'Chrome 特有，按浏览器存储',
+        'browser-firefox': 'Firefox 特有，按浏览器存储',
+        'site-specific': '网站特定，按域名存储',
+      }[patchType] || '未知类型',
+    }, null, 2);
+  },
+  {
+    name: 'classify_patch',
+    description: '判断补丁的类型（通用/浏览器特定/网站特定）',
+    schema: z.object({
+      objPath: z.string().describe('对象路径'),
+    }),
+  }
+);
+
+export const extractTools = [
+  generateExtractScript,
+  generateBatchExtractScript,
+  convertToPatchCode,
+  classifyPatch,
+];

package/src/agent/tools/extractor.js

@@ -0,0 +1,54 @@
+/**
+ * DeepSpider - 代码提取辅助工具
+ * 提供信息辅助 AI 进行代码提取
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { ASTAnalyzer } from '../../analyzer/ASTAnalyzer.js';
+
+const astAnalyzer = new ASTAnalyzer();
+
+/**
+ * 获取函数列表
+ */
+export const listFunctions = tool(
+  async ({ code }) => {
+    const functions = astAnalyzer.extractFunctions(code);
+    return JSON.stringify({
+      count: functions.length,
+      functions: functions.slice(0, 50),
+    }, null, 2);
+  },
+  {
+    name: 'list_functions',
+    description: '列出代码中的所有函数，辅助定位目标函数',
+    schema: z.object({
+      code: z.string().describe('源代码'),
+    }),
+  }
+);
+
+/**
+ * 获取函数代码片段
+ */
+export const getFunctionCode = tool(
+  async ({ code, funcName }) => {
+    const slice = astAnalyzer.extractSlice(code, funcName);
+    return JSON.stringify({
+      funcName,
+      found: !!slice,
+      code: slice || '未找到该函数',
+    }, null, 2);
+  },
+  {
+    name: 'get_function_code',
+    description: '获取指定函数的代码片段',
+    schema: z.object({
+      code: z.string().describe('源代码'),
+      funcName: z.string().describe('函数名'),
+    }),
+  }
+);
+
+export const extractorTools = [listFunctions, getFunctionCode];

package/src/agent/tools/file.js

@@ -0,0 +1,224 @@
+/**
+ * DeepSpider - 文件操作工具
+ * 统一存储到 ~/.deepspider/output/
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'fs';
+import { dirname, join, isAbsolute, relative } from 'path';
+import { PATHS, ensureDir, DEEPSPIDER_HOME } from '../../config/paths.js';
+
+const OUTPUT_DIR = PATHS.OUTPUT_DIR;
+
+function ensureFileDir(filePath) {
+  const dir = dirname(filePath);
+  ensureDir(dir);
+}
+
+function getSafePath(filePath) {
+  if (isAbsolute(filePath)) {
+    // 如果是 ~/.deepspider/ 目录下的路径，直接使用
+    if (filePath.startsWith(DEEPSPIDER_HOME)) {
+      return filePath;
+    }
+    // 其他绝对路径：放到 OUTPUT_DIR 下
+    return join(OUTPUT_DIR, filePath.replace(/^\/+/, ''));
+  }
+  return join(OUTPUT_DIR, filePath);
+}
+
+export const artifactSave = tool(
+  async ({ file_path, content }) => {
+    try {
+      const safePath = getSafePath(file_path);
+      ensureFileDir(safePath);
+      writeFileSync(safePath, content, 'utf-8');
+      console.log('[artifact_save] 已保存:', safePath);
+      return JSON.stringify({ success: true, path: safePath, size: content.length });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'artifact_save',
+    description: '保存逆向分析产出文件（代码、数据、报告等）到 ~/.deepspider/output/ 目录',
+    schema: z.object({
+      file_path: z.string().describe('文件路径（相对于 output 目录）'),
+      content: z.string().describe('文件内容'),
+    }),
+  }
+);
+
+export const artifactLoad = tool(
+  async ({ file_path }) => {
+    try {
+      const safePath = getSafePath(file_path);
+      if (!existsSync(safePath)) {
+        return JSON.stringify({ success: false, error: '文件不存在' });
+      }
+      const content = readFileSync(safePath, 'utf-8');
+      return JSON.stringify({ success: true, content, size: content.length });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'artifact_load',
+    description: '读取逆向分析产出文件（从 ~/.deepspider/output/ 目录）',
+    schema: z.object({
+      file_path: z.string().describe('文件路径（相对于 output 目录）'),
+    }),
+  }
+);
+
+export const artifactEdit = tool(
+  async ({ file_path, old_string, new_string, replace_all = false }) => {
+    try {
+      const safePath = getSafePath(file_path);
+      if (!existsSync(safePath)) {
+        return JSON.stringify({ success: false, error: '文件不存在' });
+      }
+      let content = readFileSync(safePath, 'utf-8');
+      const occurrences = content.split(old_string).length - 1;
+
+      if (occurrences === 0) {
+        return JSON.stringify({ success: false, error: '未找到要替换的字符串' });
+      }
+      if (occurrences > 1 && !replace_all) {
+        return JSON.stringify({
+          success: false,
+          error: `找到 ${occurrences} 处匹配，请使用 replace_all=true 或提供更精确的字符串`
+        });
+      }
+
+      content = replace_all
+        ? content.split(old_string).join(new_string)
+        : content.replace(old_string, new_string);
+
+      writeFileSync(safePath, content, 'utf-8');
+      return JSON.stringify({ success: true, path: safePath, replacements: replace_all ? occurrences : 1 });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'artifact_edit',
+    description: '编辑逆向分析产出文件，替换指定字符串',
+    schema: z.object({
+      file_path: z.string().describe('文件路径'),
+      old_string: z.string().describe('要替换的原字符串'),
+      new_string: z.string().describe('替换后的新字符串'),
+      replace_all: z.boolean().optional().default(false).describe('是否替换所有匹配'),
+    }),
+  }
+);
+
+/**
+ * 递归遍历目录，匹配文件
+ */
+function walkDir(dir, pattern, results = []) {
+  if (!existsSync(dir)) return results;
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      walkDir(fullPath, pattern, results);
+    } else if (entry.isFile()) {
+      // 简单的 glob 匹配：支持 * 和 **
+      const relativePath = relative(OUTPUT_DIR, fullPath);
+      if (matchGlob(relativePath, pattern)) {
+        results.push(relativePath);
+      }
+    }
+  }
+  return results;
+}
+
+/**
+ * 简单的 glob 匹配
+ */
+function matchGlob(path, pattern) {
+  // 转换 glob 为正则
+  const regex = pattern
+    .replace(/\*\*/g, '{{GLOBSTAR}}')
+    .replace(/\*/g, '[^/]*')
+    .replace(/\?/g, '.')
+    .replace(/{{GLOBSTAR}}/g, '.*');
+  return new RegExp(`^${regex}$`).test(path);
+}
+
+export const artifactGlob = tool(
+  async ({ pattern }) => {
+    try {
+      const files = walkDir(OUTPUT_DIR, pattern);
+      return JSON.stringify({ success: true, files, count: files.length });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'artifact_glob',
+    description: '在产出目录中查找匹配模式的文件（支持 * 和 ** 通配符）',
+    schema: z.object({
+      pattern: z.string().describe('文件匹配模式，如 "*.py" 或 "**/*.js"'),
+    }),
+  }
+);
+
+/**
+ * 在文件中搜索内容
+ */
+function searchInFile(filePath, pattern, isRegex) {
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+    const lines = content.split('\n');
+    const matches = [];
+    const regex = isRegex ? new RegExp(pattern, 'g') : null;
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      const found = isRegex ? regex.test(line) : line.includes(pattern);
+      if (found) {
+        matches.push({ line: i + 1, content: line.trim() });
+      }
+      if (isRegex) regex.lastIndex = 0; // 重置正则
+    }
+    return matches;
+  } catch (e) {
+    return [];
+  }
+}
+
+export const artifactGrep = tool(
+  async ({ pattern, file_pattern = '**/*', is_regex = false }) => {
+    try {
+      const files = walkDir(OUTPUT_DIR, file_pattern);
+      const results = [];
+
+      for (const file of files) {
+        const fullPath = join(OUTPUT_DIR, file);
+        const matches = searchInFile(fullPath, pattern, is_regex);
+        if (matches.length > 0) {
+          results.push({ file, matches });
+        }
+      }
+
+      return JSON.stringify({ success: true, results, total_files: files.length, matched_files: results.length });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'artifact_grep',
+    description: '在产出文件中搜索内容',
+    schema: z.object({
+      pattern: z.string().describe('搜索模式（字符串或正则表达式）'),
+      file_pattern: z.string().optional().default('**/*').describe('文件匹配模式'),
+      is_regex: z.boolean().optional().default(false).describe('是否使用正则表达式'),
+    }),
+  }
+);
+
+export const fileTools = [artifactSave, artifactLoad, artifactEdit, artifactGlob, artifactGrep];

package/src/agent/tools/hook.js

@@ -0,0 +1,84 @@
+/**
+ * DeepSpider - Hook 工具
+ * 提供网络请求和 Cookie 监控能力
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { NetworkHook } from '../../env/NetworkHook.js';
+import { CookieHook } from '../../env/CookieHook.js';
+
+const networkHook = new NetworkHook();
+const cookieHook = new CookieHook();
+
+/**
+ * 生成 XHR Hook 代码
+ */
+export const generateXHRHook = tool(
+  async ({ captureBody, captureResponse }) => {
+    const code = networkHook.generateXHRHookCode({ captureBody, captureResponse });
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('xhr') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_xhr_hook',
+    description: '生成 XMLHttpRequest Hook 代码，拦截所有 XHR 请求',
+    schema: z.object({
+      captureBody: z.boolean().default(true).describe('是否捕获请求体'),
+      captureResponse: z.boolean().default(true).describe('是否捕获响应'),
+    }),
+  }
+);
+
+/**
+ * 生成 Fetch Hook 代码
+ */
+export const generateFetchHook = tool(
+  async ({ captureBody, captureResponse }) => {
+    const code = networkHook.generateFetchHookCode({ captureBody, captureResponse });
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('fetch') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_fetch_hook',
+    description: '生成 Fetch Hook 代码，拦截所有 fetch 请求',
+    schema: z.object({
+      captureBody: z.boolean().default(true).describe('是否捕获请求体'),
+      captureResponse: z.boolean().default(true).describe('是否捕获响应'),
+    }),
+  }
+);
+
+/**
+ * 生成 Cookie Hook 代码
+ */
+export const generateCookieHook = tool(
+  async ({ trackRead, trackWrite }) => {
+    const code = cookieHook.generateCookieHookCode({ trackRead, trackWrite });
+    return JSON.stringify({
+      success: true,
+      code,
+      usage: "注入后通过 __deepspider__.getLogs('cookie') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_cookie_hook',
+    description: '生成 Cookie Hook 代码，监控 document.cookie 读写',
+    schema: z.object({
+      trackRead: z.boolean().default(true).describe('是否监控读取'),
+      trackWrite: z.boolean().default(true).describe('是否监控写入'),
+    }),
+  }
+);
+
+export const hookTools = [
+  generateXHRHook,
+  generateFetchHook,
+  generateCookieHook,
+];

package/src/agent/tools/hookManager.js

@@ -0,0 +1,178 @@
+/**
+ * DeepSpider - Hook 管理工具
+ * 供 Agent 在运行时动态控制 Hook
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { getBrowser } from '../../browser/index.js';
+
+/**
+ * 通过 CDP 执行 JS
+ */
+async function evaluateViaCDP(browser, expression) {
+  const cdp = await browser.getCDPSession();
+  if (!cdp) return null;
+  const result = await cdp.send('Runtime.evaluate', {
+    expression,
+    returnByValue: true,
+  });
+  return result.result?.value;
+}
+
+/**
+ * 列出所有可用的 Hook
+ */
+export const listHooks = tool(
+  async () => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const result = await evaluateViaCDP(
+        browser,
+        `JSON.stringify(window.__deepspider__?.listHooks?.() || [])`
+      );
+      return JSON.stringify({
+        success: true,
+        hooks: JSON.parse(result || '[]')
+      });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'list_hooks',
+    description: '列出所有可用的 Hook 及其状态',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 启用指定的 Hook
+ */
+export const enableHook = tool(
+  async ({ name }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const result = await evaluateViaCDP(
+        browser,
+        `window.__deepspider__?.enableHook?.('${name}')`
+      );
+      return JSON.stringify({ success: result === true, name });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'enable_hook',
+    description: '启用指定的 Hook',
+    schema: z.object({
+      name: z.string().describe('Hook 名称'),
+    }),
+  }
+);
+
+/**
+ * 禁用指定的 Hook
+ */
+export const disableHook = tool(
+  async ({ name }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const result = await evaluateViaCDP(
+        browser,
+        `window.__deepspider__?.disableHook?.('${name}')`
+      );
+      return JSON.stringify({ success: result === true, name });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'disable_hook',
+    description: '禁用指定的 Hook',
+    schema: z.object({
+      name: z.string().describe('要禁用的 Hook 名称'),
+    }),
+  }
+);
+
+/**
+ * 注入自定义 Hook 代码
+ */
+export const injectHook = tool(
+  async ({ code }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const escapedCode = code
+        .replace(/\\/g, '\\\\')
+        .replace(/'/g, "\\'")
+        .replace(/\n/g, '\\n');
+
+      const result = await evaluateViaCDP(
+        browser,
+        `JSON.stringify(window.__deepspider__?.injectHook?.('${escapedCode}'))`
+      );
+      return result || JSON.stringify({ success: false, error: '注入失败' });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'inject_hook',
+    description: '注入自定义 Hook 代码到浏览器',
+    schema: z.object({
+      code: z.string().describe('要注入的 JS 代码'),
+    }),
+  }
+);
+
+/**
+ * 设置 Hook 配置
+ */
+export const setHookConfig = tool(
+  async ({ key, value }) => {
+    try {
+      const browser = await getBrowser();
+      if (!browser.getPage()) {
+        return JSON.stringify({ success: false, error: '浏览器未就绪' });
+      }
+      const val = typeof value === 'string' ? `'${value}'` : value;
+      const result = await evaluateViaCDP(
+        browser,
+        `window.__deepspider__?.setConfig?.('${key}', ${val})`
+      );
+      return JSON.stringify({ success: result === true, key, value });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'set_hook_config',
+    description: '设置 Hook 配置（silent, captureStack, logToConsole 等）',
+    schema: z.object({
+      key: z.string().describe('配置项名称'),
+      value: z.union([z.boolean(), z.number(), z.string()]).describe('配置值'),
+    }),
+  }
+);
+
+// 导出所有工具
+export const hookManagerTools = [
+  listHooks,
+  enableHook,
+  disableHook,
+  injectHook,
+  setHookConfig,
+];