deepspider 0.1.0

package/src/agent/tools/debug.js

@@ -0,0 +1,246 @@
+/**
+ * DeepSpider - CDP 调试工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { getBrowser } from '../../browser/index.js';
+import { CDPSession } from '../../browser/cdp.js';
+
+let cdpSession = null;
+let isPaused = false;
+let currentCallFrames = [];
+
+/**
+ * 获取 CDP 会话
+ */
+async function getSession() {
+  if (!cdpSession) {
+    const browser = await getBrowser();
+    cdpSession = await CDPSession.fromBrowser(browser);
+
+    // 监听暂停事件
+    cdpSession.on('Debugger.paused', (params) => {
+      isPaused = true;
+      currentCallFrames = params.callFrames || [];
+      console.log('[debug] Debugger paused, callFrames:', currentCallFrames.length);
+    });
+
+    // 监听恢复事件
+    cdpSession.on('Debugger.resumed', () => {
+      isPaused = false;
+      currentCallFrames = [];
+      console.log('[debug] Debugger resumed');
+    });
+  }
+  return cdpSession;
+}
+
+/**
+ * 检查是否处于暂停状态
+ */
+function checkPaused() {
+  if (!isPaused || currentCallFrames.length === 0) {
+    return { success: false, error: '调试器未处于暂停状态，请先设置断点并触发断点' };
+  }
+  return null;
+}
+
+/**
+ * 设置断点
+ */
+export const setBreakpoint = tool(
+  async ({ url, line, column }) => {
+    const session = await getSession();
+    const result = await session.setBreakpoint(url, line, column);
+    return JSON.stringify({ success: true, breakpointId: result.breakpointId });
+  },
+  {
+    name: 'set_breakpoint',
+    description: '在指定位置设置断点',
+    schema: z.object({
+      url: z.string().describe('脚本 URL'),
+      line: z.number().describe('行号'),
+      column: z.number().default(0).describe('列号'),
+    }),
+  }
+);
+
+/**
+ * 设置 XHR 断点
+ */
+export const setXHRBreakpoint = tool(
+  async ({ urlPattern }) => {
+    const session = await getSession();
+    await session.setXHRBreakpoint(urlPattern);
+    return JSON.stringify({ success: true });
+  },
+  {
+    name: 'set_xhr_breakpoint',
+    description: '设置 XHR 请求断点',
+    schema: z.object({
+      urlPattern: z.string().default('').describe('URL 匹配模式'),
+    }),
+  }
+);
+
+/**
+ * 获取当前调用栈
+ */
+export const getCallStack = tool(
+  async () => {
+    await getSession(); // 确保事件监听已注册
+
+    const pauseError = checkPaused();
+    if (pauseError) {
+      return JSON.stringify(pauseError);
+    }
+
+    const stack = currentCallFrames.map((frame, i) => ({
+      index: i,
+      callFrameId: frame.callFrameId,
+      functionName: frame.functionName || '(anonymous)',
+      url: frame.url,
+      line: frame.location.lineNumber,
+      column: frame.location.columnNumber,
+    }));
+
+    return JSON.stringify({ success: true, stack }, null, 2);
+  },
+  {
+    name: 'get_call_stack',
+    description: '获取当前断点处的调用栈（需要先触发断点）',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 获取栈帧变量
+ */
+export const getFrameVariables = tool(
+  async ({ frameIndex }) => {
+    const session = await getSession();
+
+    const pauseError = checkPaused();
+    if (pauseError) {
+      return JSON.stringify(pauseError);
+    }
+
+    if (frameIndex >= currentCallFrames.length) {
+      return JSON.stringify({ success: false, error: `栈帧索引 ${frameIndex} 超出范围，当前共 ${currentCallFrames.length} 个栈帧` });
+    }
+
+    const callFrameId = currentCallFrames[frameIndex].callFrameId;
+
+    try {
+      const { result } = await session.send('Debugger.evaluateOnCallFrame', {
+        callFrameId,
+        expression: '(function() { var vars = {}; for (var k in this) vars[k] = typeof this[k]; return JSON.stringify(vars); })()',
+        returnByValue: true,
+      });
+
+      return JSON.stringify({ success: true, frameIndex, variables: JSON.parse(result.value || '{}') }, null, 2);
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'get_frame_variables',
+    description: '获取指定栈帧的变量列表（需要先触发断点）',
+    schema: z.object({
+      frameIndex: z.number().default(0).describe('栈帧索引'),
+    }),
+  }
+);
+
+/**
+ * 在断点处执行表达式
+ */
+export const evaluateAtBreakpoint = tool(
+  async ({ expression, frameIndex }) => {
+    const session = await getSession();
+
+    const pauseError = checkPaused();
+    if (pauseError) {
+      return JSON.stringify(pauseError);
+    }
+
+    if (frameIndex >= currentCallFrames.length) {
+      return JSON.stringify({ success: false, error: `栈帧索引 ${frameIndex} 超出范围` });
+    }
+
+    const callFrameId = currentCallFrames[frameIndex].callFrameId;
+
+    try {
+      const { result, exceptionDetails } = await session.send('Debugger.evaluateOnCallFrame', {
+        callFrameId,
+        expression,
+        returnByValue: true,
+      });
+
+      if (exceptionDetails) {
+        return JSON.stringify({ success: false, error: exceptionDetails.text || '执行出错' });
+      }
+
+      return JSON.stringify({ success: true, result: result.value }, null, 2);
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'evaluate_at_breakpoint',
+    description: '在断点处执行表达式，获取变量值（需要先触发断点）',
+    schema: z.object({
+      expression: z.string().describe('要执行的表达式'),
+      frameIndex: z.number().default(0).describe('栈帧索引'),
+    }),
+  }
+);
+
+/**
+ * 继续执行
+ */
+export const resumeExecution = tool(
+  async () => {
+    const session = await getSession();
+    try {
+      await session.send('Debugger.resume');
+      isPaused = false;
+      currentCallFrames = [];
+      return JSON.stringify({ success: true });
+    } catch (e) {
+      return JSON.stringify({ success: false, error: e.message });
+    }
+  },
+  {
+    name: 'resume_execution',
+    description: '继续执行（从断点恢复）',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 单步执行
+ */
+export const stepOver = tool(
+  async () => {
+    const session = await getSession();
+    await session.send('Debugger.stepOver');
+    return JSON.stringify({ success: true });
+  },
+  {
+    name: 'step_over',
+    description: '单步执行（跳过函数调用）',
+    schema: z.object({}),
+  }
+);
+
+export const debugTools = [
+  setBreakpoint,
+  setXHRBreakpoint,
+  getCallStack,
+  getFrameVariables,
+  evaluateAtBreakpoint,
+  resumeExecution,
+  stepOver,
+];

package/src/agent/tools/deobfuscator.js

@@ -0,0 +1,90 @@
+/**
+ * DeepSpider - 反混淆工具
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { Deobfuscator } from '../../analyzer/Deobfuscator.js';
+
+/**
+ * 反混淆工具
+ */
+export const deobfuscate = tool(
+  async ({ code, type }) => {
+    const deob = new Deobfuscator();
+    return JSON.stringify(deob.deobfuscate(code, type), null, 2);
+  },
+  {
+    name: 'deobfuscate',
+    description: '反混淆代码，还原可读性。支持 eval、string-array、hex-string 等类型。',
+    schema: z.object({
+      code: z.string().describe('混淆代码'),
+      type: z.enum(['auto', 'eval', 'string-array', 'hex-string', 'unicode'])
+        .optional().default('auto'),
+    }),
+  }
+);
+
+/**
+ * 反混淆流水线
+ */
+export const deobfuscatePipeline = tool(
+  async ({ code, steps }) => {
+    const deob = new Deobfuscator();
+    return JSON.stringify(deob.runPipeline(code, steps), null, 2);
+  },
+  {
+    name: 'deobfuscate_pipeline',
+    description: '执行反混淆流水线，按步骤还原代码。',
+    schema: z.object({
+      code: z.string().describe('混淆代码'),
+      steps: z.array(z.string()).optional().describe(
+        '执行步骤: unicode, hex-string, base64, string-array, control-flow, deadcode, simplify, rename'
+      ),
+    }),
+  }
+);
+
+/**
+ * 混淆器识别
+ */
+export const detectObfuscator = tool(
+  async ({ code }) => {
+    const deob = new Deobfuscator();
+    return JSON.stringify({
+      obfuscator: deob.detectObfuscator(code),
+      type: deob._detectType(code),
+    });
+  },
+  {
+    name: 'detect_obfuscator',
+    description: '识别代码使用的混淆器类型（obfuscator.io、sojson、jshaman等）。',
+    schema: z.object({
+      code: z.string().describe('JS代码'),
+    }),
+  }
+);
+
+/**
+ * 字符串解码
+ */
+export const decodeStrings = tool(
+  async ({ code }) => {
+    const deob = new Deobfuscator();
+    return JSON.stringify(deob.decodeStrings(code), null, 2);
+  },
+  {
+    name: 'decode_strings',
+    description: '解码代码中的加密字符串（hex、unicode、base64）。',
+    schema: z.object({
+      code: z.string().describe('JS代码'),
+    }),
+  }
+);
+
+export const deobfuscatorTools = [
+  deobfuscate,
+  deobfuscatePipeline,
+  detectObfuscator,
+  decodeStrings,
+];

package/src/agent/tools/env.js

@@ -0,0 +1,83 @@
+/**
+ * DeepSpider - 环境加载工具
+ * 提供预置浏览器环境模块的加载能力
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { modules, loadOrder } from '../../env/modules/index.js';
+
+/**
+ * 列出可用环境模块
+ */
+export const listEnvModules = tool(
+  async () => {
+    return JSON.stringify({
+      modules: Object.keys(modules),
+      loadOrder,
+      description: {
+        navigator: '浏览器信息（userAgent, platform, plugins 等）',
+        location: 'URL 信息（href, hostname, protocol 等）',
+        document: 'DOM 操作（cookie, createElement 等）',
+        screen: '屏幕信息（width, height, colorDepth）',
+        storage: 'localStorage/sessionStorage',
+        history: '浏览历史',
+        fetch: 'Fetch API',
+        xhr: 'XMLHttpRequest',
+        url: 'URL/URLSearchParams',
+        event: '事件系统',
+      },
+    }, null, 2);
+  },
+  {
+    name: 'list_env_modules',
+    description: '列出所有可用的预置浏览器环境模块。',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 加载指定环境模块
+ */
+export const loadEnvModule = tool(
+  async ({ name }) => {
+    const code = modules[name];
+    if (!code) {
+      return JSON.stringify({
+        success: false,
+        error: `模块 ${name} 不存在`,
+        available: Object.keys(modules),
+      });
+    }
+    return JSON.stringify({ success: true, name, code });
+  },
+  {
+    name: 'load_env_module',
+    description: '加载指定的预置浏览器环境模块代码。',
+    schema: z.object({
+      name: z.enum(['navigator', 'location', 'document', 'screen', 'storage', 'history', 'fetch', 'xhr', 'url', 'event'])
+        .describe('模块名称'),
+    }),
+  }
+);
+
+/**
+ * 加载全部环境模块
+ */
+export const loadAllEnvModules = tool(
+  async () => {
+    const code = loadOrder.map(n => modules[n]).filter(Boolean).join('\n\n');
+    return JSON.stringify({
+      success: true,
+      modules: loadOrder,
+      code,
+    });
+  },
+  {
+    name: 'load_all_env_modules',
+    description: '加载全部预置浏览器环境模块，按正确顺序合并。',
+    schema: z.object({}),
+  }
+);
+
+export const envTools = [listEnvModules, loadEnvModule, loadAllEnvModules];

package/src/agent/tools/envdump.js

@@ -0,0 +1,92 @@
+/**
+ * DeepSpider - 环境自吐工具
+ * 提供环境监控和自吐能力
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { EnvDumper } from '../../env/EnvDumper.js';
+
+const dumper = new EnvDumper();
+
+/**
+ * 生成环境自吐代码
+ */
+export const generateEnvDumpCode = tool(
+  async ({ targets, enableCallStack, maxValueLength }) => {
+    const code = dumper.generateDumpCode({
+      targets,
+      enableCallStack,
+      maxValueLength,
+    });
+
+    return JSON.stringify({
+      success: true,
+      code,
+      description: '环境自吐代码已生成，请在目标代码执行前注入此代码',
+      usage: "1. 先注入基础环境 2. 注入此自吐代码 3. 执行目标代码 4. 调用 __deepspider__.getLogs('env') 获取日志",
+    }, null, 2);
+  },
+  {
+    name: 'generate_env_dump_code',
+    description: '生成环境自吐注入代码。此代码会代理全局对象，记录所有环境访问。',
+    schema: z.object({
+      targets: z.array(z.string())
+        .default(['window', 'document', 'navigator', 'location'])
+        .describe('要监控的全局对象列表'),
+      enableCallStack: z.boolean()
+        .default(false)
+        .describe('是否记录调用栈'),
+      maxValueLength: z.number()
+        .default(70)
+        .describe('日志中值的最大长度'),
+    }),
+  }
+);
+
+/**
+ * 生成基础环境桩代码
+ */
+export const generateBaseEnvCode = tool(
+  async () => {
+    const code = dumper.generateBaseEnv();
+
+    return JSON.stringify({
+      success: true,
+      code,
+      description: '基础环境桩代码，提供 window/document/navigator 等基础对象',
+    }, null, 2);
+  },
+  {
+    name: 'generate_base_env_code',
+    description: '生成基础环境桩代码，在沙箱中创建 window、document、navigator 等基础全局对象。',
+    schema: z.object({}),
+  }
+);
+
+/**
+ * 解析环境日志
+ */
+export const parseEnvLogs = tool(
+  async ({ logsJson }) => {
+    const result = dumper.parseEnvLogs(logsJson);
+
+    return JSON.stringify({
+      success: !result.error,
+      ...result,
+    }, null, 2);
+  },
+  {
+    name: 'parse_env_logs',
+    description: '解析环境自吐日志，提取缺失的环境属性和调用记录。',
+    schema: z.object({
+      logsJson: z.string().describe("__deepspider__.getLogs('env') 返回的 JSON 字符串"),
+    }),
+  }
+);
+
+export const envDumpTools = [
+  generateEnvDumpCode,
+  generateBaseEnvCode,
+  parseEnvLogs,
+];

package/src/agent/tools/evolve.js

@@ -0,0 +1,164 @@
+/**
+ * DeepSpider - evolve_skill 工具
+ * 让 Agent 可以追加经验到 evolved.md
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import fs from 'fs';
+import path from 'path';
+import { parseEvolvedMd, generateEvolvedMd, MERGE_THRESHOLD } from '../skills/evolve.js';
+import { SKILLS } from '../skills/config.js';
+
+// Skills 基础目录
+const SKILLS_BASE = new URL('../skills/', import.meta.url).pathname;
+
+/**
+ * 获取 skill 路径，支持动态创建
+ */
+function getSkillPath(skillName) {
+  const skillMap = {
+    'static-analysis': SKILLS.static,
+    'dynamic-analysis': SKILLS.dynamic,
+    'sandbox': SKILLS.sandbox,
+    'env': SKILLS.env,
+    'js2python': SKILLS.js2python,
+    'report': SKILLS.report,
+    'general': SKILLS.general,
+  };
+
+  // 检查是否是动态创建
+  if (skillName.startsWith('new:')) {
+    const newName = skillName.slice(4);
+    return { path: path.join(SKILLS_BASE, newName), isNew: true, name: newName };
+  }
+
+  const skillPath = skillMap[skillName];
+  return skillPath ? { path: skillPath, isNew: false, name: skillName } : null;
+}
+
+/**
+ * 创建新 skill 目录和模板文件
+ */
+function createNewSkill(skillPath, skillName) {
+  // 创建目录
+  fs.mkdirSync(skillPath, { recursive: true });
+
+  // 创建 SKILL.md
+  const skillMd = `---
+name: ${skillName}
+description: |
+  ${skillName} 相关经验。
+---
+
+# ${skillName}
+
+自动创建的 skill 目录。
+`;
+  fs.writeFileSync(path.join(skillPath, 'SKILL.md'), skillMd, 'utf-8');
+
+  // 创建 evolved.md
+  const evolvedMd = `---
+total: 0
+last_merged: null
+---
+
+## 核心经验
+
+<!-- 经过验证的高价值经验 -->
+
+## 近期发现
+
+<!-- 最近发现，FIFO 滚动，最多保留 10 条 -->
+`;
+  fs.writeFileSync(path.join(skillPath, 'evolved.md'), evolvedMd, 'utf-8');
+}
+
+/**
+ * evolve_skill 工具
+ */
+export const evolveSkill = tool(
+  async ({ skill, title, scenario, insight, isCore }) => {
+    const skillInfo = getSkillPath(skill);
+    if (!skillInfo) {
+      return JSON.stringify({
+        success: false,
+        error: `未知的 skill: ${skill}。可用: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general。或使用 new:<name> 创建新 skill。`
+      });
+    }
+
+    const { path: skillPath, isNew, name: skillName } = skillInfo;
+
+    // 如果是新 skill，先创建目录
+    if (isNew) {
+      createNewSkill(skillPath, skillName);
+    }
+
+    const evolvedPath = path.join(skillPath, 'evolved.md');
+
+    // 读取现有内容
+    let content = '';
+    try {
+      content = fs.readFileSync(evolvedPath, 'utf-8');
+    } catch (e) {
+      // 文件不存在，使用空内容
+    }
+
+    const data = parseEvolvedMd(content);
+
+    // 生成新条目
+    const date = new Date().toISOString().split('T')[0];
+    const entry = `### [${date}] ${title}
+**场景**: ${scenario}
+**经验**: ${insight}`;
+
+    if (isCore) {
+      // 追加到核心经验
+      data.core = data.core
+        ? `${data.core}\n\n${entry}`
+        : entry;
+    } else {
+      // 追加到近期发现
+      data.recent.push(entry);
+      // 保持最多 10 条
+      if (data.recent.length > 10) {
+        data.recent = data.recent.slice(-10);
+      }
+    }
+
+    data.total += 1;
+
+    // 写回文件
+    const newContent = generateEvolvedMd(data);
+    fs.writeFileSync(evolvedPath, newContent, 'utf-8');
+
+    // 检查阈值
+    const needsMerge = data.total >= MERGE_THRESHOLD;
+
+    return JSON.stringify({
+      success: true,
+      skill: skillName,
+      total: data.total,
+      needsMerge,
+      isNew,
+      message: isNew
+        ? `已创建新 skill "${skillName}" 并记录经验。`
+        : needsMerge
+          ? `经验已记录。动态经验已达 ${data.total} 条，建议执行 /evolve:merge ${skillName}。`
+          : `经验已记录到 ${skillName}。`
+    });
+  },
+  {
+    name: 'evolve_skill',
+    description: '记录分析过程中学到的经验。支持现有 skill 或 new:<name> 创建新 skill',
+    schema: z.object({
+      skill: z.string().describe('目标 skill: static-analysis, dynamic-analysis, sandbox, env, js2python, report, general，或 new:<name> 创建新 skill'),
+      title: z.string().describe('经验标题，简短描述'),
+      scenario: z.string().describe('具体场景/案例'),
+      insight: z.string().describe('一句话总结经验'),
+      isCore: z.boolean().default(false).describe('是否为核心经验'),
+    }),
+  }
+);
+
+export const evolveTools = [evolveSkill];