deepspider 0.2.6 → 0.2.9

package/src/agent/tools/report.js

@@ -6,9 +6,9 @@
 
 import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
-import { writeFileSync } from 'fs';
-import { join } from 'path';
-import { PATHS, ensureDir, getReportDir } from '../../config/paths.js';
+import { writeFileSync, readFileSync, existsSync } from 'fs';
+import { join, basename } from 'path';
+import { PATHS, ensureDir, DEEPSPIDER_HOME } from '../../config/paths.js';
 
 const OUTPUT_DIR = PATHS.REPORTS_DIR;
 
@@ -25,6 +25,31 @@ function escapeHtml(str) {
   return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
 }
 
+/**
+ * 从文件路径读取代码内容
+ * 支持相对路径和绝对路径
+ */
+function readCodeFromFile(filePath) {
+  if (!filePath) return null;
+
+  let fullPath = filePath;
+  if (!fullPath.startsWith('/')) {
+    fullPath = join(PATHS.OUTPUT_DIR, filePath);
+  }
+
+  if (!existsSync(fullPath)) {
+    console.warn('[report] 代码文件不存在:', fullPath);
+    return null;
+  }
+
+  try {
+    return readFileSync(fullPath, 'utf-8');
+  } catch (e) {
+    console.warn('[report] 读取代码文件失败:', e.message);
+    return null;
+  }
+}
+
 /**
  * 生成 HTML 报告页面
  */
@@ -72,34 +97,57 @@ function generateHtmlPage(title, markdown, pythonCode, jsCode) {
 
 /**
  * 保存分析报告
+ * 支持两种模式：
+ * 1. 传入代码文件路径（推荐）- pythonCodeFile/jsCodeFile
+ * 2. 传入代码内容（兼容）- pythonCode/jsCode
  */
 export const saveAnalysisReport = tool(
-  async ({ domain, title, markdown, pythonCode, jsCode }) => {
+  async ({ domain, title, markdown, pythonCode, pythonCodeFile, jsCode, jsCodeFile }) => {
     try {
       const domainDir = join(OUTPUT_DIR, extractDomain(domain));
       ensureDir(domainDir);
 
       const paths = {};
 
+      // 优先从文件读取代码
+      let finalPythonCode = pythonCode;
+      let finalJsCode = jsCode;
+
+      if (pythonCodeFile) {
+        const code = readCodeFromFile(pythonCodeFile);
+        if (code) {
+          finalPythonCode = code;
+          console.log('[report] 从文件读取 Python 代码:', pythonCodeFile);
+        }
+      }
+
+      if (jsCodeFile) {
+        const code = readCodeFromFile(jsCodeFile);
+        if (code) {
+          finalJsCode = code;
+          console.log('[report] 从文件读取 JS 代码:', jsCodeFile);
+        }
+      }
+
       // 保存 Markdown
       paths.markdown = join(domainDir, 'analysis.md');
       writeFileSync(paths.markdown, markdown, 'utf-8');
 
       // 保存 Python 代码
-      if (pythonCode) {
+      if (finalPythonCode) {
         paths.python = join(domainDir, 'decrypt.py');
-        writeFileSync(paths.python, pythonCode, 'utf-8');
+        writeFileSync(paths.python, finalPythonCode, 'utf-8');
       }
 
       // 保存 JS 代码
-      if (jsCode) {
+      if (finalJsCode) {
         paths.javascript = join(domainDir, 'decrypt.js');
-        writeFileSync(paths.javascript, jsCode, 'utf-8');
+        writeFileSync(paths.javascript, finalJsCode, 'utf-8');
       }
 
       // 生成 HTML
       paths.html = join(domainDir, 'report.html');
-      const html = generateHtmlPage(title || domain, markdown, pythonCode, jsCode);
+      const html = generateHtmlPage(title || domain, markdown, finalPythonCode, finalJsCode);
       writeFileSync(paths.html, html, 'utf-8');
 
       console.log('[report] 已保存:', domainDir);
@@ -110,13 +158,15 @@ export const saveAnalysisReport = tool(
   },
   {
     name: 'save_analysis_report',
-    description: '保存加密分析报告。分析完成后必须调用，保存 Markdown、HTML 和代码文件。',
+    description: `保存分析报告。推荐先用 artifact_save 保存代码文件，再传入文件路径。`,
     schema: z.object({
-      domain: z.string().describe('网站域名或 URL'),
+      domain: z.string().describe('网站域名'),
       title: z.string().optional().describe('报告标题'),
-      markdown: z.string().describe('Markdown 分析报告'),
-      pythonCode: z.string().describe('Python 解密代码（必须提供完整可运行代码）'),
-      jsCode: z.string().optional().describe('JavaScript 解密代码'),
+      markdown: z.string().describe('Markdown 摘要'),
+      pythonCodeFile: z.string().optional().describe('Python 代码文件路径（推荐）'),
+      pythonCode: z.string().optional().describe('Python 代码内容（不推荐）'),
+      jsCodeFile: z.string().optional().describe('JS 代码文件路径'),
+      jsCode: z.string().optional().describe('JS 代码内容'),
     }),
   }
 );

package/src/agent/tools/runtime.js

@@ -12,10 +12,12 @@ let hookManager = null;
 
 /**
  * 标记 Hook 已注入（供外部调用）
+ * 注意：Hook 脚本由 browser/client.js 自动注入，此处仅标记状态
  */
 export function markHookInjected() {
   if (!hookManager) {
     hookManager = new HookManager();
+    hookManager.markInjected();
   }
 }
 
@@ -25,13 +27,13 @@ export function markHookInjected() {
 export const launchBrowser = tool(
   async ({ headless }) => {
     const browser = await getBrowser({ headless });
-    // 检查是否已经注入过 Hook
+    // Hook 已由 browser/client.js 自动注入，此处仅标记状态
     if (!hookManager) {
       hookManager = new HookManager();
-      await hookManager.inject(browser.getPage());
-      return JSON.stringify({ success: true, message: '浏览器已启动，Hook 已注入' });
+      hookManager.markInjected();
+      hookManager.bindConsole(browser.getPage());
     }
-    return JSON.stringify({ success: true, message: '浏览器已就绪' });
+    return JSON.stringify({ success: true, message: '浏览器已就绪，Hook 已注入' });
   },
   {
     name: 'launch_browser',

package/src/agent/tools/utils.js

@@ -7,7 +7,6 @@ import { join } from 'path';
 import { PATHS, ensureDir, generateFilename } from '../../config/paths.js';
 
 // 导出路径常量（兼容旧代码）
-export const OUTPUT_DIR = PATHS.OUTPUT_DIR;
 export const SCREENSHOTS_DIR = PATHS.SCREENSHOTS_DIR;
 export const REPORTS_DIR = PATHS.REPORTS_DIR;
 export const UNPACKED_DIR = PATHS.UNPACKED_DIR;

package/src/browser/defaultHooks.js

@@ -185,11 +185,14 @@ function getJSONHooks() {
 
   const OriginalParse = JSON.parse;
   const OriginalStringify = JSON.stringify;
-  const MIN_LOG_LENGTH = 50; // 只记录长度超过 50 的数据
+  const MIN_LOG_LENGTH = 50;
+
+  // 内部数据标记 - 跳过 DeepSpider 内部操作
+  const INTERNAL_MARKER = '"__ds__":true';
 
   JSON.parse = deepspider.native(function(text, reviver) {
     const textStr = String(text);
-    if (textStr.length >= MIN_LOG_LENGTH) {
+    if (textStr.length >= MIN_LOG_LENGTH && !textStr.includes(INTERNAL_MARKER)) {
       deepspider.log('json', {
         action: 'parse',
         input: textStr.slice(0, 200),
@@ -201,7 +204,7 @@ function getJSONHooks() {
 
   JSON.stringify = deepspider.native(function(value, replacer, space) {
     const result = OriginalStringify.call(JSON, value, replacer, space);
-    if (result && result.length >= MIN_LOG_LENGTH) {
+    if (result && result.length >= MIN_LOG_LENGTH && !result.includes(INTERNAL_MARKER)) {
       deepspider.log('json', {
         action: 'stringify',
         output: result.slice(0, 200),
@@ -297,18 +300,27 @@ function getStorageHooks() {
   const deepspider = window.__deepspider__;
   if (!deepspider) return;
 
+  // 内部使用的 key 前缀，不记录日志
+  const INTERNAL_PREFIX = 'deepspider_';
+
   function hookStorage(storage, name) {
     const origGet = storage.getItem.bind(storage);
     const origSet = storage.setItem.bind(storage);
 
     storage.getItem = deepspider.native(function(key) {
       const value = origGet(key);
-      deepspider.log('storage', { action: 'get', storage: name, key: key, value: value?.slice(0, 100) });
+      // 跳过内部 key
+      if (!key.startsWith(INTERNAL_PREFIX)) {
+        deepspider.log('storage', { action: 'get', storage: name, key: key, value: value?.slice(0, 100) });
+      }
       return value;
     }, origGet);
 
     storage.setItem = deepspider.native(function(key, value) {
-      deepspider.log('storage', { action: 'set', storage: name, key: key, value: String(value).slice(0, 100) });
+      // 跳过内部 key
+      if (!key.startsWith(INTERNAL_PREFIX)) {
+        deepspider.log('storage', { action: 'set', storage: name, key: key, value: String(value).slice(0, 100) });
+      }
       return origSet(key, value);
     }, origSet);
   }
@@ -380,41 +392,327 @@ function getWebSocketHooks() {
 
 /**
  * Webpack 模块 Hook - 检测闭包内的加密库
+ * 通过特征检测而非变量名来识别加密库
  */
 function getWebpackHooks() {
   return `
-// === Webpack Hook ===
+// === Webpack Module Hook ===
 (function() {
   const deepspider = window.__deepspider__;
   if (!deepspider) return;
 
-  // 加密特征检测
-  const cryptoPatterns = [
-    { name: 'CryptoJS', pattern: /CryptoJS|crypto-js/i },
-    { name: 'MD5', pattern: /\\bmd5\\b/i },
-    { name: 'SHA', pattern: /\\bsha(1|256|512)\\b/i },
-    { name: 'AES', pattern: /\\baes\\b/i },
-    { name: 'RSA', pattern: /\\brsa\\b/i },
-    { name: 'Base64', pattern: /base64|btoa|atob/i }
-  ];
+  // 已 Hook 的对象集合
+  const hookedObjects = new WeakSet();
+
+  // === 特征检测函数 ===
+
+  // 检测 CryptoJS 特征
+  function isCryptoJS(obj) {
+    if (!obj || typeof obj !== 'object') return false;
+    // CryptoJS 特征：有 AES/DES/MD5 等属性，且有 enc.Utf8
+    return (obj.AES && obj.AES.encrypt && obj.AES.decrypt) ||
+           (obj.enc && obj.enc.Utf8 && obj.enc.Hex) ||
+           (obj.MD5 && typeof obj.MD5 === 'function') ||
+           (obj.SHA256 && typeof obj.SHA256 === 'function');
+  }
+
+  // 检测 JSEncrypt 特征
+  function isJSEncrypt(obj) {
+    if (!obj || typeof obj !== 'function') return false;
+    const proto = obj.prototype;
+    return proto && proto.encrypt && proto.decrypt && proto.setPublicKey;
+  }
+
+  // 检测 SM2/SM3/SM4 国密特征
+  function isSMCrypto(obj) {
+    if (!obj || typeof obj !== 'object') return false;
+    return (obj.doEncrypt && obj.doDecrypt) ||
+           (obj.sm2 && obj.sm3) ||
+           (typeof obj.encrypt === 'function' && obj.cipherMode !== undefined);
+  }
+
+  // 检测 node-forge 特征
+  function isForge(obj) {
+    if (!obj || typeof obj !== 'object') return false;
+    return (obj.cipher && obj.md && obj.util) ||
+           (obj.pki && obj.pki.rsa);
+  }
+
+  // === Hook 函数 ===
+
+  // Hook CryptoJS 对象
+  function hookCryptoJSObject(obj, source) {
+    if (hookedObjects.has(obj)) return;
+    hookedObjects.add(obj);
+
+    // Hook 对称加密
+    ['AES', 'DES', 'TripleDES', 'RC4', 'Rabbit'].forEach(function(cipher) {
+      if (!obj[cipher]) return;
+      ['encrypt', 'decrypt'].forEach(function(method) {
+        if (!obj[cipher][method]) return;
+        const original = obj[cipher][method];
+        obj[cipher][method] = deepspider.native(function(data, key, options) {
+          const entry = deepspider.log('crypto', {
+            algo: cipher + '.' + method,
+            source: source,
+            data: String(data).slice(0, 100),
+            keyLen: key ? String(key).length : 0
+          });
+          deepspider.linkCrypto(entry);
+          return original.apply(this, arguments);
+        }, original);
+      });
+    });
+
+    // Hook 哈希算法
+    ['MD5', 'SHA1', 'SHA256', 'SHA512', 'SHA3', 'RIPEMD160'].forEach(function(algo) {
+      if (!obj[algo] || typeof obj[algo] !== 'function') return;
+      const original = obj[algo];
+      obj[algo] = deepspider.native(function() {
+        const entry = deepspider.log('crypto', {
+          algo: algo,
+          source: source,
+          inputLen: arguments[0] ? String(arguments[0]).length : 0
+        });
+        deepspider.linkCrypto(entry);
+        return original.apply(this, arguments);
+      }, original);
+    });
+
+    // Hook HMAC
+    ['HmacMD5', 'HmacSHA1', 'HmacSHA256', 'HmacSHA512'].forEach(function(algo) {
+      if (!obj[algo] || typeof obj[algo] !== 'function') return;
+      const original = obj[algo];
+      obj[algo] = deepspider.native(function() {
+        const entry = deepspider.log('crypto', {
+          algo: algo,
+          source: source,
+          inputLen: arguments[0] ? String(arguments[0]).length : 0
+        });
+        deepspider.linkCrypto(entry);
+        return original.apply(this, arguments);
+      }, original);
+    });
+
+    console.log('[DeepSpider] CryptoJS Hook 已启用 (来源: ' + source + ')');
+  }
+
+  // Hook JSEncrypt 构造函数
+  function hookJSEncryptObject(JSEncrypt, source) {
+    if (hookedObjects.has(JSEncrypt)) return;
+    hookedObjects.add(JSEncrypt);
+
+    const proto = JSEncrypt.prototype;
+    if (proto.encrypt) {
+      const origEnc = proto.encrypt;
+      proto.encrypt = deepspider.native(function(data) {
+        const entry = deepspider.log('crypto', {
+          algo: 'RSA.encrypt',
+          source: source,
+          data: String(data).slice(0, 100)
+        });
+        deepspider.linkCrypto(entry);
+        return origEnc.apply(this, arguments);
+      }, origEnc);
+    }
+    if (proto.decrypt) {
+      const origDec = proto.decrypt;
+      proto.decrypt = deepspider.native(function(data) {
+        const entry = deepspider.log('crypto', {
+          algo: 'RSA.decrypt',
+          source: source
+        });
+        deepspider.linkCrypto(entry);
+        return origDec.apply(this, arguments);
+      }, origDec);
+    }
+    console.log('[DeepSpider] JSEncrypt Hook 已启用 (来源: ' + source + ')');
+  }
+
+  // Hook SM 国密对象
+  function hookSMCryptoObject(obj, source) {
+    if (hookedObjects.has(obj)) return;
+    hookedObjects.add(obj);
+
+    if (obj.doEncrypt) {
+      const origEnc = obj.doEncrypt;
+      obj.doEncrypt = deepspider.native(function(msg, pubKey) {
+        const entry = deepspider.log('crypto', {
+          algo: 'SM2.encrypt',
+          source: source,
+          msg: String(msg).slice(0, 100)
+        });
+        deepspider.linkCrypto(entry);
+        return origEnc.apply(this, arguments);
+      }, origEnc);
+    }
+    console.log('[DeepSpider] SM Crypto Hook 已启用 (来源: ' + source + ')');
+  }
+
+  // === 扫描并 Hook 模块导出 ===
+  function scanAndHook(exports, source) {
+    if (!exports || typeof exports !== 'object') return;
+
+    try {
+      // 直接检测导出对象
+      if (isCryptoJS(exports)) {
+        hookCryptoJSObject(exports, source);
+        return;
+      }
+      if (isJSEncrypt(exports)) {
+        hookJSEncryptObject(exports, source);
+        return;
+      }
+      if (isSMCrypto(exports)) {
+        hookSMCryptoObject(exports, source);
+        return;
+      }
+
+      // 检测 default 导出
+      if (exports.default) {
+        if (isCryptoJS(exports.default)) {
+          hookCryptoJSObject(exports.default, source + '.default');
+        } else if (isJSEncrypt(exports.default)) {
+          hookJSEncryptObject(exports.default, source + '.default');
+        }
+      }
 
-  // Hook webpackJsonp
-  function hookWebpack() {
-    if (window.webpackJsonp && !window.webpackJsonp.__hooked__) {
-      const orig = window.webpackJsonp.push;
-      window.webpackJsonp.push = function(chunk) {
-        deepspider.log('env', { action: 'webpack.chunk', id: chunk[0] });
-        return orig.apply(this, arguments);
+      // 遍历导出的属性（限制深度避免性能问题）
+      const keys = Object.keys(exports);
+      for (let i = 0; i < Math.min(keys.length, 20); i++) {
+        const key = keys[i];
+        const val = exports[key];
+        if (val && typeof val === 'object' && !hookedObjects.has(val)) {
+          if (isCryptoJS(val)) {
+            hookCryptoJSObject(val, source + '.' + key);
+          }
+        }
+        if (val && typeof val === 'function') {
+          if (isJSEncrypt(val)) {
+            hookJSEncryptObject(val, source + '.' + key);
+          }
+        }
+      }
+    } catch (e) {
+      // 忽略访问错误
+    }
+  }
+
+  // === Hook Webpack 模块系统 ===
+
+  // Hook Webpack 4 webpackJsonp
+  function hookWebpackJsonp() {
+    const jsonp = window.webpackJsonp;
+    if (!jsonp || jsonp.__deepspider_hooked__) return;
+
+    const origPush = jsonp.push;
+    jsonp.push = function(chunk) {
+      const result = origPush.apply(this, arguments);
+
+      // chunk[1] 是模块对象 { moduleId: function(module, exports, require) {} }
+      if (chunk && chunk[1]) {
+        const modules = chunk[1];
+        Object.keys(modules).forEach(function(moduleId) {
+          // 延迟扫描，等模块执行完
+          setTimeout(function() {
+            try {
+              // 尝试获取模块导出
+              if (window.__webpack_require__ && window.__webpack_require__.c) {
+                const mod = window.__webpack_require__.c[moduleId];
+                if (mod && mod.exports) {
+                  scanAndHook(mod.exports, 'webpack:' + moduleId);
+                }
+              }
+            } catch (e) {}
+          }, 10);
+        });
+      }
+      return result;
+    };
+    jsonp.__deepspider_hooked__ = true;
+    console.log('[DeepSpider] Webpack4 jsonp Hook 已启用');
+  }
+
+  // Hook Webpack 5 webpackChunk
+  function hookWebpackChunk() {
+    // Webpack 5 使用 self["webpackChunk" + name]
+    const chunkNames = Object.keys(self).filter(function(k) {
+      return k.startsWith('webpackChunk');
+    });
+
+    chunkNames.forEach(function(name) {
+      const chunk = self[name];
+      if (!chunk || chunk.__deepspider_hooked__) return;
+
+      const origPush = chunk.push.bind(chunk);
+      chunk.push = function(data) {
+        const result = origPush(data);
+
+        // data[1] 是模块对象
+        if (data && data[1]) {
+          Object.keys(data[1]).forEach(function(moduleId) {
+            setTimeout(function() {
+              try {
+                // Webpack 5 的 require cache
+                const cache = window.__webpack_require__ && window.__webpack_require__.c;
+                if (cache && cache[moduleId] && cache[moduleId].exports) {
+                  scanAndHook(cache[moduleId].exports, 'webpack5:' + moduleId);
+                }
+              } catch (e) {}
+            }, 10);
+          });
+        }
+        return result;
       };
-      window.webpackJsonp.__hooked__ = true;
+      chunk.__deepspider_hooked__ = true;
+      console.log('[DeepSpider] Webpack5 chunk Hook 已启用: ' + name);
+    });
+  }
+
+  // Hook __webpack_require__ 直接拦截模块加载
+  function hookWebpackRequire() {
+    if (!window.__webpack_require__ || window.__webpack_require__.__deepspider_hooked__) return;
+
+    const origRequire = window.__webpack_require__;
+    window.__webpack_require__ = function(moduleId) {
+      const result = origRequire.apply(this, arguments);
+      // 扫描返回的模块
+      setTimeout(function() {
+        scanAndHook(result, 'require:' + moduleId);
+      }, 0);
+      return result;
+    };
+    // 复制原有属性
+    Object.keys(origRequire).forEach(function(key) {
+      window.__webpack_require__[key] = origRequire[key];
+    });
+    window.__webpack_require__.__deepspider_hooked__ = true;
+    console.log('[DeepSpider] __webpack_require__ Hook 已启用');
+  }
+
+  // 定期检查并 Hook
+  function checkAndHook() {
+    hookWebpackJsonp();
+    hookWebpackChunk();
+    hookWebpackRequire();
+
+    // 扫描已加载的模块缓存
+    if (window.__webpack_require__ && window.__webpack_require__.c) {
+      const cache = window.__webpack_require__.c;
+      Object.keys(cache).forEach(function(moduleId) {
+        if (cache[moduleId] && cache[moduleId].exports) {
+          scanAndHook(cache[moduleId].exports, 'cache:' + moduleId);
+        }
+      });
     }
   }
 
-  // 定期检查
-  const check = setInterval(hookWebpack, 100);
-  setTimeout(function() { clearInterval(check); }, 5000);
+  // 启动检查
+  checkAndHook();
+  const interval = setInterval(checkAndHook, 200);
+  setTimeout(function() { clearInterval(interval); }, 10000);
 
-  console.log('[DeepSpider] Webpack Hook 已启用');
+  console.log('[DeepSpider] Webpack Module Hook 已启用');
 })();
 `;
 }

package/src/browser/hooks/index.js

@@ -1,38 +1,34 @@
 /**
  * DeepSpider - Hook 管理器
+ * 注意：Hook 脚本已由 browser/client.js 通过 defaultHooks.js 自动注入
+ * 此类仅用于日志管理和状态跟踪
  */
 
-import { cryptoHook } from './crypto.js';
-import { networkHook } from './network.js';
-import { nativeProtect } from './native.js';
-
 export class HookManager {
   constructor() {
     this.logs = [];
     this.onLog = null;
+    this.injected = false;
   }
 
   /**
-   * 获取完整的 Hook 脚本
+   * 标记 Hook 已注入（由 client.js 调用）
    */
-  getCombinedScript() {
-    return [
-      nativeProtect,
-      cryptoHook,
-      networkHook,
-    ].join('\n\n');
+  markInjected() {
+    this.injected = true;
   }
 
   /**
-   * 注入 Hook 到页面
+   * 检查是否已注入
    */
-  async inject(page) {
-    const script = this.getCombinedScript();
-
-    // 在新文档加载前注入
-    await page.addInitScript(script);
+  isInjected() {
+    return this.injected;
+  }
 
-    // 监听 console 输出
+  /**
+   * 绑定页面 console 监听（用于收集 Hook 日志）
+   */
+  bindConsole(page) {
     page.on('console', (msg) => {
       const text = msg.text();
       if (text.includes('[DeepSpider:')) {