deepspider 0.3.0 → 0.3.2

package/src/agent/tools/verifyAlgorithm.js

@@ -0,0 +1,117 @@
+/**
+ * DeepSpider - 统一算法验证工具
+ * 合并 verify_md5/sha256/hmac/aes + identify_encryption
+ */
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import crypto from 'crypto';
+
+/**
+ * 识别密文特征
+ */
+function identifyPattern(ciphertext) {
+  const features = [];
+  const len = ciphertext.length;
+
+  if (len === 32) features.push('可能是 MD5');
+  if (len === 40) features.push('可能是 SHA1');
+  if (len === 64) features.push('可能是 SHA256');
+  if (len === 128) features.push('可能是 SHA512');
+  if (/^[A-Za-z0-9+/]+=*$/.test(ciphertext)) features.push('Base64 编码');
+  if (/^[0-9a-fA-F]+$/.test(ciphertext)) features.push('Hex 编码');
+  if (/^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/.test(ciphertext)) features.push('JWT Token');
+
+  return features;
+}
+
+export const verifyAlgorithm = tool(
+  async ({ algorithm, input, expected, key, iv, hmacHash, aesMode }) => {
+    // 识别模式：不传 algorithm，只传 expected
+    if (!algorithm) {
+      return JSON.stringify({
+        ciphertext: expected.slice(0, 50) + (expected.length > 50 ? '...' : ''),
+        length: expected.length,
+        features: identifyPattern(expected),
+      }, null, 2);
+    }
+
+    // 验证模式：需要 input + expected
+    if (!input) {
+      return JSON.stringify({ error: `验证 ${algorithm} 需要 input 参数` });
+    }
+
+    let computed;
+    let algoLabel;
+
+    switch (algorithm) {
+      case 'md5':
+      case 'sha1':
+      case 'sha256':
+      case 'sha512': {
+        computed = crypto.createHash(algorithm).update(input).digest('hex');
+        algoLabel = algorithm.toUpperCase();
+        break;
+      }
+      case 'hmac': {
+        if (!key) return JSON.stringify({ error: 'HMAC 需要 key 参数' });
+        const hash = hmacHash || 'sha256';
+        computed = crypto.createHmac(hash, key).update(input).digest('hex');
+        algoLabel = `HMAC-${hash.toUpperCase()}`;
+        break;
+      }
+      case 'aes': {
+        if (!key) return JSON.stringify({ error: 'AES 需要 key 参数' });
+        try {
+          const keyBuf = Buffer.from(key, 'utf8');
+          const ivBuf = iv ? Buffer.from(iv, 'utf8') : Buffer.alloc(16, 0);
+          const mode = aesMode || 'cbc';
+          const cipher = crypto.createCipheriv(
+            `aes-${keyBuf.length * 8}-${mode}`,
+            keyBuf,
+            mode === 'ecb' ? null : ivBuf
+          );
+          computed = cipher.update(input, 'utf8', 'base64') + cipher.final('base64');
+          algoLabel = `AES-${keyBuf.length * 8}-${mode.toUpperCase()}`;
+        } catch (e) {
+          return JSON.stringify({ error: e.message });
+        }
+        break;
+      }
+      default:
+        return JSON.stringify({ error: `未知算法: ${algorithm}` });
+    }
+
+    const match = algorithm === 'aes'
+      ? computed === expected
+      : computed.toLowerCase() === expected.toLowerCase();
+
+    return JSON.stringify({
+      algorithm: algoLabel,
+      input,
+      computed,
+      expected,
+      match,
+      conclusion: match ? `标准 ${algoLabel}` : '可能魔改或参数不同',
+    }, null, 2);
+  },
+  {
+    name: 'verify_algorithm',
+    description: `验证是否为标准加密算法，或根据密文特征识别算法类型。
+
+验证模式：传入 algorithm + input + expected，对比计算结果
+识别模式：只传 expected（密文），自动识别可能的算法类型`,
+    schema: z.object({
+      algorithm: z.enum(['md5', 'sha1', 'sha256', 'sha512', 'hmac', 'aes']).optional()
+        .describe('算法类型。不传则进入识别模式'),
+      input: z.string().optional().describe('原始输入'),
+      expected: z.string().describe('目标加密结果（验证模式）或待识别的密文（识别模式）'),
+      key: z.string().optional().describe('密钥（HMAC/AES 需要）'),
+      iv: z.string().optional().describe('IV 向量（AES 可选）'),
+      hmacHash: z.enum(['md5', 'sha1', 'sha256', 'sha512']).optional().describe('HMAC 哈希算法，默认 sha256'),
+      aesMode: z.enum(['cbc', 'ecb']).optional().describe('AES 模式，默认 cbc'),
+    }),
+  }
+);
+
+export const verifyAlgorithmTools = [verifyAlgorithm];

package/src/analyzer/EncryptionAnalyzer.js

@@ -51,7 +51,7 @@ export class EncryptionAnalyzer {
   }
 
   traceParam(code, paramName) {
-    const ast = this.astAnalyzer.parse(code);
+    const _ast = this.astAnalyzer.parse(code);
     const traces = [];
 
     // 简化实现：查找参数使用位置
@@ -80,7 +80,7 @@ export class EncryptionAnalyzer {
   // 使用模式库进行深度检测
   detectWithPatterns(code) {
     const detected = [];
-    for (const [key, pattern] of Object.entries(cryptoPatterns)) {
+    for (const [_key, pattern] of Object.entries(cryptoPatterns)) {
       for (const sig of pattern.signatures) {
         if (sig.test(code)) {
           detected.push({

package/src/browser/EnvBridge.js

@@ -26,24 +26,38 @@ export class EnvBridge {
 
     for (const path of missingPaths) {
       try {
-        // 1. 从真实浏览器采集
+        // 1. 尝试从真实浏览器采集
         const collected = await this.collector.collect(path, { depth: 2 });
 
-        if (!collected.success) {
-          results.failed.push({ path, reason: 'collect_failed', error: collected.error });
-          continue;
-        }
+        if (collected.success) {
+          results.collected.push(path);
+          this.collectedData.set(path, collected);
 
-        results.collected.push(path);
-        this.collectedData.set(path, collected);
+          // 2. 生成补丁代码
+          const patch = this._generatePatch(path, collected);
+          if (patch) {
+            results.patched.push({ path, code: patch });
+            continue;
+          }
+        }
 
-        // 2. 生成补丁代码
-        const patch = this._generatePatch(path, collected);
-        if (patch) {
-          results.patched.push({ path, code: patch });
+        // 3. 采集失败时 fallback 到 PatchGenerator
+        const fallback = await this.patchGenerator.generate(path);
+        if (fallback.code) {
+          results.patched.push({ path, code: fallback.code, source: fallback.source });
+        } else {
+          results.failed.push({ path, reason: 'no_patch' });
         }
 
       } catch (e) {
+        // 浏览器不可用时也 fallback
+        try {
+          const fallback = await this.patchGenerator.generate(path);
+          if (fallback.code) {
+            results.patched.push({ path, code: fallback.code, source: fallback.source });
+            continue;
+          }
+        } catch { /* ignore */ }
         results.failed.push({ path, reason: 'error', error: e.message });
       }
     }
@@ -66,7 +80,7 @@ export class EnvBridge {
     // 根据数据类型生成不同的补丁
     switch (data.type) {
       case 'string':
-        return `${parentPath}.${propName} = "${data.value}";`;
+        return `${parentPath}.${propName} = ${JSON.stringify(data.value)};`;
 
       case 'number':
         return `${parentPath}.${propName} = ${data.value};`;
@@ -123,7 +137,7 @@ export class EnvBridge {
 
   _serializeValue(data) {
     switch (data.type) {
-      case 'string': return `"${data.value}"`;
+      case 'string': return JSON.stringify(data.value);
       case 'number': return data.value;
       case 'boolean': return data.value;
       case 'null': return 'null';

package/src/browser/client.js

@@ -8,6 +8,7 @@ import { EventEmitter } from 'events';
 import { getDefaultHookScript } from './defaultHooks.js';
 import { NetworkInterceptor } from './interceptors/NetworkInterceptor.js';
 import { ScriptInterceptor } from './interceptors/ScriptInterceptor.js';
+import { AntiDebugInterceptor } from './interceptors/AntiDebugInterceptor.js';
 import { getDataStore } from '../store/DataStore.js';
 
 export class BrowserClient extends EventEmitter {
@@ -20,9 +21,13 @@ export class BrowserClient extends EventEmitter {
     this.cdpSession = null;
     this.networkInterceptor = null;
     this.scriptInterceptor = null;
+    this.antiDebugInterceptor = null;
     this.hookScript = null;
     this.onMessage = null;
     this._isCleaningUp = false;
+    // CDP session 健康检查节流
+    this._cdpLastCheck = 0;
+    this._cdpCheckInterval = 5000; // 5秒内不重复检查
   }
 
   /**
@@ -37,6 +42,7 @@ export class BrowserClient extends EventEmitter {
       headless = false,
       executablePath = null,
       args = [],
+      userDataDir = null,
     } = options;
 
     const launchOptions = {
@@ -53,12 +59,22 @@ export class BrowserClient extends EventEmitter {
       launchOptions.executablePath = executablePath;
     }
 
-    this.browser = await chromium.launch(launchOptions);
-    this.emit('launched', { headless });
-
-    this.context = await this.browser.newContext({
-      ignoreHTTPSErrors: true,
-    });
+    this._persistent = !!userDataDir;
+
+    if (userDataDir) {
+      // 持久化模式：launchPersistentContext 返回 BrowserContext
+      launchOptions.ignoreHTTPSErrors = true;
+      this.context = await chromium.launchPersistentContext(userDataDir, launchOptions);
+      this.browser = this.context.browser();
+      this.emit('launched', { headless, persistent: true });
+    } else {
+      // 临时模式（原有逻辑）
+      this.browser = await chromium.launch(launchOptions);
+      this.emit('launched', { headless });
+      this.context = await this.browser.newContext({
+        ignoreHTTPSErrors: true,
+      });
+    }
 
     // 保存 hook 脚本
     this.hookScript = getDefaultHookScript();
@@ -66,11 +82,22 @@ export class BrowserClient extends EventEmitter {
     // 使用 addInitScript 在 context 级别注入
     await this.context.addInitScript(this.hookScript);
 
-    this.page = await this.context.newPage();
+    // 持久化上下文自带默认页面，临时模式需要新建
+    this.page = this._persistent
+      ? (this.context.pages()[0] || await this.context.newPage())
+      : await this.context.newPage();
 
     // 监听新页面创建（弹窗、新标签页）
     this.context.on('page', async (newPage) => {
       console.log('[BrowserClient] 检测到新页面');
+
+      // 清理旧页面的 CDP session（避免泄漏）
+      if (this.cdpSession && this._cdpSessionPage && this._cdpSessionPage !== newPage) {
+        await this.cdpSession.detach().catch(() => {});
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+
       this.pages.push(newPage);
       this.page = newPage;  // 切换到新页面
       await this.setupPage(newPage);
@@ -96,6 +123,13 @@ export class BrowserClient extends EventEmitter {
    */
   async setupPage(page) {
     try {
+      // 如果这是当前页面的重新设置，先清理旧的 session
+      if (page === this.page && this.cdpSession && this._cdpSessionPage === page) {
+        await this.cdpSession.detach().catch(() => {});
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+
       const cdp = await page.context().newCDPSession(page);
 
       // 1. 启用 Runtime 域
@@ -125,11 +159,22 @@ export class BrowserClient extends EventEmitter {
       await networkInterceptor.start();
       await scriptInterceptor.start();
 
-      // 保存引用
+      // 反无限 debugger：必须在 ScriptInterceptor 之后（Debugger 域已启用）
+      const antiDebugInterceptor = new AntiDebugInterceptor(cdp);
+      await antiDebugInterceptor.start();
+
+      // ScriptInterceptor 拉取源码后通知 AntiDebugInterceptor，避免重复 CDP 调用
+      scriptInterceptor.onSource = (scriptId, source) => {
+        antiDebugInterceptor.checkScript(scriptId, source);
+      };
+
+      // 保存引用（仅对当前活动页面）
       if (page === this.page) {
         this.cdpSession = cdp;
+        this._cdpSessionPage = page;  // 关键：设置标记，让 getCDPSession 知道这是当前页面的 session
         this.networkInterceptor = networkInterceptor;
         this.scriptInterceptor = scriptInterceptor;
+        this.antiDebugInterceptor = antiDebugInterceptor;
       }
 
       // 监听页面导航
@@ -146,17 +191,52 @@ export class BrowserClient extends EventEmitter {
   }
 
   /**
-   * 获取 CDP 会话（始终使用当前页面的 session）
+   * 获取 CDP 会话（复用已有 session，仅在 page 变化时重建）
    */
   async getCDPSession() {
-    // 每次都为当前页面创建新的 CDP session，确保上下文正确
-    if (this.page) {
+    if (!this.page) return this.cdpSession;
+
+    // page 未变且 session 存在 → 复用
+    if (this.cdpSession && this._cdpSessionPage === this.page) {
+      // 节流：避免频繁健康检查
+      const now = Date.now();
+      if (now - this._cdpLastCheck < this._cdpCheckInterval) {
+        return this.cdpSession;
+      }
+
+      try {
+        // 通过简单的 Runtime.evaluate 验证 session 是否还活着
+        await this.cdpSession.send('Runtime.evaluate', { expression: '1' });
+        this._cdpLastCheck = now;
+        return this.cdpSession;
+      } catch {
+        // session 已失效，需要重新创建
+        console.log('[BrowserClient] CDP session 已失效，重新创建');
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+    }
+
+    // page 变了或 session 失效 → detach 旧 session，创建新的
+    if (this.cdpSession) {
       try {
-        this.cdpSession = await this.page.context().newCDPSession(this.page);
-      } catch (e) {
-        console.error('[BrowserClient] 创建 CDP session 失败:', e.message);
-        return null;
+        await this.cdpSession.detach();
+      } catch {
+        // 忽略 detach 错误（session 可能已断开）
       }
+      this.cdpSession = null;
+    }
+
+    try {
+      this.cdpSession = await this.page.context().newCDPSession(this.page);
+      this._cdpSessionPage = this.page;
+      this._cdpLastCheck = Date.now();
+      console.log('[BrowserClient] CDP session 已创建');
+    } catch (e) {
+      console.error('[BrowserClient] 创建 CDP session 失败:', e.message);
+      this.cdpSession = null;
+      this._cdpSessionPage = null;
+      return null;
     }
     return this.cdpSession;
   }
@@ -165,8 +245,19 @@ export class BrowserClient extends EventEmitter {
    * 导航到 URL
    */
   async navigate(url, options = {}) {
-    const { waitUntil = 'domcontentloaded' } = options;
-    await this.page.goto(url, { waitUntil });
+    const { waitUntil = 'domcontentloaded', timeout = 30000 } = options;
+    try {
+      await this.page.goto(url, { waitUntil, timeout });
+    } catch (e) {
+      // 超时不一定是错误，页面可能仍在加载，继续执行
+      if (e.message?.includes('timeout')) {
+        console.log('[BrowserClient] 导航超时，继续等待页面稳定...');
+        // 等待一小段时间让页面尽可能完成加载
+        await this.page.waitForTimeout(2000);
+      } else {
+        throw e;
+      }
+    }
     return this.page.url();
   }
 
@@ -208,6 +299,9 @@ export class BrowserClient extends EventEmitter {
         await this.scriptInterceptor.stop?.().catch(() => {});
         this.scriptInterceptor = null;
       }
+      if (this.antiDebugInterceptor) {
+        this.antiDebugInterceptor = null;
+      }
 
       // 分离 CDP session
       if (this.cdpSession) {
@@ -216,7 +310,16 @@ export class BrowserClient extends EventEmitter {
       }
 
       // 关闭浏览器
-      if (this.browser) {
+      if (this._persistent) {
+        // 持久化模式：关闭 context 即保存数据并关闭浏览器
+        if (this.context) {
+          await this.context.close();
+          this.context = null;
+          this.browser = null;
+          this.page = null;
+          this.pages = [];
+        }
+      } else if (this.browser) {
         await this.browser.close();
         this.browser = null;
         this.context = null;
@@ -229,6 +332,9 @@ export class BrowserClient extends EventEmitter {
       this.emit('error', e);
     } finally {
       this._isCleaningUp = false;
+      // 重置 CDP 相关状态
+      this._cdpLastCheck = 0;
+      this._cdpSessionPage = null;
     }
   }
 }

package/src/browser/collector.js

@@ -15,13 +15,17 @@ export class EnvCollector {
    * @param {object} options - 采集选项
    */
   async collect(path, options = {}) {
-    const { depth = 1, includeProto = false, useCache = true } = options;
+    const { depth = 1, includeProto = false, useCache = true, timeout = 5000 } = options;
 
     if (useCache && this.cache.has(path)) {
       return this.cache.get(path);
     }
 
-    const result = await this.page.evaluate(({ path, depth, includeProto }) => {
+    // 使用 Promise.race 添加超时保护
+    const evaluatePromise = this.page.evaluate(({ path, depth, includeProto: _includeProto }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
+
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -40,29 +44,55 @@ export class EnvCollector {
           return { type, value: val };
         }
 
+        // 检测循环引用
+        if (seen.has(val)) {
+          return { type: 'object', value: '[Circular]', circular: true };
+        }
+
         if (currentDepth >= maxDepth) {
           return { type: 'object', value: '[Object]', truncated: true };
         }
 
+        seen.add(val);
+
         if (Array.isArray(val)) {
           return {
             type: 'array',
-            value: val.map(v => serialize(v, currentDepth + 1, maxDepth))
+            length: val.length,
+            value: val.slice(0, 20).map(v => serialize(v, currentDepth + 1, maxDepth))
           };
         }
 
         const result = { type: 'object', properties: {} };
-        const keys = Object.getOwnPropertyNames(val);
+        let keys;
+        try {
+          keys = Object.getOwnPropertyNames(val);
+        } catch (e) {
+          return { type: 'object', value: '[Error accessing keys]', error: e.message };
+        }
 
-        for (const key of keys.slice(0, 50)) {
+        for (const key of keys.slice(0, 30)) {
           try {
             const desc = Object.getOwnPropertyDescriptor(val, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
             if (desc.get) {
+              // 对于 getter，只记录描述符信息，不执行 getter
+              result.properties[key] = {
+                type: 'getter',
+                hasGetter: true,
+                enumerable: desc.enumerable,
+                configurable: desc.configurable
+              };
+            } else if (desc.set && desc.value === undefined) {
+              // 只有 setter 没有 getter
               result.properties[key] = {
-                ...serialize(val[key], currentDepth + 1, maxDepth),
-                hasGetter: true
+                type: 'setter',
+                hasSetter: true
               };
             } else {
+              // 普通值
               result.properties[key] = serialize(desc.value, currentDepth + 1, maxDepth);
             }
           } catch (e) {
@@ -89,15 +119,19 @@ export class EnvCollector {
 
         let descriptor = null;
         if (parent) {
-          const desc = Object.getOwnPropertyDescriptor(parent, propName);
-          if (desc) {
-            descriptor = {
-              configurable: desc.configurable,
-              enumerable: desc.enumerable,
-              writable: desc.writable,
-              hasGetter: !!desc.get,
-              hasSetter: !!desc.set
-            };
+          try {
+            const desc = Object.getOwnPropertyDescriptor(parent, propName);
+            if (desc) {
+              descriptor = {
+                configurable: desc.configurable,
+                enumerable: desc.enumerable,
+                writable: desc.writable,
+                hasGetter: !!desc.get,
+                hasSetter: !!desc.set
+              };
+            }
+          } catch (e) {
+            // 忽略描述符读取错误
           }
         }
 
@@ -112,7 +146,19 @@ export class EnvCollector {
       }
     }, { path, depth, includeProto });
 
-    if (result.success && useCache) {
+    // 添加超时
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('采集超时')), timeout)
+    );
+
+    let result;
+    try {
+      result = await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      result = { success: false, error: e.message };
+    }
+
+    if (result?.success && useCache) {
       this.cache.set(path, result);
     }
 
@@ -154,9 +200,12 @@ export class EnvCollector {
    * 深度采集整个对象
    */
   async collectDeep(rootPath, options = {}) {
-    const { maxDepth = 3, maxProps = 100 } = options;
+    const { maxDepth = 3, maxProps = 100, timeout = 5000 } = options;
+
+    const evaluatePromise = this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
 
-    return await this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -165,19 +214,38 @@ export class EnvCollector {
         if (depth > maxDepth || collected.size > maxProps) return;
         if (!obj || typeof obj !== 'object') return;
 
+        // 检测循环引用
+        if (seen.has(obj)) return;
+        seen.add(obj);
+
         const keys = Object.getOwnPropertyNames(obj);
-        for (const key of keys) {
+        for (const key of keys.slice(0, 30)) {
           if (collected.size > maxProps) break;
 
           const fullPath = path ? `${path}.${key}` : key;
           try {
-            const val = obj[key];
-            const type = typeof val;
+            const desc = Object.getOwnPropertyDescriptor(obj, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
+            let val;
+            let type;
+            if (desc.get) {
+              type = 'getter';
+              val = '[Getter]';
+            } else if (desc.set && desc.value === undefined) {
+              type = 'setter';
+              val = '[Setter]';
+            } else {
+              val = desc.value;
+              type = typeof val;
+            }
 
             collected.set(fullPath, {
               type,
               value: type === 'function' ? '[Function]' :
                      type === 'object' ? '[Object]' :
+                     type === 'getter' || type === 'setter' ? val :
                      val
             });
 
@@ -204,6 +272,17 @@ export class EnvCollector {
         properties: Object.fromEntries(collected)
       };
     }, { rootPath, maxDepth, maxProps });
+
+    // 添加超时保护
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('collectDeep timeout')), timeout)
+    );
+
+    try {
+      return await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      return { success: false, error: e.message };
+    }
   }
 
   // === 特殊环境采集 ===

package/src/browser/defaultHooks.js

@@ -160,7 +160,9 @@ function getCookieHook() {
         return value;
       },
       set: function(val) {
-        deepspider.log('cookie', { action: 'write', value: val });
+        // 解析 cookie name（cookie 格式: "name=value; expires=...; path=..."）
+        const cookieName = val?.split('=')[0]?.trim();
+        deepspider.log('cookie', { action: 'write', name: cookieName, value: val });
         return cookieDesc.set.call(document, val);
       },
       configurable: true

package/src/browser/hooks/index.js

@@ -7,6 +7,7 @@
 export class HookManager {
   constructor() {
     this.logs = [];
+    this.maxLogs = 5000;
     this.onLog = null;
     this.injected = false;
   }
@@ -37,6 +38,10 @@ export class HookManager {
           text,
           timestamp: Date.now(),
         });
+        // 超过上限时丢弃最旧的 20%
+        if (this.logs.length > this.maxLogs) {
+          this.logs = this.logs.slice(Math.floor(this.maxLogs * 0.2));
+        }
         if (this.onLog) {
           this.onLog({ type: msg.type(), text });
         }