deepspider 0.3.1 → 0.3.2

package/src/browser/client.js

@@ -8,6 +8,7 @@ import { EventEmitter } from 'events';
 import { getDefaultHookScript } from './defaultHooks.js';
 import { NetworkInterceptor } from './interceptors/NetworkInterceptor.js';
 import { ScriptInterceptor } from './interceptors/ScriptInterceptor.js';
+import { AntiDebugInterceptor } from './interceptors/AntiDebugInterceptor.js';
 import { getDataStore } from '../store/DataStore.js';
 
 export class BrowserClient extends EventEmitter {
@@ -20,9 +21,13 @@ export class BrowserClient extends EventEmitter {
     this.cdpSession = null;
     this.networkInterceptor = null;
     this.scriptInterceptor = null;
+    this.antiDebugInterceptor = null;
     this.hookScript = null;
     this.onMessage = null;
     this._isCleaningUp = false;
+    // CDP session 健康检查节流
+    this._cdpLastCheck = 0;
+    this._cdpCheckInterval = 5000; // 5秒内不重复检查
   }
 
   /**
@@ -37,6 +42,7 @@ export class BrowserClient extends EventEmitter {
       headless = false,
       executablePath = null,
       args = [],
+      userDataDir = null,
     } = options;
 
     const launchOptions = {
@@ -53,12 +59,22 @@ export class BrowserClient extends EventEmitter {
       launchOptions.executablePath = executablePath;
     }
 
-    this.browser = await chromium.launch(launchOptions);
-    this.emit('launched', { headless });
-
-    this.context = await this.browser.newContext({
-      ignoreHTTPSErrors: true,
-    });
+    this._persistent = !!userDataDir;
+
+    if (userDataDir) {
+      // 持久化模式：launchPersistentContext 返回 BrowserContext
+      launchOptions.ignoreHTTPSErrors = true;
+      this.context = await chromium.launchPersistentContext(userDataDir, launchOptions);
+      this.browser = this.context.browser();
+      this.emit('launched', { headless, persistent: true });
+    } else {
+      // 临时模式（原有逻辑）
+      this.browser = await chromium.launch(launchOptions);
+      this.emit('launched', { headless });
+      this.context = await this.browser.newContext({
+        ignoreHTTPSErrors: true,
+      });
+    }
 
     // 保存 hook 脚本
     this.hookScript = getDefaultHookScript();
@@ -66,11 +82,22 @@ export class BrowserClient extends EventEmitter {
     // 使用 addInitScript 在 context 级别注入
     await this.context.addInitScript(this.hookScript);
 
-    this.page = await this.context.newPage();
+    // 持久化上下文自带默认页面，临时模式需要新建
+    this.page = this._persistent
+      ? (this.context.pages()[0] || await this.context.newPage())
+      : await this.context.newPage();
 
     // 监听新页面创建（弹窗、新标签页）
     this.context.on('page', async (newPage) => {
       console.log('[BrowserClient] 检测到新页面');
+
+      // 清理旧页面的 CDP session（避免泄漏）
+      if (this.cdpSession && this._cdpSessionPage && this._cdpSessionPage !== newPage) {
+        await this.cdpSession.detach().catch(() => {});
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+
       this.pages.push(newPage);
       this.page = newPage;  // 切换到新页面
       await this.setupPage(newPage);
@@ -96,6 +123,13 @@ export class BrowserClient extends EventEmitter {
    */
   async setupPage(page) {
     try {
+      // 如果这是当前页面的重新设置，先清理旧的 session
+      if (page === this.page && this.cdpSession && this._cdpSessionPage === page) {
+        await this.cdpSession.detach().catch(() => {});
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+
       const cdp = await page.context().newCDPSession(page);
 
       // 1. 启用 Runtime 域
@@ -125,11 +159,22 @@ export class BrowserClient extends EventEmitter {
       await networkInterceptor.start();
       await scriptInterceptor.start();
 
-      // 保存引用
+      // 反无限 debugger：必须在 ScriptInterceptor 之后（Debugger 域已启用）
+      const antiDebugInterceptor = new AntiDebugInterceptor(cdp);
+      await antiDebugInterceptor.start();
+
+      // ScriptInterceptor 拉取源码后通知 AntiDebugInterceptor，避免重复 CDP 调用
+      scriptInterceptor.onSource = (scriptId, source) => {
+        antiDebugInterceptor.checkScript(scriptId, source);
+      };
+
+      // 保存引用（仅对当前活动页面）
       if (page === this.page) {
         this.cdpSession = cdp;
+        this._cdpSessionPage = page;  // 关键：设置标记，让 getCDPSession 知道这是当前页面的 session
         this.networkInterceptor = networkInterceptor;
         this.scriptInterceptor = scriptInterceptor;
+        this.antiDebugInterceptor = antiDebugInterceptor;
       }
 
       // 监听页面导航
@@ -146,17 +191,52 @@ export class BrowserClient extends EventEmitter {
   }
 
   /**
-   * 获取 CDP 会话（始终使用当前页面的 session）
+   * 获取 CDP 会话（复用已有 session，仅在 page 变化时重建）
    */
   async getCDPSession() {
-    // 每次都为当前页面创建新的 CDP session，确保上下文正确
-    if (this.page) {
+    if (!this.page) return this.cdpSession;
+
+    // page 未变且 session 存在 → 复用
+    if (this.cdpSession && this._cdpSessionPage === this.page) {
+      // 节流：避免频繁健康检查
+      const now = Date.now();
+      if (now - this._cdpLastCheck < this._cdpCheckInterval) {
+        return this.cdpSession;
+      }
+
+      try {
+        // 通过简单的 Runtime.evaluate 验证 session 是否还活着
+        await this.cdpSession.send('Runtime.evaluate', { expression: '1' });
+        this._cdpLastCheck = now;
+        return this.cdpSession;
+      } catch {
+        // session 已失效，需要重新创建
+        console.log('[BrowserClient] CDP session 已失效，重新创建');
+        this.cdpSession = null;
+        this._cdpSessionPage = null;
+      }
+    }
+
+    // page 变了或 session 失效 → detach 旧 session，创建新的
+    if (this.cdpSession) {
       try {
-        this.cdpSession = await this.page.context().newCDPSession(this.page);
-      } catch (e) {
-        console.error('[BrowserClient] 创建 CDP session 失败:', e.message);
-        return null;
+        await this.cdpSession.detach();
+      } catch {
+        // 忽略 detach 错误（session 可能已断开）
       }
+      this.cdpSession = null;
+    }
+
+    try {
+      this.cdpSession = await this.page.context().newCDPSession(this.page);
+      this._cdpSessionPage = this.page;
+      this._cdpLastCheck = Date.now();
+      console.log('[BrowserClient] CDP session 已创建');
+    } catch (e) {
+      console.error('[BrowserClient] 创建 CDP session 失败:', e.message);
+      this.cdpSession = null;
+      this._cdpSessionPage = null;
+      return null;
     }
     return this.cdpSession;
   }
@@ -165,8 +245,19 @@ export class BrowserClient extends EventEmitter {
    * 导航到 URL
    */
   async navigate(url, options = {}) {
-    const { waitUntil = 'domcontentloaded' } = options;
-    await this.page.goto(url, { waitUntil });
+    const { waitUntil = 'domcontentloaded', timeout = 30000 } = options;
+    try {
+      await this.page.goto(url, { waitUntil, timeout });
+    } catch (e) {
+      // 超时不一定是错误，页面可能仍在加载，继续执行
+      if (e.message?.includes('timeout')) {
+        console.log('[BrowserClient] 导航超时，继续等待页面稳定...');
+        // 等待一小段时间让页面尽可能完成加载
+        await this.page.waitForTimeout(2000);
+      } else {
+        throw e;
+      }
+    }
     return this.page.url();
   }
 
@@ -208,6 +299,9 @@ export class BrowserClient extends EventEmitter {
         await this.scriptInterceptor.stop?.().catch(() => {});
         this.scriptInterceptor = null;
       }
+      if (this.antiDebugInterceptor) {
+        this.antiDebugInterceptor = null;
+      }
 
       // 分离 CDP session
       if (this.cdpSession) {
@@ -216,7 +310,16 @@ export class BrowserClient extends EventEmitter {
       }
 
       // 关闭浏览器
-      if (this.browser) {
+      if (this._persistent) {
+        // 持久化模式：关闭 context 即保存数据并关闭浏览器
+        if (this.context) {
+          await this.context.close();
+          this.context = null;
+          this.browser = null;
+          this.page = null;
+          this.pages = [];
+        }
+      } else if (this.browser) {
         await this.browser.close();
         this.browser = null;
         this.context = null;
@@ -229,6 +332,9 @@ export class BrowserClient extends EventEmitter {
       this.emit('error', e);
     } finally {
       this._isCleaningUp = false;
+      // 重置 CDP 相关状态
+      this._cdpLastCheck = 0;
+      this._cdpSessionPage = null;
     }
   }
 }

package/src/browser/collector.js

@@ -15,13 +15,17 @@ export class EnvCollector {
    * @param {object} options - 采集选项
    */
   async collect(path, options = {}) {
-    const { depth = 1, includeProto = false, useCache = true } = options;
+    const { depth = 1, includeProto = false, useCache = true, timeout = 5000 } = options;
 
     if (useCache && this.cache.has(path)) {
       return this.cache.get(path);
     }
 
-    const result = await this.page.evaluate(({ path, depth, includeProto: _includeProto }) => {
+    // 使用 Promise.race 添加超时保护
+    const evaluatePromise = this.page.evaluate(({ path, depth, includeProto: _includeProto }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
+
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -40,29 +44,55 @@ export class EnvCollector {
           return { type, value: val };
         }
 
+        // 检测循环引用
+        if (seen.has(val)) {
+          return { type: 'object', value: '[Circular]', circular: true };
+        }
+
         if (currentDepth >= maxDepth) {
           return { type: 'object', value: '[Object]', truncated: true };
         }
 
+        seen.add(val);
+
         if (Array.isArray(val)) {
           return {
             type: 'array',
-            value: val.map(v => serialize(v, currentDepth + 1, maxDepth))
+            length: val.length,
+            value: val.slice(0, 20).map(v => serialize(v, currentDepth + 1, maxDepth))
           };
         }
 
         const result = { type: 'object', properties: {} };
-        const keys = Object.getOwnPropertyNames(val);
+        let keys;
+        try {
+          keys = Object.getOwnPropertyNames(val);
+        } catch (e) {
+          return { type: 'object', value: '[Error accessing keys]', error: e.message };
+        }
 
-        for (const key of keys.slice(0, 50)) {
+        for (const key of keys.slice(0, 30)) {
           try {
             const desc = Object.getOwnPropertyDescriptor(val, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
             if (desc.get) {
+              // 对于 getter，只记录描述符信息，不执行 getter
+              result.properties[key] = {
+                type: 'getter',
+                hasGetter: true,
+                enumerable: desc.enumerable,
+                configurable: desc.configurable
+              };
+            } else if (desc.set && desc.value === undefined) {
+              // 只有 setter 没有 getter
               result.properties[key] = {
-                ...serialize(val[key], currentDepth + 1, maxDepth),
-                hasGetter: true
+                type: 'setter',
+                hasSetter: true
               };
             } else {
+              // 普通值
               result.properties[key] = serialize(desc.value, currentDepth + 1, maxDepth);
             }
           } catch (e) {
@@ -89,15 +119,19 @@ export class EnvCollector {
 
         let descriptor = null;
         if (parent) {
-          const desc = Object.getOwnPropertyDescriptor(parent, propName);
-          if (desc) {
-            descriptor = {
-              configurable: desc.configurable,
-              enumerable: desc.enumerable,
-              writable: desc.writable,
-              hasGetter: !!desc.get,
-              hasSetter: !!desc.set
-            };
+          try {
+            const desc = Object.getOwnPropertyDescriptor(parent, propName);
+            if (desc) {
+              descriptor = {
+                configurable: desc.configurable,
+                enumerable: desc.enumerable,
+                writable: desc.writable,
+                hasGetter: !!desc.get,
+                hasSetter: !!desc.set
+              };
+            }
+          } catch (e) {
+            // 忽略描述符读取错误
           }
         }
 
@@ -112,7 +146,19 @@ export class EnvCollector {
       }
     }, { path, depth, includeProto });
 
-    if (result.success && useCache) {
+    // 添加超时
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('采集超时')), timeout)
+    );
+
+    let result;
+    try {
+      result = await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      result = { success: false, error: e.message };
+    }
+
+    if (result?.success && useCache) {
       this.cache.set(path, result);
     }
 
@@ -154,9 +200,12 @@ export class EnvCollector {
    * 深度采集整个对象
    */
   async collectDeep(rootPath, options = {}) {
-    const { maxDepth = 3, maxProps = 100 } = options;
+    const { maxDepth = 3, maxProps = 100, timeout = 5000 } = options;
+
+    const evaluatePromise = this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
 
-    return await this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -165,19 +214,38 @@ export class EnvCollector {
         if (depth > maxDepth || collected.size > maxProps) return;
         if (!obj || typeof obj !== 'object') return;
 
+        // 检测循环引用
+        if (seen.has(obj)) return;
+        seen.add(obj);
+
         const keys = Object.getOwnPropertyNames(obj);
-        for (const key of keys) {
+        for (const key of keys.slice(0, 30)) {
           if (collected.size > maxProps) break;
 
           const fullPath = path ? `${path}.${key}` : key;
           try {
-            const val = obj[key];
-            const type = typeof val;
+            const desc = Object.getOwnPropertyDescriptor(obj, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
+            let val;
+            let type;
+            if (desc.get) {
+              type = 'getter';
+              val = '[Getter]';
+            } else if (desc.set && desc.value === undefined) {
+              type = 'setter';
+              val = '[Setter]';
+            } else {
+              val = desc.value;
+              type = typeof val;
+            }
 
             collected.set(fullPath, {
               type,
               value: type === 'function' ? '[Function]' :
                      type === 'object' ? '[Object]' :
+                     type === 'getter' || type === 'setter' ? val :
                      val
             });
 
@@ -204,6 +272,17 @@ export class EnvCollector {
         properties: Object.fromEntries(collected)
       };
     }, { rootPath, maxDepth, maxProps });
+
+    // 添加超时保护
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('collectDeep timeout')), timeout)
+    );
+
+    try {
+      return await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      return { success: false, error: e.message };
+    }
   }
 
   // === 特殊环境采集 ===

package/src/browser/defaultHooks.js

@@ -160,7 +160,9 @@ function getCookieHook() {
         return value;
       },
       set: function(val) {
-        deepspider.log('cookie', { action: 'write', value: val });
+        // 解析 cookie name（cookie 格式: "name=value; expires=...; path=..."）
+        const cookieName = val?.split('=')[0]?.trim();
+        deepspider.log('cookie', { action: 'write', name: cookieName, value: val });
         return cookieDesc.set.call(document, val);
       },
       configurable: true

package/src/browser/hooks/index.js

@@ -7,6 +7,7 @@
 export class HookManager {
   constructor() {
     this.logs = [];
+    this.maxLogs = 5000;
     this.onLog = null;
     this.injected = false;
   }
@@ -37,6 +38,10 @@ export class HookManager {
           text,
           timestamp: Date.now(),
         });
+        // 超过上限时丢弃最旧的 20%
+        if (this.logs.length > this.maxLogs) {
+          this.logs = this.logs.slice(Math.floor(this.maxLogs * 0.2));
+        }
         if (this.onLog) {
           this.onLog({ type: msg.type(), text });
         }

package/src/browser/interceptors/AntiDebugInterceptor.js

@@ -0,0 +1,132 @@
+/**
+ * DeepSpider - 反无限 debugger 拦截器
+ * 通过 CDP Debugger.setBlackboxedRanges 跳过包含 debugger 语句的脚本
+ * 零运行时开销，不修改源码，不触发完整性校验
+ *
+ * 已知限制：/\bdebugger\b/ 会匹配字符串/注释中的 debugger，
+ * 对反爬场景可接受（误 blackbox 的脚本仍正常执行，只是不可调试）
+ */
+
+export class AntiDebugInterceptor {
+  constructor(cdpClient) {
+    this.client = cdpClient;
+    this.blackboxedScripts = new Set();
+    // 高频 debugger 检测
+    this.pausedCount = 0;
+    this.pausedWindowStart = 0;
+    this.PAUSED_WINDOW_MS = 1000;  // 1秒窗口
+    this.PAUSED_THRESHOLD = 5;     // 1秒内超过5次paused认为是debugger风暴
+    this.stormMode = false;        // 风暴模式：跳过所有断点
+    this.stormTimer = null;        // 风暴模式自动退出定时器
+  }
+
+  async start() {
+    // 兜底：对于 blackbox 来不及处理的同步 debugger（时序竞争），自动 resume
+    // reason 可能是 'other' 或 'debugCommand'（不同 Chrome 版本），
+    // 只要不是我们主动设的断点（hitBreakpoints 非空 / reason=breakpoint）就 resume
+    this.client.on('Debugger.paused', (params) => {
+      // 手动设置的断点（除非在风暴模式）
+      if (!this.stormMode && params.reason === 'breakpoint') return;
+      if (!this.stormMode && params.hitBreakpoints?.length > 0) return;
+
+      // 风暴模式下直接 resume，不参与计数
+      if (this.stormMode) {
+        this.client.send('Debugger.resume').catch(() => {});
+        return;
+      }
+
+      // 高频 debugger 检测
+      const now = Date.now();
+      if (now - this.pausedWindowStart > this.PAUSED_WINDOW_MS) {
+        // 新窗口
+        this.pausedWindowStart = now;
+        this.pausedCount = 1;
+      } else {
+        this.pausedCount++;
+      }
+
+      // 触发风暴模式
+      if (this.pausedCount > this.PAUSED_THRESHOLD) {
+        console.log('[AntiDebugInterceptor] 检测到 debugger 风暴，启用风暴模式');
+        this.stormMode = true;
+        // 清除之前的定时器
+        if (this.stormTimer) {
+          clearTimeout(this.stormTimer);
+        }
+        // 3秒后退出风暴模式
+        this.stormTimer = setTimeout(() => {
+          console.log('[AntiDebugInterceptor] 退出风暴模式');
+          this.stormMode = false;
+          this.pausedCount = 0;
+          this.stormTimer = null;
+        }, 3000);
+      }
+
+      // 自动 resume
+      this.client.send('Debugger.resume').catch(() => {});
+    });
+
+    console.log('[AntiDebugInterceptor] 已启动');
+  }
+
+  /**
+   * 检查脚本源码，包含 debugger 则 blackbox 整个脚本
+   * 由 ScriptInterceptor.onSource 回调驱动，避免重复拉取源码
+   */
+  checkScript(scriptId, scriptSource) {
+    if (/\bdebugger\b/.test(scriptSource)) {
+      this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [{ lineNumber: 0, columnNumber: 0 }],
+      }).then(() => {
+        this.blackboxedScripts.add(scriptId);
+      }).catch(() => {});
+    }
+  }
+
+  /**
+   * 取消指定脚本的 blackbox（供断点工具调用）
+   */
+  async unblackbox(scriptId) {
+    if (this.blackboxedScripts.has(scriptId)) {
+      await this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [],
+      });
+      this.blackboxedScripts.delete(scriptId);
+    }
+  }
+
+  /**
+   * 手动启用/禁用风暴模式
+   * 用于绕过强反调试场景
+   */
+  setStormMode(enabled) {
+    // 清除之前的定时器
+    if (this.stormTimer) {
+      clearTimeout(this.stormTimer);
+      this.stormTimer = null;
+    }
+
+    this.stormMode = enabled;
+    if (enabled) {
+      console.log('[AntiDebugInterceptor] 手动启用风暴模式');
+      // 自动退出
+      this.stormTimer = setTimeout(() => {
+        this.stormMode = false;
+        this.stormTimer = null;
+        console.log('[AntiDebugInterceptor] 自动退出风暴模式');
+      }, 5000);
+    } else {
+      console.log('[AntiDebugInterceptor] 手动禁用风暴模式');
+      this.pausedCount = 0;
+    }
+  }
+
+  /**
+   * 检查当前是否在风暴模式
+   */
+  isStormMode() {
+    return this.stormMode;
+  }
+}