deepspider 0.3.1 → 0.4.0

package/src/browser/collector.js

@@ -15,13 +15,17 @@ export class EnvCollector {
    * @param {object} options - 采集选项
    */
   async collect(path, options = {}) {
-    const { depth = 1, includeProto = false, useCache = true } = options;
+    const { depth = 1, includeProto = false, useCache = true, timeout = 5000 } = options;
 
     if (useCache && this.cache.has(path)) {
       return this.cache.get(path);
     }
 
-    const result = await this.page.evaluate(({ path, depth, includeProto: _includeProto }) => {
+    // 使用 Promise.race 添加超时保护
+    const evaluatePromise = this.page.evaluate(({ path, depth, includeProto: _includeProto }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
+
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -40,29 +44,55 @@ export class EnvCollector {
           return { type, value: val };
         }
 
+        // 检测循环引用
+        if (seen.has(val)) {
+          return { type: 'object', value: '[Circular]', circular: true };
+        }
+
         if (currentDepth >= maxDepth) {
           return { type: 'object', value: '[Object]', truncated: true };
         }
 
+        seen.add(val);
+
         if (Array.isArray(val)) {
           return {
             type: 'array',
-            value: val.map(v => serialize(v, currentDepth + 1, maxDepth))
+            length: val.length,
+            value: val.slice(0, 20).map(v => serialize(v, currentDepth + 1, maxDepth))
           };
         }
 
         const result = { type: 'object', properties: {} };
-        const keys = Object.getOwnPropertyNames(val);
+        let keys;
+        try {
+          keys = Object.getOwnPropertyNames(val);
+        } catch (e) {
+          return { type: 'object', value: '[Error accessing keys]', error: e.message };
+        }
 
-        for (const key of keys.slice(0, 50)) {
+        for (const key of keys.slice(0, 30)) {
           try {
             const desc = Object.getOwnPropertyDescriptor(val, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
             if (desc.get) {
+              // 对于 getter，只记录描述符信息，不执行 getter
+              result.properties[key] = {
+                type: 'getter',
+                hasGetter: true,
+                enumerable: desc.enumerable,
+                configurable: desc.configurable
+              };
+            } else if (desc.set && desc.value === undefined) {
+              // 只有 setter 没有 getter
               result.properties[key] = {
-                ...serialize(val[key], currentDepth + 1, maxDepth),
-                hasGetter: true
+                type: 'setter',
+                hasSetter: true
               };
             } else {
+              // 普通值
               result.properties[key] = serialize(desc.value, currentDepth + 1, maxDepth);
             }
           } catch (e) {
@@ -89,15 +119,19 @@ export class EnvCollector {
 
         let descriptor = null;
         if (parent) {
-          const desc = Object.getOwnPropertyDescriptor(parent, propName);
-          if (desc) {
-            descriptor = {
-              configurable: desc.configurable,
-              enumerable: desc.enumerable,
-              writable: desc.writable,
-              hasGetter: !!desc.get,
-              hasSetter: !!desc.set
-            };
+          try {
+            const desc = Object.getOwnPropertyDescriptor(parent, propName);
+            if (desc) {
+              descriptor = {
+                configurable: desc.configurable,
+                enumerable: desc.enumerable,
+                writable: desc.writable,
+                hasGetter: !!desc.get,
+                hasSetter: !!desc.set
+              };
+            }
+          } catch (e) {
+            // 忽略描述符读取错误
           }
         }
 
@@ -112,7 +146,19 @@ export class EnvCollector {
       }
     }, { path, depth, includeProto });
 
-    if (result.success && useCache) {
+    // 添加超时
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('采集超时')), timeout)
+    );
+
+    let result;
+    try {
+      result = await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      result = { success: false, error: e.message };
+    }
+
+    if (result?.success && useCache) {
       this.cache.set(path, result);
     }
 
@@ -154,9 +200,12 @@ export class EnvCollector {
    * 深度采集整个对象
    */
   async collectDeep(rootPath, options = {}) {
-    const { maxDepth = 3, maxProps = 100 } = options;
+    const { maxDepth = 3, maxProps = 100, timeout = 5000 } = options;
+
+    const evaluatePromise = this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
+      // 用于检测循环引用的 WeakSet
+      const seen = new WeakSet();
 
-    return await this.page.evaluate(({ rootPath, maxDepth, maxProps }) => {
       function getByPath(obj, path) {
         return path.split('.').reduce((o, k) => o && o[k], obj);
       }
@@ -165,19 +214,38 @@ export class EnvCollector {
         if (depth > maxDepth || collected.size > maxProps) return;
         if (!obj || typeof obj !== 'object') return;
 
+        // 检测循环引用
+        if (seen.has(obj)) return;
+        seen.add(obj);
+
         const keys = Object.getOwnPropertyNames(obj);
-        for (const key of keys) {
+        for (const key of keys.slice(0, 30)) {
           if (collected.size > maxProps) break;
 
           const fullPath = path ? `${path}.${key}` : key;
           try {
-            const val = obj[key];
-            const type = typeof val;
+            const desc = Object.getOwnPropertyDescriptor(obj, key);
+            if (!desc) continue;
+
+            // 安全处理：避免触发有副作用的 getter
+            let val;
+            let type;
+            if (desc.get) {
+              type = 'getter';
+              val = '[Getter]';
+            } else if (desc.set && desc.value === undefined) {
+              type = 'setter';
+              val = '[Setter]';
+            } else {
+              val = desc.value;
+              type = typeof val;
+            }
 
             collected.set(fullPath, {
               type,
               value: type === 'function' ? '[Function]' :
                      type === 'object' ? '[Object]' :
+                     type === 'getter' || type === 'setter' ? val :
                      val
             });
 
@@ -204,6 +272,17 @@ export class EnvCollector {
         properties: Object.fromEntries(collected)
       };
     }, { rootPath, maxDepth, maxProps });
+
+    // 添加超时保护
+    const timeoutPromise = new Promise((_, reject) =>
+      setTimeout(() => reject(new Error('collectDeep timeout')), timeout)
+    );
+
+    try {
+      return await Promise.race([evaluatePromise, timeoutPromise]);
+    } catch (e) {
+      return { success: false, error: e.message };
+    }
   }
 
   // === 特殊环境采集 ===

package/src/browser/defaultHooks.js

@@ -160,7 +160,9 @@ function getCookieHook() {
         return value;
       },
       set: function(val) {
-        deepspider.log('cookie', { action: 'write', value: val });
+        // 解析 cookie name（cookie 格式: "name=value; expires=...; path=..."）
+        const cookieName = val?.split('=')[0]?.trim();
+        deepspider.log('cookie', { action: 'write', name: cookieName, value: val });
         return cookieDesc.set.call(document, val);
       },
       configurable: true

package/src/browser/hooks/index.js

@@ -7,6 +7,7 @@
 export class HookManager {
   constructor() {
     this.logs = [];
+    this.maxLogs = 5000;
     this.onLog = null;
     this.injected = false;
   }
@@ -37,6 +38,10 @@ export class HookManager {
           text,
           timestamp: Date.now(),
         });
+        // 超过上限时丢弃最旧的 20%
+        if (this.logs.length > this.maxLogs) {
+          this.logs = this.logs.slice(Math.floor(this.maxLogs * 0.2));
+        }
         if (this.onLog) {
           this.onLog({ type: msg.type(), text });
         }

package/src/browser/interceptors/AntiDebugInterceptor.js

@@ -0,0 +1,132 @@
+/**
+ * DeepSpider - 反无限 debugger 拦截器
+ * 通过 CDP Debugger.setBlackboxedRanges 跳过包含 debugger 语句的脚本
+ * 零运行时开销，不修改源码，不触发完整性校验
+ *
+ * 已知限制：/\bdebugger\b/ 会匹配字符串/注释中的 debugger，
+ * 对反爬场景可接受（误 blackbox 的脚本仍正常执行，只是不可调试）
+ */
+
+export class AntiDebugInterceptor {
+  constructor(cdpClient) {
+    this.client = cdpClient;
+    this.blackboxedScripts = new Set();
+    // 高频 debugger 检测
+    this.pausedCount = 0;
+    this.pausedWindowStart = 0;
+    this.PAUSED_WINDOW_MS = 1000;  // 1秒窗口
+    this.PAUSED_THRESHOLD = 5;     // 1秒内超过5次paused认为是debugger风暴
+    this.stormMode = false;        // 风暴模式：跳过所有断点
+    this.stormTimer = null;        // 风暴模式自动退出定时器
+  }
+
+  async start() {
+    // 兜底：对于 blackbox 来不及处理的同步 debugger（时序竞争），自动 resume
+    // reason 可能是 'other' 或 'debugCommand'（不同 Chrome 版本），
+    // 只要不是我们主动设的断点（hitBreakpoints 非空 / reason=breakpoint）就 resume
+    this.client.on('Debugger.paused', (params) => {
+      // 手动设置的断点（除非在风暴模式）
+      if (!this.stormMode && params.reason === 'breakpoint') return;
+      if (!this.stormMode && params.hitBreakpoints?.length > 0) return;
+
+      // 风暴模式下直接 resume，不参与计数
+      if (this.stormMode) {
+        this.client.send('Debugger.resume').catch(() => {});
+        return;
+      }
+
+      // 高频 debugger 检测
+      const now = Date.now();
+      if (now - this.pausedWindowStart > this.PAUSED_WINDOW_MS) {
+        // 新窗口
+        this.pausedWindowStart = now;
+        this.pausedCount = 1;
+      } else {
+        this.pausedCount++;
+      }
+
+      // 触发风暴模式
+      if (this.pausedCount > this.PAUSED_THRESHOLD) {
+        console.log('[AntiDebugInterceptor] 检测到 debugger 风暴，启用风暴模式');
+        this.stormMode = true;
+        // 清除之前的定时器
+        if (this.stormTimer) {
+          clearTimeout(this.stormTimer);
+        }
+        // 3秒后退出风暴模式
+        this.stormTimer = setTimeout(() => {
+          console.log('[AntiDebugInterceptor] 退出风暴模式');
+          this.stormMode = false;
+          this.pausedCount = 0;
+          this.stormTimer = null;
+        }, 3000);
+      }
+
+      // 自动 resume
+      this.client.send('Debugger.resume').catch(() => {});
+    });
+
+    console.log('[AntiDebugInterceptor] 已启动');
+  }
+
+  /**
+   * 检查脚本源码，包含 debugger 则 blackbox 整个脚本
+   * 由 ScriptInterceptor.onSource 回调驱动，避免重复拉取源码
+   */
+  checkScript(scriptId, scriptSource) {
+    if (/\bdebugger\b/.test(scriptSource)) {
+      this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [{ lineNumber: 0, columnNumber: 0 }],
+      }).then(() => {
+        this.blackboxedScripts.add(scriptId);
+      }).catch(() => {});
+    }
+  }
+
+  /**
+   * 取消指定脚本的 blackbox（供断点工具调用）
+   */
+  async unblackbox(scriptId) {
+    if (this.blackboxedScripts.has(scriptId)) {
+      await this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [],
+      });
+      this.blackboxedScripts.delete(scriptId);
+    }
+  }
+
+  /**
+   * 手动启用/禁用风暴模式
+   * 用于绕过强反调试场景
+   */
+  setStormMode(enabled) {
+    // 清除之前的定时器
+    if (this.stormTimer) {
+      clearTimeout(this.stormTimer);
+      this.stormTimer = null;
+    }
+
+    this.stormMode = enabled;
+    if (enabled) {
+      console.log('[AntiDebugInterceptor] 手动启用风暴模式');
+      // 自动退出
+      this.stormTimer = setTimeout(() => {
+        this.stormMode = false;
+        this.stormTimer = null;
+        console.log('[AntiDebugInterceptor] 自动退出风暴模式');
+      }, 5000);
+    } else {
+      console.log('[AntiDebugInterceptor] 手动禁用风暴模式');
+      this.pausedCount = 0;
+    }
+  }
+
+  /**
+   * 检查当前是否在风暴模式
+   */
+  isStormMode() {
+    return this.stormMode;
+  }
+}

package/src/browser/interceptors/NetworkInterceptor.js

@@ -46,11 +46,16 @@ export class NetworkInterceptor {
       this.onLoadingFinished(params);
     });
 
+    // 监听加载失败（清理 pendingRequests，防止内存泄漏）
+    this.client.on('Network.loadingFailed', (params) => {
+      this.pendingRequests.delete(params.requestId);
+    });
+
     console.log('[NetworkInterceptor] 已启动');
   }
 
   onRequest(params) {
-    const { requestId, request, timestamp } = params;
+    const { requestId, request, timestamp, initiator } = params;
 
     // 只记录 XHR/Fetch 请求
     const type = params.type;
@@ -62,10 +67,37 @@ export class NetworkInterceptor {
       headers: request.headers,
       postData: request.postData,
       timestamp: timestamp * 1000,
-      pageUrl: this.getPageUrl()  // 记录请求时的页面 URL
+      pageUrl: this.getPageUrl(),
+      initiator: this.formatInitiator(initiator),
     });
   }
 
+  /**
+   * 精简 initiator 调用栈（只保留前 5 帧，过滤内部帧）
+   */
+  formatInitiator(initiator) {
+    if (!initiator) return null;
+    const result = { type: initiator.type };
+    if (initiator.url) {
+      result.url = initiator.url;
+      result.lineNumber = initiator.lineNumber;
+    }
+    if (initiator.stack?.callFrames) {
+      result.callFrames = initiator.stack.callFrames
+        .filter(f => f.url && !f.url.includes('patchright') && !f.url.includes('__playwright'))
+        .slice(0, 5)
+        .map(f => ({
+          functionName: f.functionName || '(anonymous)',
+          url: f.url,
+          lineNumber: f.lineNumber,
+          columnNumber: f.columnNumber,
+        }));
+    }
+    // 只有 type 没有实际定位信息时返回 null
+    if (!result.url && !result.callFrames?.length) return null;
+    return result;
+  }
+
   onResponse(params) {
     const { requestId, response } = params;
     const pending = this.pendingRequests.get(requestId);
@@ -81,15 +113,31 @@ export class NetworkInterceptor {
     if (!pending) return;
 
     try {
-      // 获取响应体
-      const { body, base64Encoded } = await this.client.send(
-        'Network.getResponseBody',
-        { requestId }
+      // 获取响应体，添加超时保护防止 CDP 命令挂起
+      const bodyPromise = this.client.send('Network.getResponseBody', { requestId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getResponseBody timeout')), 5000)
       );
-
-      const responseBody = base64Encoded
-        ? Buffer.from(body, 'base64').toString('utf-8')
-        : body;
+      const { body, base64Encoded } = await Promise.race([bodyPromise, timeoutPromise]);
+
+      // 处理响应体：检测二进制内容，避免损坏
+      let responseBody;
+      const contentType = pending.responseHeaders?.['content-type'] || '';
+
+      if (this.isBinaryContent(contentType)) {
+        // 二进制内容：存储元数据而非原始内容
+        // base64 长度计算：每 4 个字符 = 3 字节，考虑 padding
+        const binarySize = base64Encoded
+          ? Math.floor(body.length * 0.75) - (body.match(/=*$/)?.[0].length || 0)
+          : body.length;
+        responseBody = `[Binary: ${contentType}, ${binarySize} bytes]`;
+      } else {
+        // 文本内容：安全地转换为字符串
+        const rawBody = base64Encoded
+          ? Buffer.from(body, 'base64').toString('utf-8')
+          : body;
+        responseBody = rawBody.slice(0, 50000);
+      }
 
       // 异步存储到文件
       this.store.saveResponse({
@@ -98,9 +146,10 @@ export class NetworkInterceptor {
         status: pending.status,
         requestHeaders: pending.headers,
         requestBody: pending.postData,
-        responseBody: responseBody.slice(0, 50000),
+        responseBody,
         timestamp: pending.timestamp,
-        pageUrl: pending.pageUrl  // 传递页面 URL 用于分站点存储
+        pageUrl: pending.pageUrl,
+        initiator: pending.initiator,
       }).catch(e => {
         console.error('[NetworkInterceptor] 保存失败:', e.message);
       });
@@ -111,6 +160,21 @@ export class NetworkInterceptor {
 
     this.pendingRequests.delete(requestId);
   }
+
+  /**
+   * 检测是否为二进制内容类型
+   */
+  isBinaryContent(contentType) {
+    if (!contentType) return false;
+    const binaryTypes = [
+      'image/', 'audio/', 'video/', 'application/pdf',
+      'application/octet-stream', 'application/zip',
+      'application/gzip', 'application/x-protobuf',
+      'font/', 'application/vnd.'
+    ];
+    const lowerType = contentType.toLowerCase();
+    return binaryTypes.some(type => lowerType.includes(type));
+  }
 }
 
 export default NetworkInterceptor;

package/src/browser/interceptors/ScriptInterceptor.js

@@ -11,6 +11,7 @@ export class ScriptInterceptor {
     this.page = page;  // Playwright page 对象
     this.store = getDataStore();
     this.scriptIds = new Set();
+    this.onSource = null;  // 回调: (scriptId, scriptSource) => void
   }
 
   /**
@@ -40,22 +41,46 @@ export class ScriptInterceptor {
   async onScriptParsed(params) {
     const { scriptId, url, length: _length } = params;
 
-    // 跳过扩展和空脚本
-    if (!url || url.startsWith('chrome-extension://')) return;
+    // 跳过扩展脚本
+    if (url?.startsWith('chrome-extension://')) return;
     if (this.scriptIds.has(scriptId)) return;
 
     this.scriptIds.add(scriptId);
 
-    // 异步获取并存储源码
-    this.fetchAndSave(scriptId, url).catch(() => {});
+    if (url) {
+      // 有 URL 的脚本：获取源码、通知订阅者、存储
+      this.fetchAndSave(scriptId, url).catch(() => {});
+    } else if (this.onSource) {
+      // 无 URL 脚本（eval/new Function 生成）：仅通知订阅者用于 debugger 检测，不存储
+      this.fetchAndNotify(scriptId).catch(() => {});
+    }
+  }
+
+  async fetchAndNotify(scriptId) {
+    try {
+      // 添加超时保护防止 CDP 命令挂起
+      const sourcePromise = this.client.send('Debugger.getScriptSource', { scriptId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getScriptSource timeout')), 5000)
+      );
+      const { scriptSource } = await Promise.race([sourcePromise, timeoutPromise]);
+      try { this.onSource(scriptId, scriptSource); } catch { /* 订阅者异常不影响主流程 */ }
+    } catch {
+      // 获取失败（脚本已卸载等），忽略
+    }
   }
 
   async fetchAndSave(scriptId, url) {
     try {
-      const { scriptSource } = await this.client.send(
-        'Debugger.getScriptSource',
-        { scriptId }
+      // 添加超时保护防止 CDP 命令挂起
+      const sourcePromise = this.client.send('Debugger.getScriptSource', { scriptId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getScriptSource timeout')), 5000)
       );
+      const { scriptSource } = await Promise.race([sourcePromise, timeoutPromise]);
+
+      // 通知订阅者（AntiDebugInterceptor 等）
+      try { this.onSource?.(scriptId, scriptSource); } catch { /* 订阅者异常不影响主流程 */ }
 
       // 限制大小，超大脚本只保存部分
       const source = scriptSource.slice(0, 500000);

package/src/browser/interceptors/index.js

@@ -4,3 +4,4 @@
 
 export { NetworkInterceptor } from './NetworkInterceptor.js';
 export { ScriptInterceptor } from './ScriptInterceptor.js';
+export { AntiDebugInterceptor } from './AntiDebugInterceptor.js';