deepspider 0.3.0 → 0.3.2

package/src/browser/interceptors/AntiDebugInterceptor.js

@@ -0,0 +1,132 @@
+/**
+ * DeepSpider - 反无限 debugger 拦截器
+ * 通过 CDP Debugger.setBlackboxedRanges 跳过包含 debugger 语句的脚本
+ * 零运行时开销，不修改源码，不触发完整性校验
+ *
+ * 已知限制：/\bdebugger\b/ 会匹配字符串/注释中的 debugger，
+ * 对反爬场景可接受（误 blackbox 的脚本仍正常执行，只是不可调试）
+ */
+
+export class AntiDebugInterceptor {
+  constructor(cdpClient) {
+    this.client = cdpClient;
+    this.blackboxedScripts = new Set();
+    // 高频 debugger 检测
+    this.pausedCount = 0;
+    this.pausedWindowStart = 0;
+    this.PAUSED_WINDOW_MS = 1000;  // 1秒窗口
+    this.PAUSED_THRESHOLD = 5;     // 1秒内超过5次paused认为是debugger风暴
+    this.stormMode = false;        // 风暴模式：跳过所有断点
+    this.stormTimer = null;        // 风暴模式自动退出定时器
+  }
+
+  async start() {
+    // 兜底：对于 blackbox 来不及处理的同步 debugger（时序竞争），自动 resume
+    // reason 可能是 'other' 或 'debugCommand'（不同 Chrome 版本），
+    // 只要不是我们主动设的断点（hitBreakpoints 非空 / reason=breakpoint）就 resume
+    this.client.on('Debugger.paused', (params) => {
+      // 手动设置的断点（除非在风暴模式）
+      if (!this.stormMode && params.reason === 'breakpoint') return;
+      if (!this.stormMode && params.hitBreakpoints?.length > 0) return;
+
+      // 风暴模式下直接 resume，不参与计数
+      if (this.stormMode) {
+        this.client.send('Debugger.resume').catch(() => {});
+        return;
+      }
+
+      // 高频 debugger 检测
+      const now = Date.now();
+      if (now - this.pausedWindowStart > this.PAUSED_WINDOW_MS) {
+        // 新窗口
+        this.pausedWindowStart = now;
+        this.pausedCount = 1;
+      } else {
+        this.pausedCount++;
+      }
+
+      // 触发风暴模式
+      if (this.pausedCount > this.PAUSED_THRESHOLD) {
+        console.log('[AntiDebugInterceptor] 检测到 debugger 风暴，启用风暴模式');
+        this.stormMode = true;
+        // 清除之前的定时器
+        if (this.stormTimer) {
+          clearTimeout(this.stormTimer);
+        }
+        // 3秒后退出风暴模式
+        this.stormTimer = setTimeout(() => {
+          console.log('[AntiDebugInterceptor] 退出风暴模式');
+          this.stormMode = false;
+          this.pausedCount = 0;
+          this.stormTimer = null;
+        }, 3000);
+      }
+
+      // 自动 resume
+      this.client.send('Debugger.resume').catch(() => {});
+    });
+
+    console.log('[AntiDebugInterceptor] 已启动');
+  }
+
+  /**
+   * 检查脚本源码，包含 debugger 则 blackbox 整个脚本
+   * 由 ScriptInterceptor.onSource 回调驱动，避免重复拉取源码
+   */
+  checkScript(scriptId, scriptSource) {
+    if (/\bdebugger\b/.test(scriptSource)) {
+      this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [{ lineNumber: 0, columnNumber: 0 }],
+      }).then(() => {
+        this.blackboxedScripts.add(scriptId);
+      }).catch(() => {});
+    }
+  }
+
+  /**
+   * 取消指定脚本的 blackbox（供断点工具调用）
+   */
+  async unblackbox(scriptId) {
+    if (this.blackboxedScripts.has(scriptId)) {
+      await this.client.send('Debugger.setBlackboxedRanges', {
+        scriptId,
+        positions: [],
+      });
+      this.blackboxedScripts.delete(scriptId);
+    }
+  }
+
+  /**
+   * 手动启用/禁用风暴模式
+   * 用于绕过强反调试场景
+   */
+  setStormMode(enabled) {
+    // 清除之前的定时器
+    if (this.stormTimer) {
+      clearTimeout(this.stormTimer);
+      this.stormTimer = null;
+    }
+
+    this.stormMode = enabled;
+    if (enabled) {
+      console.log('[AntiDebugInterceptor] 手动启用风暴模式');
+      // 自动退出
+      this.stormTimer = setTimeout(() => {
+        this.stormMode = false;
+        this.stormTimer = null;
+        console.log('[AntiDebugInterceptor] 自动退出风暴模式');
+      }, 5000);
+    } else {
+      console.log('[AntiDebugInterceptor] 手动禁用风暴模式');
+      this.pausedCount = 0;
+    }
+  }
+
+  /**
+   * 检查当前是否在风暴模式
+   */
+  isStormMode() {
+    return this.stormMode;
+  }
+}

package/src/browser/interceptors/NetworkInterceptor.js

@@ -46,11 +46,16 @@ export class NetworkInterceptor {
       this.onLoadingFinished(params);
     });
 
+    // 监听加载失败（清理 pendingRequests，防止内存泄漏）
+    this.client.on('Network.loadingFailed', (params) => {
+      this.pendingRequests.delete(params.requestId);
+    });
+
     console.log('[NetworkInterceptor] 已启动');
   }
 
   onRequest(params) {
-    const { requestId, request, timestamp } = params;
+    const { requestId, request, timestamp, initiator } = params;
 
     // 只记录 XHR/Fetch 请求
     const type = params.type;
@@ -62,10 +67,37 @@ export class NetworkInterceptor {
       headers: request.headers,
       postData: request.postData,
       timestamp: timestamp * 1000,
-      pageUrl: this.getPageUrl()  // 记录请求时的页面 URL
+      pageUrl: this.getPageUrl(),
+      initiator: this.formatInitiator(initiator),
     });
   }
 
+  /**
+   * 精简 initiator 调用栈（只保留前 5 帧，过滤内部帧）
+   */
+  formatInitiator(initiator) {
+    if (!initiator) return null;
+    const result = { type: initiator.type };
+    if (initiator.url) {
+      result.url = initiator.url;
+      result.lineNumber = initiator.lineNumber;
+    }
+    if (initiator.stack?.callFrames) {
+      result.callFrames = initiator.stack.callFrames
+        .filter(f => f.url && !f.url.includes('patchright') && !f.url.includes('__playwright'))
+        .slice(0, 5)
+        .map(f => ({
+          functionName: f.functionName || '(anonymous)',
+          url: f.url,
+          lineNumber: f.lineNumber,
+          columnNumber: f.columnNumber,
+        }));
+    }
+    // 只有 type 没有实际定位信息时返回 null
+    if (!result.url && !result.callFrames?.length) return null;
+    return result;
+  }
+
   onResponse(params) {
     const { requestId, response } = params;
     const pending = this.pendingRequests.get(requestId);
@@ -81,15 +113,31 @@ export class NetworkInterceptor {
     if (!pending) return;
 
     try {
-      // 获取响应体
-      const { body, base64Encoded } = await this.client.send(
-        'Network.getResponseBody',
-        { requestId }
+      // 获取响应体，添加超时保护防止 CDP 命令挂起
+      const bodyPromise = this.client.send('Network.getResponseBody', { requestId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getResponseBody timeout')), 5000)
       );
-
-      const responseBody = base64Encoded
-        ? Buffer.from(body, 'base64').toString('utf-8')
-        : body;
+      const { body, base64Encoded } = await Promise.race([bodyPromise, timeoutPromise]);
+
+      // 处理响应体：检测二进制内容，避免损坏
+      let responseBody;
+      const contentType = pending.responseHeaders?.['content-type'] || '';
+
+      if (this.isBinaryContent(contentType)) {
+        // 二进制内容：存储元数据而非原始内容
+        // base64 长度计算：每 4 个字符 = 3 字节，考虑 padding
+        const binarySize = base64Encoded
+          ? Math.floor(body.length * 0.75) - (body.match(/=*$/)?.[0].length || 0)
+          : body.length;
+        responseBody = `[Binary: ${contentType}, ${binarySize} bytes]`;
+      } else {
+        // 文本内容：安全地转换为字符串
+        const rawBody = base64Encoded
+          ? Buffer.from(body, 'base64').toString('utf-8')
+          : body;
+        responseBody = rawBody.slice(0, 50000);
+      }
 
       // 异步存储到文件
       this.store.saveResponse({
@@ -98,19 +146,35 @@ export class NetworkInterceptor {
         status: pending.status,
         requestHeaders: pending.headers,
         requestBody: pending.postData,
-        responseBody: responseBody.slice(0, 50000),
+        responseBody,
         timestamp: pending.timestamp,
-        pageUrl: pending.pageUrl  // 传递页面 URL 用于分站点存储
+        pageUrl: pending.pageUrl,
+        initiator: pending.initiator,
       }).catch(e => {
         console.error('[NetworkInterceptor] 保存失败:', e.message);
       });
 
-    } catch (e) {
+    } catch {
       // 某些响应无法获取 body
     }
 
     this.pendingRequests.delete(requestId);
   }
+
+  /**
+   * 检测是否为二进制内容类型
+   */
+  isBinaryContent(contentType) {
+    if (!contentType) return false;
+    const binaryTypes = [
+      'image/', 'audio/', 'video/', 'application/pdf',
+      'application/octet-stream', 'application/zip',
+      'application/gzip', 'application/x-protobuf',
+      'font/', 'application/vnd.'
+    ];
+    const lowerType = contentType.toLowerCase();
+    return binaryTypes.some(type => lowerType.includes(type));
+  }
 }
 
 export default NetworkInterceptor;

package/src/browser/interceptors/ScriptInterceptor.js

@@ -11,6 +11,7 @@ export class ScriptInterceptor {
     this.page = page;  // Playwright page 对象
     this.store = getDataStore();
     this.scriptIds = new Set();
+    this.onSource = null;  // 回调: (scriptId, scriptSource) => void
   }
 
   /**
@@ -38,24 +39,48 @@ export class ScriptInterceptor {
   }
 
   async onScriptParsed(params) {
-    const { scriptId, url, length } = params;
+    const { scriptId, url, length: _length } = params;
 
-    // 跳过扩展和空脚本
-    if (!url || url.startsWith('chrome-extension://')) return;
+    // 跳过扩展脚本
+    if (url?.startsWith('chrome-extension://')) return;
     if (this.scriptIds.has(scriptId)) return;
 
     this.scriptIds.add(scriptId);
 
-    // 异步获取并存储源码
-    this.fetchAndSave(scriptId, url).catch(() => {});
+    if (url) {
+      // 有 URL 的脚本：获取源码、通知订阅者、存储
+      this.fetchAndSave(scriptId, url).catch(() => {});
+    } else if (this.onSource) {
+      // 无 URL 脚本（eval/new Function 生成）：仅通知订阅者用于 debugger 检测，不存储
+      this.fetchAndNotify(scriptId).catch(() => {});
+    }
+  }
+
+  async fetchAndNotify(scriptId) {
+    try {
+      // 添加超时保护防止 CDP 命令挂起
+      const sourcePromise = this.client.send('Debugger.getScriptSource', { scriptId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getScriptSource timeout')), 5000)
+      );
+      const { scriptSource } = await Promise.race([sourcePromise, timeoutPromise]);
+      try { this.onSource(scriptId, scriptSource); } catch { /* 订阅者异常不影响主流程 */ }
+    } catch {
+      // 获取失败（脚本已卸载等），忽略
+    }
   }
 
   async fetchAndSave(scriptId, url) {
     try {
-      const { scriptSource } = await this.client.send(
-        'Debugger.getScriptSource',
-        { scriptId }
+      // 添加超时保护防止 CDP 命令挂起
+      const sourcePromise = this.client.send('Debugger.getScriptSource', { scriptId });
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('getScriptSource timeout')), 5000)
       );
+      const { scriptSource } = await Promise.race([sourcePromise, timeoutPromise]);
+
+      // 通知订阅者（AntiDebugInterceptor 等）
+      try { this.onSource?.(scriptId, scriptSource); } catch { /* 订阅者异常不影响主流程 */ }
 
       // 限制大小，超大脚本只保存部分
       const source = scriptSource.slice(0, 500000);
@@ -67,7 +92,7 @@ export class ScriptInterceptor {
         timestamp: Date.now(),
         pageUrl: this.getPageUrl()  // 传递页面 URL
       });
-    } catch (e) {
+    } catch {
       // 获取失败，跳过
     }
   }

package/src/browser/interceptors/index.js

@@ -4,3 +4,4 @@
 
 export { NetworkInterceptor } from './NetworkInterceptor.js';
 export { ScriptInterceptor } from './ScriptInterceptor.js';
+export { AntiDebugInterceptor } from './AntiDebugInterceptor.js';