deepspider 0.1.0

package/src/analyzer/EncryptionAnalyzer.js

@@ -0,0 +1,99 @@
+/**
+ * DeepSpider - 加密分析器
+ * 算法识别、参数追踪、密钥提取
+ */
+
+import { ASTAnalyzer } from './ASTAnalyzer.js';
+import { cryptoPatterns } from '../config/patterns/crypto.js';
+
+export class EncryptionAnalyzer {
+  constructor() {
+    this.astAnalyzer = new ASTAnalyzer();
+    this.cryptoPatterns = [
+      { name: 'MD5', pattern: /md5|MD5/, type: 'hash' },
+      { name: 'SHA1', pattern: /sha1|SHA1/, type: 'hash' },
+      { name: 'SHA256', pattern: /sha256|SHA256/, type: 'hash' },
+      { name: 'AES', pattern: /\baes\b|AES|CryptoJS\.AES/, type: 'symmetric' },
+      { name: 'DES', pattern: /\bdes\b|DES|CryptoJS\.DES/, type: 'symmetric' },
+      { name: 'Base64', pattern: /btoa|atob|base64/i, type: 'encoding' },
+      { name: 'RSA', pattern: /\brsa\b|RSA|JSEncrypt/, type: 'asymmetric' },
+      { name: 'HMAC', pattern: /hmac|HMAC/, type: 'mac' }
+    ];
+  }
+
+  analyze(code) {
+    const functions = this.astAnalyzer.extractFunctions(code);
+    const calls = this.astAnalyzer.extractCalls(code);
+    const detected = [];
+
+    // 检测加密模式
+    for (const p of this.cryptoPatterns) {
+      if (p.pattern.test(code)) {
+        detected.push({ name: p.name, type: p.type });
+      }
+    }
+
+    // 查找可疑的加密函数
+    const suspiciousFuncs = functions.filter(f =>
+      /encrypt|decrypt|sign|hash|encode|decode|cipher/i.test(f.name)
+    );
+
+    // 查找密钥相关变量
+    const keys = this._findKeys(code);
+
+    return {
+      detectedAlgorithms: detected,
+      suspiciousFunctions: suspiciousFuncs,
+      possibleKeys: keys,
+      totalFunctions: functions.length,
+      totalCalls: calls.length
+    };
+  }
+
+  traceParam(code, paramName) {
+    const ast = this.astAnalyzer.parse(code);
+    const traces = [];
+
+    // 简化实现：查找参数使用位置
+    const regex = new RegExp(`\\b${paramName}\\b`, 'g');
+    let match;
+    while ((match = regex.exec(code)) !== null) {
+      traces.push({ position: match.index });
+    }
+
+    return { param: paramName, traces };
+  }
+
+  _findKeys(code) {
+    const keys = [];
+    const keyPattern = /(?:key|secret|password|iv|salt)\s*[=:]\s*['"]([^'"]+)['"]/gi;
+    let match;
+    while ((match = keyPattern.exec(code)) !== null) {
+      keys.push({
+        value: match[1],
+        position: match.index
+      });
+    }
+    return keys;
+  }
+
+  // 使用模式库进行深度检测
+  detectWithPatterns(code) {
+    const detected = [];
+    for (const [key, pattern] of Object.entries(cryptoPatterns)) {
+      for (const sig of pattern.signatures) {
+        if (sig.test(code)) {
+          detected.push({
+            algorithm: pattern.name,
+            type: pattern.type,
+            confidence: 0.9
+          });
+          break;
+        }
+      }
+    }
+    return detected;
+  }
+}
+
+export default EncryptionAnalyzer;

package/src/analyzer/index.js

@@ -0,0 +1,22 @@
+/**
+ * DeepSpider - 分析器注册表
+ */
+
+import { ASTAnalyzer } from './ASTAnalyzer.js';
+import { CallStackAnalyzer } from './CallStackAnalyzer.js';
+import { EncryptionAnalyzer } from './EncryptionAnalyzer.js';
+import { Deobfuscator } from './Deobfuscator.js';
+
+export const analyzers = {
+  ast: ASTAnalyzer,
+  callstack: CallStackAnalyzer,
+  encryption: EncryptionAnalyzer,
+  deobfuscator: Deobfuscator
+};
+
+export function getAnalyzer(name) {
+  const Analyzer = analyzers[name];
+  return Analyzer ? new Analyzer() : null;
+}
+
+export default analyzers;

package/src/browser/EnvBridge.js

@@ -0,0 +1,186 @@
+/**
+ * DeepSpider - 环境补全桥接器
+ * 连接 EnvDumper（发现缺失）与 EnvCollector（采集真实数据）
+ */
+
+import { EnvCollector } from './collector.js';
+import { PatchGenerator } from '../core/PatchGenerator.js';
+
+export class EnvBridge {
+  constructor(page) {
+    this.collector = new EnvCollector(page);
+    this.patchGenerator = new PatchGenerator();
+    this.collectedData = new Map();
+  }
+
+  /**
+   * 根据缺失列表自动采集并生成补丁
+   * @param {string[]} missingPaths - EnvDumper 发现的缺失属性路径
+   */
+  async autoFix(missingPaths) {
+    const results = {
+      collected: [],
+      patched: [],
+      failed: []
+    };
+
+    for (const path of missingPaths) {
+      try {
+        // 1. 从真实浏览器采集
+        const collected = await this.collector.collect(path, { depth: 2 });
+
+        if (!collected.success) {
+          results.failed.push({ path, reason: 'collect_failed', error: collected.error });
+          continue;
+        }
+
+        results.collected.push(path);
+        this.collectedData.set(path, collected);
+
+        // 2. 生成补丁代码
+        const patch = this._generatePatch(path, collected);
+        if (patch) {
+          results.patched.push({ path, code: patch });
+        }
+
+      } catch (e) {
+        results.failed.push({ path, reason: 'error', error: e.message });
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * 根据采集数据生成补丁代码
+   */
+  _generatePatch(path, collected) {
+    const { data, descriptor } = collected;
+
+    if (!data) return null;
+
+    const parts = path.split('.');
+    const propName = parts.pop();
+    const parentPath = parts.join('.') || 'window';
+
+    // 根据数据类型生成不同的补丁
+    switch (data.type) {
+      case 'string':
+        return `${parentPath}.${propName} = "${data.value}";`;
+
+      case 'number':
+        return `${parentPath}.${propName} = ${data.value};`;
+
+      case 'boolean':
+        return `${parentPath}.${propName} = ${data.value};`;
+
+      case 'array':
+        return `${parentPath}.${propName} = ${JSON.stringify(
+          data.value.map(v => v.value)
+        )};`;
+
+      case 'object':
+        if (descriptor?.hasGetter) {
+          // 有 getter 的属性用 defineProperty
+          return this._generateDefineProperty(parentPath, propName, data);
+        }
+        return this._generateObjectPatch(parentPath, propName, data);
+
+      case 'function':
+        return `${parentPath}.${propName} = function ${data.name || ''}() {};`;
+
+      default:
+        return `${parentPath}.${propName} = undefined;`;
+    }
+  }
+
+  _generateDefineProperty(parentPath, propName, data) {
+    const value = this._serializeValue(data);
+    return `Object.defineProperty(${parentPath}, "${propName}", {
+  get: function() { return ${value}; },
+  configurable: true
+});`;
+  }
+
+  _generateObjectPatch(parentPath, propName, data) {
+    if (!data.properties) {
+      return `${parentPath}.${propName} = {};`;
+    }
+
+    const props = [];
+    for (const [key, val] of Object.entries(data.properties)) {
+      if (val.type === 'function') {
+        props.push(`${key}: function() {}`);
+      } else if (val.type === 'object') {
+        props.push(`${key}: {}`);
+      } else {
+        props.push(`${key}: ${this._serializeValue(val)}`);
+      }
+    }
+
+    return `${parentPath}.${propName} = { ${props.join(', ')} };`;
+  }
+
+  _serializeValue(data) {
+    switch (data.type) {
+      case 'string': return `"${data.value}"`;
+      case 'number': return data.value;
+      case 'boolean': return data.value;
+      case 'null': return 'null';
+      case 'undefined': return 'undefined';
+      case 'array': return JSON.stringify(data.value?.map(v => v.value) || []);
+      default: return '{}';
+    }
+  }
+
+  /**
+   * 生成合并后的完整补丁代码
+   */
+  generateMergedPatch(patches) {
+    const grouped = new Map();
+
+    for (const { path, code } of patches) {
+      const root = path.split('.')[0];
+      if (!grouped.has(root)) {
+        grouped.set(root, []);
+      }
+      grouped.get(root).push(code);
+    }
+
+    const lines = ['// DeepSpider Auto-Generated Patch'];
+    for (const [root, codes] of grouped) {
+      lines.push(`\n// === ${root} ===`);
+      lines.push(...codes);
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * 完整的补环境流程
+   */
+  async runFullPipeline(missingPaths) {
+    console.log(`[DeepSpider:bridge] 开始补环境，共 ${missingPaths.length} 个缺失属性`);
+
+    // 1. 采集并生成补丁
+    const result = await this.autoFix(missingPaths);
+
+    // 2. 合并补丁
+    const mergedCode = this.generateMergedPatch(result.patched);
+
+    // 3. 返回结果
+    return {
+      success: result.failed.length === 0,
+      stats: {
+        total: missingPaths.length,
+        collected: result.collected.length,
+        patched: result.patched.length,
+        failed: result.failed.length
+      },
+      patchCode: mergedCode,
+      failed: result.failed
+    };
+  }
+}
+
+export default EnvBridge;

package/src/browser/cdp.js

@@ -0,0 +1,168 @@
+/**
+ * DeepSpider - CDP 会话管理
+ * 封装 Chrome DevTools Protocol 操作
+ */
+
+export class CDPSession {
+  constructor(client) {
+    this.client = client;
+    this.breakpoints = new Map();
+    this.scriptSources = new Map();
+    this.pausedHandler = null;
+  }
+
+  /**
+   * 从 BrowserClient 创建 CDP 会话
+   */
+  static async fromBrowser(browserClient) {
+    const client = await browserClient.getCDPSession();
+    const session = new CDPSession(client);
+    await session.enable();
+    return session;
+  }
+
+  /**
+   * 启用必要的 CDP 域
+   */
+  async enable() {
+    await Promise.all([
+      this.client.send('Debugger.enable'),
+      this.client.send('Runtime.enable'),
+      this.client.send('Page.enable'),
+      this.client.send('Network.enable'),
+    ]);
+
+    // 设置异步调用栈深度
+    await this.client.send('Debugger.setAsyncCallStackDepth', { maxDepth: 32 });
+
+    // 监听脚本解析事件
+    this.client.on('Debugger.scriptParsed', (event) => {
+      if (event.url) {
+        this.scriptSources.set(event.scriptId, event.url);
+      }
+    });
+  }
+
+  /**
+   * 设置断点
+   */
+  async setBreakpoint(url, lineNumber, columnNumber = 0) {
+    const result = await this.client.send('Debugger.setBreakpointByUrl', {
+      url,
+      lineNumber,
+      columnNumber,
+    });
+    if (result.breakpointId) {
+      this.breakpoints.set(result.breakpointId, { url, lineNumber, columnNumber });
+    }
+    return result;
+  }
+
+  /**
+   * 移除断点
+   */
+  async removeBreakpoint(breakpointId) {
+    await this.client.send('Debugger.removeBreakpoint', { breakpointId });
+    this.breakpoints.delete(breakpointId);
+  }
+
+  /**
+   * 设置 XHR 断点
+   */
+  async setXHRBreakpoint(urlPattern = '') {
+    await this.client.send('DOMDebugger.setXHRBreakpoint', { url: urlPattern });
+  }
+
+  /**
+   * 恢复执行
+   */
+  async resume() {
+    await this.client.send('Debugger.resume');
+  }
+
+  /**
+   * 单步执行
+   */
+  async stepOver() {
+    await this.client.send('Debugger.stepOver');
+  }
+
+  async stepInto() {
+    await this.client.send('Debugger.stepInto');
+  }
+
+  async stepOut() {
+    await this.client.send('Debugger.stepOut');
+  }
+
+  /**
+   * 获取脚本源码
+   */
+  async getScriptSource(scriptId) {
+    const result = await this.client.send('Debugger.getScriptSource', { scriptId });
+    return result.scriptSource;
+  }
+
+  /**
+   * 获取作用域变量
+   */
+  async getScopeVariables(callFrameId) {
+    const result = await this.client.send('Debugger.evaluateOnCallFrame', {
+      callFrameId,
+      expression: 'JSON.stringify(Object.keys(this))',
+      returnByValue: true,
+    });
+    return result;
+  }
+
+  /**
+   * 在调用帧上执行表达式
+   */
+  async evaluate(callFrameId, expression) {
+    const result = await this.client.send('Debugger.evaluateOnCallFrame', {
+      callFrameId,
+      expression,
+      returnByValue: true,
+    });
+    return result.result?.value;
+  }
+
+  /**
+   * 监听断点暂停事件
+   */
+  onPaused(handler) {
+    this.pausedHandler = handler;
+    this.client.on('Debugger.paused', handler);
+  }
+
+  /**
+   * 移除暂停监听
+   */
+  offPaused() {
+    if (this.pausedHandler) {
+      this.client.off('Debugger.paused', this.pausedHandler);
+      this.pausedHandler = null;
+    }
+  }
+
+  /**
+   * 发送 CDP 命令（代理到 client）
+   */
+  send(method, params = {}) {
+    return this.client.send(method, params);
+  }
+
+  /**
+   * 监听 CDP 事件（代理到 client）
+   */
+  on(event, handler) {
+    return this.client.on(event, handler);
+  }
+
+  /**
+   * 移除 CDP 事件监听（代理到 client）
+   */
+  off(event, handler) {
+    return this.client.off(event, handler);
+  }
+}

package/src/browser/client.js

@@ -0,0 +1,197 @@
+/**
+ * DeepSpider - Patchright 浏览器客户端
+ * 使用反检测版 Playwright
+ */
+
+import { chromium } from 'patchright';
+import { getDefaultHookScript } from './defaultHooks.js';
+import { NetworkInterceptor } from './interceptors/NetworkInterceptor.js';
+import { ScriptInterceptor } from './interceptors/ScriptInterceptor.js';
+import { getDataStore } from '../store/DataStore.js';
+
+export class BrowserClient {
+  constructor() {
+    this.browser = null;
+    this.context = null;
+    this.page = null;
+    this.pages = [];
+    this.cdpSession = null;
+    this.networkInterceptor = null;
+    this.scriptInterceptor = null;
+    this.hookScript = null;
+    this.onMessage = null;
+  }
+
+  /**
+   * 启动浏览器
+   */
+  async launch(options = {}) {
+    // 启动新会话
+    const dataStore = getDataStore();
+    dataStore.startSession();
+
+    const {
+      headless = false,
+      executablePath = null,
+      args = [],
+    } = options;
+
+    const launchOptions = {
+      headless,
+      args: [
+        '--disable-blink-features=AutomationControlled',
+        '--disable-web-security',
+        '--ignore-certificate-errors',
+        ...args,
+      ],
+    };
+
+    if (executablePath) {
+      launchOptions.executablePath = executablePath;
+    }
+
+    this.browser = await chromium.launch(launchOptions);
+    this.context = await this.browser.newContext({
+      ignoreHTTPSErrors: true,
+    });
+
+    // 保存 hook 脚本
+    this.hookScript = getDefaultHookScript();
+
+    // 使用 addInitScript 在 context 级别注入
+    await this.context.addInitScript(this.hookScript);
+
+    this.page = await this.context.newPage();
+
+    // 监听新页面创建（弹窗、新标签页）
+    this.context.on('page', async (newPage) => {
+      console.log('[BrowserClient] 检测到新页面');
+      this.pages.push(newPage);
+      this.page = newPage;  // 切换到新页面
+      await this.setupPage(newPage);
+
+      // 监听页面关闭
+      newPage.on('close', () => {
+        this.pages = this.pages.filter(p => p !== newPage);
+        if (this.page === newPage && this.pages.length > 0) {
+          this.page = this.pages[this.pages.length - 1];
+        }
+      });
+    });
+
+    // 设置当前页面
+    this.pages.push(this.page);
+    await this.setupPage(this.page);
+
+    return this;
+  }
+
+  /**
+   * 设置页面（CDP 拦截器 + 消息绑定）
+   */
+  async setupPage(page) {
+    try {
+      const cdp = await page.context().newCDPSession(page);
+
+      // 1. 启用 Runtime 域
+      await cdp.send('Runtime.enable');
+
+      // 2. 添加 CDP binding（前端调用此函数，后端接收）
+      await cdp.send('Runtime.addBinding', { name: '__deepspider_send__' });
+
+      // 3. 监听 binding 调用
+      cdp.on('Runtime.bindingCalled', (event) => {
+        if (event.name === '__deepspider_send__') {
+          try {
+            const data = JSON.parse(event.payload);
+            console.log('[BrowserClient] 收到消息:', data.type);
+            if (this.onMessage) {
+              this.onMessage(data, page);
+            }
+          } catch (e) {
+            console.error('[BrowserClient] 解析消息失败:', e.message);
+          }
+        }
+      });
+
+      // 4. 启动拦截器
+      const networkInterceptor = new NetworkInterceptor(cdp, page);
+      const scriptInterceptor = new ScriptInterceptor(cdp, page);
+      await networkInterceptor.start();
+      await scriptInterceptor.start();
+
+      // 保存引用
+      if (page === this.page) {
+        this.cdpSession = cdp;
+        this.networkInterceptor = networkInterceptor;
+        this.scriptInterceptor = scriptInterceptor;
+      }
+
+      // 监听页面导航
+      page.on('framenavigated', (frame) => {
+        if (frame === page.mainFrame()) {
+          console.log('[BrowserClient] 页面导航到:', frame.url());
+        }
+      });
+
+      console.log('[BrowserClient] 页面已设置:', page.url() || '(空白页)');
+    } catch (e) {
+      console.error('[BrowserClient] 设置页面失败:', e.message);
+    }
+  }
+
+  /**
+   * 获取 CDP 会话（始终使用当前页面的 session）
+   */
+  async getCDPSession() {
+    // 每次都为当前页面创建新的 CDP session，确保上下文正确
+    if (this.page) {
+      try {
+        this.cdpSession = await this.page.context().newCDPSession(this.page);
+      } catch (e) {
+        console.error('[BrowserClient] 创建 CDP session 失败:', e.message);
+        return null;
+      }
+    }
+    return this.cdpSession;
+  }
+
+  /**
+   * 导航到 URL
+   */
+  async navigate(url, options = {}) {
+    const { waitUntil = 'domcontentloaded' } = options;
+    await this.page.goto(url, { waitUntil });
+    return this.page.url();
+  }
+
+  /**
+   * 获取当前页面
+   */
+  getPage() {
+    return this.page;
+  }
+
+  /**
+   * 获取浏览器上下文
+   */
+  getContext() {
+    return this.context;
+  }
+
+  /**
+   * 关闭浏览器
+   */
+  async close() {
+    if (this.cdpSession) {
+      await this.cdpSession.detach().catch(() => {});
+      this.cdpSession = null;
+    }
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+      this.context = null;
+      this.page = null;
+    }
+  }
+}