deepspider 0.1.0

package/src/browser/hooks/crypto.js

@@ -0,0 +1,55 @@
+/**
+ * DeepSpider - 加密库 Hook
+ * 已废弃，请使用 src/env/CryptoHook.js
+ */
+
+export const cryptoHook = `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider) return;
+
+  // Hook Function.prototype.apply (CryptoJS)
+  const _apply = Function.prototype.apply;
+  const applyHook = function() {
+    const result = _apply.call(this, ...arguments);
+    try {
+      if (arguments.length === 2 && arguments[1]?.[0]) {
+        const cfg = arguments[1][0];
+        if (cfg.ciphertext && cfg.key && cfg.algorithm) {
+          deepspider.log('crypto', {
+            algo: 'CryptoJS',
+            key: cfg.key?.toString?.() || '',
+            iv: cfg.iv?.toString?.() || '',
+            mode: cfg.mode?.name || 'unknown'
+          });
+        }
+      }
+    } catch (e) {}
+    return result;
+  };
+  Function.prototype.apply = deepspider.native(applyHook, _apply);
+
+  // Hook RSA
+  const _call = Function.prototype.call;
+  const callHook = function() {
+    const result = _call.call(this, ...arguments);
+    try {
+      const arg = arguments[0];
+      if (arg?.__proto__?.getPublicKey && arg?.__proto__?.encrypt) {
+        const proto = arg.__proto__.__proto__;
+        if (proto?.encrypt && !proto.__hooked__) {
+          proto.__hooked__ = true;
+          const _enc = proto.encrypt;
+          proto.encrypt = deepspider.native(function(data) {
+            const enc = _enc.call(this, data);
+            deepspider.log('crypto', { algo: 'RSA', data, encrypted: enc });
+            return enc;
+          }, _enc);
+        }
+      }
+    } catch (e) {}
+    return result;
+  };
+  Function.prototype.call = deepspider.native(callHook, _call);
+})();
+`;

package/src/browser/hooks/index.js

@@ -0,0 +1,64 @@
+/**
+ * DeepSpider - Hook 管理器
+ */
+
+import { cryptoHook } from './crypto.js';
+import { networkHook } from './network.js';
+import { nativeProtect } from './native.js';
+
+export class HookManager {
+  constructor() {
+    this.logs = [];
+    this.onLog = null;
+  }
+
+  /**
+   * 获取完整的 Hook 脚本
+   */
+  getCombinedScript() {
+    return [
+      nativeProtect,
+      cryptoHook,
+      networkHook,
+    ].join('\n\n');
+  }
+
+  /**
+   * 注入 Hook 到页面
+   */
+  async inject(page) {
+    const script = this.getCombinedScript();
+
+    // 在新文档加载前注入
+    await page.addInitScript(script);
+
+    // 监听 console 输出
+    page.on('console', (msg) => {
+      const text = msg.text();
+      if (text.includes('[DeepSpider:')) {
+        this.logs.push({
+          type: msg.type(),
+          text,
+          timestamp: Date.now(),
+        });
+        if (this.onLog) {
+          this.onLog({ type: msg.type(), text });
+        }
+      }
+    });
+  }
+
+  /**
+   * 获取捕获的日志
+   */
+  getLogs() {
+    return this.logs;
+  }
+
+  /**
+   * 清空日志
+   */
+  clearLogs() {
+    this.logs = [];
+  }
+}

package/src/browser/hooks/native.js

@@ -0,0 +1,9 @@
+/**
+ * DeepSpider - Hook 反检测工具
+ * 已废弃，请使用 src/env/HookBase.js
+ */
+
+import { HookBase } from '../../env/HookBase.js';
+
+// 兼容旧代码
+export const nativeProtect = HookBase.getBaseCode();

package/src/browser/hooks/network.js

@@ -0,0 +1,33 @@
+/**
+ * DeepSpider - 网络请求 Hook
+ * 已废弃，请使用 src/env/NetworkHook.js
+ */
+
+export const networkHook = `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider) return;
+
+  // Hook fetch
+  const _fetch = window.fetch;
+  window.fetch = deepspider.native(async function(url, options = {}) {
+    deepspider.log('fetch', { url, body: options.body });
+    return _fetch.call(this, url, options);
+  }, _fetch);
+
+  // Hook XHR
+  const _open = XMLHttpRequest.prototype.open;
+  const _send = XMLHttpRequest.prototype.send;
+
+  XMLHttpRequest.prototype.open = deepspider.native(function(method, url) {
+    this._url = url;
+    this._method = method;
+    return _open.apply(this, arguments);
+  }, _open);
+
+  XMLHttpRequest.prototype.send = deepspider.native(function(body) {
+    deepspider.log('xhr', { method: this._method, url: this._url, body });
+    return _send.apply(this, arguments);
+  }, _send);
+})();
+`;

package/src/browser/index.js

@@ -0,0 +1,42 @@
+/**
+ * DeepSpider - 浏览器基础设施
+ * 提供真实浏览器环境，作为动态分析的基础
+ */
+
+export { BrowserClient } from './client.js';
+export { CDPSession } from './cdp.js';
+export { HookManager } from './hooks/index.js';
+export { EnvCollector } from './collector.js';
+export { EnvBridge } from './EnvBridge.js';
+
+// 单例实例
+let browserInstance = null;
+
+/**
+ * 获取浏览器实例（单例）
+ */
+export async function getBrowser(options = {}) {
+  if (!browserInstance) {
+    const { BrowserClient } = await import('./client.js');
+    browserInstance = new BrowserClient();
+    await browserInstance.launch(options);
+  }
+  return browserInstance;
+}
+
+/**
+ * 关闭浏览器
+ */
+export async function closeBrowser() {
+  if (browserInstance) {
+    await browserInstance.close();
+    browserInstance = null;
+  }
+}
+
+/**
+ * 获取当前浏览器客户端（不创建新实例）
+ */
+export function getBrowserClient() {
+  return browserInstance;
+}

package/src/browser/interceptors/NetworkInterceptor.js

@@ -0,0 +1,116 @@
+/**
+ * DeepSpider - CDP 网络拦截器
+ * 通过 CDP 捕获网络请求/响应，按站点存储到文件系统
+ */
+
+import { getDataStore } from '../../store/DataStore.js';
+
+export class NetworkInterceptor {
+  constructor(cdpClient, page) {
+    this.client = cdpClient;
+    this.page = page;  // Playwright page 对象
+    this.store = getDataStore();
+    this.pendingRequests = new Map();
+  }
+
+  /**
+   * 获取当前页面 URL
+   */
+  getPageUrl() {
+    try {
+      return this.page?.url() || '';
+    } catch {
+      return '';
+    }
+  }
+
+  /**
+   * 启动拦截
+   */
+  async start() {
+    // 启用网络域
+    await this.client.send('Network.enable');
+
+    // 监听请求
+    this.client.on('Network.requestWillBeSent', (params) => {
+      this.onRequest(params);
+    });
+
+    // 监听响应
+    this.client.on('Network.responseReceived', (params) => {
+      this.onResponse(params);
+    });
+
+    // 监听加载完成
+    this.client.on('Network.loadingFinished', (params) => {
+      this.onLoadingFinished(params);
+    });
+
+    console.log('[NetworkInterceptor] 已启动');
+  }
+
+  onRequest(params) {
+    const { requestId, request, timestamp } = params;
+
+    // 只记录 XHR/Fetch 请求
+    const type = params.type;
+    if (type !== 'XHR' && type !== 'Fetch') return;
+
+    this.pendingRequests.set(requestId, {
+      url: request.url,
+      method: request.method,
+      headers: request.headers,
+      postData: request.postData,
+      timestamp: timestamp * 1000,
+      pageUrl: this.getPageUrl()  // 记录请求时的页面 URL
+    });
+  }
+
+  onResponse(params) {
+    const { requestId, response } = params;
+    const pending = this.pendingRequests.get(requestId);
+    if (!pending) return;
+
+    pending.status = response.status;
+    pending.responseHeaders = response.headers;
+  }
+
+  async onLoadingFinished(params) {
+    const { requestId } = params;
+    const pending = this.pendingRequests.get(requestId);
+    if (!pending) return;
+
+    try {
+      // 获取响应体
+      const { body, base64Encoded } = await this.client.send(
+        'Network.getResponseBody',
+        { requestId }
+      );
+
+      const responseBody = base64Encoded
+        ? Buffer.from(body, 'base64').toString('utf-8')
+        : body;
+
+      // 异步存储到文件
+      this.store.saveResponse({
+        url: pending.url,
+        method: pending.method,
+        status: pending.status,
+        requestHeaders: pending.headers,
+        requestBody: pending.postData,
+        responseBody: responseBody.slice(0, 50000),
+        timestamp: pending.timestamp,
+        pageUrl: pending.pageUrl  // 传递页面 URL 用于分站点存储
+      }).catch(e => {
+        console.error('[NetworkInterceptor] 保存失败:', e.message);
+      });
+
+    } catch (e) {
+      // 某些响应无法获取 body
+    }
+
+    this.pendingRequests.delete(requestId);
+  }
+}
+
+export default NetworkInterceptor;

package/src/browser/interceptors/ScriptInterceptor.js

@@ -0,0 +1,76 @@
+/**
+ * DeepSpider - CDP 脚本拦截器
+ * 通过 CDP 捕获 JS 脚本源码，按站点存储到文件系统
+ */
+
+import { getDataStore } from '../../store/DataStore.js';
+
+export class ScriptInterceptor {
+  constructor(cdpClient, page) {
+    this.client = cdpClient;
+    this.page = page;  // Playwright page 对象
+    this.store = getDataStore();
+    this.scriptIds = new Set();
+  }
+
+  /**
+   * 获取当前页面 URL
+   */
+  getPageUrl() {
+    try {
+      return this.page?.url() || '';
+    } catch {
+      return '';
+    }
+  }
+
+  /**
+   * 启动拦截
+   */
+  async start() {
+    await this.client.send('Debugger.enable');
+
+    this.client.on('Debugger.scriptParsed', (params) => {
+      this.onScriptParsed(params);
+    });
+
+    console.log('[ScriptInterceptor] 已启动');
+  }
+
+  async onScriptParsed(params) {
+    const { scriptId, url, length } = params;
+
+    // 跳过扩展和空脚本
+    if (!url || url.startsWith('chrome-extension://')) return;
+    if (this.scriptIds.has(scriptId)) return;
+
+    this.scriptIds.add(scriptId);
+
+    // 异步获取并存储源码
+    this.fetchAndSave(scriptId, url).catch(() => {});
+  }
+
+  async fetchAndSave(scriptId, url) {
+    try {
+      const { scriptSource } = await this.client.send(
+        'Debugger.getScriptSource',
+        { scriptId }
+      );
+
+      // 限制大小，超大脚本只保存部分
+      const source = scriptSource.slice(0, 500000);
+
+      await this.store.saveScript({
+        url,
+        type: 'external',
+        source,
+        timestamp: Date.now(),
+        pageUrl: this.getPageUrl()  // 传递页面 URL
+      });
+    } catch (e) {
+      // 获取失败，跳过
+    }
+  }
+}
+
+export default ScriptInterceptor;

package/src/browser/interceptors/index.js

@@ -0,0 +1,6 @@
+/**
+ * DeepSpider - 拦截器索引
+ */
+
+export { NetworkInterceptor } from './NetworkInterceptor.js';
+export { ScriptInterceptor } from './ScriptInterceptor.js';