deepspider 0.1.0

package/src/env/NetworkHook.js

@@ -0,0 +1,159 @@
+/**
+ * DeepSpider - 网络请求 Hook 模块
+ * 拦截 XHR 和 Fetch 请求/响应
+ */
+
+import { HookBase } from './HookBase.js';
+
+export class NetworkHook {
+  constructor() {
+    this.logs = [];
+  }
+
+  /**
+   * 生成 XHR Hook 代码
+   */
+  generateXHRHookCode(options = {}) {
+    const { captureBody = true, captureResponse = true } = options;
+
+    return HookBase.getBaseCode() + `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider) return;
+
+  const OriginalXHR = XMLHttpRequest;
+
+  XMLHttpRequest = function() {
+    const xhr = new OriginalXHR();
+    const log = {
+      method: '',
+      url: '',
+      requestHeaders: {},
+      requestBody: null,
+      response: null,
+      status: 0
+    };
+
+    // Hook open
+    const originalOpen = xhr.open;
+    xhr.open = deepspider.native(function(method, url) {
+      log.method = method;
+      log.url = url;
+      return originalOpen.apply(xhr, arguments);
+    }, originalOpen);
+
+    // Hook setRequestHeader
+    const originalSetHeader = xhr.setRequestHeader;
+    xhr.setRequestHeader = deepspider.native(function(name, value) {
+      log.requestHeaders[name] = value;
+      return originalSetHeader.apply(xhr, arguments);
+    }, originalSetHeader);
+
+    // Hook send
+    const originalSend = xhr.send;
+    xhr.send = deepspider.native(function(body) {
+      // 开始请求上下文
+      log.requestId = deepspider.startRequest(log.url, log.method);
+      ${captureBody ? `log.requestBody = body;` : ''}
+      deepspider.log('xhr', { action: 'send', ...log, body: body?.toString().slice(0, 200) });
+      return originalSend.apply(xhr, arguments);
+    }, originalSend);
+
+    ${captureResponse ? `
+    xhr.addEventListener('load', function() {
+      log.status = xhr.status;
+      log.response = xhr.responseText?.slice(0, 500);
+      // 结束请求上下文，获取关联的加密调用
+      const ctx = deepspider.endRequest();
+      deepspider.log('xhr', {
+        action: 'response',
+        url: log.url,
+        status: log.status,
+        response: log.response?.slice(0, 100),
+        linkedCrypto: ctx?.cryptoCalls || []
+      });
+    });
+    ` : ''}
+
+    return xhr;
+  };
+
+  XMLHttpRequest.prototype = OriginalXHR.prototype;
+  console.log('[DeepSpider:xhr] XHR Hook 已启用');
+})();
+`;
+  }
+
+  /**
+   * 生成 Fetch Hook 代码
+   */
+  generateFetchHookCode(options = {}) {
+    const { captureBody = true, captureResponse = true } = options;
+
+    return HookBase.getBaseCode() + `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider) return;
+
+  const OriginalFetch = fetch;
+
+  fetch = deepspider.native(async function(url, options = {}) {
+    const log = {
+      url: typeof url === 'string' ? url : url.url,
+      method: options.method || 'GET',
+      headers: options.headers || {},
+      body: null,
+      response: null,
+      status: 0
+    };
+
+    // 开始请求上下文
+    log.requestId = deepspider.startRequest(log.url, log.method);
+
+    ${captureBody ? `
+    if (options.body) {
+      log.body = typeof options.body === 'string' ? options.body.slice(0, 500) : '[FormData/Blob]';
+    }
+    ` : ''}
+
+    deepspider.log('fetch', { action: 'request', ...log });
+
+    const response = await OriginalFetch.apply(this, arguments);
+    log.status = response.status;
+
+    ${captureResponse ? `
+    try {
+      const cloned = response.clone();
+      const text = await cloned.text();
+      log.response = text.slice(0, 500);
+      // 结束请求上下文
+      const ctx = deepspider.endRequest();
+      deepspider.log('fetch', {
+        action: 'response',
+        url: log.url,
+        status: log.status,
+        response: log.response.slice(0, 100),
+        linkedCrypto: ctx?.cryptoCalls || []
+      });
+    } catch (e) {
+      deepspider.endRequest();
+    }
+    ` : 'deepspider.endRequest();'}
+
+    return response;
+  }, OriginalFetch);
+
+  console.log('[DeepSpider:fetch] Fetch Hook 已启用');
+})();
+`;
+  }
+
+  /**
+   * 生成完整的网络 Hook 代码
+   */
+  generateFullNetworkHookCode(options = {}) {
+    return this.generateXHRHookCode(options) + '\n' + this.generateFetchHookCode(options);
+  }
+}
+
+export default NetworkHook;

package/src/env/modules/bom/history.js

@@ -0,0 +1,29 @@
+/**
+ * DeepSpider - History 环境模块
+ */
+
+export const historyCode = `
+(function() {
+  const _history = [];
+  let _index = 0;
+
+  window.history = {
+    get length() { return _history.length || 1; },
+    get state() { return _history[_index]?.state || null; },
+    scrollRestoration: 'auto',
+    back: function() { if (_index > 0) _index--; },
+    forward: function() { if (_index < _history.length - 1) _index++; },
+    go: function(delta) { _index = Math.max(0, Math.min(_history.length - 1, _index + delta)); },
+    pushState: function(state, title, url) {
+      _history.splice(_index + 1);
+      _history.push({ state, title, url });
+      _index = _history.length - 1;
+    },
+    replaceState: function(state, title, url) {
+      _history[_index] = { state, title, url };
+    }
+  };
+})();
+`;
+
+export default historyCode;

package/src/env/modules/bom/location.js

@@ -0,0 +1,26 @@
+/**
+ * DeepSpider - Location 环境模块
+ */
+
+export const locationCode = `
+(function() {
+  const location = {
+    href: 'https://example.com/',
+    protocol: 'https:',
+    host: 'example.com',
+    hostname: 'example.com',
+    port: '',
+    pathname: '/',
+    search: '',
+    hash: '',
+    origin: 'https://example.com',
+    assign: function(url) { this.href = url; },
+    replace: function(url) { this.href = url; },
+    reload: function() {},
+    toString: function() { return this.href; }
+  };
+  window.location = location;
+})();
+`;
+
+export default locationCode;

package/src/env/modules/bom/navigator.js

@@ -0,0 +1,70 @@
+/**
+ * DeepSpider - Navigator 环境模块
+ */
+
+export const navigatorCode = `
+(function() {
+  const navigator = {
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',
+    appCodeName: 'Mozilla',
+    appName: 'Netscape',
+    appVersion: '5.0 (Windows NT 10.0; Win64; x64)',
+    platform: 'Win32',
+    product: 'Gecko',
+    vendor: 'Google Inc.',
+    language: 'zh-CN',
+    languages: ['zh-CN', 'zh', 'en'],
+    onLine: true,
+    cookieEnabled: true,
+    hardwareConcurrency: 8,
+    maxTouchPoints: 0,
+    deviceMemory: 8,
+    webdriver: false,
+    doNotTrack: null,
+
+    plugins: { length: 0, item: () => null, namedItem: () => null, refresh: () => {} },
+    mimeTypes: { length: 0, item: () => null, namedItem: () => null },
+
+    connection: {
+      downlink: 10, effectiveType: '4g', rtt: 50, saveData: false,
+      addEventListener: () => {}, removeEventListener: () => {}
+    },
+
+    geolocation: {
+      getCurrentPosition: (s, e) => e && e({ code: 1, message: 'Denied' }),
+      watchPosition: () => 0,
+      clearWatch: () => {}
+    },
+
+    permissions: {
+      query: (d) => Promise.resolve({ name: d.name, state: 'prompt' })
+    },
+
+    javaEnabled: () => false,
+    vibrate: () => true,
+    sendBeacon: () => true,
+
+    getBattery: () => Promise.resolve({
+      charging: true, chargingTime: 0, dischargingTime: Infinity, level: 1
+    }),
+
+    clipboard: {
+      read: () => Promise.reject(new Error('NotAllowed')),
+      readText: () => Promise.reject(new Error('NotAllowed')),
+      write: () => Promise.reject(new Error('NotAllowed')),
+      writeText: () => Promise.reject(new Error('NotAllowed'))
+    },
+
+    userAgentData: {
+      brands: [{ brand: 'Chromium', version: '120' }],
+      mobile: false,
+      platform: 'Windows',
+      getHighEntropyValues: () => Promise.resolve({ platform: 'Windows', architecture: 'x86' })
+    }
+  };
+
+  window.navigator = navigator;
+})();
+`;
+
+export default navigatorCode;

package/src/env/modules/bom/screen.js

@@ -0,0 +1,26 @@
+/**
+ * DeepSpider - Screen 环境模块
+ */
+
+export const screenCode = `
+(function() {
+  window.screen = {
+    width: 1920,
+    height: 1080,
+    availWidth: 1920,
+    availHeight: 1040,
+    colorDepth: 24,
+    pixelDepth: 24,
+    availLeft: 0,
+    availTop: 0,
+    orientation: {
+      type: 'landscape-primary',
+      angle: 0,
+      addEventListener: () => {},
+      removeEventListener: () => {}
+    }
+  };
+})();
+`;
+
+export default screenCode;

package/src/env/modules/bom/storage.js

@@ -0,0 +1,23 @@
+/**
+ * DeepSpider - Storage 环境模块
+ */
+
+export const storageCode = `
+(function() {
+  function createStorage() {
+    const data = {};
+    return {
+      get length() { return Object.keys(data).length; },
+      key: function(i) { return Object.keys(data)[i] || null; },
+      getItem: function(k) { return data[k] ?? null; },
+      setItem: function(k, v) { data[k] = String(v); },
+      removeItem: function(k) { delete data[k]; },
+      clear: function() { for (let k in data) delete data[k]; }
+    };
+  }
+  window.localStorage = createStorage();
+  window.sessionStorage = createStorage();
+})();
+`;
+
+export default storageCode;

package/src/env/modules/dom/document.js

@@ -0,0 +1,110 @@
+/**
+ * DeepSpider - Document 环境模块（基础版）
+ */
+
+export const documentCode = `
+(function() {
+  // 元素ID生成器
+  let __elId = 0;
+  const __elements = new Map();
+
+  // 基础元素类
+  function Element(tagName) {
+    this.__id__ = 'el_' + (++__elId);
+    this.tagName = tagName.toUpperCase();
+    this.nodeName = this.tagName;
+    this.nodeType = 1;
+    this.children = [];
+    this.childNodes = [];
+    this.parentNode = null;
+    this.parentElement = null;
+    this.attributes = {};
+    this._attrs = {};
+    this.style = {};
+    this.classList = {
+      _list: [],
+      add: function(...c) { c.forEach(x => !this._list.includes(x) && this._list.push(x)); },
+      remove: function(...c) { c.forEach(x => { const i = this._list.indexOf(x); if (i > -1) this._list.splice(i, 1); }); },
+      contains: function(c) { return this._list.includes(c); },
+      toggle: function(c) { this.contains(c) ? this.remove(c) : this.add(c); }
+    };
+    this.innerHTML = '';
+    this.innerText = '';
+    this.textContent = '';
+    this.id = '';
+    this.className = '';
+    this.dataset = {};
+    __elements.set(this.__id__, this);
+  }
+
+  Element.prototype = {
+    getAttribute: function(n) { return this._attrs[n] ?? null; },
+    setAttribute: function(n, v) { this._attrs[n] = String(v); if (n === 'id') this.id = v; },
+    removeAttribute: function(n) { delete this._attrs[n]; },
+    hasAttribute: function(n) { return n in this._attrs; },
+    appendChild: function(c) { this.children.push(c); this.childNodes.push(c); c.parentNode = this; return c; },
+    removeChild: function(c) {
+      const i = this.children.indexOf(c);
+      if (i > -1) { this.children.splice(i, 1); this.childNodes.splice(i, 1); }
+      return c;
+    },
+    querySelector: function(s) { return null; },
+    querySelectorAll: function(s) { return []; },
+    getElementsByTagName: function(t) { return []; },
+    getElementsByClassName: function(c) { return []; },
+    addEventListener: function(t, h, o) {},
+    removeEventListener: function(t, h, o) {},
+    dispatchEvent: function(e) { return true; },
+    cloneNode: function(deep) { return new Element(this.tagName); },
+    getBoundingClientRect: function() {
+      return { top: 0, left: 0, right: 0, bottom: 0, width: 0, height: 0, x: 0, y: 0 };
+    }
+  };
+
+  // Document 对象
+  const document = {
+    nodeType: 9,
+    nodeName: '#document',
+    documentElement: new Element('HTML'),
+    head: new Element('HEAD'),
+    body: new Element('BODY'),
+    title: '',
+    cookie: '',
+    domain: 'example.com',
+    URL: 'https://example.com/',
+    referrer: '',
+    readyState: 'complete',
+    hidden: false,
+    visibilityState: 'visible',
+
+    createElement: function(tag) { return new Element(tag); },
+    createTextNode: function(text) { return { nodeType: 3, textContent: text }; },
+    createDocumentFragment: function() { return new Element('fragment'); },
+    createComment: function(text) { return { nodeType: 8, textContent: text }; },
+
+    getElementById: function(id) { return null; },
+    getElementsByTagName: function(tag) { return []; },
+    getElementsByClassName: function(cls) { return []; },
+    getElementsByName: function(name) { return []; },
+    querySelector: function(sel) { return null; },
+    querySelectorAll: function(sel) { return []; },
+
+    addEventListener: function() {},
+    removeEventListener: function() {},
+    dispatchEvent: function() { return true; },
+
+    write: function() {},
+    writeln: function() {},
+    open: function() {},
+    close: function() {}
+  };
+
+  document.documentElement.appendChild(document.head);
+  document.documentElement.appendChild(document.body);
+
+  window.document = document;
+  window.Element = Element;
+})();
+`;
+
+export default documentCode;

package/src/env/modules/dom/event.js

@@ -0,0 +1,51 @@
+/**
+ * DeepSpider - Event 环境模块
+ */
+
+export const eventCode = `
+(function() {
+  function Event(type, options = {}) {
+    this.type = type;
+    this.bubbles = options.bubbles || false;
+    this.cancelable = options.cancelable || false;
+    this.defaultPrevented = false;
+    this.target = null;
+    this.currentTarget = null;
+    this.timeStamp = Date.now();
+  }
+  Event.prototype = {
+    preventDefault: function() { this.defaultPrevented = true; },
+    stopPropagation: function() {},
+    stopImmediatePropagation: function() {}
+  };
+
+  function CustomEvent(type, options = {}) {
+    Event.call(this, type, options);
+    this.detail = options.detail || null;
+  }
+  CustomEvent.prototype = Object.create(Event.prototype);
+
+  function MouseEvent(type, options = {}) {
+    Event.call(this, type, options);
+    this.clientX = options.clientX || 0;
+    this.clientY = options.clientY || 0;
+    this.button = options.button || 0;
+  }
+  MouseEvent.prototype = Object.create(Event.prototype);
+
+  function KeyboardEvent(type, options = {}) {
+    Event.call(this, type, options);
+    this.key = options.key || '';
+    this.code = options.code || '';
+    this.keyCode = options.keyCode || 0;
+  }
+  KeyboardEvent.prototype = Object.create(Event.prototype);
+
+  window.Event = Event;
+  window.CustomEvent = CustomEvent;
+  window.MouseEvent = MouseEvent;
+  window.KeyboardEvent = KeyboardEvent;
+})();
+`;
+
+export default eventCode;

package/src/env/modules/index.js

@@ -0,0 +1,34 @@
+/**
+ * DeepSpider - 环境模块索引
+ */
+
+import { navigatorCode } from './bom/navigator.js';
+import { locationCode } from './bom/location.js';
+import { screenCode } from './bom/screen.js';
+import { historyCode } from './bom/history.js';
+import { storageCode } from './bom/storage.js';
+import { documentCode } from './dom/document.js';
+import { eventCode } from './dom/event.js';
+import { fetchCode } from './webapi/fetch.js';
+import { xhrCode } from './webapi/xhr.js';
+import { urlCode } from './webapi/url.js';
+
+export const modules = {
+  navigator: navigatorCode,
+  location: locationCode,
+  screen: screenCode,
+  history: historyCode,
+  storage: storageCode,
+  document: documentCode,
+  event: eventCode,
+  fetch: fetchCode,
+  xhr: xhrCode,
+  url: urlCode
+};
+
+export const loadOrder = [
+  'event', 'document', 'navigator', 'location',
+  'screen', 'history', 'storage', 'url', 'fetch', 'xhr'
+];
+
+export default modules;

package/src/env/modules/webapi/fetch.js

@@ -0,0 +1,46 @@
+/**
+ * DeepSpider - Fetch 环境模块
+ */
+
+export const fetchCode = `
+(function() {
+  window.fetch = function(url, options = {}) {
+    console.log('[fetch]', url);
+    return Promise.resolve({
+      ok: true,
+      status: 200,
+      statusText: 'OK',
+      url: url,
+      headers: new Map(),
+      json: () => Promise.resolve({}),
+      text: () => Promise.resolve(''),
+      blob: () => Promise.resolve(new Blob()),
+      arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)),
+      clone: function() { return this; }
+    });
+  };
+
+  window.Headers = function(init) {
+    this._h = new Map();
+    if (init) Object.entries(init).forEach(([k,v]) => this._h.set(k,v));
+  };
+  window.Headers.prototype = {
+    get: function(k) { return this._h.get(k); },
+    set: function(k,v) { this._h.set(k,v); },
+    has: function(k) { return this._h.has(k); },
+    delete: function(k) { this._h.delete(k); }
+  };
+
+  window.Request = function(url, options) {
+    this.url = url;
+    this.method = options?.method || 'GET';
+  };
+
+  window.Response = function(body, options) {
+    this.body = body;
+    this.status = options?.status || 200;
+  };
+})();
+`;
+
+export default fetchCode;

package/src/env/modules/webapi/url.js

@@ -0,0 +1,47 @@
+/**
+ * DeepSpider - URL 环境模块
+ */
+
+export const urlCode = `
+(function() {
+  function URL(url, base) {
+    const full = base ? new URL(base).origin + url : url;
+    const m = full.match(/^(\\w+):\\/\\/([^/:]+)(:\\d+)?([^?#]*)(\\?[^#]*)?(#.*)?$/);
+    this.href = full;
+    this.protocol = (m?.[1] || 'https') + ':';
+    this.host = (m?.[2] || '') + (m?.[3] || '');
+    this.hostname = m?.[2] || '';
+    this.port = (m?.[3] || '').slice(1);
+    this.pathname = m?.[4] || '/';
+    this.search = m?.[5] || '';
+    this.hash = m?.[6] || '';
+    this.origin = this.protocol + '//' + this.host;
+    this.searchParams = new URLSearchParams(this.search);
+  }
+  URL.prototype.toString = function() { return this.href; };
+
+  function URLSearchParams(init) {
+    this._p = new Map();
+    if (typeof init === 'string') {
+      init.replace(/^\\?/, '').split('&').forEach(p => {
+        const [k,v] = p.split('=');
+        if (k) this._p.set(decodeURIComponent(k), decodeURIComponent(v || ''));
+      });
+    }
+  }
+  URLSearchParams.prototype = {
+    get: function(k) { return this._p.get(k) || null; },
+    set: function(k,v) { this._p.set(k,v); },
+    has: function(k) { return this._p.has(k); },
+    delete: function(k) { this._p.delete(k); },
+    toString: function() {
+      return Array.from(this._p).map(([k,v]) => k+'='+encodeURIComponent(v)).join('&');
+    }
+  };
+
+  window.URL = URL;
+  window.URLSearchParams = URLSearchParams;
+})();
+`;
+
+export default urlCode;

package/src/env/modules/webapi/xhr.js

@@ -0,0 +1,48 @@
+/**
+ * DeepSpider - XMLHttpRequest 环境模块
+ */
+
+export const xhrCode = `
+(function() {
+  function XMLHttpRequest() {
+    this.readyState = 0;
+    this.status = 0;
+    this.statusText = '';
+    this.responseText = '';
+    this.responseXML = null;
+    this.response = '';
+    this.responseType = '';
+    this.timeout = 0;
+    this.withCredentials = false;
+    this._headers = {};
+    this._url = '';
+    this._method = 'GET';
+  }
+
+  XMLHttpRequest.prototype = {
+    UNSENT: 0, OPENED: 1, HEADERS_RECEIVED: 2, LOADING: 3, DONE: 4,
+    open: function(method, url) {
+      this._method = method;
+      this._url = url;
+      this.readyState = 1;
+    },
+    setRequestHeader: function(k, v) { this._headers[k] = v; },
+    getResponseHeader: function(k) { return null; },
+    getAllResponseHeaders: function() { return ''; },
+    send: function(data) {
+      console.log('[DeepSpider:xhr]', this._method, this._url);
+      this.readyState = 4;
+      this.status = 200;
+      if (this.onreadystatechange) this.onreadystatechange();
+      if (this.onload) this.onload();
+    },
+    abort: function() { this.readyState = 0; },
+    addEventListener: function() {},
+    removeEventListener: function() {}
+  };
+
+  window.XMLHttpRequest = XMLHttpRequest;
+})();
+`;
+
+export default xhrCode;