deepspider 0.1.0

package/src/config/profiles/firefox.json

@@ -0,0 +1,44 @@
+{
+  "name": "Firefox 121 Windows",
+  "browser": "firefox",
+  "version": "121.0",
+  "platform": "Win32",
+
+  "navigator": {
+    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
+    "appVersion": "5.0 (Windows)",
+    "platform": "Win32",
+    "language": "zh-CN",
+    "languages": ["zh-CN", "zh", "en-US", "en"],
+    "cookieEnabled": true,
+    "doNotTrack": "unspecified",
+    "maxTouchPoints": 0,
+    "hardwareConcurrency": 8,
+    "webdriver": false,
+    "vendor": "",
+    "vendorSub": "",
+    "productSub": "20100101"
+  },
+
+  "screen": {
+    "width": 1920,
+    "height": 1080,
+    "availWidth": 1920,
+    "availHeight": 1040,
+    "colorDepth": 24,
+    "pixelDepth": 24
+  },
+
+  "window": {
+    "innerWidth": 1903,
+    "innerHeight": 937,
+    "outerWidth": 1920,
+    "outerHeight": 1040,
+    "devicePixelRatio": 1
+  },
+
+  "webgl": {
+    "vendor": "Mozilla",
+    "renderer": "NVIDIA GeForce GTX 1060/PCIe/SSE2"
+  }
+}

package/src/config/profiles/safari.json

@@ -0,0 +1,38 @@
+{
+  "name": "Safari 17 macOS",
+  "browser": "safari",
+  "version": "17.0",
+  "platform": "MacIntel",
+
+  "navigator": {
+    "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
+    "appVersion": "5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
+    "platform": "MacIntel",
+    "language": "zh-CN",
+    "languages": ["zh-CN", "zh"],
+    "cookieEnabled": true,
+    "maxTouchPoints": 0,
+    "hardwareConcurrency": 8,
+    "webdriver": false,
+    "vendor": "Apple Computer, Inc.",
+    "vendorSub": "",
+    "productSub": "20030107"
+  },
+
+  "screen": {
+    "width": 1440,
+    "height": 900,
+    "availWidth": 1440,
+    "availHeight": 875,
+    "colorDepth": 30,
+    "pixelDepth": 30
+  },
+
+  "window": {
+    "innerWidth": 1440,
+    "innerHeight": 789,
+    "outerWidth": 1440,
+    "outerHeight": 875,
+    "devicePixelRatio": 2
+  }
+}

package/src/core/EnvMonitor.js

@@ -0,0 +1,200 @@
+/**
+ * DeepSpider - 环境监控系统
+ * Node.js 端监控核心，API 风格与浏览器端 __deepspider__ 保持一致
+ */
+
+export class EnvMonitor {
+  constructor() {
+    this.config = {
+      enabled: true,
+      logLevel: 'all',
+      maxLogSize: 10000,
+      trackPropertyAccess: true,
+      trackMethodCalls: true,
+      mockEnabled: true
+    };
+
+    // 统一日志结构，与浏览器端 __deepspider__ 保持一致
+    this.logs = {
+      env: [],      // 环境访问
+      call: [],     // 方法调用
+      missing: [],  // 缺失属性
+    };
+
+    this.mocks = {
+      properties: {},
+      methods: {},
+      returnValues: {}
+    };
+
+    this._logId = 0;
+  }
+
+  _generateId() {
+    return `log_${++this._logId}_${Date.now()}`;
+  }
+
+  // 统一日志方法，与 __deepspider__.log() 风格一致
+  log(type, data) {
+    if (!this.config.enabled) return;
+
+    const entry = {
+      id: this._generateId(),
+      timestamp: Date.now(),
+      ...data
+    };
+
+    if (this.logs[type]) {
+      this.logs[type].push(entry);
+      if (this.logs[type].length > this.config.maxLogSize) {
+        this.logs[type].shift();
+      }
+    }
+
+    return entry;
+  }
+
+  // 记录缺失属性访问
+  logMissing(path, context = {}) {
+    const exists = this.logs.missing.find(l => l.path === path);
+    if (exists) return exists;
+
+    return this.log('missing', { path, context, fixed: false });
+  }
+
+  // 记录方法调用
+  logCall(path, args, result) {
+    if (!this.config.enabled || !this.config.trackMethodCalls) return;
+
+    return this.log('call', {
+      path,
+      args: this._serializeArgs(args),
+      result: this._serializeValue(result)
+    });
+  }
+
+  // 记录属性访问
+  logAccess(path, value) {
+    if (!this.config.enabled || !this.config.trackPropertyAccess) return;
+
+    return this.log('env', {
+      path,
+      value: this._serializeValue(value)
+    });
+  }
+
+  // Mock 管理
+  setMock(type, path, value) {
+    switch (type) {
+      case 'property':
+        this.mocks.properties[path] = value;
+        break;
+      case 'method':
+        this.mocks.methods[path] = value;
+        break;
+      case 'returnValue':
+        this.mocks.returnValues[path] = value;
+        break;
+    }
+  }
+
+  getMock(path) {
+    if (path in this.mocks.properties) {
+      return { type: 'property', value: this.mocks.properties[path] };
+    }
+    if (path in this.mocks.methods) {
+      return { type: 'method', value: this.mocks.methods[path] };
+    }
+    return null;
+  }
+
+  hasMock(path) {
+    return path in this.mocks.properties ||
+           path in this.mocks.methods ||
+           path in this.mocks.returnValues;
+  }
+
+  // 查询方法 - 与 __deepspider__.getLogs() 风格一致
+  getLogs(type) {
+    if (type) {
+      return this.logs[type] || [];
+    }
+    // 返回所有日志，按时间排序
+    return Object.entries(this.logs)
+      .flatMap(([t, logs]) => logs.map(l => ({ type: t, ...l })))
+      .sort((a, b) => a.timestamp - b.timestamp);
+  }
+
+  getMissingLogs(options = {}) {
+    let logs = this.logs.missing;
+    if (options.unfixedOnly) {
+      logs = logs.filter(l => !l.fixed);
+    }
+    if (options.limit) {
+      logs = logs.slice(-options.limit);
+    }
+    return logs;
+  }
+
+  getCallLogs(options = {}) {
+    let logs = this.logs.call;
+    if (options.path) {
+      logs = logs.filter(l => l.path.includes(options.path));
+    }
+    if (options.limit) {
+      logs = logs.slice(-options.limit);
+    }
+    return logs;
+  }
+
+  // 标记已修复
+  markFixed(path) {
+    const log = this.logs.missing.find(l => l.path === path);
+    if (log) {
+      log.fixed = true;
+      log.fixedAt = Date.now();
+    }
+  }
+
+  // 统计信息
+  getStats() {
+    return {
+      totalEnv: this.logs.env.length,
+      totalCalls: this.logs.call.length,
+      totalMissing: this.logs.missing.length,
+      unfixedMissing: this.logs.missing.filter(l => !l.fixed).length,
+      mockCount: {
+        properties: Object.keys(this.mocks.properties).length,
+        methods: Object.keys(this.mocks.methods).length
+      }
+    };
+  }
+
+  // 清空日志
+  clearLogs(type) {
+    if (type && this.logs[type]) {
+      this.logs[type] = [];
+    } else if (!type) {
+      this.logs = { env: [], call: [], missing: [] };
+    }
+  }
+
+  // 序列化辅助
+  _serializeArgs(args) {
+    if (!args) return [];
+    return Array.from(args).map(a => this._serializeValue(a));
+  }
+
+  _serializeValue(val) {
+    if (val === undefined) return 'undefined';
+    if (val === null) return 'null';
+    if (typeof val === 'function') return '[Function]';
+    if (typeof val === 'object') {
+      try { return JSON.stringify(val).slice(0, 200); }
+      catch { return '[Object]'; }
+    }
+    return String(val).slice(0, 200);
+  }
+}
+
+export default EnvMonitor;

package/src/core/PatchGenerator.js

@@ -0,0 +1,278 @@
+/**
+ * DeepSpider - 补丁生成器
+ * 多级生成策略：模式规则 → 模板生成
+ */
+
+import { Store } from '../store/Store.js';
+
+// 属性类型推断规则
+const TYPE_RULES = {
+  // 字符串类型
+  string: [
+    /userAgent$/, /platform$/, /language$/, /vendor$/,
+    /href$/, /hostname$/, /pathname$/, /protocol$/,
+    /cookie$/, /title$/, /referrer$/, /domain$/
+  ],
+  // 数字类型
+  number: [
+    /width$/, /height$/, /devicePixelRatio$/, /colorDepth$/,
+    /hardwareConcurrency$/, /maxTouchPoints$/, /length$/
+  ],
+  // 布尔类型
+  boolean: [
+    /cookieEnabled$/, /onLine$/, /hidden$/, /webdriver$/
+  ],
+  // 数组类型
+  array: [
+    /languages$/, /plugins$/, /mimeTypes$/
+  ],
+  // 对象类型
+  object: [
+    /connection$/, /geolocation$/, /mediaDevices$/,
+    /style$/, /dataset$/, /classList$/
+  ],
+  // 函数类型
+  function: [
+    /getElementById$/, /querySelector/, /createElement$/,
+    /addEventListener$/, /getAttribute$/, /setAttribute$/
+  ]
+};
+
+export class PatchGenerator {
+  constructor() {
+    this.store = new Store();
+    this.patterns = this._initPatterns();
+    this.generated = new Map();
+  }
+
+  async generate(property, context = {}) {
+    // 检查缓存
+    if (this.generated.has(property)) {
+      return { ...this.generated.get(property), cached: true };
+    }
+
+    // 1. 知识库精确匹配
+    const libResult = this._searchLibrary(property);
+    if (libResult) {
+      this.generated.set(property, libResult);
+      return libResult;
+    }
+
+    // 2. 模式规则匹配
+    const patternResult = this._matchPattern(property);
+    if (patternResult) {
+      this.generated.set(property, patternResult);
+      return patternResult;
+    }
+
+    // 3. 智能模板生成
+    const templateResult = this._generateSmartTemplate(property, context);
+    this.generated.set(property, templateResult);
+    return templateResult;
+  }
+
+  _searchLibrary(property) {
+    // 精确匹配
+    const results = this.store.query('env-module', property);
+    if (results.length > 0) {
+      return {
+        source: 'store',
+        code: results[0].code,
+        property,
+        confidence: 1.0,
+        entry: results[0].name
+      };
+    }
+
+    // 模糊匹配
+    const parts = property.split('.');
+    const rootObj = parts[0];
+    const rootResults = this.store.query('env-module', rootObj);
+
+    for (const result of rootResults) {
+      if (result.code?.includes(property)) {
+        return {
+          source: 'store-partial',
+          code: result.code,
+          property,
+          confidence: 0.8,
+          entry: result.name
+        };
+      }
+    }
+
+    return null;
+  }
+
+  _initPatterns() {
+    return {
+      // Navigator
+      'navigator.userAgent': `navigator.userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";`,
+      'navigator.platform': `navigator.platform = "Win32";`,
+      'navigator.language': `navigator.language = "zh-CN";`,
+      'navigator.languages': `navigator.languages = ["zh-CN", "en"];`,
+      'navigator.cookieEnabled': `navigator.cookieEnabled = true;`,
+      'navigator.onLine': `navigator.onLine = true;`,
+      'navigator.hardwareConcurrency': `navigator.hardwareConcurrency = 8;`,
+      'navigator.webdriver': `Object.defineProperty(navigator, 'webdriver', { get: () => false });`,
+      // Location
+      'location.href': `location.href = "https://example.com/";`,
+      'location.hostname': `location.hostname = "example.com";`,
+      'location.protocol': `location.protocol = "https:";`,
+      // Document
+      'document.cookie': `document.cookie = "";`,
+      'document.referrer': `document.referrer = "";`,
+      'document.domain': `document.domain = "example.com";`,
+      // Screen
+      'screen.width': `screen.width = 1920;`,
+      'screen.height': `screen.height = 1080;`,
+      'screen.colorDepth': `screen.colorDepth = 24;`,
+      // Window
+      'window.innerWidth': `window.innerWidth = 1920;`,
+      'window.innerHeight': `window.innerHeight = 1080;`,
+      'window.devicePixelRatio': `window.devicePixelRatio = 1;`
+    };
+  }
+
+  _matchPattern(property) {
+    const pattern = this.patterns[property];
+    if (pattern) {
+      return {
+        source: 'pattern',
+        code: pattern,
+        property,
+        confidence: 0.9
+      };
+    }
+    return null;
+  }
+
+  _inferType(property) {
+    for (const [type, patterns] of Object.entries(TYPE_RULES)) {
+      if (patterns.some(p => p.test(property))) {
+        return type;
+      }
+    }
+    return 'unknown';
+  }
+
+  _getDefaultValue(type) {
+    const defaults = {
+      string: '""',
+      number: '0',
+      boolean: 'false',
+      array: '[]',
+      object: '{}',
+      function: 'function() {}',
+      unknown: 'undefined'
+    };
+    return defaults[type] || 'undefined';
+  }
+
+  _generateSmartTemplate(property, context = {}) {
+    const parts = property.split('.');
+    const propName = parts[parts.length - 1];
+    const type = this._inferType(property);
+    const defaultVal = this._getDefaultValue(type);
+
+    let code;
+    if (parts.length === 1) {
+      code = `window.${property} = ${defaultVal};`;
+    } else {
+      const parent = parts.slice(0, -1).join('.');
+      code = `${parent}.${propName} = ${defaultVal};`;
+    }
+
+    return {
+      source: 'template',
+      code,
+      property,
+      inferredType: type,
+      confidence: 0.5,
+      needsLLM: true
+    };
+  }
+
+  // 批量生成补丁
+  async generateBatch(properties) {
+    const results = [];
+    const conflicts = this._detectConflicts(properties);
+
+    for (const prop of properties) {
+      const result = await this.generate(prop);
+      result.hasConflict = conflicts.has(prop);
+      results.push(result);
+    }
+
+    return {
+      patches: results,
+      conflicts: Array.from(conflicts),
+      stats: {
+        total: results.length,
+        fromLibrary: results.filter(r => r.source === 'library').length,
+        fromPattern: results.filter(r => r.source === 'pattern').length,
+        needsLLM: results.filter(r => r.needsLLM).length
+      }
+    };
+  }
+
+  // 冲突检测
+  _detectConflicts(properties) {
+    const conflicts = new Set();
+    const roots = new Map();
+
+    for (const prop of properties) {
+      const parts = prop.split('.');
+      const root = parts[0];
+
+      if (!roots.has(root)) {
+        roots.set(root, []);
+      }
+      roots.get(root).push(prop);
+    }
+
+    // 检测同一对象的重复定义
+    for (const [root, props] of roots) {
+      if (props.length > 1) {
+        const seen = new Set();
+        for (const p of props) {
+          if (seen.has(p)) {
+            conflicts.add(p);
+          }
+          seen.add(p);
+        }
+      }
+    }
+
+    return conflicts;
+  }
+
+  // 合并补丁代码
+  mergePatchCode(patches) {
+    const grouped = new Map();
+
+    for (const patch of patches) {
+      const root = patch.property.split('.')[0];
+      if (!grouped.has(root)) {
+        grouped.set(root, []);
+      }
+      grouped.get(root).push(patch);
+    }
+
+    const merged = [];
+    for (const [root, group] of grouped) {
+      merged.push(`// ${root} patches`);
+      merged.push(...group.map(p => p.code));
+      merged.push('');
+    }
+
+    return merged.join('\n');
+  }
+
+  // 清除缓存
+  clearCache() {
+    this.generated.clear();
+  }
+}
+
+export default PatchGenerator;