deepspider 0.3.0 → 0.3.2

package/src/cli/commands/config.js

@@ -36,7 +36,7 @@ function list() {
   const effective = getEffectiveConfig();
   console.log('配置项:');
   for (const [key, { value, source }] of Object.entries(effective)) {
-    const display = key === 'apiKey' && value ? maskKey(value) : value || '(未设置)';
+    const display = key === 'apiKey' && value ? maskKey(value) : formatValue(value);
     console.log(`  ${key} = ${display}  [${source}]`);
   }
 }
@@ -52,7 +52,7 @@ function get(key) {
   }
   const effective = getEffectiveConfig();
   const { value, source } = effective[key];
-  const display = key === 'apiKey' && value ? maskKey(value) : value || '(未设置)';
+  const display = key === 'apiKey' && value ? maskKey(value) : formatValue(value);
   console.log(`${key} = ${display}  [${source}]`);
 }
 
@@ -66,7 +66,10 @@ function set(key, value) {
     process.exit(1);
   }
   const config = loadConfig();
-  config[key] = value;
+  // 布尔类型配置项：CLI 字符串转布尔值存储
+  config[key] = typeof DEFAULTS[key] === 'boolean'
+    ? (value === 'true' || value === '1')
+    : value;
   saveConfig(config);
 
   const envVar = ENV_MAP[key];
@@ -88,6 +91,11 @@ function path() {
   console.log(CONFIG_FILE);
 }
 
+function formatValue(value) {
+  if (value === undefined || value === null || value === '') return '(未设置)';
+  return String(value);
+}
+
 function maskKey(key) {
   if (key.length <= 8) return '****';
   return key.slice(0, 4) + '****' + key.slice(-4);

package/src/config/paths.js

@@ -27,6 +27,9 @@ export const PATHS = {
 
   // 配置（预留）
   CONFIG_DIR: join(DEEPSPIDER_HOME, 'config'),
+
+  // 浏览器持久化数据（按需创建，不加入 initDirectories）
+  BROWSER_DATA_DIR: join(DEEPSPIDER_HOME, 'browser-data'),
 };
 
 /**
@@ -41,8 +44,13 @@ export function ensureDir(dir) {
 /**
  * 初始化所有目录
  */
+// 按需创建的目录，不随 initDirectories 自动创建
+const ON_DEMAND_DIRS = new Set([PATHS.BROWSER_DATA_DIR]);
+
 export function initDirectories() {
-  Object.values(PATHS).forEach(dir => ensureDir(dir));
+  Object.values(PATHS).forEach(dir => {
+    if (!ON_DEMAND_DIRS.has(dir)) ensureDir(dir);
+  });
 }
 
 /**

package/src/config/settings.js

@@ -16,12 +16,14 @@ export const DEFAULTS = {
   apiKey: '',
   baseUrl: 'https://api.openai.com/v1',
   model: 'gpt-4o',
+  persistBrowserData: false,
 };
 
 export const ENV_MAP = {
   apiKey: 'DEEPSPIDER_API_KEY',
   baseUrl: 'DEEPSPIDER_BASE_URL',
   model: 'DEEPSPIDER_MODEL',
+  persistBrowserData: 'DEEPSPIDER_PERSIST_BROWSER',
 };
 
 /**
@@ -57,7 +59,11 @@ export function getEffectiveConfig() {
     const envVal = process.env[envVar];
 
     if (envVal !== undefined && envVal !== '') {
-      result[key] = { value: envVal, source: 'env' };
+      // 布尔类型配置项：环境变量字符串转布尔
+      const value = typeof DEFAULTS[key] === 'boolean'
+        ? (envVal === 'true' || envVal === '1')
+        : envVal;
+      result[key] = { value, source: 'env' };
     } else if (fileConfig[key] !== undefined && fileConfig[key] !== '') {
       result[key] = { value: fileConfig[key], source: 'file' };
     } else {

package/src/core/PatchGenerator.js

@@ -4,6 +4,7 @@
  */
 
 import { Store } from '../store/Store.js';
+import { coveredAPIs } from '../env/modules/index.js';
 
 // 属性类型推断规则
 const TYPE_RULES = {
@@ -46,9 +47,26 @@ export class PatchGenerator {
   }
 
   async generate(property, context = {}) {
-    // 检查缓存
+    // 检查缓存（skipCoveredCheck 时忽略 skipped 缓存）
     if (this.generated.has(property)) {
-      return { ...this.generated.get(property), cached: true };
+      const cached = this.generated.get(property);
+      if (!(context.skipCoveredCheck && cached.skipped)) {
+        return { ...cached, cached: true };
+      }
+    }
+
+    // 0. 预置模块已覆盖，跳过（模块已加载到沙箱，无需重复补丁）
+    //    当模块未加载时（skipCoveredCheck=true），不跳过
+    if (!context.skipCoveredCheck && coveredAPIs.has(property)) {
+      const result = {
+        source: 'module-covered',
+        code: '',
+        property,
+        confidence: 1.0,
+        skipped: true,
+      };
+      this.generated.set(property, result);
+      return result;
     }
 
     // 1. 知识库精确匹配
@@ -169,7 +187,7 @@ export class PatchGenerator {
     return defaults[type] || 'undefined';
   }
 
-  _generateSmartTemplate(property, context = {}) {
+  _generateSmartTemplate(property, _context = {}) {
     const parts = property.split('.');
     const propName = parts[parts.length - 1];
     const type = this._inferType(property);
@@ -194,12 +212,12 @@ export class PatchGenerator {
   }
 
   // 批量生成补丁
-  async generateBatch(properties) {
+  async generateBatch(properties, context = {}) {
     const results = [];
     const conflicts = this._detectConflicts(properties);
 
     for (const prop of properties) {
-      const result = await this.generate(prop);
+      const result = await this.generate(prop, context);
       result.hasConflict = conflicts.has(prop);
       results.push(result);
     }
@@ -232,7 +250,7 @@ export class PatchGenerator {
     }
 
     // 检测同一对象的重复定义
-    for (const [root, props] of roots) {
+    for (const [_root, props] of roots) {
       if (props.length > 1) {
         const seen = new Set();
         for (const p of props) {
@@ -252,6 +270,8 @@ export class PatchGenerator {
     const grouped = new Map();
 
     for (const patch of patches) {
+      // 跳过已被模块覆盖或无代码的补丁
+      if (patch.skipped || !patch.code) continue;
       const root = patch.property.split('.')[0];
       if (!grouped.has(root)) {
         grouped.set(root, []);

package/src/core/Sandbox.js

@@ -5,6 +5,8 @@
 
 import ivm from 'isolated-vm';
 import { EnvMonitor } from './EnvMonitor.js';
+import { PatchGenerator } from './PatchGenerator.js';
+import { modules, loadOrder } from '../env/modules/index.js';
 
 export class Sandbox {
   constructor() {
@@ -12,6 +14,7 @@ export class Sandbox {
     this.context = null;
     this.missingEnv = [];
     this.monitor = new EnvMonitor();
+    this.patchGenerator = new PatchGenerator();
     this.envLoaded = false;
   }
 
@@ -74,17 +77,26 @@ export class Sandbox {
       global.window = window;
       global.self = window;
 
+      const __proxyCache__ = new WeakMap();
       global.__createProxy__ = function(obj, path) {
-        return new Proxy(obj, {
+        if (__proxyCache__.has(obj)) return __proxyCache__.get(obj);
+        const proxy = new Proxy(obj, {
           get(t, p) {
             if (typeof p === 'symbol') return t[p];
             const fullPath = path ? path + '.' + p : String(p);
-            if (t[p] === undefined && !(p in t)) {
+            const val = t[p];
+            if (val === undefined && !(p in t)) {
               __recordMissing__(fullPath);
             }
-            return t[p];
+            // 递归代理子对象
+            if (val !== null && typeof val === 'object' && !__proxyCache__.has(val)) {
+              return __createProxy__(val, fullPath);
+            }
+            return val;
           }
         });
+        __proxyCache__.set(obj, proxy);
+        return proxy;
       };
     `;
 
@@ -138,6 +150,131 @@ export class Sandbox {
     }
   }
 
+  /**
+   * 自动补环境闭环执行
+   * 加载预置模块 → Proxy 监控 → 迭代补丁 → 返回结果
+   */
+  async executeWithAutoFix(code, options = {}) {
+    const {
+      timeout = 5000,
+      maxIterations = 10,
+      loadModules = true,
+    } = options;
+
+    // 1. 加载预置环境模块
+    if (loadModules && !this.envLoaded) {
+      const envCode = loadOrder.map(n => modules[n]).filter(Boolean).join('\n\n');
+      await this.loadEnv(envCode);
+    }
+
+    // 2. 用 __createProxy__ 包裹全局对象，启用 missing 监控
+    await this._wrapGlobalsWithProxy();
+
+    const appliedPatches = [];
+    let lastMissingKey = '';
+
+    for (let i = 0; i < maxIterations; i++) {
+      const result = await this.execute(code, { timeout });
+
+      // 成功且无缺失，直接返回
+      if (result.success && result.missingEnv.length === 0) {
+        return { ...result, iterations: i + 1, appliedPatches };
+      }
+
+      // 没有新的缺失可补，退出
+      const currentKey = result.missingEnv.sort().join('|');
+      if (result.missingEnv.length === 0 || currentKey === lastMissingKey) {
+        return {
+          ...result,
+          iterations: i + 1,
+          appliedPatches,
+          remainingMissing: result.missingEnv,
+          stalled: currentKey === lastMissingKey,
+        };
+      }
+      lastMissingKey = currentKey;
+
+      // 非环境类错误（timeout / runtime-error），不继续补
+      if (!result.success) {
+        const envErrors = ['undefined-reference', 'not-a-function', 'null-access'];
+        if (!envErrors.includes(result.errorType)) {
+          return {
+            ...result,
+            iterations: i + 1,
+            appliedPatches,
+            remainingMissing: result.missingEnv,
+          };
+        }
+      }
+
+      // 批量生成补丁并注入（过滤掉 skipped 和低置信度的）
+      // 未加载模块时，不跳过 coveredAPIs 中的属性
+      const genContext = this.envLoaded ? {} : { skipCoveredCheck: true };
+      const batch = await this.patchGenerator.generateBatch(result.missingEnv, genContext);
+      const effective = batch.patches.filter(p => p.confidence >= 0.5 && !p.skipped);
+      const patchCode = this.patchGenerator.mergePatchCode(effective);
+
+      if (!patchCode.trim()) {
+        return {
+          ...result,
+          iterations: i + 1,
+          appliedPatches,
+          remainingMissing: result.missingEnv,
+        };
+      }
+
+      const injectResult = await this.inject(patchCode);
+      if (!injectResult.success) {
+        return {
+          ...result,
+          iterations: i + 1,
+          appliedPatches,
+          remainingMissing: result.missingEnv,
+          patchError: injectResult.error,
+        };
+      }
+
+      appliedPatches.push(...effective);
+
+      // 补丁注入后重新包裹全局对象，恢复 Proxy 监控
+      await this._wrapGlobalsWithProxy();
+    }
+
+    // 达到最大迭代次数
+    const finalResult = await this.execute(code, { timeout });
+    return {
+      ...finalResult,
+      iterations: maxIterations,
+      appliedPatches,
+      remainingMissing: finalResult.missingEnv,
+      maxIterationsReached: true,
+    };
+  }
+
+  /**
+   * 将全局环境对象用 __createProxy__ 包裹，启用缺失属性监控
+   */
+  async _wrapGlobalsWithProxy() {
+    const wrapCode = `
+      (function() {
+        var targets = ['navigator', 'document', 'screen', 'location', 'history'];
+        for (var i = 0; i < targets.length; i++) {
+          var name = targets[i];
+          // 全局对象不存在时先创建空壳，确保 Proxy 可监控
+          if (typeof global[name] === 'undefined') {
+            global[name] = {};
+          }
+          if (typeof global[name] === 'object' && global[name] !== null) {
+            try {
+              global[name] = __createProxy__(global[name], name);
+            } catch(e) {}
+          }
+        }
+      })();
+    `;
+    await this.context.eval(wrapCode);
+  }
+
   // 错误分类（帮助判断是环境缺失还是代码错误）
   _classifyError(e) {
     const msg = e.message || '';

package/src/env/EnvCodeGenerator.js

@@ -1,10 +1,9 @@
 /**
  * DeepSpider - 环境代码生成器
- * 借鉴 v_jstools 的 make_env_1_v3.js
- * 根据 Hook 收集的数据生成可在 Node.js 运行的浏览器环境代码
+ * 基于预置模块 + Hook 日志数据生成可在 Node.js 运行的浏览器环境代码
  */
 
-import { GETSET_LIST, FUNC_LIST, HTML_TAG_MAP } from './BrowserAPIList.js';
+import { modules, loadOrder } from './modules/index.js';
 
 export class EnvCodeGenerator {
   constructor() {
@@ -20,21 +19,26 @@ export class EnvCodeGenerator {
   }
 
   /**
-   * 解析环境日志
+   * 解析环境日志（支持任意深度路径）
    */
   parseEnvLogs(logs) {
     for (const log of logs) {
       if (log._type === 'env' && log.path) {
         const parts = log.path.split('.');
-        const className = parts[0];
-        const propName = parts[1];
-        if (!this.envData[className]) {
-          this.envData[className] = { props: {}, methods: {} };
+        const rootObj = parts[0];
+
+        if (!this.envData[rootObj]) {
+          this.envData[rootObj] = { props: {}, methods: {} };
         }
+
+        // 完整路径作为 key，保留深层访问信息
+        const fullPath = parts.slice(1).join('.');
+        if (!fullPath) continue;
+
         if (log.type === 'get' || log.type === 'set') {
-          this.envData[className].props[propName] = log.value;
+          this.envData[rootObj].props[fullPath] = log.value;
         } else if (log.type === 'call') {
-          this.envData[className].methods[propName] = log.result;
+          this.envData[rootObj].methods[fullPath] = log.result;
         }
       }
     }
@@ -49,8 +53,8 @@ export class EnvCodeGenerator {
     lines.push('// 生成时间: ' + new Date().toISOString());
     lines.push('');
     lines.push(this.generateSafeWrapper());
-    lines.push(this.generateBaseClasses());
-    lines.push(this.generateGlobalObjects());
+    lines.push(this.generateModules());
+    lines.push(this.generateOverrides());
     return lines.join('\n');
   }
 
@@ -69,88 +73,56 @@ function v_saf(fn) {
   }
 
   /**
-   * 生成基础类
+   * 加载预置模块代码
    */
-  generateBaseClasses() {
-    return `
-// EventTarget 基类
-class EventTarget {
-  constructor() { this._listeners = {}; }
-  addEventListener(type, fn) {
-    if (!this._listeners[type]) this._listeners[type] = [];
-    this._listeners[type].push(fn);
-  }
-  removeEventListener(type, fn) {
-    if (!this._listeners[type]) return;
-    this._listeners[type] = this._listeners[type].filter(f => f !== fn);
-  }
-  dispatchEvent(event) {
-    const fns = this._listeners[event.type] || [];
-    fns.forEach(fn => fn.call(this, event));
-    return true;
-  }
-}
-
-// Node 基类
-class Node extends EventTarget {
-  constructor() {
-    super();
-    this.childNodes = [];
-    this.parentNode = null;
-  }
-  appendChild(node) { node.parentNode = this; this.childNodes.push(node); return node; }
-  removeChild(node) {
-    const idx = this.childNodes.indexOf(node);
-    if (idx > -1) this.childNodes.splice(idx, 1);
-    node.parentNode = null;
-    return node;
-  }
-}`;
+  generateModules() {
+    return loadOrder
+      .map(name => modules[name])
+      .filter(Boolean)
+      .join('\n\n');
   }
 
   /**
-   * 生成全局对象
+   * 根据 Hook 日志数据生成覆盖补丁
+   * 用真实采集值覆盖预置模块的默认值
    */
-  generateGlobalObjects() {
-    const nav = this.envData.navigator?.props || {};
-    const scr = this.envData.screen?.props || {};
+  generateOverrides() {
+    const lines = ['\n// Hook 数据覆盖（真实环境值）'];
+    let hasOverrides = false;
 
-    return `
-// Navigator
-var navigator = {
-  userAgent: ${JSON.stringify(nav.userAgent || 'Mozilla/5.0')},
-  platform: ${JSON.stringify(nav.platform || 'Win32')},
-  language: ${JSON.stringify(nav.language || 'zh-CN')},
-  cookieEnabled: true,
-  onLine: true,
-  webdriver: false,
-};
-
-// Screen
-var screen = {
-  width: ${scr.width || 1920},
-  height: ${scr.height || 1080},
-  colorDepth: 24,
-};
-
-// Location
-var location = { href: '', protocol: 'https:', host: '', pathname: '/' };
-
-// Document
-var document = {
-  cookie: '',
-  createElement: function(tag) { return {}; },
-  getElementById: function(id) { return null; },
-  querySelector: function(sel) { return null; },
-};
-
-// Window
-var window = this;
-window.navigator = navigator;
-window.screen = screen;
-window.location = location;
-window.document = document;
-`;
+    for (const [rootObj, data] of Object.entries(this.envData)) {
+      // 属性覆盖
+      for (const [path, value] of Object.entries(data.props)) {
+        if (value === undefined || value === '[Object]') continue;
+        const fullPath = `${rootObj}.${path}`;
+        const parts = fullPath.split('.');
+
+        // 深层路径需要确保中间对象存在
+        if (parts.length > 2) {
+          for (let i = 2; i < parts.length; i++) {
+            const parentPath = parts.slice(0, i).join('.');
+            lines.push(`try { if (typeof ${parentPath} === 'undefined') ${parentPath} = {}; } catch(e) {}`);
+          }
+        }
+
+        lines.push(`try { ${fullPath} = ${this._serializeValue(value)}; } catch(e) {}`);
+        hasOverrides = true;
+      }
+    }
+
+    return hasOverrides ? lines.join('\n') : '';
+  }
+
+  _serializeValue(value) {
+    if (value === null) return 'null';
+    if (value === undefined) return 'undefined';
+    if (typeof value === 'string') return JSON.stringify(value);
+    if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+    try {
+      return JSON.stringify(value);
+    } catch {
+      return 'undefined';
+    }
   }
 }
 

package/src/env/modules/bom/history.js

@@ -26,4 +26,10 @@ export const historyCode = `
 })();
 `;
 
+export const historyCovers = [
+  'history.length', 'history.state', 'history.scrollRestoration',
+  'history.back', 'history.forward', 'history.go',
+  'history.pushState', 'history.replaceState',
+];
+
 export default historyCode;

package/src/env/modules/bom/location.js

@@ -23,4 +23,10 @@ export const locationCode = `
 })();
 `;
 
+export const locationCovers = [
+  'location.href', 'location.protocol', 'location.host', 'location.hostname',
+  'location.port', 'location.pathname', 'location.search', 'location.hash',
+  'location.origin', 'location.assign', 'location.replace', 'location.reload',
+];
+
 export default locationCode;

package/src/env/modules/bom/navigator.js

@@ -67,4 +67,17 @@ export const navigatorCode = `
 })();
 `;
 
+export const navigatorCovers = [
+  'navigator.userAgent', 'navigator.appCodeName', 'navigator.appName',
+  'navigator.appVersion', 'navigator.platform', 'navigator.product',
+  'navigator.vendor', 'navigator.language', 'navigator.languages',
+  'navigator.onLine', 'navigator.cookieEnabled', 'navigator.hardwareConcurrency',
+  'navigator.maxTouchPoints', 'navigator.deviceMemory', 'navigator.webdriver',
+  'navigator.doNotTrack', 'navigator.plugins', 'navigator.mimeTypes',
+  'navigator.connection', 'navigator.geolocation', 'navigator.permissions',
+  'navigator.clipboard', 'navigator.userAgentData',
+  'navigator.javaEnabled', 'navigator.vibrate', 'navigator.sendBeacon',
+  'navigator.getBattery',
+];
+
 export default navigatorCode;

package/src/env/modules/bom/screen.js

@@ -23,4 +23,10 @@ export const screenCode = `
 })();
 `;
 
+export const screenCovers = [
+  'screen.width', 'screen.height', 'screen.availWidth', 'screen.availHeight',
+  'screen.colorDepth', 'screen.pixelDepth', 'screen.availLeft', 'screen.availTop',
+  'screen.orientation',
+];
+
 export default screenCode;

package/src/env/modules/bom/storage.js

@@ -20,4 +20,11 @@ export const storageCode = `
 })();
 `;
 
+export const storageCovers = [
+  'localStorage.length', 'localStorage.key', 'localStorage.getItem',
+  'localStorage.setItem', 'localStorage.removeItem', 'localStorage.clear',
+  'sessionStorage.length', 'sessionStorage.key', 'sessionStorage.getItem',
+  'sessionStorage.setItem', 'sessionStorage.removeItem', 'sessionStorage.clear',
+];
+
 export default storageCode;

package/src/env/modules/dom/document.js

@@ -107,4 +107,18 @@ export const documentCode = `
 })();
 `;
 
+export const documentCovers = [
+  'document.nodeType', 'document.nodeName', 'document.documentElement',
+  'document.head', 'document.body', 'document.title', 'document.cookie',
+  'document.domain', 'document.URL', 'document.referrer', 'document.readyState',
+  'document.hidden', 'document.visibilityState',
+  'document.createElement', 'document.createTextNode',
+  'document.createDocumentFragment', 'document.createComment',
+  'document.getElementById', 'document.getElementsByTagName',
+  'document.getElementsByClassName', 'document.getElementsByName',
+  'document.querySelector', 'document.querySelectorAll',
+  'document.addEventListener', 'document.removeEventListener',
+  'document.dispatchEvent', 'document.write', 'document.writeln',
+];
+
 export default documentCode;

package/src/env/modules/dom/event.js

@@ -48,4 +48,8 @@ export const eventCode = `
 })();
 `;
 
+export const eventCovers = [
+  'Event', 'CustomEvent', 'MouseEvent', 'KeyboardEvent',
+];
+
 export default eventCode;