deepspider 0.3.0 → 0.3.2

package/src/env/modules/index.js

@@ -2,16 +2,16 @@
  * DeepSpider - 环境模块索引
  */
 
-import { navigatorCode } from './bom/navigator.js';
-import { locationCode } from './bom/location.js';
-import { screenCode } from './bom/screen.js';
-import { historyCode } from './bom/history.js';
-import { storageCode } from './bom/storage.js';
-import { documentCode } from './dom/document.js';
-import { eventCode } from './dom/event.js';
-import { fetchCode } from './webapi/fetch.js';
-import { xhrCode } from './webapi/xhr.js';
-import { urlCode } from './webapi/url.js';
+import { navigatorCode, navigatorCovers } from './bom/navigator.js';
+import { locationCode, locationCovers } from './bom/location.js';
+import { screenCode, screenCovers } from './bom/screen.js';
+import { historyCode, historyCovers } from './bom/history.js';
+import { storageCode, storageCovers } from './bom/storage.js';
+import { documentCode, documentCovers } from './dom/document.js';
+import { eventCode, eventCovers } from './dom/event.js';
+import { fetchCode, fetchCovers } from './webapi/fetch.js';
+import { xhrCode, xhrCovers } from './webapi/xhr.js';
+import { urlCode, urlCovers } from './webapi/url.js';
 
 export const modules = {
   navigator: navigatorCode,
@@ -31,4 +31,21 @@ export const loadOrder = [
   'screen', 'history', 'storage', 'url', 'fetch', 'xhr'
 ];
 
+/**
+ * 所有预置模块覆盖的 API 集合
+ * 供 PatchGenerator 查询：已有模块覆盖的属性不需要生成低质量 template 补丁
+ */
+export const coveredAPIs = new Set([
+  ...navigatorCovers,
+  ...locationCovers,
+  ...screenCovers,
+  ...historyCovers,
+  ...storageCovers,
+  ...documentCovers,
+  ...eventCovers,
+  ...fetchCovers,
+  ...xhrCovers,
+  ...urlCovers,
+]);
+
 export default modules;

package/src/env/modules/webapi/fetch.js

@@ -43,4 +43,8 @@ export const fetchCode = `
 })();
 `;
 
+export const fetchCovers = [
+  'fetch', 'Headers', 'Request', 'Response',
+];
+
 export default fetchCode;

package/src/env/modules/webapi/url.js

@@ -44,4 +44,8 @@ export const urlCode = `
 })();
 `;
 
+export const urlCovers = [
+  'URL', 'URLSearchParams',
+];
+
 export default urlCode;

package/src/env/modules/webapi/xhr.js

@@ -45,4 +45,12 @@ export const xhrCode = `
 })();
 `;
 
+export const xhrCovers = [
+  'XMLHttpRequest', 'XMLHttpRequest.prototype.open',
+  'XMLHttpRequest.prototype.send', 'XMLHttpRequest.prototype.abort',
+  'XMLHttpRequest.prototype.setRequestHeader',
+  'XMLHttpRequest.prototype.getResponseHeader',
+  'XMLHttpRequest.prototype.getAllResponseHeaders',
+];
+
 export default xhrCode;

package/src/store/DataStore.js

@@ -4,8 +4,8 @@
  * 支持会话隔离、内容去重、自动清理
  */
 
-import { mkdirSync, existsSync, readFileSync, statSync } from 'fs';
-import { writeFile, readFile, readdir, rm, stat } from 'fs/promises';
+import { existsSync, readFileSync } from 'fs';
+import { writeFile, readFile, rm } from 'fs/promises';
 import { join } from 'path';
 import { createHash } from 'crypto';
 import { PATHS, ensureDir } from '../config/paths.js';
@@ -128,12 +128,68 @@ export class DataStore {
     this.sessionId = null;
     // 上次清理时间
     this.lastCleanup = 0;
+    // 文件锁：防止并发写入同一站点索引
+    this.siteLocks = new Map();
 
     ensureDir(DATA_DIR);
     ensureDir(SITES_DIR);
     this.loadGlobalIndex();
   }
 
+  /**
+   * 获取站点锁（带超时和队列）
+   */
+  async acquireLock(site, timeout = 30000) {
+    // 初始化该站点的锁队列
+    if (!this.siteLocks.has(site)) {
+      this.siteLocks.set(site, { locked: false, queue: [] });
+    }
+
+    const lockState = this.siteLocks.get(site);
+
+    // 如果当前未锁定，直接获取锁
+    if (!lockState.locked) {
+      lockState.locked = true;
+      let released = false;
+      return () => {
+        if (released) return;
+        released = true;
+        lockState.locked = false;
+        // 唤醒队列中的下一个
+        const next = lockState.queue.shift();
+        if (next) next.resolve();
+      };
+    }
+
+    // 当前已锁定，加入等待队列
+    return new Promise((resolve, reject) => {
+      const id = Date.now().toString(36) + Math.random().toString(36).slice(2, 5);
+      const timer = setTimeout(() => {
+        // 从队列中移除
+        const idx = lockState.queue.findIndex(item => item.id === id);
+        if (idx > -1) lockState.queue.splice(idx, 1);
+        reject(new Error(`获取站点 ${site} 的锁超时`));
+      }, timeout);
+
+      lockState.queue.push({
+        id,
+        resolve: () => {
+          clearTimeout(timer);
+          lockState.locked = true;
+          let released = false;
+          resolve(() => {
+            if (released) return;
+            released = true;
+            lockState.locked = false;
+            // 唤醒队列中的下一个
+            const next = lockState.queue.shift();
+            if (next) next.resolve();
+          });
+        }
+      });
+    });
+  }
+
   /**
    * 创建新会话
    */
@@ -201,7 +257,7 @@ export class DataStore {
       if (existsSync(indexFile)) {
         index = JSON.parse(readFileSync(indexFile, 'utf-8'));
       }
-    } catch (e) {
+    } catch {
       // 使用默认索引
     }
 
@@ -241,61 +297,87 @@ export class DataStore {
       this.globalIndex.sites.push(stats);
     }
 
-    this.saveGlobalIndex().catch(() => {});
+    await this.saveGlobalIndex();
   }
 
   /**
-   * 保存响应数据（带去重）
+   * 保存响应数据（带去重，带锁防止竞态条件）
    */
   async saveResponse(data) {
-    const { url, method, status, requestHeaders, requestBody, responseBody, timestamp, pageUrl } = data;
+    const { url, method, status, requestHeaders, requestBody, responseBody, timestamp, pageUrl, initiator } = data;
     const { site, path } = parseUrl(pageUrl || url);
 
-    // 生成去重 hash
-    const hash = requestHash(url, method, requestBody);
-    const index = await this.getSiteIndex(site);
-
-    // 检查是否已存在相同内容
-    const existing = index.responses.find(r => r.hash === hash);
-    if (existing) {
-      // 更新时间戳和会话，不重复存储
-      existing.timestamp = timestamp || Date.now();
-      existing.sessionId = this.getSessionId();
-      await this.saveSiteIndex(site);
-      return { id: existing.id, site, path, deduplicated: true };
-    }
-
-    const siteDir = this.getSiteDir(site);
-    const responsesDir = join(siteDir, 'responses', path);
-    ensureDir(responsesDir);
-
-    // 生成可读文件名
-    const readableName = getReadableFilename(url, 'response', method);
-    const seq = String(index.responses.length).padStart(3, '0');
-    const id = `${readableName}_${seq}`;
-    const file = join(responsesDir, `${id}.json`);
+    // 获取站点锁，防止并发写入
+    const releaseLock = await this.acquireLock(site);
+    let result;
 
-    const content = JSON.stringify({
-      url, method, status,
-      requestHeaders, requestBody, responseBody,
-      pageUrl, timestamp
-    });
+    try {
+      // 生成去重 hash
+      const hash = requestHash(url, method, requestBody);
 
-    await writeFile(file, content);
+      // 重新加载索引（获取最新状态）
+      this.siteIndexCache.delete(site);
+      const index = await this.getSiteIndex(site);
 
-    index.responses.push({
-      id, url, path, method, status,
-      timestamp: timestamp || Date.now(),
-      file, size: content.length,
-      hash,
-      sessionId: this.getSessionId()
-    });
+      // 检查是否已存在相同内容
+      const existing = index.responses.find(r => r.hash === hash);
+      if (existing) {
+        // 更新时间戳和会话，不重复存储
+        existing.timestamp = timestamp || Date.now();
+        existing.sessionId = this.getSessionId();
+        // 更新 initiator（不同代码路径可能调用同一 API）
+        if (initiator) {
+          existing.hasInitiator = true;
+          // 同步更新详情文件中的 initiator
+          try {
+            const detail = JSON.parse(await readFile(existing.file, 'utf-8'));
+            detail.initiator = initiator;
+            await writeFile(existing.file, JSON.stringify(detail));
+          } catch { /* 文件读写失败不影响主流程 */ }
+        }
+        await this.saveSiteIndex(site);
+        result = { id: existing.id, site, path, deduplicated: true };
+      } else {
+        const siteDir = this.getSiteDir(site);
+        const responsesDir = join(siteDir, 'responses', path);
+        ensureDir(responsesDir);
+
+        // 生成可读文件名（使用当前索引长度作为序号）
+        const readableName = getReadableFilename(url, 'response', method);
+        const seq = String(index.responses.length).padStart(3, '0');
+        const id = `${readableName}_${seq}`;
+        const file = join(responsesDir, `${id}.json`);
+
+        const content = JSON.stringify({
+          url, method, status,
+          requestHeaders, requestBody, responseBody,
+          pageUrl, timestamp, initiator,
+        });
+
+        await writeFile(file, content);
+
+        index.responses.push({
+          id, url, path, method, status,
+          timestamp: timestamp || Date.now(),
+          file, size: content.length,
+          hash, hasInitiator: !!initiator,
+          sessionId: this.getSessionId()
+        });
+
+        await this.saveSiteIndex(site);
+        result = { id, site, path };
+      }
+    } finally {
+      // 确保锁被释放
+      releaseLock();
+    }
 
-    await this.saveSiteIndex(site);
-    await this.updateSiteStats(site);
+    if (!result.deduplicated) {
+      await this.updateSiteStats(site);
+    }
     this.maybeCleanup();
 
-    return { id, site, path };
+    return result;
   }
 
   /**
@@ -374,6 +456,7 @@ export class DataStore {
         id: r.id, url: r.url, path: r.path,
         method: r.method, status: r.status,
         timestamp: r.timestamp, size: r.size,
+        hasInitiator: !!r.hasInitiator,
         sessionId: r.sessionId
       }));
     }
@@ -446,7 +529,7 @@ export class DataStore {
               timestamp: meta.timestamp
             });
           }
-        } catch (e) { /* skip */ }
+        } catch { /* skip */ }
       }
     }
     return results;
@@ -477,7 +560,7 @@ export class DataStore {
               timestamp: meta.timestamp
             });
           }
-        } catch (e) { /* skip */ }
+        } catch { /* skip */ }
       }
     }
     return results;

package/src/store/Store.js

@@ -37,7 +37,8 @@ export class Store {
   }
 
   _getFilePath(type, name) {
-    const typeDir = path.join(this.baseDir, type);
+    const safeType = type.replace(/[^a-zA-Z0-9_.-]/g, '_');
+    const typeDir = path.join(this.baseDir, safeType);
     if (!fs.existsSync(typeDir)) {
       fs.mkdirSync(typeDir, { recursive: true });
     }

package/src/agent/subagents/dynamic.js

@@ -1,64 +0,0 @@
-/**
- * DeepSpider - 动态分析子代理
- */
-
-import { createSkillsMiddleware } from 'deepagents';
-import { SKILLS, skillsBackend } from '../skills/config.js';
-import { createFilterToolsMiddleware } from '../middleware/filterTools.js';
-
-import { runtimeTools } from '../tools/runtime.js';
-import { debugTools } from '../tools/debug.js';
-import { captureTools } from '../tools/capture.js';
-import { browserTools } from '../tools/browser.js';
-import { cryptoHookTools } from '../tools/cryptohook.js';
-import { correlateTools } from '../tools/correlate.js';
-import { tracingTools } from '../tools/tracing.js';
-import { evolveTools } from '../tools/evolve.js';
-
-export const dynamicSubagent = {
-  name: 'dynamic-agent',
-  description: '动态分析专家。当需要在浏览器中调试分析时使用，适用于：设置断点捕获运行时数据、分析请求与加密的关联、采集真实环境数据。',
-  systemPrompt: `你是 DeepSpider 的动态分析专家。
-
-## 职责
-- 控制浏览器执行
-- 设置断点捕获运行时数据
-- 采集真实环境数据
-- 收集 Hook 日志
-- 分析请求与加密的关联
-
-## 浏览器状态检查
-**在执行任何操作前，先判断浏览器状态：**
-- 如果任务描述中包含"浏览器已就绪"等关键词，不要调用 launch_browser
-- 先使用 get_hook_logs 检查是否有数据
-- 只有确认浏览器未启动时，才执行启动流程
-
-## 工作流程
-1. 检查浏览器状态
-2. 如需启动：launch_browser → navigate_to
-3. 等待 Hook 捕获加密调用
-4. 分析请求与加密的关联
-5. 必要时设置断点深入分析
-6. 采集环境数据
-
-## 经验记录
-完成分析后，如发现有价值的经验，使用 evolve_skill 记录：
-- skill: "dynamic-analysis"`,
-  tools: [
-    ...runtimeTools,
-    ...debugTools,
-    ...captureTools,
-    ...browserTools,
-    ...cryptoHookTools,
-    ...correlateTools,
-    ...tracingTools,
-    ...evolveTools,
-  ],
-  middleware: [
-    createFilterToolsMiddleware(),
-    createSkillsMiddleware({
-      backend: skillsBackend,
-      sources: [SKILLS.dynamic],
-    }),
-  ],
-};

package/src/agent/subagents/env-agent.js

@@ -1,82 +0,0 @@
-/**
- * DeepSpider - 补环境子代理
- * 方向：通过补全浏览器环境让代码直接运行
- */
-
-import { createSkillsMiddleware } from 'deepagents';
-import { SKILLS, skillsBackend } from '../skills/config.js';
-import { createFilterToolsMiddleware } from '../middleware/filterTools.js';
-
-import { sandboxTools } from '../tools/sandbox.js';
-import { nodejsTools } from '../tools/nodejs.js';
-import { envDumpTools } from '../tools/envdump.js';
-import { extractTools } from '../tools/extract.js';
-import { patchTools } from '../tools/patch.js';
-import { envTools } from '../tools/env.js';
-import { profileTools } from '../tools/profile.js';
-import { storeTools } from '../tools/store.js';
-import { hookTools } from '../tools/hook.js';
-import { antiDebugTools } from '../tools/antidebug.js';
-import { asyncTools } from '../tools/async.js';
-import { evolveTools } from '../tools/evolve.js';
-
-export const envAgentSubagent = {
-  name: 'env-agent',
-  description: '补环境专家。当需要让混淆代码在沙箱中直接运行时使用，适用于：环境检测多、算法复杂难还原、需要快速获取结果的场景。',
-  systemPrompt: `你是 DeepSpider 的补环境专家。
-
-## 分析方向
-补环境是 JS 逆向的黑盒方向，目标是让混淆代码在沙箱中直接运行，无需理解算法逻辑。
-
-## 核心流程
-1. **环境自吐** - 发现代码访问了哪些环境
-2. **浏览器提取** - 从真实浏览器获取环境值
-3. **生成补丁** - 转换为可注入的代码
-4. **沙箱执行** - 运行并获取结果
-
-## 判断标准
-适合补环境的场景：
-- 环境检测多（webdriver、chrome对象等）
-- 算法复杂难以还原
-- 需要快速获取结果
-- 代码频繁更新
-
-## 快速模式
-如果只需快速验证代码能否运行：
-1. list_env_modules 查看预置模块
-2. load_all_env_modules 加载全部
-3. sandbox_inject 注入
-4. sandbox_execute 执行
-
-## 执行工具选择
-- sandbox_execute: 隔离沙箱，适合补环境后的代码执行
-- run_node_code: Node.js 执行，适合需要 require npm 包的场景
-
-## 失败处理
-如果补环境多次失败，建议切换到纯算分析方向。
-
-## 经验记录
-完成分析后，如发现有价值的经验，使用 evolve_skill 记录：
-- skill: "env"`,
-  tools: [
-    ...sandboxTools,
-    ...nodejsTools,
-    ...envDumpTools,
-    ...extractTools,
-    ...patchTools,
-    ...envTools,
-    ...profileTools,
-    ...hookTools,
-    ...antiDebugTools,
-    ...asyncTools,
-    ...storeTools,
-    ...evolveTools,
-  ],
-  middleware: [
-    createFilterToolsMiddleware(),
-    createSkillsMiddleware({
-      backend: skillsBackend,
-      sources: [SKILLS.env],
-    }),
-  ],
-};

package/src/agent/subagents/sandbox.js

@@ -1,55 +0,0 @@
-/**
- * DeepSpider - 沙箱验证子代理
- */
-
-import { createSkillsMiddleware } from 'deepagents';
-import { SKILLS, skillsBackend } from '../skills/config.js';
-import { createFilterToolsMiddleware } from '../middleware/filterTools.js';
-
-import { sandboxTools } from '../tools/sandbox.js';
-import { nodejsTools } from '../tools/nodejs.js';
-import { patchTools } from '../tools/patch.js';
-import { envTools } from '../tools/env.js';
-import { verifyTools } from '../tools/verify.js';
-import { fileTools } from '../tools/file.js';
-import { evolveTools } from '../tools/evolve.js';
-
-export const sandboxSubagent = {
-  name: 'sandbox-agent',
-  description: '沙箱验证专家。当需要验证提取的代码能否正确执行时使用，适用于：验证加密算法、补全缺失环境、生成可独立运行的脚本。',
-  systemPrompt: `你是 DeepSpider 的验证执行专家。
-
-## 职责
-- 在沙箱中验证提取的加密算法
-- 补全缺失的环境
-- 生成可独立运行的脚本
-- 验证加密结果是否正确
-
-## 执行工具选择
-- sandbox_execute: 隔离沙箱，适合不需要外部依赖的代码
-- run_node_code: Node.js 执行，适合需要 require npm 包的代码（如 crypto-js）
-
-## 输出
-- 验证结果
-- 可执行的 JS 模块
-
-## 经验记录
-完成验证后，如发现有价值的经验，使用 evolve_skill 记录：
-- skill: "sandbox"`,
-  tools: [
-    ...sandboxTools,
-    ...nodejsTools,
-    ...patchTools,
-    ...envTools,
-    ...verifyTools,
-    ...fileTools,
-    ...evolveTools,
-  ],
-  middleware: [
-    createFilterToolsMiddleware(),
-    createSkillsMiddleware({
-      backend: skillsBackend,
-      sources: [SKILLS.sandbox],
-    }),
-  ],
-};

package/src/agent/subagents/static.js

@@ -1,66 +0,0 @@
-/**
- * DeepSpider - 静态分析子代理
- */
-
-import { createSkillsMiddleware } from 'deepagents';
-import { SKILLS, skillsBackend } from '../skills/config.js';
-import { createFilterToolsMiddleware } from '../middleware/filterTools.js';
-
-import { analyzerTools } from '../tools/analyzer.js';
-import { deobfuscatorTools } from '../tools/deobfuscator.js';
-import { traceTools } from '../tools/trace.js';
-import { webcrackTools } from '../tools/webcrack.js';
-import { preprocessTools } from '../tools/preprocess.js';
-import { extractorTools } from '../tools/extractor.js';
-import { storeTools } from '../tools/store.js';
-import { verifyTools } from '../tools/verify.js';
-import { correlateTools } from '../tools/correlate.js';
-import { evolveTools } from '../tools/evolve.js';
-
-export const staticSubagent = {
-  name: 'static-agent',
-  description: '静态代码分析专家。当需要分析混淆代码、还原加密算法时使用，适用于：Webpack解包、反混淆、定位加密入口、算法还原验证。',
-  systemPrompt: `你是 DeepSpider 的静态分析专家。
-
-## 职责
-- 预处理打包代码（Webpack/Vite/Rollup）
-- 反混淆处理
-- 定位加密函数入口
-- 还原算法逻辑
-- 验证算法正确性
-
-## 工作流程
-1. preprocess_code 预处理
-2. 如有 bundle 则解包
-3. deobfuscate 反混淆
-4. analyze_encryption 定位入口
-5. 验证算法
-
-## 输出
-- 加密函数位置
-- 断点建议
-- 算法分析结果
-
-## 经验记录
-完成分析后，如发现有价值的经验，使用 evolve_skill 记录：
-- skill: "static-analysis"`,
-  tools: [
-    ...preprocessTools,
-    ...webcrackTools,
-    ...analyzerTools,
-    ...deobfuscatorTools,
-    ...traceTools,
-    ...extractorTools,
-    ...verifyTools,
-    ...correlateTools,
-    ...storeTools,
-    ...evolveTools,
-  ],
-  middleware: [
-    createFilterToolsMiddleware(),
-    createSkillsMiddleware({
-      backend: skillsBackend,
-      sources: [SKILLS.static],
-    }),
-  ],
-};