deepspider 0.1.0

package/src/store/Store.js

@@ -0,0 +1,158 @@
+/**
+ * DeepSpider - 知识库存储
+ * 文件系统持久化，支持分类索引
+ * 统一存储到 ~/.deepspider/store/
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { PATHS, ensureDir } from '../config/paths.js';
+
+export class Store {
+  constructor(options = {}) {
+    this.baseDir = options.baseDir || PATHS.STORE_DIR;
+    this.cache = new Map();
+    this.indexFile = path.join(this.baseDir, 'index.json');
+    this.index = { types: {} };
+    this._ensureDir();
+    this._loadIndex();
+  }
+
+  _ensureDir() {
+    ensureDir(this.baseDir);
+  }
+
+  _loadIndex() {
+    try {
+      if (fs.existsSync(this.indexFile)) {
+        this.index = JSON.parse(fs.readFileSync(this.indexFile, 'utf-8'));
+      }
+    } catch {
+      this.index = { types: {} };
+    }
+  }
+
+  _saveIndex() {
+    fs.writeFileSync(this.indexFile, JSON.stringify(this.index, null, 2));
+  }
+
+  _getFilePath(type, name) {
+    const typeDir = path.join(this.baseDir, type);
+    if (!fs.existsSync(typeDir)) {
+      fs.mkdirSync(typeDir, { recursive: true });
+    }
+    const safeName = name.replace(/[^a-zA-Z0-9_.-]/g, '_');
+    return path.join(typeDir, `${safeName}.json`);
+  }
+
+  save(type, name, entry) {
+    const key = `${type}:${name}`;
+    const data = { ...entry, type, name, updatedAt: Date.now() };
+
+    // 写入文件
+    const filePath = this._getFilePath(type, name);
+    fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
+
+    // 更新索引
+    if (!this.index.types[type]) {
+      this.index.types[type] = [];
+    }
+    if (!this.index.types[type].includes(name)) {
+      this.index.types[type].push(name);
+    }
+    this._saveIndex();
+
+    // 更新缓存
+    this.cache.set(key, data);
+
+    return { success: true, path: filePath };
+  }
+
+  get(type, name) {
+    const key = `${type}:${name}`;
+
+    // 检查缓存
+    if (this.cache.has(key)) {
+      return this.cache.get(key);
+    }
+
+    // 从文件读取
+    const filePath = this._getFilePath(type, name);
+    if (fs.existsSync(filePath)) {
+      const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+      this.cache.set(key, data);
+      return data;
+    }
+
+    return null;
+  }
+
+  query(type, keyword) {
+    const results = [];
+    const names = this.index.types[type] || [];
+
+    for (const name of names) {
+      if (name.includes(keyword)) {
+        const entry = this.get(type, name);
+        if (entry) results.push(entry);
+      }
+    }
+
+    // 也搜索内容
+    if (results.length === 0) {
+      for (const name of names) {
+        const entry = this.get(type, name);
+        if (entry?.code?.includes(keyword)) {
+          results.push(entry);
+        }
+      }
+    }
+
+    return results;
+  }
+
+  // 列出某类型下所有条目
+  list(type) {
+    return this.index.types[type] || [];
+  }
+
+  // 删除条目
+  delete(type, name) {
+    const key = `${type}:${name}`;
+    const filePath = this._getFilePath(type, name);
+
+    if (fs.existsSync(filePath)) {
+      fs.unlinkSync(filePath);
+    }
+
+    this.cache.delete(key);
+
+    if (this.index.types[type]) {
+      this.index.types[type] = this.index.types[type].filter(n => n !== name);
+      this._saveIndex();
+    }
+  }
+
+  // 导出所有数据
+  exportAll() {
+    const all = {};
+    for (const [type, names] of Object.entries(this.index.types)) {
+      all[type] = names.map(name => this.get(type, name));
+    }
+    return all;
+  }
+
+  // 导入数据
+  import(data) {
+    let count = 0;
+    for (const [type, entries] of Object.entries(data)) {
+      for (const entry of entries) {
+        this.save(type, entry.name, entry);
+        count++;
+      }
+    }
+    return { imported: count };
+  }
+}
+
+export default Store;

package/src/store/Validator.js

@@ -0,0 +1,24 @@
+/**
+ * DeepSpider - 补丁验证器
+ */
+
+export class Validator {
+  constructor(sandbox) {
+    this.sandbox = sandbox;
+  }
+
+  async validate(patch, testCode) {
+    try {
+      await this.sandbox.inject(patch.code);
+      const result = await this.sandbox.execute(testCode);
+      return {
+        valid: result.success,
+        error: result.error
+      };
+    } catch (e) {
+      return { valid: false, error: e.message };
+    }
+  }
+}
+
+export default Validator;

package/test/analyze.test.js

@@ -0,0 +1,90 @@
+/**
+ * DeepSpider 分析能力测试
+ */
+
+import { ASTAnalyzer } from '../src/analyzer/ASTAnalyzer.js';
+import { CallStackAnalyzer } from '../src/analyzer/CallStackAnalyzer.js';
+import { EncryptionAnalyzer } from '../src/analyzer/EncryptionAnalyzer.js';
+import { Deobfuscator } from '../src/analyzer/Deobfuscator.js';
+import { Sandbox } from '../src/core/Sandbox.js';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// 读取测试样本
+const originalCode = fs.readFileSync(path.join(__dirname, 'samples/original.js'), 'utf-8');
+const obfuscatedCode = fs.readFileSync(path.join(__dirname, 'samples/obfuscated.js'), 'utf-8');
+
+console.log('=== DeepSpider 分析能力测试 ===\n');
+
+// 测试 1: 混淆器识别
+console.log('【测试1】混淆器识别');
+const deob = new Deobfuscator();
+const obfuscatorType = deob.detectObfuscator(obfuscatedCode);
+console.log('识别结果:', obfuscatorType);
+console.log('混淆类型:', deob._detectType(obfuscatedCode));
+console.log('');
+
+// 测试 2: 反混淆流水线
+console.log('【测试2】反混淆流水线');
+const pipelineResult = deob.runPipeline(obfuscatedCode);
+console.log('应用的步骤:', pipelineResult.applied);
+console.log('代码长度变化:', obfuscatedCode.length, '->', pipelineResult.code.length);
+console.log('');
+
+// 测试 3: AST 分析
+console.log('【测试3】AST 分析');
+const astAnalyzer = new ASTAnalyzer();
+const functions = astAnalyzer.extractFunctions(obfuscatedCode);
+console.log('提取到函数数量:', functions.length);
+functions.forEach(f => console.log('  -', f.name, `(${f.params.join(', ')})`));
+console.log('');
+
+// 测试 4: 调用链分析
+console.log('【测试4】调用链分析');
+const callAnalyzer = new CallStackAnalyzer();
+const entryPoints = callAnalyzer.findEntryPoints(obfuscatedCode);
+console.log('入口点:', entryPoints.length);
+entryPoints.forEach(e => console.log('  -', e.type, e.name || ''));
+
+const callGraph = callAnalyzer.buildCallGraph(obfuscatedCode);
+console.log('调用图节点数:', callGraph.size);
+console.log('');
+
+// 测试 5: 加密分析
+console.log('【测试5】加密分析');
+const encAnalyzer = new EncryptionAnalyzer();
+const cryptoResult = encAnalyzer.analyze(obfuscatedCode);
+console.log('检测到算法:', cryptoResult.detectedAlgorithms.map(a => a.name));
+console.log('可疑函数:', cryptoResult.suspiciousFunctions.length);
+console.log('');
+
+// 测试 6: 字符串提取
+console.log('【测试6】字符串提取');
+const strings = astAnalyzer.extractStrings(obfuscatedCode);
+console.log('字符串数量:', strings.length);
+const keyStrings = strings.filter(s =>
+  s.value.includes('key') ||
+  s.value.includes('sign') ||
+  s.value.includes('app')
+);
+console.log('关键字符串:', keyStrings.map(s => s.value));
+console.log('');
+
+// 测试 7: 沙箱执行
+console.log('【测试7】沙箱执行');
+const sandbox = new Sandbox();
+
+const execResult = await sandbox.execute(obfuscatedCode);
+console.log('执行成功:', execResult.success);
+if (execResult.success) {
+  console.log('执行结果:', execResult.result);
+} else {
+  console.log('错误:', execResult.error);
+  console.log('错误类型:', execResult.errorType);
+  console.log('缺失环境:', execResult.missingEnv);
+}
+
+console.log('\n=== 测试完成 ===');

package/test/envdump.test.js

@@ -0,0 +1,74 @@
+/**
+ * 环境自吐功能测试
+ */
+
+import { generateEnvDumpCode, generateBaseEnvCode } from '../src/agent/tools/envdump.js';
+import { sandboxExecute, sandboxInject, sandboxReset } from '../src/agent/tools/sandbox.js';
+
+async function test() {
+  console.log('=== 环境自吐测试 ===\n');
+
+  // 1. 生成基础环境代码
+  console.log('【Step 1】生成基础环境');
+  const baseResult = JSON.parse(await generateBaseEnvCode.invoke({}));
+  console.log('基础环境代码长度:', baseResult.code.length, '字符\n');
+
+  // 2. 生成环境自吐代码
+  console.log('【Step 2】生成环境自吐代码');
+  const dumpResult = JSON.parse(await generateEnvDumpCode.invoke({
+    targets: ['window', 'document', 'navigator'],
+    enableCallStack: false,
+    maxValueLength: 50,
+  }));
+  console.log('自吐代码长度:', dumpResult.code.length, '字符\n');
+
+  // 3. 注入基础环境
+  console.log('【Step 3】注入基础环境');
+  await sandboxReset.invoke({});
+  const injectBase = JSON.parse(await sandboxInject.invoke({ code: baseResult.code }));
+  console.log('注入结果:', injectBase.success ? '✅' : '❌', '\n');
+
+  // 4. 注入自吐代码
+  console.log('【Step 4】注入自吐代码');
+  const injectDump = JSON.parse(await sandboxInject.invoke({ code: dumpResult.code }));
+  console.log('注入结果:', injectDump.success ? '✅' : '❌', '\n');
+
+  // 5. 执行测试代码
+  console.log('【Step 5】执行测试代码');
+  const testCode = `
+    // 模拟目标代码访问环境
+    var ua = navigator.userAgent;
+    var platform = navigator.platform;
+    var cookie = document.cookie;
+    document.createElement('div');
+    window.innerWidth;
+
+    // 获取日志
+    __deepspider__.getLogs('env');
+  `;
+
+  const execResult = JSON.parse(await sandboxExecute.invoke({ code: testCode, timeout: 3000 }));
+  console.log('执行结果:', execResult.success ? '✅' : '❌');
+
+  if (execResult.success && execResult.result) {
+    console.log('\n【环境访问日志】');
+    try {
+      const logs = JSON.parse(execResult.result);
+      console.log('记录条数:', logs.length);
+      logs.slice(0, 10).forEach((log, i) => {
+        console.log(`  ${i + 1}. [${log.type}] ${log.path}`);
+      });
+      if (logs.length > 10) {
+        console.log(`  ... 还有 ${logs.length - 10} 条`);
+      }
+    } catch (e) {
+      console.log('日志解析失败:', e.message);
+    }
+  } else {
+    console.log('错误:', execResult.error);
+  }
+
+  console.log('\n=== 测试完成 ===');
+}
+
+test().catch(console.error);

package/test/flow.test.js

@@ -0,0 +1,90 @@
+/**
+ * DeepSpider 完整流程测试 - 单样本深度分析
+ */
+
+import { ASTAnalyzer } from '../src/analyzer/ASTAnalyzer.js';
+import { CallStackAnalyzer } from '../src/analyzer/CallStackAnalyzer.js';
+import { EncryptionAnalyzer } from '../src/analyzer/EncryptionAnalyzer.js';
+import { Deobfuscator } from '../src/analyzer/Deobfuscator.js';
+import { Sandbox } from '../src/core/Sandbox.js';
+import { Store } from '../src/store/Store.js';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// 选择 v2_ob_advanced.js 进行深度分析
+const code = fs.readFileSync(
+  path.join(__dirname, 'samples/v2_ob_advanced.js'),
+  'utf-8'
+);
+
+console.log('=== DeepSpider 完整流程测试 ===\n');
+console.log('目标: v2_ob_advanced.js\n');
+
+// Step 1: 混淆识别
+console.log('【Step 1】混淆识别');
+const deob = new Deobfuscator();
+console.log('  混淆器:', deob.detectObfuscator(code));
+console.log('  类型:', deob._detectType(code));
+
+// Step 2: 反混淆
+console.log('\n【Step 2】反混淆流水线');
+const deobResult = deob.runPipeline(code);
+console.log('  应用步骤:', deobResult.applied.join(' → '));
+console.log('  代码压缩:', code.length, '→', deobResult.code.length);
+
+// Step 3: AST 分析
+console.log('\n【Step 3】AST 分析');
+const ast = new ASTAnalyzer();
+const funcs = ast.extractFunctions(code);
+console.log('  函数列表:');
+funcs.forEach(f => console.log(`    - ${f.name}(${f.params.join(', ')})`));
+
+// Step 4: 调用链分析
+console.log('\n【Step 4】调用链分析');
+const call = new CallStackAnalyzer();
+const graph = call.buildCallGraph(code);
+console.log('  调用图:');
+for (const [func, calls] of graph) {
+  if (calls.length > 0) {
+    console.log(`    ${func} → ${calls.map(c => c.callee).join(', ')}`);
+  }
+}
+
+// Step 5: 加密分析
+console.log('\n【Step 5】加密分析');
+const enc = new EncryptionAnalyzer();
+const crypto = enc.analyze(code);
+console.log('  检测算法:', crypto.detectedAlgorithms.map(a => a.name).join(', ') || '无');
+console.log('  可疑函数:', crypto.suspiciousFunctions.length);
+
+// Step 6: 字符串提取
+console.log('\n【Step 6】字符串提取');
+const strings = ast.extractStrings(code);
+const keywords = strings.filter(s =>
+  /sign|key|secret|app|encrypt/i.test(s.value)
+);
+console.log('  总字符串:', strings.length);
+console.log('  关键字符串:', keywords.map(s => `"${s.value}"`).join(', '));
+
+// Step 7: 沙箱执行
+console.log('\n【Step 7】沙箱执行');
+const sandbox = new Sandbox();
+const result = await sandbox.execute(code, { timeout: 5000 });
+console.log('  执行状态:', result.success ? '✅ 成功' : '❌ 失败');
+console.log('  结果:', result.result);
+
+// Step 8: 存储结果
+console.log('\n【Step 8】存储分析结果');
+const store = new Store();
+store.save('analysis', 'v2_ob_advanced', {
+  obfuscator: deob.detectObfuscator(code),
+  functions: funcs.length,
+  crypto: crypto.detectedAlgorithms.map(a => a.name),
+  executed: result.success
+});
+console.log('  已保存到 Store');
+
+console.log('\n=== 流程测试完成 ===');

package/test/hooks.test.js

@@ -0,0 +1,138 @@
+/**
+ * DeepSpider Hook 系统测试
+ */
+
+import { HookBase } from '../src/env/HookBase.js';
+import { getDefaultHookScript } from '../src/browser/defaultHooks.js';
+
+console.log('=== DeepSpider Hook 系统测试 ===\n');
+
+// 测试 1: HookBase 代码生成
+console.log('【测试1】HookBase 代码生成');
+const baseCode = HookBase.getBaseCode();
+console.log('基础代码长度:', baseCode.length);
+console.log('包含 __deepspider__:', baseCode.includes('window.__deepspider__'));
+console.log('包含日志限制:', baseCode.includes('LOG_LIMIT'));
+console.log('包含配置管理:', baseCode.includes('getConfig'));
+console.log('');
+
+// 测试 2: 完整 Hook 脚本生成
+console.log('【测试2】完整 Hook 脚本生成');
+const fullScript = getDefaultHookScript();
+console.log('完整脚本长度:', fullScript.length);
+console.log('包含 XHR Hook:', fullScript.includes('XHR Hook'));
+console.log('包含 Fetch Hook:', fullScript.includes('Fetch Hook'));
+console.log('包含 Cookie Hook:', fullScript.includes('Cookie Hook'));
+console.log('包含 JSON Hook:', fullScript.includes('JSON Hook'));
+console.log('包含 Eval Hook:', fullScript.includes('Eval/Function Hook'));
+console.log('包含 Crypto Hook:', fullScript.includes('Crypto Hook'));
+console.log('包含 DOM Hook:', fullScript.includes('DOM Hook'));
+console.log('包含 Debugger Bypass:', fullScript.includes('debugger bypassed'));
+console.log('包含 Encoding Hook:', fullScript.includes('Encoding Hook'));
+console.log('包含 Storage Hook:', fullScript.includes('Storage Hook'));
+console.log('包含 WebSocket Hook:', fullScript.includes('WebSocket Hook'));
+console.log('包含 Web Crypto API:', fullScript.includes('WebCrypto'));
+console.log('包含 Canvas Hook:', fullScript.includes('Canvas Hook'));
+console.log('包含 Navigator Hook:', fullScript.includes('Navigator Hook'));
+console.log('包含 Webpack Hook:', fullScript.includes('Webpack Hook'));
+console.log('包含 Forge Hook:', fullScript.includes('Forge Hook'));
+console.log('包含 jsrsasign Hook:', fullScript.includes('jsrsasign Hook'));
+console.log('包含 Proxy Hook:', fullScript.includes('Proxy Hook'));
+console.log('包含 Error Stack Hook:', fullScript.includes('Error Stack Hook'));
+console.log('');
+
+// 测试 6: 验证新增 API
+console.log('【测试6】验证新增 API');
+const newAPIs = [
+  'searchLogs',
+  'traceValue',
+  'correlateParams',
+  'getRecentCrypto',
+  'getRecentRequests',
+  'exportLogs',
+  'recordPerf',
+  'getPerf',
+  'getCaller'
+];
+newAPIs.forEach(api => {
+  const found = fullScript.includes(api);
+  console.log(`  ${found ? '✓' : '✗'} ${api}`);
+});
+console.log('');
+
+// 测试 3: 验证 Hook 代码语法
+console.log('【测试3】验证 Hook 代码语法');
+try {
+  new Function(fullScript);
+  console.log('语法检查: 通过');
+} catch (e) {
+  console.error('语法检查: 失败 -', e.message);
+}
+console.log('');
+
+// 测试 4: 验证关键功能存在
+console.log('【测试4】验证关键功能');
+const features = [
+  { name: 'JSON.parse Hook', pattern: /JSON\.parse\s*=\s*deepspider\.native/ },
+  { name: 'JSON.stringify Hook', pattern: /JSON\.stringify\s*=\s*deepspider\.native/ },
+  { name: 'eval Hook', pattern: /window\.eval\s*=\s*deepspider\.native/ },
+  { name: 'Function Hook', pattern: /window\.Function\s*=\s*deepspider\.native/ },
+  { name: 'setTimeout 字符串检测', pattern: /typeof handler === 'string'/ },
+  { name: 'CryptoJS 即时 Hook', pattern: /watchGlobal\('CryptoJS'/ },
+  { name: 'DOM querySelector Hook', pattern: /m\.obj\[m\.name\]\s*=\s*deepspider\.native/ },
+  { name: 'debugger 绕过', pattern: /debugger bypassed/ },
+  { name: '日志限制', pattern: /logCounts\[countKey\].*config\.logLimit/ },
+];
+
+features.forEach(f => {
+  const found = f.pattern.test(fullScript);
+  console.log(`  ${found ? '✓' : '✗'} ${f.name}`);
+});
+console.log('');
+
+// 测试 5: 验证配置项
+console.log('【测试5】验证配置项');
+const configItems = [
+  'json', 'eval', 'crypto', 'cookie', 'xhr', 'fetch', 'dom', 'logLimit',
+  'captureStack', 'stackDepth', 'protectDescriptor', 'protectKeys',
+  'silent', 'logToConsole'
+];
+configItems.forEach(item => {
+  const found = fullScript.includes(`${item}:`);
+  console.log(`  ${found ? '✓' : '✗'} config.${item}`);
+});
+console.log('');
+
+// 测试 7: 验证反检测功能
+console.log('【测试7】验证反检测功能');
+const antiDetectFeatures = [
+  { name: 'toString 伪装', pattern: /hookedFns\.has\(this\)/ },
+  { name: 'getOwnPropertyDescriptor 保护', pattern: /Object\.getOwnPropertyDescriptor\s*=\s*function/ },
+  { name: 'Object.keys 保护', pattern: /Object\.keys\s*=\s*function/ },
+  { name: 'getOwnPropertyNames 保护', pattern: /Object\.getOwnPropertyNames\s*=\s*function/ },
+  { name: 'Error.stack 过滤', pattern: /__deepspider__|DeepSpider|deepspider\\.native/ },
+  { name: 'Proxy 监控', pattern: /Proxy\.create/ },
+];
+antiDetectFeatures.forEach(f => {
+  const found = f.pattern.test(fullScript);
+  console.log(`  ${found ? '✓' : '✗'} ${f.name}`);
+});
+console.log('');
+
+// 测试 8: 验证 Hook 管理 API
+console.log('【测试8】验证 Hook 管理 API');
+const hookMgmtAPIs = [
+  'registerHook',
+  'enableHook',
+  'disableHook',
+  'listHooks',
+  'injectHook',
+  'setHooks'
+];
+hookMgmtAPIs.forEach(api => {
+  const found = fullScript.includes(api);
+  console.log(`  ${found ? '✓' : '✗'} ${api}`);
+});
+console.log('');
+
+console.log('=== Hook 系统测试完成 ===');

package/test/plugin.test.js

@@ -0,0 +1,35 @@
+/**
+ * Plugin 可用性测试
+ */
+
+import { allTools } from '../src/agent/tools/index.js';
+
+console.log('=== Plugin 工具验证 ===\n');
+
+console.log('【工具数量】', allTools.length, '个\n');
+
+// 验证每个工具
+let valid = 0;
+let invalid = 0;
+
+for (const tool of allTools) {
+  const hasName = !!tool.name;
+  const hasDesc = !!tool.description;
+  const hasSchema = !!tool.schema;
+
+  if (hasName && hasDesc && hasSchema) {
+    valid++;
+  } else {
+    invalid++;
+    console.log('❌', tool.name || 'unknown', '- 缺少必要属性');
+  }
+}
+
+console.log(`\n验证结果: ${valid} 个有效, ${invalid} 个无效`);
+
+if (invalid === 0) {
+  console.log('\n✅ 所有工具验证通过!');
+} else {
+  console.log('\n❌ 存在无效工具');
+  process.exit(1);
+}

package/test/refactor-full.test.js

@@ -0,0 +1,30 @@
+/**
+ * DeepSpider 完整重构测试
+ */
+
+import { allTools } from '../src/agent/tools/index.js';
+import { allSubagents } from '../src/agent/subagents/index.js';
+
+async function testTools() {
+  console.log('=== 测试工具导入 ===');
+  console.log('工具数量:', allTools.length);
+  console.log('工具列表:', allTools.map(t => t.name).join(', '));
+  console.log('');
+}
+
+async function testSubagents() {
+  console.log('=== 测试子代理 ===');
+  console.log('子代理数量:', allSubagents.length);
+  allSubagents.forEach(s => {
+    console.log(`- ${s.name}: ${s.tools?.length || 0} 个工具`);
+  });
+  console.log('');
+}
+
+async function main() {
+  await testTools();
+  await testSubagents();
+  console.log('完整重构测试通过!');
+}
+
+main().catch(console.error);

package/test/refactor.test.js

@@ -0,0 +1,21 @@
+/**
+ * DeepSpider 重构测试
+ */
+
+import { Store } from '../src/store/Store.js';
+
+async function testStore() {
+  console.log('=== 测试知识库 ===');
+  const store = new Store();
+  store.save('env', 'navigator', { code: 'test' });
+  const result = store.get('env', 'navigator');
+  console.log('存储成功:', result !== undefined);
+  console.log('知识库测试通过!\n');
+}
+
+async function main() {
+  await testStore();
+  console.log('重构基础测试通过!');
+}
+
+main().catch(console.error);