deepspider 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.env.example +3 -0
  2. package/README.md +13 -13
  3. package/package.json +6 -6
  4. package/src/agent/core/PanelBridge.js +29 -77
  5. package/src/agent/core/StreamHandler.js +139 -14
  6. package/src/agent/index.js +51 -12
  7. package/src/agent/logger.js +184 -9
  8. package/src/agent/middleware/report.js +42 -16
  9. package/src/agent/middleware/subagent.js +233 -0
  10. package/src/agent/middleware/toolGuard.js +77 -0
  11. package/src/agent/middleware/validationWorkflow.js +171 -0
  12. package/src/agent/prompts/system.js +181 -59
  13. package/src/agent/run.js +41 -6
  14. package/src/agent/skills/crawler/SKILL.md +64 -3
  15. package/src/agent/skills/crawler/evolved.md +9 -1
  16. package/src/agent/skills/dynamic-analysis/SKILL.md +74 -7
  17. package/src/agent/skills/env/SKILL.md +75 -0
  18. package/src/agent/skills/evolve.js +0 -3
  19. package/src/agent/skills/sandbox/SKILL.md +35 -0
  20. package/src/agent/skills/static-analysis/SKILL.md +98 -2
  21. package/src/agent/subagents/anti-detect.js +10 -20
  22. package/src/agent/subagents/captcha.js +7 -19
  23. package/src/agent/subagents/crawler.js +25 -37
  24. package/src/agent/subagents/factory.js +109 -9
  25. package/src/agent/subagents/index.js +4 -13
  26. package/src/agent/subagents/js2python.js +7 -19
  27. package/src/agent/subagents/reverse.js +180 -0
  28. package/src/agent/tools/analysis.js +84 -1
  29. package/src/agent/tools/anti-detect.js +5 -2
  30. package/src/agent/tools/browser.js +160 -0
  31. package/src/agent/tools/captcha.js +1 -1
  32. package/src/agent/tools/capture.js +24 -3
  33. package/src/agent/tools/correlate.js +129 -15
  34. package/src/agent/tools/crawler.js +2 -1
  35. package/src/agent/tools/crawlerGenerator.js +90 -0
  36. package/src/agent/tools/debug.js +43 -6
  37. package/src/agent/tools/evolve.js +6 -3
  38. package/src/agent/tools/extractor.js +5 -1
  39. package/src/agent/tools/file.js +16 -7
  40. package/src/agent/tools/generateHook.js +66 -0
  41. package/src/agent/tools/hookManager.js +19 -9
  42. package/src/agent/tools/index.js +33 -20
  43. package/src/agent/tools/nodejs.js +41 -6
  44. package/src/agent/tools/python.js +4 -4
  45. package/src/agent/tools/report.js +2 -2
  46. package/src/agent/tools/runtime.js +1 -1
  47. package/src/agent/tools/sandbox.js +21 -1
  48. package/src/agent/tools/scratchpad.js +70 -0
  49. package/src/agent/tools/tracing.js +26 -0
  50. package/src/agent/tools/verifyAlgorithm.js +117 -0
  51. package/src/analyzer/EncryptionAnalyzer.js +2 -2
  52. package/src/browser/EnvBridge.js +27 -13
  53. package/src/browser/client.js +124 -18
  54. package/src/browser/collector.js +101 -22
  55. package/src/browser/defaultHooks.js +3 -1
  56. package/src/browser/hooks/index.js +5 -0
  57. package/src/browser/interceptors/AntiDebugInterceptor.js +132 -0
  58. package/src/browser/interceptors/NetworkInterceptor.js +77 -13
  59. package/src/browser/interceptors/ScriptInterceptor.js +34 -9
  60. package/src/browser/interceptors/index.js +1 -0
  61. package/src/browser/ui/analysisPanel.js +469 -464
  62. package/src/cli/commands/config.js +11 -3
  63. package/src/config/paths.js +9 -1
  64. package/src/config/settings.js +7 -1
  65. package/src/core/PatchGenerator.js +26 -6
  66. package/src/core/Sandbox.js +140 -3
  67. package/src/env/EnvCodeGenerator.js +60 -88
  68. package/src/env/modules/bom/history.js +6 -0
  69. package/src/env/modules/bom/location.js +6 -0
  70. package/src/env/modules/bom/navigator.js +13 -0
  71. package/src/env/modules/bom/screen.js +6 -0
  72. package/src/env/modules/bom/storage.js +7 -0
  73. package/src/env/modules/dom/document.js +14 -0
  74. package/src/env/modules/dom/event.js +4 -0
  75. package/src/env/modules/index.js +27 -10
  76. package/src/env/modules/webapi/fetch.js +4 -0
  77. package/src/env/modules/webapi/url.js +4 -0
  78. package/src/env/modules/webapi/xhr.js +8 -0
  79. package/src/store/DataStore.js +130 -47
  80. package/src/store/Store.js +2 -1
  81. package/src/agent/subagents/dynamic.js +0 -64
  82. package/src/agent/subagents/env-agent.js +0 -82
  83. package/src/agent/subagents/sandbox.js +0 -55
  84. package/src/agent/subagents/static.js +0 -66
@@ -5,8 +5,8 @@
5
5
 
6
6
  import { z } from 'zod';
7
7
  import { tool } from '@langchain/core/tools';
8
- import { writeFileSync, readFileSync, existsSync, readdirSync, statSync } from 'fs';
9
- import { dirname, join, isAbsolute, relative } from 'path';
8
+ import { writeFileSync, readFileSync, existsSync, readdirSync } from 'fs';
9
+ import { dirname, join, isAbsolute, relative, resolve } from 'path';
10
10
  import { PATHS, ensureDir, DEEPSPIDER_HOME } from '../../config/paths.js';
11
11
 
12
12
  const OUTPUT_DIR = PATHS.OUTPUT_DIR;
@@ -17,15 +17,24 @@ function ensureFileDir(filePath) {
17
17
  }
18
18
 
19
19
  function getSafePath(filePath) {
20
+ let resolved;
20
21
  if (isAbsolute(filePath)) {
21
22
  // 如果是 ~/.deepspider/ 目录下的路径,直接使用
22
23
  if (filePath.startsWith(DEEPSPIDER_HOME)) {
23
- return filePath;
24
+ resolved = filePath;
25
+ } else {
26
+ // 其他绝对路径:放到 OUTPUT_DIR 下
27
+ resolved = join(OUTPUT_DIR, filePath.replace(/^\/+/, ''));
24
28
  }
25
- // 其他绝对路径:放到 OUTPUT_DIR 下
26
- return join(OUTPUT_DIR, filePath.replace(/^\/+/, ''));
29
+ } else {
30
+ resolved = join(OUTPUT_DIR, filePath);
27
31
  }
28
- return join(OUTPUT_DIR, filePath);
32
+ // 防止 ../ 穿越到 DEEPSPIDER_HOME 之外
33
+ const normalized = resolve(resolved);
34
+ if (!normalized.startsWith(DEEPSPIDER_HOME)) {
35
+ throw new Error(`路径不允许超出 ${DEEPSPIDER_HOME}: ${filePath}`);
36
+ }
37
+ return normalized;
29
38
  }
30
39
 
31
40
  export const artifactSave = tool(
@@ -186,7 +195,7 @@ function searchInFile(filePath, pattern, isRegex) {
186
195
  if (isRegex) regex.lastIndex = 0; // 重置正则
187
196
  }
188
197
  return matches;
189
- } catch (e) {
198
+ } catch {
190
199
  return [];
191
200
  }
192
201
  }
@@ -0,0 +1,66 @@
1
+ /**
2
+ * DeepSpider - 统一 Hook 代码生成工具
3
+ * 合并 hookTools + cryptoHookTools + asyncTools + antiDebugTools
4
+ */
5
+
6
+ import { z } from 'zod';
7
+ import { tool } from '@langchain/core/tools';
8
+ import { NetworkHook } from '../../env/NetworkHook.js';
9
+ import { CookieHook } from '../../env/CookieHook.js';
10
+ import { CryptoHook } from '../../env/CryptoHook.js';
11
+ import { AsyncHook } from '../../env/AsyncHook.js';
12
+ import { AntiAntiDebug } from '../../env/AntiAntiDebug.js';
13
+
14
+ const networkHook = new NetworkHook();
15
+ const cookieHook = new CookieHook();
16
+ const cryptoHook = new CryptoHook();
17
+ const asyncHook = new AsyncHook();
18
+ const antiDebug = new AntiAntiDebug();
19
+
20
+ const HOOK_TYPES = {
21
+ // 网络
22
+ xhr: { gen: () => networkHook.generateXHRHookCode({ captureBody: true, captureResponse: true }), usage: "getLogs('xhr')" },
23
+ fetch: { gen: () => networkHook.generateFetchHookCode({ captureBody: true, captureResponse: true }), usage: "getLogs('fetch')" },
24
+ cookie: { gen: () => cookieHook.generateCookieHookCode({ trackRead: true, trackWrite: true }), usage: "getLogs('cookie')" },
25
+ // 加密
26
+ cryptojs: { gen: () => cryptoHook.generateCryptoJSHookCode(), usage: "getLogs('crypto')" },
27
+ sm_crypto: { gen: () => cryptoHook.generateSMCryptoHookCode(), usage: "getLogs('crypto')" },
28
+ rsa: { gen: () => cryptoHook.generateRSAHookCode(), usage: "getLogs('crypto')" },
29
+ generic_crypto: { gen: () => cryptoHook.generateGenericCryptoHookCode(), usage: "getLogs('crypto')" },
30
+ // 异步
31
+ promise: { gen: () => asyncHook.generatePromiseHookCode(), usage: "getLogs('async')" },
32
+ timer: { gen: () => asyncHook.generateTimerHookCode(), usage: "getLogs('timer')" },
33
+ // 反反调试
34
+ anti_debugger: { gen: () => antiDebug.generateAntiDebuggerCode(), usage: '绕过无限 debugger' },
35
+ anti_console: { gen: () => antiDebug.generateAntiConsoleDetectCode(), usage: '绕过控制台检测' },
36
+ anti_cdp: { gen: () => antiDebug.generateAntiCDPDetectCode(), usage: '绕过 CDP 检测' },
37
+ anti_debug_full: { gen: () => antiDebug.generateFullAntiDebugCode(), usage: '完整反反调试(包含以上所有)' },
38
+ };
39
+
40
+ const typeEnum = /** @type {[string, ...string[]]} */ (Object.keys(HOOK_TYPES));
41
+
42
+ export const generateHook = tool(
43
+ async ({ type }) => {
44
+ const entry = HOOK_TYPES[type];
45
+ if (!entry) {
46
+ return JSON.stringify({ success: false, error: `未知类型: ${type},可选: ${typeEnum.join(', ')}` });
47
+ }
48
+ const code = entry.gen();
49
+ return JSON.stringify({ success: true, type, code, usage: entry.usage }, null, 2);
50
+ },
51
+ {
52
+ name: 'generate_hook',
53
+ description: `生成 Hook 代码。生成后需通过 inject_hook 注入浏览器。
54
+
55
+ 类型:
56
+ - 网络: xhr, fetch, cookie
57
+ - 加密: cryptojs(CryptoJS), sm_crypto(国密), rsa(JSEncrypt/node-forge), generic_crypto(通用)
58
+ - 异步: promise, timer
59
+ - 反反调试: anti_debugger, anti_console, anti_cdp, anti_debug_full(完整)`,
60
+ schema: z.object({
61
+ type: z.enum(typeEnum).describe('Hook 类型'),
62
+ }),
63
+ }
64
+ );
65
+
66
+ export const generateHookTools = [generateHook];
@@ -8,16 +8,28 @@ import { tool } from '@langchain/core/tools';
8
8
  import { getBrowser } from '../../browser/index.js';
9
9
 
10
10
  /**
11
- * 通过 CDP 执行 JS
11
+ * 通过 CDP 执行 JS(带超时保护)
12
12
  */
13
- async function evaluateViaCDP(browser, expression) {
13
+ async function evaluateViaCDP(browser, expression, timeout = 5000) {
14
14
  const cdp = await browser.getCDPSession();
15
15
  if (!cdp) return null;
16
- const result = await cdp.send('Runtime.evaluate', {
16
+
17
+ const evaluatePromise = cdp.send('Runtime.evaluate', {
17
18
  expression,
18
19
  returnByValue: true,
19
20
  });
20
- return result.result?.value;
21
+
22
+ const timeoutPromise = new Promise((_, reject) =>
23
+ setTimeout(() => reject(new Error('CDP evaluate timeout')), timeout)
24
+ );
25
+
26
+ try {
27
+ const result = await Promise.race([evaluatePromise, timeoutPromise]);
28
+ return result.result?.value;
29
+ } catch (e) {
30
+ console.error('[evaluateViaCDP] 超时或错误:', e.message);
31
+ return null;
32
+ }
21
33
  }
22
34
 
23
35
  /**
@@ -115,14 +127,12 @@ export const injectHook = tool(
115
127
  if (!browser.getPage()) {
116
128
  return JSON.stringify({ success: false, error: '浏览器未就绪' });
117
129
  }
118
- const escapedCode = code
119
- .replace(/\\/g, '\\\\')
120
- .replace(/'/g, "\\'")
121
- .replace(/\n/g, '\\n');
130
+ // JSON.stringify 安全转义,避免手动转义遗漏特殊字符
131
+ const safeCode = JSON.stringify(code);
122
132
 
123
133
  const result = await evaluateViaCDP(
124
134
  browser,
125
- `JSON.stringify(window.__deepspider__?.injectHook?.('${escapedCode}'))`
135
+ `JSON.stringify(window.__deepspider__?.injectHook?.(${safeCode}))`
126
136
  );
127
137
  return result || JSON.stringify({ success: false, error: '注入失败' });
128
138
  } catch (e) {
@@ -11,9 +11,9 @@ export { patchTools, generatePatch, matchModule } from './patch.js';
11
11
  export { envTools, listEnvModules, loadEnvModule, loadAllEnvModules } from './env.js';
12
12
  export { profileTools, listProfiles, loadProfile, generateProfileCode } from './profile.js';
13
13
  export { runtimeTools, launchBrowser, navigateTo, browserClose, addInitScript, clearCookies } from './runtime.js';
14
- export { debugTools, setBreakpoint, setXHRBreakpoint, getCallStack, getFrameVariables, evaluateAtBreakpoint, resumeExecution, stepOver } from './debug.js';
14
+ export { debugTools, setBreakpoint, setXHRBreakpoint, getCallStack, getFrameVariables, evaluateAtBreakpoint, resumeExecution, stepOver, getAgentLogs } from './debug.js';
15
15
  export { captureTools, collectEnv, collectProperty, autoFixEnv, getHookLogs } from './capture.js';
16
- export { browserTools, clickElement, fillInput, waitForSelector } from './browser.js';
16
+ export { browserTools, clickElement, fillInput, waitForSelector, takeScreenshot, reloadPage, goBack, goForward, scrollPage, pressKey, hoverElement, getPageInfo, getPageSource, getElementHtml, getCookies, getInteractiveElements } from './browser.js';
17
17
  export { reportTools, saveAnalysisReport } from './report.js';
18
18
  export { webcrackTools, unpackBundle, analyzeBundle } from './webcrack.js';
19
19
  export { preprocessTools, preprocessCode } from './preprocess.js';
@@ -24,17 +24,22 @@ export { asyncTools, generatePromiseHook, generateTimerHook } from './async.js';
24
24
  export { antiDebugTools, generateAntiDebugger, generateAntiConsoleDetect, generateAntiCDP, generateFullAntiDebug } from './antidebug.js';
25
25
  export { verifyTools, verifyMD5, verifySHA256, verifyHMAC, verifyAES, identifyEncryption } from './verify.js';
26
26
  export { cryptoHookTools, generateCryptoJSHook, generateRSAHook } from './cryptohook.js';
27
- export { correlateTools, analyzeCorrelation, locateCryptoSource, analyzeHeaderEncryption, analyzeCookieEncryption, analyzeResponseDecryption } from './correlate.js';
27
+ // 合并工具(reverse-agent 使用)
28
+ export { generateHookTools, generateHook } from './generateHook.js';
29
+ export { verifyAlgorithmTools, verifyAlgorithm } from './verifyAlgorithm.js';
30
+ export { correlateTools, analyzeCorrelation, locateCryptoSource, analyzeHeaderEncryption, analyzeCookieEncryption, analyzeResponseDecryption, analyzeRequestParams } from './correlate.js';
28
31
  export { extractorTools, listFunctions, getFunctionCode } from './extractor.js';
29
- export { tracingTools, getSiteList, searchInResponses, getRequestDetail, getRequestList, getScriptList, getScriptSource, searchInScripts, clearSiteData, clearAllData } from './tracing.js';
32
+ export { tracingTools, getSiteList, searchInResponses, getRequestDetail, getRequestList, getRequestInitiator, getScriptList, getScriptSource, searchInScripts, clearSiteData, clearAllData } from './tracing.js';
30
33
  export { analysisTools, getPendingAnalysis, getPendingChat, sendPanelMessage, startSelector } from './analysis.js';
31
34
  export { fileTools, artifactSave, artifactLoad, artifactEdit, artifactGlob, artifactGrep } from './file.js';
32
35
  export { evolveTools, evolveSkill } from './evolve.js';
33
36
  export { captchaTools } from './captcha.js';
34
37
  export { antiDetectTools } from './anti-detect.js';
35
38
  export { crawlerTools } from './crawler.js';
39
+ export { crawlerGeneratorTools, generateCrawlerWithConfirm, delegateCrawlerGeneration } from './crawlerGenerator.js';
36
40
  export { nodejsTools, runNodeCode } from './nodejs.js';
37
41
  export { hookManagerTools, listHooks, enableHook, disableHook, injectHook, setHookConfig } from './hookManager.js';
42
+ export { scratchpadTools, saveMemo, loadMemo, listMemo } from './scratchpad.js';
38
43
  // pythonTools 只在 js2python 子代理中使用,不导出到主工具集
39
44
 
40
45
  // 所有工具
@@ -62,15 +67,17 @@ import { verifyTools } from './verify.js';
62
67
  import { cryptoHookTools } from './cryptohook.js';
63
68
  import { correlateTools } from './correlate.js';
64
69
  import { extractorTools } from './extractor.js';
65
- import { tracingTools } from './tracing.js';
70
+ import { tracingTools, getSiteList, getRequestList, searchInResponses, getRequestDetail, getRequestInitiator } from './tracing.js';
66
71
  import { analysisTools } from './analysis.js';
67
72
  import { fileTools } from './file.js';
68
73
  import { evolveTools } from './evolve.js';
69
74
  import { captchaTools } from './captcha.js';
70
75
  import { antiDetectTools } from './anti-detect.js';
71
76
  import { crawlerTools } from './crawler.js';
77
+ import { crawlerGeneratorTools } from './crawlerGenerator.js';
72
78
  import { nodejsTools } from './nodejs.js';
73
79
  import { hookManagerTools } from './hookManager.js';
80
+ import { scratchpadTools } from './scratchpad.js';
74
81
 
75
82
  export const allTools = [
76
83
  ...sandboxTools,
@@ -104,34 +111,40 @@ export const allTools = [
104
111
  ...captchaTools,
105
112
  ...antiDetectTools,
106
113
  ...crawlerTools,
114
+ ...crawlerGeneratorTools,
107
115
  ...nodejsTools,
108
116
  ...hookManagerTools,
117
+ ...scratchpadTools,
109
118
  ];
110
119
 
111
120
  /**
112
121
  * 主 Agent 核心工具
113
- * 只包含必要的运行时、交互和数据查询工具
114
- * pythonTools 只在 js2python 子代理中使用
122
+ * 职责:浏览器生命周期、简单页面交互、数据查询、委托调度
123
+ *
124
+ * 以下工具有意不包含在主 agent 中,由专属子代理持有:
125
+ * - hookManagerTools (inject_hook 等) → reverse-agent
126
+ * - captureTools (collect_env, get_hook_logs 等) → reverse-agent
127
+ * - sandboxTools → reverse-agent
128
+ * - debugTools → reverse-agent
129
+ * - 静态分析工具 → reverse-agent
115
130
  */
116
131
  export const coreTools = [
117
- // 浏览器运行时
132
+ // 浏览器运行时(生命周期管理)
118
133
  ...runtimeTools,
119
- // 页面交互
120
- ...browserTools,
121
- // 浏览器分析交互
134
+ // 浏览器分析面板交互
122
135
  ...analysisTools,
123
- // 数据溯源查询
124
- ...tracingTools,
125
- // 沙箱执行(验证代码)
126
- ...sandboxTools,
127
- // Hook 日志
128
- ...captureTools,
136
+ // 数据查询(仅调度所需的最小集:列表、搜索、详情、initiator)
137
+ getSiteList, getRequestList, searchInResponses, getRequestDetail, getRequestInitiator,
138
+ // 报告生成
139
+ ...reportTools,
129
140
  // 文件操作
130
141
  ...fileTools,
131
142
  // 经验进化
132
143
  ...evolveTools,
133
- // Node.js 执行(支持 require)
144
+ // Node.js 执行(委托前快速验证假设)- 已添加网络请求防护
134
145
  ...nodejsTools,
135
- // Hook 动态管理
136
- ...hookManagerTools,
146
+ // 工作记忆
147
+ ...scratchpadTools,
148
+ // 爬虫代码生成(带 HITL 确认)
149
+ ...crawlerGeneratorTools,
137
150
  ];
@@ -17,10 +17,19 @@ const PROJECT_ROOT = join(__dirname, '../../..');
17
17
  // 输出大小限制
18
18
  const MAX_OUTPUT_SIZE = 100000;
19
19
 
20
+ // 超时上限(防止 LLM 传入过大值)
21
+ const MAX_TIMEOUT = 30000;
22
+
23
+ // 连续超时计数器
24
+ let consecutiveTimeouts = 0;
25
+ const MAX_CONSECUTIVE_TIMEOUTS = 3;
26
+
20
27
  /**
21
28
  * 执行 Node.js 代码
22
29
  */
23
30
  async function executeNode(code, timeout = 10000) {
31
+ const effectiveTimeout = Math.min(timeout, MAX_TIMEOUT);
32
+
24
33
  return new Promise((resolve) => {
25
34
  const proc = spawn('node', ['-e', code], {
26
35
  env: { ...process.env },
@@ -30,12 +39,17 @@ async function executeNode(code, timeout = 10000) {
30
39
  let stdout = '';
31
40
  let stderr = '';
32
41
  let killed = false;
42
+ let killTimer = null;
33
43
 
34
- // 手动实现超时
44
+ // SIGTERM 超时
35
45
  const timer = setTimeout(() => {
36
46
  killed = true;
37
47
  proc.kill('SIGTERM');
38
- }, timeout);
48
+ // SIGKILL 兜底:环境检测死循环可能忽略 SIGTERM
49
+ killTimer = setTimeout(() => {
50
+ try { proc.kill('SIGKILL'); } catch { /* already dead */ }
51
+ }, 2000);
52
+ }, effectiveTimeout);
39
53
 
40
54
  proc.stdout.on('data', (data) => {
41
55
  if (stdout.length < MAX_OUTPUT_SIZE) {
@@ -51,21 +65,25 @@ async function executeNode(code, timeout = 10000) {
51
65
 
52
66
  proc.on('close', (exitCode) => {
53
67
  clearTimeout(timer);
68
+ if (killTimer) clearTimeout(killTimer);
54
69
  resolve({
55
70
  success: !killed && exitCode === 0,
56
71
  stdout: stdout.trim(),
57
- stderr: killed ? 'Timeout: process killed' : stderr.trim(),
72
+ stderr: killed ? `Timeout after ${effectiveTimeout}ms: process killed` : stderr.trim(),
58
73
  exitCode: killed ? -1 : exitCode,
74
+ timedOut: killed,
59
75
  });
60
76
  });
61
77
 
62
78
  proc.on('error', (err) => {
63
79
  clearTimeout(timer);
80
+ if (killTimer) clearTimeout(killTimer);
64
81
  resolve({
65
82
  success: false,
66
83
  stdout: '',
67
84
  stderr: err.message,
68
85
  exitCode: -1,
86
+ timedOut: false,
69
87
  });
70
88
  });
71
89
  });
@@ -76,8 +94,25 @@ async function executeNode(code, timeout = 10000) {
76
94
  */
77
95
  export const runNodeCode = tool(
78
96
  async ({ code, timeout }) => {
79
- const result = await executeNode(code, timeout || 10000);
80
- return JSON.stringify(result);
97
+ // 连续超时保护:降级为短超时探测,而非完全拒绝
98
+ const isBlocked = consecutiveTimeouts >= MAX_CONSECUTIVE_TIMEOUTS;
99
+ const effectiveTimeout = isBlocked ? 5000 : (timeout || 10000);
100
+ const result = await executeNode(code, effectiveTimeout);
101
+
102
+ // 更新连续超时计数
103
+ if (result.timedOut) {
104
+ consecutiveTimeouts++;
105
+ if (consecutiveTimeouts >= MAX_CONSECUTIVE_TIMEOUTS) {
106
+ result.stderr += `\n\n⚠️ 已连续超时 ${consecutiveTimeouts} 次,后续调用将降级为 5s 短超时。代码很可能存在死循环或触发了环境检测。请停止重试相同逻辑,改用 sandbox_execute(沙箱执行)、断点调试或静态分析。`;
107
+ } else {
108
+ result.stderr += `\n\n⚠️ 连续超时 ${consecutiveTimeouts}/${MAX_CONSECUTIVE_TIMEOUTS} 次。如果代码包含从网站提取的混淆代码,可能存在环境检测导致死循环。建议:1) 检查代码是否有 while(true)/setInterval 等循环 2) 改用 sandbox_execute 在受控环境中执行`;
109
+ }
110
+ } else {
111
+ consecutiveTimeouts = 0; // 成功执行或非超时失败,重置计数
112
+ }
113
+
114
+ const { timedOut: _, ...output } = result;
115
+ return JSON.stringify(output);
81
116
  },
82
117
  {
83
118
  name: 'run_node_code',
@@ -93,7 +128,7 @@ export const runNodeCode = tool(
93
128
  示例:const CryptoJS = require('crypto-js'); console.log(CryptoJS.MD5('test').toString());`,
94
129
  schema: z.object({
95
130
  code: z.string().describe('要执行的 JS 代码,可使用 require 引入加密库'),
96
- timeout: z.number().optional().default(10000).describe('超时时间(毫秒)'),
131
+ timeout: z.number().optional().default(10000).describe('超时时间(毫秒),上限 30000'),
97
132
  }),
98
133
  }
99
134
  );
@@ -53,7 +53,7 @@ async function executePython(code, timeout = 10000) {
53
53
  * 生成 Python 验证代码
54
54
  */
55
55
  function generateVerifyCode(algorithm, params) {
56
- const { plaintext, ciphertext, key, iv, mode, format, hmacKey, digestmod } = params;
56
+ const { plaintext, ciphertext, key, iv, _mode, format, hmacKey, digestmod } = params;
57
57
 
58
58
  const templates = {
59
59
  // AES
@@ -223,7 +223,7 @@ print('MATCH' if result == expected else f'MISMATCH: got {result}')
223
223
  * 生成可复用的 Python 代码片段
224
224
  */
225
225
  function generatePythonSnippet(algorithm, params) {
226
- const { key, iv, mode, format, hmacKey, digestmod } = params;
226
+ const { key, iv, _mode, _format, hmacKey, digestmod } = params;
227
227
 
228
228
  const snippets = {
229
229
  'AES-CBC': `
@@ -488,7 +488,7 @@ export const executePythonCode = tool(
488
488
  export const generateExecjsCode = tool(
489
489
  async ({ jsCode, functionName, description }) => {
490
490
  // 转义 JS 代码中的特殊字符
491
- const escapedJs = jsCode
491
+ const _escapedJs = jsCode
492
492
  .replace(/\\/g, '\\\\')
493
493
  .replace(/"""/g, '\\"\\"\\"')
494
494
  .replace(/\n/g, '\\n');
@@ -539,7 +539,7 @@ def ${functionName || 'execute_js'}(*args):
539
539
  * 分析 JS 代码,判断转换策略
540
540
  */
541
541
  export const analyzeJsForPython = tool(
542
- async ({ jsCode, cryptoPatterns }) => {
542
+ async ({ jsCode, cryptoPatterns: _cryptoPatterns }) => {
543
543
  const analysis = {
544
544
  canPureRewrite: true,
545
545
  reasons: [],
@@ -7,8 +7,8 @@
7
7
  import { z } from 'zod';
8
8
  import { tool } from '@langchain/core/tools';
9
9
  import { writeFileSync, readFileSync, existsSync } from 'fs';
10
- import { join, basename } from 'path';
11
- import { PATHS, ensureDir, DEEPSPIDER_HOME } from '../../config/paths.js';
10
+ import { join } from 'path';
11
+ import { PATHS, ensureDir } from '../../config/paths.js';
12
12
 
13
13
  const OUTPUT_DIR = PATHS.REPORTS_DIR;
14
14
 
@@ -110,7 +110,7 @@ export const addInitScript = tool(
110
110
  * 清除 Cookie
111
111
  */
112
112
  export const clearCookies = tool(
113
- async ({ domain }) => {
113
+ async ({ domain: _domain }) => {
114
114
  const browser = await getBrowser();
115
115
  const context = browser.getContext();
116
116
  await context.clearCookies();
@@ -76,4 +76,24 @@ export const sandboxReset = tool(
76
76
  }
77
77
  );
78
78
 
79
- export const sandboxTools = [sandboxExecute, sandboxInject, sandboxReset];
79
+ /**
80
+ * 自动补环境执行工具
81
+ */
82
+ export const sandboxAutoFix = tool(
83
+ async ({ code, timeout, maxIterations }) => {
84
+ const sb = await getSandbox();
85
+ const result = await sb.executeWithAutoFix(code, { timeout, maxIterations });
86
+ return JSON.stringify(result, null, 2);
87
+ },
88
+ {
89
+ name: 'sandbox_auto_fix',
90
+ description: '自动补环境闭环执行:加载预置模块 → 执行代码 → 发现缺失环境 → 自动生成补丁 → 重试,直到成功或无法继续。适合快速验证混淆代码能否在沙箱中运行。',
91
+ schema: z.object({
92
+ code: z.string().describe('要执行的目标JS代码'),
93
+ timeout: z.number().optional().default(5000).describe('单次执行超时时间(ms)'),
94
+ maxIterations: z.number().optional().default(10).describe('最大迭代次数'),
95
+ }),
96
+ }
97
+ );
98
+
99
+ export const sandboxTools = [sandboxExecute, sandboxInject, sandboxReset, sandboxAutoFix];
@@ -0,0 +1,70 @@
1
+ /**
2
+ * DeepSpider - 工作记忆工具(Scratchpad)
3
+ * 保存/读取关键发现,防止多步推理中丢失上下文
4
+ */
5
+
6
+ import { z } from 'zod';
7
+ import { tool } from '@langchain/core/tools';
8
+ import { writeFileSync, readFileSync, existsSync, readdirSync } from 'fs';
9
+ import { join } from 'path';
10
+ import { DEEPSPIDER_HOME, ensureDir } from '../../config/paths.js';
11
+
12
+ const MEMO_DIR = join(DEEPSPIDER_HOME, 'memo');
13
+
14
+ /** 清理 key:只保留字母、数字、连字符、下划线,防止路径穿越 */
15
+ function sanitizeKey(key) {
16
+ return key.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 100);
17
+ }
18
+
19
+ export const saveMemo = tool(
20
+ async ({ key, content }) => {
21
+ ensureDir(MEMO_DIR);
22
+ const safeKey = sanitizeKey(key);
23
+ const filePath = join(MEMO_DIR, `${safeKey}.txt`);
24
+ writeFileSync(filePath, content, 'utf-8');
25
+ return JSON.stringify({ success: true, key: safeKey, size: content.length });
26
+ },
27
+ {
28
+ name: 'save_memo',
29
+ description: '保存工作记忆。用于记录关键发现、中间结果、待验证假设等,防止多步推理中丢失上下文',
30
+ schema: z.object({
31
+ key: z.string().describe('记忆键名,如 "encryption-analysis"、"key-source"'),
32
+ content: z.string().describe('要保存的内容'),
33
+ }),
34
+ }
35
+ );
36
+
37
+ export const loadMemo = tool(
38
+ async ({ key }) => {
39
+ const safeKey = sanitizeKey(key);
40
+ const filePath = join(MEMO_DIR, `${safeKey}.txt`);
41
+ if (!existsSync(filePath)) {
42
+ return JSON.stringify({ success: false, error: `memo "${safeKey}" not found` });
43
+ }
44
+ const content = readFileSync(filePath, 'utf-8');
45
+ return JSON.stringify({ success: true, key: safeKey, content });
46
+ },
47
+ {
48
+ name: 'load_memo',
49
+ description: '读取之前保存的工作记忆',
50
+ schema: z.object({
51
+ key: z.string().describe('记忆键名'),
52
+ }),
53
+ }
54
+ );
55
+
56
+ export const listMemo = tool(
57
+ async () => {
58
+ ensureDir(MEMO_DIR);
59
+ const files = readdirSync(MEMO_DIR).filter(f => f.endsWith('.txt'));
60
+ const memos = files.map(f => f.replace('.txt', ''));
61
+ return JSON.stringify({ success: true, memos, count: memos.length });
62
+ },
63
+ {
64
+ name: 'list_memo',
65
+ description: '列出所有已保存的工作记忆',
66
+ schema: z.object({}),
67
+ }
68
+ );
69
+
70
+ export const scratchpadTools = [saveMemo, loadMemo, listMemo];
@@ -188,11 +188,37 @@ export const clearAllData = tool(
188
188
  }
189
189
  );
190
190
 
191
+ /**
192
+ * 获取请求的 initiator(调用栈)
193
+ */
194
+ export const getRequestInitiator = tool(
195
+ async ({ site, id }) => {
196
+ const store = getDataStore();
197
+ const result = await store.getResponse(site, id);
198
+ if (!result) {
199
+ return JSON.stringify({ error: '未找到该请求' });
200
+ }
201
+ if (!result.initiator) {
202
+ return JSON.stringify({ error: '该请求无 initiator 信息(可能是旧数据或浏览器内部请求)' });
203
+ }
204
+ return JSON.stringify(result.initiator, null, 2);
205
+ },
206
+ {
207
+ name: 'get_request_initiator',
208
+ description: '获取发起该请求的 JS 代码位置(脚本URL + 行号 + 函数名)。返回调用栈的前5帧。用于从目标请求反向定位加密函数入口,是逆向分析的第一步。',
209
+ schema: z.object({
210
+ site: z.string().describe('站点 hostname'),
211
+ id: z.string().describe('请求 ID'),
212
+ }),
213
+ }
214
+ );
215
+
191
216
  export const tracingTools = [
192
217
  getSiteList,
193
218
  searchInResponses,
194
219
  getRequestDetail,
195
220
  getRequestList,
221
+ getRequestInitiator,
196
222
  getScriptList,
197
223
  getScriptSource,
198
224
  searchInScripts,