deepspider 0.1.0

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "deepspider",
+  "version": "0.1.0",
+  "description": "智能爬虫工程平台 - 基于 DeepAgents + Patchright 的 AI 爬虫 Agent",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "deepspider": "./src/agent/run.js"
+  },
+  "scripts": {
+    "start": "node src/index.js",
+    "dev": "node --watch src/index.js",
+    "cli": "node bin/cli.js",
+    "mcp": "node src/mcp/server.js",
+    "agent": "node src/agent/run.js",
+    "test": "node --test test/",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
+    "setup:crypto": "uv venv .venv --python 3.11 2>/dev/null || true && uv pip install -r requirements-crypto.txt",
+    "prepare": "husky"
+  },
+  "keywords": [
+    "crawler",
+    "web-scraping",
+    "javascript",
+    "reverse-engineering",
+    "deobfuscation",
+    "captcha",
+    "anti-detect",
+    "automation"
+  ],
+  "author": "pony-ma",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ma-pony/deepspider.git"
+  },
+  "homepage": "https://github.com/ma-pony/deepspider#readme",
+  "bugs": {
+    "url": "https://github.com/ma-pony/deepspider/issues"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@babel/generator": "^7.28.6",
+    "@babel/parser": "^7.28.6",
+    "@babel/traverse": "^7.28.6",
+    "@babel/types": "^7.28.6",
+    "@langchain/anthropic": "^1.3.12",
+    "@langchain/core": "^1.1.17",
+    "@langchain/langgraph": "^1.1.2",
+    "@langchain/openai": "^1.2.3",
+    "@modelcontextprotocol/sdk": "^1.25.3",
+    "crypto-js": "^4.2.0",
+    "deepagents": "^1.6.0",
+    "dotenv": "^17.2.3",
+    "hono": "4.11.7",
+    "isolated-vm": "^6.0.2",
+    "js-md5": "^0.8.3",
+    "js-sha256": "^0.11.1",
+    "jsencrypt": "^3.5.4",
+    "langchain": "^1.2.15",
+    "marked": "^17.0.1",
+    "patchright": "^1.57.0",
+    "sm-crypto": "^0.4.0",
+    "webcrack": "^2.15.1",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "eslint": "^9.39.2",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.2.7"
+  },
+  "lint-staged": {
+    "*.js": "eslint --fix"
+  }
+}

package/requirements-crypto.txt

@@ -0,0 +1,14 @@
+# JSForge Python 加密库依赖
+# 安装: uv pip install -r requirements-crypto.txt
+
+# 通用加密库
+pycryptodome>=3.20.0    # AES/DES/RSA/MD5/SHA 等
+
+# 国密算法
+gmssl>=3.2.2            # SM2/SM3/SM4
+
+# RSA
+rsa>=4.9
+
+# 其他常用
+pyDes>=2.0.1            # DES/3DES

package/src/agent/index.js

@@ -0,0 +1,97 @@
+/**
+ * DeepSpider - DeepAgent 主入口
+ * 基于 DeepAgents 最佳实践重构
+ */
+
+import 'dotenv/config';
+import { createDeepAgent, StateBackend, FilesystemBackend } from 'deepagents';
+import { ChatOpenAI } from '@langchain/openai';
+import { MemorySaver } from '@langchain/langgraph';
+
+import { coreTools } from './tools/index.js';
+import { allSubagents } from './subagents/index.js';
+import { systemPrompt } from './prompts/system.js';
+import { createReportMiddleware } from './middleware/report.js';
+import { createFilterToolsMiddleware } from './middleware/filterTools.js';
+
+// 从环境变量读取配置
+const config = {
+  apiKey: process.env.LLM_API_KEY,
+  baseUrl: process.env.LLM_BASE_URL,
+  model: process.env.LLM_MODEL || 'gpt-4o',
+};
+
+/**
+ * 创建 LLM 模型实例
+ * 使用 ChatOpenAI 兼容 OpenAI 格式的任意供应商
+ */
+function createModel(options = {}) {
+  const {
+    model = config.model,
+    apiKey = config.apiKey,
+    baseUrl = config.baseUrl,
+  } = options;
+
+  return new ChatOpenAI({
+    model,
+    apiKey,
+    configuration: baseUrl ? { baseURL: baseUrl } : undefined,
+    temperature: 0,
+  });
+}
+
+/**
+ * 创建 DeepSpider Agent
+ */
+export function createDeepSpiderAgent(options = {}) {
+  const {
+    model = config.model,
+    apiKey = config.apiKey,
+    baseUrl = config.baseUrl,
+    enableMemory = true,
+    enableInterrupt = false,
+    onReportReady = null,  // 报告就绪回调
+  } = options;
+
+  // 创建 LLM 模型实例
+  const llm = createModel({ model, apiKey, baseUrl });
+
+  // 后端配置：使用文件系统持久化
+  const backend = enableMemory
+    ? new FilesystemBackend({ rootDir: './.deepspider-agent' })
+    : new StateBackend();
+
+  // Checkpointer：保存对话状态，支持断点恢复
+  const checkpointer = new MemorySaver();
+
+  // 人机交互配置
+  const interruptOn = enableInterrupt
+    ? {
+        sandbox_execute: { allowedDecisions: ['approve', 'reject', 'edit'] },
+        sandbox_inject: { allowedDecisions: ['approve', 'reject'] },
+      }
+    : undefined;
+
+  // 中间件配置
+  const middleware = [
+    createFilterToolsMiddleware(),  // 过滤内置的 write_file/read_file
+    createReportMiddleware({ onReportReady }),
+  ];
+
+  return createDeepAgent({
+    name: 'deepspider',
+    model: llm,
+    tools: coreTools,
+    subagents: allSubagents,
+    systemPrompt,
+    backend,
+    checkpointer,
+    interruptOn,
+    middleware,
+  });
+}
+
+// 默认导出
+export const agent = createDeepSpiderAgent();
+
+export default agent;

package/src/agent/logger.js

@@ -0,0 +1,164 @@
+/**
+ * DeepSpider - 日志回调处理器
+ * 记录 AI 交互、工具调用等详细日志
+ */
+
+import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
+import { appendFileSync, mkdirSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { DEEPSPIDER_HOME } from '../config/paths.js';
+
+const LOG_DIR = join(DEEPSPIDER_HOME, 'logs');
+const LOG_FILE = join(LOG_DIR, 'agent.log');
+
+function ensureLogDir() {
+  if (!existsSync(LOG_DIR)) {
+    mkdirSync(LOG_DIR, { recursive: true });
+  }
+}
+
+function formatTime() {
+  return new Date().toISOString();
+}
+
+function truncate(str, maxLen = 500) {
+  if (!str) return '';
+  const s = typeof str === 'string' ? str : JSON.stringify(str);
+  return s.length > maxLen ? s.slice(0, maxLen) + '...' : s;
+}
+
+/**
+ * 文件日志回调处理器
+ */
+export class FileLoggerCallback extends BaseCallbackHandler {
+  name = 'FileLoggerCallback';
+
+  constructor(options = {}) {
+    super();
+    this.logFile = options.logFile || LOG_FILE;
+    this.verbose = options.verbose || false;
+    ensureLogDir();
+  }
+
+  log(level, category, message, data = null) {
+    const line = JSON.stringify({
+      time: formatTime(),
+      level,
+      category,
+      message,
+      data,
+    }) + '\n';
+
+    appendFileSync(this.logFile, line);
+
+    if (this.verbose) {
+      console.log(`[${level}] [${category}] ${message}`);
+    }
+  }
+
+  // ========== LLM 事件 ==========
+  handleLLMStart(llm, prompts, runId) {
+    this.log('INFO', 'LLM', 'LLM 调用开始', {
+      runId,
+      model: llm?.id?.[2] || llm?.name,
+      promptCount: prompts?.length,
+      promptPreview: truncate(prompts?.[0], 200),
+    });
+  }
+
+  handleLLMEnd(output, runId) {
+    const content = output?.generations?.[0]?.[0]?.text
+      || output?.generations?.[0]?.[0]?.message?.content;
+    this.log('INFO', 'LLM', 'LLM 调用结束', {
+      runId,
+      outputPreview: truncate(content, 300),
+      tokenUsage: output?.llmOutput?.tokenUsage,
+    });
+  }
+
+  handleLLMError(error, runId) {
+    this.log('ERROR', 'LLM', 'LLM 调用错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  // ========== 工具事件 ==========
+  handleToolStart(tool, input, runId) {
+    this.log('INFO', 'TOOL', `工具调用: ${tool?.name || 'unknown'}`, {
+      runId,
+      toolName: tool?.name,
+      input: truncate(input, 500),
+    });
+  }
+
+  handleToolEnd(output, runId) {
+    this.log('INFO', 'TOOL', '工具返回', {
+      runId,
+      output: truncate(output, 500),
+    });
+  }
+
+  handleToolError(error, runId) {
+    this.log('ERROR', 'TOOL', '工具错误', {
+      runId,
+      error: error?.message || String(error),
+      stack: error?.stack?.split('\n').slice(0, 5),
+    });
+  }
+
+  // ========== Chain 事件 ==========
+  handleChainStart(chain, inputs, runId) {
+    this.log('DEBUG', 'CHAIN', `Chain 开始: ${chain?.name || 'unknown'}`, {
+      runId,
+      chainName: chain?.name,
+      inputKeys: Object.keys(inputs || {}),
+    });
+  }
+
+  handleChainEnd(outputs, runId) {
+    this.log('DEBUG', 'CHAIN', 'Chain 结束', {
+      runId,
+      outputKeys: Object.keys(outputs || {}),
+    });
+  }
+
+  handleChainError(error, runId) {
+    this.log('ERROR', 'CHAIN', 'Chain 错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  // ========== Agent 事件 ==========
+  handleAgentAction(action, runId) {
+    this.log('INFO', 'AGENT', `Agent 动作: ${action?.tool}`, {
+      runId,
+      tool: action?.tool,
+      toolInput: truncate(action?.toolInput, 300),
+      log: truncate(action?.log, 200),
+    });
+  }
+
+  handleAgentEnd(action, runId) {
+    this.log('INFO', 'AGENT', 'Agent 结束', {
+      runId,
+      returnValues: truncate(action?.returnValues, 300),
+    });
+  }
+}
+
+/**
+ * 创建日志回调实例
+ */
+export function createLogger(options = {}) {
+  const enabled = process.env.DEBUG === 'true' || options.enabled;
+  if (!enabled) return null;
+
+  return new FileLoggerCallback({
+    verbose: options.verbose || false,
+    logFile: options.logFile || LOG_FILE,
+  });
+}
+
+export default FileLoggerCallback;

package/src/agent/middleware/filterTools.js

@@ -0,0 +1,64 @@
+/**
+ * DeepSpider - 工具过滤中间件
+ * 过滤掉 DeepAgents 内置的文件工具，避免与自定义工具冲突
+ * 同时替换框架提示词中的工具名称
+ */
+
+import { createMiddleware } from 'langchain';
+
+// 需要过滤的内置工具名称
+const FILTERED_TOOLS = ['write_file', 'read_file', 'edit_file', 'glob', 'grep'];
+
+// 提示词替换规则
+const PROMPT_REPLACEMENTS = [
+  // 工具名称替换
+  { from: /\bwrite_file\b/g, to: 'artifact_save' },
+  { from: /\bread_file\b/g, to: 'artifact_load' },
+  { from: /\bedit_file\b/g, to: 'artifact_edit' },
+  { from: /\bglob\b/g, to: 'artifact_glob' },
+  { from: /\bgrep\b/g, to: 'artifact_grep' },
+];
+
+/**
+ * 替换提示词中的工具名称
+ */
+function replacePromptContent(prompt) {
+  if (!prompt) return prompt;
+
+  let result = prompt;
+  for (const rule of PROMPT_REPLACEMENTS) {
+    result = result.replace(rule.from, rule.to);
+  }
+  return result;
+}
+
+/**
+ * 创建工具过滤中间件
+ * 在模型调用前过滤掉指定的工具，并替换提示词
+ */
+export function createFilterToolsMiddleware(options = {}) {
+  const { filteredTools = FILTERED_TOOLS } = options;
+
+  return createMiddleware({
+    name: 'filterToolsMiddleware',
+
+    // 在模型调用前过滤工具并替换提示词
+    wrapModelCall: async (request, handler) => {
+      // 过滤工具
+      const tools = request.tools?.filter(
+        (t) => !filteredTools.includes(t.name)
+      ) || [];
+
+      // 替换系统提示词
+      const systemPrompt = replacePromptContent(request.systemPrompt);
+
+      return handler({
+        ...request,
+        tools,
+        systemPrompt,
+      });
+    },
+  });
+}
+
+export default createFilterToolsMiddleware;

package/src/agent/middleware/report.js

@@ -0,0 +1,79 @@
+/**
+ * DeepSpider - 报告中间件
+ * 在 Agent 执行完成后自动检测并准备报告
+ */
+
+import { createMiddleware } from 'langchain';
+import { ToolMessage } from '@langchain/core/messages';
+import { z } from 'zod';
+
+// 报告状态 schema
+const reportStateSchema = z.object({
+  lastWrittenMdFile: z.string().optional(),
+  reportReady: z.boolean().default(false),
+});
+
+/**
+ * 创建报告中间件
+ * 在 afterModel 中检测 artifact_save 工具调用结果
+ * 在 afterAgent 中触发报告显示回调
+ */
+export function createReportMiddleware(options = {}) {
+  const { onReportReady } = options;
+
+  return createMiddleware({
+    name: 'reportMiddleware',
+    stateSchema: reportStateSchema,
+
+    // 模型调用后，检测工具调用结果
+    afterModel: (state) => {
+      const messages = state.messages;
+      if (!messages || messages.length === 0) return undefined;
+
+      // 查找最近的 ToolMessage
+      for (let i = messages.length - 1; i >= 0; i--) {
+        const msg = messages[i];
+        if (ToolMessage.isInstance(msg)) {
+          try {
+            const content = typeof msg.content === 'string'
+              ? JSON.parse(msg.content)
+              : msg.content;
+
+            // 检测是否是 artifact_save 写入的 .md 文件
+            if (content.success && content.path?.endsWith('.md')) {
+              console.log('[reportMiddleware] 检测到 .md 文件:', content.path);
+              return { lastWrittenMdFile: content.path };
+            }
+          } catch (e) {
+            // 解析失败，忽略
+          }
+        }
+      }
+      return undefined;
+    },
+
+    // Agent 执行完成后
+    afterAgent: async (state) => {
+      const mdFile = state.lastWrittenMdFile;
+
+      if (mdFile) {
+        console.log('[reportMiddleware] afterAgent: 准备显示报告:', mdFile);
+
+        // 调用回调通知外部
+        if (onReportReady) {
+          try {
+            await onReportReady(mdFile);
+          } catch (e) {
+            console.error('[reportMiddleware] onReportReady 失败:', e.message);
+          }
+        }
+
+        return { reportReady: true };
+      }
+
+      return undefined;
+    },
+  });
+}
+
+export default createReportMiddleware;