deepspider 0.3.1 → 0.4.0

package/src/agent/logger.js

@@ -4,12 +4,14 @@
  */
 
 import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
-import { appendFileSync, mkdirSync, existsSync } from 'fs';
+import { appendFileSync, mkdirSync, existsSync, statSync, renameSync, unlinkSync } from 'fs';
 import { join } from 'path';
 import { DEEPSPIDER_HOME } from '../config/paths.js';
 
 const LOG_DIR = join(DEEPSPIDER_HOME, 'logs');
 const LOG_FILE = join(LOG_DIR, 'agent.log');
+const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB
+const MAX_LOG_FILES = 3;              // agent.log, agent.log.1, agent.log.2
 
 function ensureLogDir() {
   if (!existsSync(LOG_DIR)) {
@@ -17,6 +19,32 @@ function ensureLogDir() {
   }
 }
 
+/**
+ * 日志滚动：agent.log 超过 MAX_LOG_SIZE 时轮转
+ * agent.log → agent.log.1 → agent.log.2 → 删除
+ */
+function rotateIfNeeded(logFile) {
+  try {
+    if (!existsSync(logFile)) return;
+    const { size } = statSync(logFile);
+    if (size < MAX_LOG_SIZE) return;
+
+    // 删除最老的
+    const oldest = `${logFile}.${MAX_LOG_FILES - 1}`;
+    if (existsSync(oldest)) unlinkSync(oldest);
+
+    // 依次轮转
+    for (let i = MAX_LOG_FILES - 2; i >= 1; i--) {
+      const from = `${logFile}.${i}`;
+      const to = `${logFile}.${i + 1}`;
+      if (existsSync(from)) renameSync(from, to);
+    }
+
+    // 当前文件变为 .1
+    renameSync(logFile, `${logFile}.1`);
+  } catch { /* 滚动失败不影响主流程 */ }
+}
+
 function formatTime() {
   return new Date().toISOString();
 }
@@ -27,6 +55,147 @@ function truncate(str, maxLen = 500) {
   return s.length > maxLen ? s.slice(0, maxLen) + '...' : s;
 }
 
+/**
+ * 内存日志环形缓冲区
+ * 始终可用，不依赖 DEBUG 开关
+ */
+export class InMemoryLogStore {
+  constructor(maxSize = 500) {
+    this.logs = [];
+    this.maxSize = maxSize;
+    this.startTime = Date.now();
+  }
+
+  add(entry) {
+    this.logs.push(entry);
+    if (this.logs.length > this.maxSize) this.logs.shift();
+  }
+
+  query({ category, level, limit = 50, toolName } = {}) {
+    let result = this.logs;
+    if (category) result = result.filter(l => l.category === category);
+    if (level) result = result.filter(l => l.level === level);
+    if (toolName) result = result.filter(l => l.data?.toolName === toolName);
+    return result.slice(-limit);
+  }
+
+  getStats() {
+    const cats = {};
+    for (const l of this.logs) {
+      cats[l.category] = (cats[l.category] || 0) + 1;
+    }
+    return {
+      total: this.logs.length,
+      categories: cats,
+      errors: this.logs.filter(l => l.level === 'ERROR').length,
+      uptimeMs: Date.now() - this.startTime,
+    };
+  }
+
+  clear() { this.logs = []; }
+}
+
+export const logStore = new InMemoryLogStore();
+
+/**
+ * 内存日志回调处理器
+ * 始终启用，将日志写入 logStore
+ */
+export class InMemoryLoggerCallback extends BaseCallbackHandler {
+  name = 'InMemoryLoggerCallback';
+
+  log(level, category, message, data = null) {
+    logStore.add({ time: formatTime(), level, category, message, data });
+  }
+
+  handleLLMStart(llm, prompts, runId) {
+    this.log('INFO', 'LLM', 'LLM 调用开始', {
+      runId,
+      model: llm?.id?.[2] || llm?.name,
+      promptCount: prompts?.length,
+      promptPreview: truncate(prompts?.[0], 200),
+    });
+  }
+
+  handleLLMEnd(output, runId) {
+    const content = output?.generations?.[0]?.[0]?.text
+      || output?.generations?.[0]?.[0]?.message?.content;
+    this.log('INFO', 'LLM', 'LLM 调用结束', {
+      runId,
+      outputPreview: truncate(content, 300),
+      tokenUsage: output?.llmOutput?.tokenUsage,
+    });
+  }
+
+  handleLLMError(error, runId) {
+    this.log('ERROR', 'LLM', 'LLM 调用错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  handleToolStart(tool, input, runId) {
+    this.log('INFO', 'TOOL', `工具调用: ${tool?.name || 'unknown'}`, {
+      runId,
+      toolName: tool?.name,
+      input: truncate(input, 500),
+    });
+  }
+
+  handleToolEnd(output, runId) {
+    this.log('INFO', 'TOOL', '工具返回', {
+      runId,
+      output: truncate(output, 500),
+    });
+  }
+
+  handleToolError(error, runId) {
+    this.log('ERROR', 'TOOL', '工具错误', {
+      runId,
+      error: error?.message || String(error),
+      stack: error?.stack?.split('\n').slice(0, 5),
+    });
+  }
+
+  handleChainStart(chain, inputs, runId) {
+    this.log('DEBUG', 'CHAIN', `Chain 开始: ${chain?.name || 'unknown'}`, {
+      runId,
+      chainName: chain?.name,
+      inputKeys: Object.keys(inputs || {}),
+    });
+  }
+
+  handleChainEnd(outputs, runId) {
+    this.log('DEBUG', 'CHAIN', 'Chain 结束', {
+      runId,
+      outputKeys: Object.keys(outputs || {}),
+    });
+  }
+
+  handleChainError(error, runId) {
+    this.log('ERROR', 'CHAIN', 'Chain 错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  handleAgentAction(action, runId) {
+    this.log('INFO', 'AGENT', `Agent 动作: ${action?.tool}`, {
+      runId,
+      tool: action?.tool,
+      toolInput: truncate(action?.toolInput, 300),
+      log: truncate(action?.log, 200),
+    });
+  }
+
+  handleAgentEnd(action, runId) {
+    this.log('INFO', 'AGENT', 'Agent 结束', {
+      runId,
+      returnValues: truncate(action?.returnValues, 300),
+    });
+  }
+}
+
 /**
  * 文件日志回调处理器
  */
@@ -41,6 +210,8 @@ export class FileLoggerCallback extends BaseCallbackHandler {
   }
 
   log(level, category, message, data = null) {
+    rotateIfNeeded(this.logFile);
+
     const line = JSON.stringify({
       time: formatTime(),
       level,
@@ -149,16 +320,20 @@ export class FileLoggerCallback extends BaseCallbackHandler {
 }
 
 /**
- * 创建日志回调实例
+ * 创建日志回调实例数组
+ * 始终包含 InMemoryLoggerCallback + FileLoggerCallback
+ * DEBUG=true 时额外开启 verbose（控制台输出）
  */
 export function createLogger(options = {}) {
-  const enabled = process.env.DEBUG === 'true' || options.enabled;
-  if (!enabled) return null;
+  const verbose = process.env.DEBUG === 'true' || options.verbose || false;
 
-  return new FileLoggerCallback({
-    verbose: options.verbose || false,
-    logFile: options.logFile || LOG_FILE,
-  });
+  return [
+    new InMemoryLoggerCallback(),
+    new FileLoggerCallback({
+      verbose,
+      logFile: options.logFile || LOG_FILE,
+    }),
+  ];
 }
 
 export default FileLoggerCallback;

package/src/agent/middleware/memoryFlush.js

@@ -0,0 +1,48 @@
+/**
+ * DeepSpider - Memory Flush 中间件
+ * 在 summarization 触发前（85k token），注入 SystemMessage 提醒 Agent 保存关键进度
+ */
+
+import { createMiddleware, countTokensApproximately } from 'langchain';
+import { SystemMessage } from '@langchain/core/messages';
+
+const FLUSH_THRESHOLD = 85000;
+
+const FLUSH_REMINDER = `⚠️ 上下文即将被压缩（当前接近 token 上限）。
+请立即使用 save_memo 工具保存以下关键信息，否则压缩后将丢失：
+1. 当前分析目标和已完成的步骤
+2. 已发现的关键参数、加密逻辑、请求链路
+3. 下一步计划
+
+保存后继续正常工作。`;
+
+export function createMemoryFlushMiddleware() {
+  let flushed = false;
+
+  return createMiddleware({
+    name: 'memoryFlushMiddleware',
+
+    beforeModel: async (state) => {
+      const tokens = countTokensApproximately(state.messages);
+
+      // token 骤降（summarization 已执行），重置标记
+      if (flushed && tokens < FLUSH_THRESHOLD * 0.5) {
+        flushed = false;
+      }
+
+      // 达到阈值且未提醒过，注入提醒
+      if (!flushed && tokens >= FLUSH_THRESHOLD) {
+        flushed = true;
+        return {
+          ...state,
+          messages: [
+            ...state.messages,
+            new SystemMessage(FLUSH_REMINDER),
+          ],
+        };
+      }
+
+      return state;
+    },
+  });
+}

package/src/agent/middleware/report.js

@@ -1,6 +1,6 @@
 /**
  * DeepSpider - 报告中间件
- * 在 Agent 执行完成后自动检测并准备报告
+ * 检测文件保存事件，触发报告显示和面板通知
  */
 
 import { createMiddleware } from 'langchain';
@@ -13,64 +13,122 @@ const reportStateSchema = z.object({
   reportReady: z.boolean().default(false),
 });
 
+/**
+ * 从工具返回值中提取 .md 文件路径
+ * 兼容两种工具的返回格式：
+ *   artifact_save: { success, path: "/xxx/analysis.md" }
+ *   save_analysis_report: { success, paths: { markdown: "/xxx/analysis.md" }, dir }
+ */
+function extractMdPath(content) {
+  if (!content?.success) return null;
+
+  // artifact_save 格式
+  if (content.path?.endsWith('.md')) return content.path;
+
+  // save_analysis_report 格式
+  if (content.paths?.markdown) return content.paths.markdown;
+
+  return null;
+}
+
+/**
+ * 从工具返回值中提取已保存的文件信息（用于面板通知）
+ * 返回 { path, type } 或 null
+ */
+function extractSavedFile(content) {
+  if (!content?.success) return null;
+
+  // artifact_save: 单文件
+  if (content.path) {
+    const ext = content.path.split('.').pop();
+    return { path: content.path, type: ext };
+  }
+
+  // save_analysis_report: 多文件
+  if (content.paths) {
+    return { path: content.dir || content.paths.markdown, type: 'report' };
+  }
+
+  return null;
+}
+
+function parseContent(result) {
+  try {
+    return typeof result?.content === 'string'
+      ? JSON.parse(result.content)
+      : result?.content;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * 创建报告中间件
- * 在 afterModel 中检测 artifact_save 工具调用结果
- * 在 afterAgent 中触发报告显示回调
+ * 监听 artifact_save 和 save_analysis_report，触发报告显示 + 面板通知
  */
 export function createReportMiddleware(options = {}) {
-  const { onReportReady } = options;
+  const { onReportReady, onFileSaved } = options;
+
+  const WATCHED_TOOLS = new Set(['artifact_save', 'save_analysis_report']);
 
   return createMiddleware({
     name: 'reportMiddleware',
     stateSchema: reportStateSchema,
 
-    // 模型调用后，检测工具调用结果
+    wrapToolCall: async (request, handler) => {
+      const toolName = request.tool?.name ?? request.toolCall?.name;
+      const result = await handler(request);
+
+      if (!WATCHED_TOOLS.has(toolName)) return result;
+
+      const content = parseContent(result);
+      if (!content) return result;
+
+      // 检测 .md 文件 → 触发报告显示
+      const mdPath = extractMdPath(content);
+      if (mdPath) {
+        console.log('[reportMiddleware] 检测到报告文件:', mdPath);
+        if (onReportReady) {
+          await onReportReady(mdPath);
+        }
+      }
+
+      // 通知文件已保存（面板可显示提示）
+      const saved = extractSavedFile(content);
+      if (saved && onFileSaved) {
+        await onFileSaved(saved);
+      }
+
+      return result;
+    },
+
+    // 备选：afterModel 检测 ToolMessage 中的报告文件
     afterModel: (state) => {
       const messages = state.messages;
-      if (!messages || messages.length === 0) return undefined;
+      if (!messages?.length) return undefined;
 
-      // 查找最近的 ToolMessage
       for (let i = messages.length - 1; i >= 0; i--) {
         const msg = messages[i];
-        if (ToolMessage.isInstance(msg)) {
-          try {
-            const content = typeof msg.content === 'string'
-              ? JSON.parse(msg.content)
-              : msg.content;
-
-            // 检测是否是 artifact_save 写入的 .md 文件
-            if (content.success && content.path?.endsWith('.md')) {
-              console.log('[reportMiddleware] 检测到 .md 文件:', content.path);
-              return { lastWrittenMdFile: content.path };
-            }
-          } catch {
-            // 解析失败，忽略
-          }
+        if (!ToolMessage.isInstance(msg)) continue;
+
+        const content = parseContent(msg);
+        if (!content) continue;
+
+        const mdPath = extractMdPath(content);
+        if (mdPath) {
+          console.log('[reportMiddleware] afterModel 检测到报告:', mdPath);
+          return { lastWrittenMdFile: mdPath };
         }
       }
       return undefined;
     },
 
-    // Agent 执行完成后
+    // streamEvents 模式下可能不被调用
     afterAgent: async (state) => {
-      const mdFile = state.lastWrittenMdFile;
-
-      if (mdFile) {
-        console.log('[reportMiddleware] afterAgent: 准备显示报告:', mdFile);
-
-        // 调用回调通知外部
-        if (onReportReady) {
-          try {
-            await onReportReady(mdFile);
-          } catch (e) {
-            console.error('[reportMiddleware] onReportReady 失败:', e.message);
-          }
-        }
-
+      if (state.lastWrittenMdFile) {
+        console.log('[reportMiddleware] afterAgent: 报告就绪:', state.lastWrittenMdFile);
         return { reportReady: true };
       }
-
       return undefined;
     },
   });

package/src/agent/middleware/subagent.js

@@ -0,0 +1,236 @@
+/**
+ * DeepSpider - 自定义子代理中间件
+ * 复刻 deepagents 内置的 createSubAgentMiddleware，增加 context 结构化传递
+ *
+ * 与内置版本的唯一区别：task tool schema 新增 context 字段（z.record(z.string(), z.string()).optional()），
+ * LLM 按需填写 key-value 对，子代理收到的 HumanMessage 中 context 以 <context> 块拼接在 description 之后。
+ */
+
+import { createMiddleware, createAgent, tool, humanInTheLoopMiddleware } from 'langchain';
+import { HumanMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';
+import { getCurrentTaskInput, Command } from '@langchain/langgraph';
+import { TASK_SYSTEM_PROMPT } from 'deepagents';
+import { z } from 'zod';
+
+// 子代理 state 中需要排除的 key（与 deepagents 内部一致）
+const EXCLUDED_STATE_KEYS = [
+  'messages',
+  'todos',
+  'structuredResponse',
+  'skillsMetadata',
+  'memoryContents',
+];
+
+/**
+ * 过滤 state，排除不应传递给子代理的 key
+ */
+function filterStateForSubagent(state) {
+  const filtered = {};
+  for (const [key, value] of Object.entries(state)) {
+    if (!EXCLUDED_STATE_KEYS.includes(key)) filtered[key] = value;
+  }
+  return filtered;
+}
+
+/**
+ * 构造 Command 返回，将子代理结果的 state 更新 + 最后一条消息作为 ToolMessage 返回
+ */
+const TRUST_SIGNAL = `\n\n---\n⚠️ 子代理已完成任务。请直接使用子代理输出的文件和结论，不要重复执行 artifact_load / artifact_glob / ls 等文件读取操作来检查子代理已保存的文件。如果需要对生成的代码做端到端验证，那是你的职责，请正常执行。`;
+
+function returnCommandWithStateUpdate(result, toolCallId) {
+  const stateUpdate = filterStateForSubagent(result);
+  const messages = result.messages;
+  const lastMessage = messages?.[messages.length - 1];
+  const content = (lastMessage?.content || 'Task completed') + TRUST_SIGNAL;
+  return new Command({
+    update: {
+      ...stateUpdate,
+      messages: [new ToolMessage({
+        content,
+        tool_call_id: toolCallId,
+        name: 'task',
+      })],
+    },
+  });
+}
+
+/**
+ * 生成 task tool 的 description（复刻 deepagents 内部的 getTaskToolDescription）
+ */
+function getTaskToolDescription(subagentDescriptions) {
+  return `
+Launch an ephemeral subagent to handle complex, multi-step independent tasks with isolated context windows.
+
+Available agent types and the tools they have access to:
+${subagentDescriptions.join('\n')}
+
+When using the Task tool, you must specify a subagent_type parameter to select which agent type to use.
+
+## Usage notes:
+1. Launch multiple agents concurrently whenever possible, to maximize performance; to do that, use a single message with multiple tool uses
+2. When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.
+3. Each agent invocation is stateless. You will not be able to send additional messages to the agent, nor will the agent be able to communicate with you outside of its final report. Therefore, your prompt should contain a highly detailed task description for the agent to perform autonomously and you should specify exactly what information the agent should return back to you in its final and only message to you.
+4. The agent's outputs should generally be trusted
+5. Clearly tell the agent whether you expect it to create content, perform analysis, or just do research (search, file reads, web fetches, etc.), since it is not aware of the user's intent
+6. If the agent description mentions that it should be used proactively, then you should try your best to use it without the user having to ask for it first. Use your judgement.
+
+## context 参数
+委托子代理时，使用 context 参数传递结构化上下文（key-value 对），如站点标识、请求 ID、目标参数名等。context 会注入到子代理的初始消息中，确保关键信息不丢失。
+  `.trim();
+}
+
+/**
+ * 编译子代理：遍历 subagents 数组，用 createAgent 编译为可运行实例
+ */
+function getSubagents(options) {
+  const {
+    defaultModel,
+    defaultTools,
+    defaultMiddleware,
+    generalPurposeMiddleware: gpMiddleware,
+    defaultInterruptOn,
+    subagents,
+    generalPurposeAgent,
+  } = options;
+
+  const defaultSubagentMiddleware = defaultMiddleware || [];
+  const generalPurposeMiddlewareBase = gpMiddleware || defaultSubagentMiddleware;
+  const agents = {};
+  const descriptions = [];
+
+  // 通用子代理（DeepSpider 默认不启用，但保留能力）
+  if (generalPurposeAgent) {
+    const generalPurposeMiddleware = [...generalPurposeMiddlewareBase];
+    if (defaultInterruptOn) generalPurposeMiddleware.push(humanInTheLoopMiddleware({ interruptOn: defaultInterruptOn }));
+    agents['general-purpose'] = createAgent({
+      model: defaultModel,
+      systemPrompt: 'In order to complete the objective that the user asks of you, you have access to a number of standard tools.',
+      tools: defaultTools,
+      middleware: generalPurposeMiddleware,
+      name: 'general-purpose',
+    });
+    descriptions.push('- general-purpose: General-purpose agent for researching complex questions, searching for files and content, and executing multi-step tasks.');
+  }
+
+  // 自定义子代理
+  for (const agentParams of subagents) {
+    descriptions.push(`- ${agentParams.name}: ${agentParams.description}`);
+
+    if ('runnable' in agentParams) {
+      // CompiledSubAgent — 已编译，直接使用
+      agents[agentParams.name] = agentParams.runnable;
+    } else {
+      const middleware = agentParams.middleware
+        ? [...defaultSubagentMiddleware, ...agentParams.middleware]
+        : [...defaultSubagentMiddleware];
+      const interruptOn = agentParams.interruptOn || defaultInterruptOn;
+      if (interruptOn) middleware.push(humanInTheLoopMiddleware({ interruptOn }));
+
+      agents[agentParams.name] = createAgent({
+        model: agentParams.model ?? defaultModel,
+        systemPrompt: agentParams.systemPrompt,
+        tools: agentParams.tools ?? defaultTools,
+        middleware,
+        name: agentParams.name,
+      });
+    }
+  }
+
+  return { agents, descriptions };
+}
+
+/**
+ * 创建增强版 task tool：schema 增加 context 字段
+ */
+function createEnhancedTaskTool(options) {
+  const { agents: subagentGraphs, descriptions: subagentDescriptions } = getSubagents(options);
+  const availableTypes = Object.keys(subagentGraphs).join(', ');
+
+  return tool(
+    async (input, config) => {
+      const { description, subagent_type, context } = input;
+
+      if (!(subagent_type in subagentGraphs)) {
+        const allowedTypes = Object.keys(subagentGraphs).map((k) => `\`${k}\``).join(', ');
+        throw new Error(`Error: invoked agent of type ${subagent_type}, the only allowed types are ${allowedTypes}`);
+      }
+
+      // 构造子代理的初始消息：description + context 块
+      let content = description;
+      if (context && Object.keys(context).length > 0) {
+        content += `\n\n<context>\n${JSON.stringify(context)}\n</context>`;
+      }
+
+      const subagent = subagentGraphs[subagent_type];
+      const subagentState = filterStateForSubagent(getCurrentTaskInput());
+      subagentState.messages = [new HumanMessage({ content })];
+
+      const result = await subagent.invoke(subagentState, config);
+      if (!config.toolCall?.id) throw new Error('Tool call ID is required for subagent invocation');
+      return returnCommandWithStateUpdate(result, config.toolCall.id);
+    },
+    {
+      name: 'task',
+      description: getTaskToolDescription(subagentDescriptions),
+      schema: z.object({
+        description: z.string().describe('The task to execute with the selected agent'),
+        subagent_type: z.string().describe(`Name of the agent to use. Available: ${availableTypes}`),
+        // NOTE: 不用 z.record() 因为 Zod v4 toJSONSchema 会生成 propertyNames，
+        // 而 Anthropic API 不支持 propertyNames 关键字
+        // 改用 z.object({}) + additionalProperties 模式
+        context: z.object({}).passthrough().optional().describe('Structured key-value context to pass to the subagent (e.g. site, requestId, targetParam)'),
+      }),
+    },
+  );
+}
+
+/**
+ * 创建自定义子代理中间件
+ * 替换 deepagents 内置的 createSubAgentMiddleware，增加 context 结构化传递
+ *
+ * @param {Object} options
+ * @param {LanguageModelLike} options.defaultModel - LLM 实例
+ * @param {StructuredTool[]} options.defaultTools - 默认工具集
+ * @param {SubAgent[]} options.subagents - 子代理配置数组
+ * @param {AgentMiddleware[]} options.defaultMiddleware - 子代理默认中间件
+ * @param {boolean} [options.generalPurposeAgent=false] - 是否创建通用子代理
+ * @param {Object} [options.defaultInterruptOn] - HITL 配置
+ */
+export function createCustomSubAgentMiddleware(options) {
+  const {
+    defaultModel,
+    defaultTools = [],
+    subagents = [],
+    defaultMiddleware = null,
+    generalPurposeMiddleware = null,
+    generalPurposeAgent = false,
+    defaultInterruptOn = null,
+  } = options;
+
+  const taskToolOptions = {
+    defaultModel,
+    defaultTools,
+    subagents,
+    defaultMiddleware,
+    generalPurposeMiddleware,
+    generalPurposeAgent,
+    defaultInterruptOn,
+  };
+
+  const enhancedTaskTool = createEnhancedTaskTool(taskToolOptions);
+
+  // context 使用说明，拼接到 TASK_SYSTEM_PROMPT 末尾
+  const contextGuide = `\n\n委托子代理时，使用 context 参数传递结构化上下文（key-value 对），如站点标识、请求 ID、目标参数名等。context 会注入到子代理的初始消息中，确保关键信息不丢失。`;
+  const fullSystemPrompt = TASK_SYSTEM_PROMPT + contextGuide;
+
+  return createMiddleware({
+    name: 'subAgentMiddleware',
+    tools: [enhancedTaskTool],
+    wrapModelCall: async (request, handler) => {
+      return handler({
+        ...request,
+        systemMessage: request.systemMessage.concat(new SystemMessage({ content: fullSystemPrompt })),
+      });
+    },
+  });
+}