deepspider 0.3.0 → 0.3.2

package/src/agent/logger.js

@@ -4,12 +4,14 @@
  */
 
 import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
-import { appendFileSync, mkdirSync, existsSync } from 'fs';
-import { join, dirname } from 'path';
+import { appendFileSync, mkdirSync, existsSync, statSync, renameSync, unlinkSync } from 'fs';
+import { join } from 'path';
 import { DEEPSPIDER_HOME } from '../config/paths.js';
 
 const LOG_DIR = join(DEEPSPIDER_HOME, 'logs');
 const LOG_FILE = join(LOG_DIR, 'agent.log');
+const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB
+const MAX_LOG_FILES = 3;              // agent.log, agent.log.1, agent.log.2
 
 function ensureLogDir() {
   if (!existsSync(LOG_DIR)) {
@@ -17,6 +19,32 @@ function ensureLogDir() {
   }
 }
 
+/**
+ * 日志滚动：agent.log 超过 MAX_LOG_SIZE 时轮转
+ * agent.log → agent.log.1 → agent.log.2 → 删除
+ */
+function rotateIfNeeded(logFile) {
+  try {
+    if (!existsSync(logFile)) return;
+    const { size } = statSync(logFile);
+    if (size < MAX_LOG_SIZE) return;
+
+    // 删除最老的
+    const oldest = `${logFile}.${MAX_LOG_FILES - 1}`;
+    if (existsSync(oldest)) unlinkSync(oldest);
+
+    // 依次轮转
+    for (let i = MAX_LOG_FILES - 2; i >= 1; i--) {
+      const from = `${logFile}.${i}`;
+      const to = `${logFile}.${i + 1}`;
+      if (existsSync(from)) renameSync(from, to);
+    }
+
+    // 当前文件变为 .1
+    renameSync(logFile, `${logFile}.1`);
+  } catch { /* 滚动失败不影响主流程 */ }
+}
+
 function formatTime() {
   return new Date().toISOString();
 }
@@ -27,6 +55,147 @@ function truncate(str, maxLen = 500) {
   return s.length > maxLen ? s.slice(0, maxLen) + '...' : s;
 }
 
+/**
+ * 内存日志环形缓冲区
+ * 始终可用，不依赖 DEBUG 开关
+ */
+export class InMemoryLogStore {
+  constructor(maxSize = 500) {
+    this.logs = [];
+    this.maxSize = maxSize;
+    this.startTime = Date.now();
+  }
+
+  add(entry) {
+    this.logs.push(entry);
+    if (this.logs.length > this.maxSize) this.logs.shift();
+  }
+
+  query({ category, level, limit = 50, toolName } = {}) {
+    let result = this.logs;
+    if (category) result = result.filter(l => l.category === category);
+    if (level) result = result.filter(l => l.level === level);
+    if (toolName) result = result.filter(l => l.data?.toolName === toolName);
+    return result.slice(-limit);
+  }
+
+  getStats() {
+    const cats = {};
+    for (const l of this.logs) {
+      cats[l.category] = (cats[l.category] || 0) + 1;
+    }
+    return {
+      total: this.logs.length,
+      categories: cats,
+      errors: this.logs.filter(l => l.level === 'ERROR').length,
+      uptimeMs: Date.now() - this.startTime,
+    };
+  }
+
+  clear() { this.logs = []; }
+}
+
+export const logStore = new InMemoryLogStore();
+
+/**
+ * 内存日志回调处理器
+ * 始终启用，将日志写入 logStore
+ */
+export class InMemoryLoggerCallback extends BaseCallbackHandler {
+  name = 'InMemoryLoggerCallback';
+
+  log(level, category, message, data = null) {
+    logStore.add({ time: formatTime(), level, category, message, data });
+  }
+
+  handleLLMStart(llm, prompts, runId) {
+    this.log('INFO', 'LLM', 'LLM 调用开始', {
+      runId,
+      model: llm?.id?.[2] || llm?.name,
+      promptCount: prompts?.length,
+      promptPreview: truncate(prompts?.[0], 200),
+    });
+  }
+
+  handleLLMEnd(output, runId) {
+    const content = output?.generations?.[0]?.[0]?.text
+      || output?.generations?.[0]?.[0]?.message?.content;
+    this.log('INFO', 'LLM', 'LLM 调用结束', {
+      runId,
+      outputPreview: truncate(content, 300),
+      tokenUsage: output?.llmOutput?.tokenUsage,
+    });
+  }
+
+  handleLLMError(error, runId) {
+    this.log('ERROR', 'LLM', 'LLM 调用错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  handleToolStart(tool, input, runId) {
+    this.log('INFO', 'TOOL', `工具调用: ${tool?.name || 'unknown'}`, {
+      runId,
+      toolName: tool?.name,
+      input: truncate(input, 500),
+    });
+  }
+
+  handleToolEnd(output, runId) {
+    this.log('INFO', 'TOOL', '工具返回', {
+      runId,
+      output: truncate(output, 500),
+    });
+  }
+
+  handleToolError(error, runId) {
+    this.log('ERROR', 'TOOL', '工具错误', {
+      runId,
+      error: error?.message || String(error),
+      stack: error?.stack?.split('\n').slice(0, 5),
+    });
+  }
+
+  handleChainStart(chain, inputs, runId) {
+    this.log('DEBUG', 'CHAIN', `Chain 开始: ${chain?.name || 'unknown'}`, {
+      runId,
+      chainName: chain?.name,
+      inputKeys: Object.keys(inputs || {}),
+    });
+  }
+
+  handleChainEnd(outputs, runId) {
+    this.log('DEBUG', 'CHAIN', 'Chain 结束', {
+      runId,
+      outputKeys: Object.keys(outputs || {}),
+    });
+  }
+
+  handleChainError(error, runId) {
+    this.log('ERROR', 'CHAIN', 'Chain 错误', {
+      runId,
+      error: error?.message || String(error),
+    });
+  }
+
+  handleAgentAction(action, runId) {
+    this.log('INFO', 'AGENT', `Agent 动作: ${action?.tool}`, {
+      runId,
+      tool: action?.tool,
+      toolInput: truncate(action?.toolInput, 300),
+      log: truncate(action?.log, 200),
+    });
+  }
+
+  handleAgentEnd(action, runId) {
+    this.log('INFO', 'AGENT', 'Agent 结束', {
+      runId,
+      returnValues: truncate(action?.returnValues, 300),
+    });
+  }
+}
+
 /**
  * 文件日志回调处理器
  */
@@ -41,6 +210,8 @@ export class FileLoggerCallback extends BaseCallbackHandler {
   }
 
   log(level, category, message, data = null) {
+    rotateIfNeeded(this.logFile);
+
     const line = JSON.stringify({
       time: formatTime(),
       level,
@@ -149,16 +320,20 @@ export class FileLoggerCallback extends BaseCallbackHandler {
 }
 
 /**
- * 创建日志回调实例
+ * 创建日志回调实例数组
+ * 始终包含 InMemoryLoggerCallback + FileLoggerCallback
+ * DEBUG=true 时额外开启 verbose（控制台输出）
  */
 export function createLogger(options = {}) {
-  const enabled = process.env.DEBUG === 'true' || options.enabled;
-  if (!enabled) return null;
+  const verbose = process.env.DEBUG === 'true' || options.verbose || false;
 
-  return new FileLoggerCallback({
-    verbose: options.verbose || false,
-    logFile: options.logFile || LOG_FILE,
-  });
+  return [
+    new InMemoryLoggerCallback(),
+    new FileLoggerCallback({
+      verbose,
+      logFile: options.logFile || LOG_FILE,
+    }),
+  ];
 }
 
 export default FileLoggerCallback;

package/src/agent/middleware/report.js

@@ -13,10 +13,33 @@ const reportStateSchema = z.object({
   reportReady: z.boolean().default(false),
 });
 
+/**
+ * 检测并触发报告显示
+ */
+async function detectAndTriggerReport(result, onReportReady) {
+  try {
+    const content = typeof result?.content === 'string'
+      ? JSON.parse(result.content)
+      : result?.content;
+
+    if (content?.success && content?.path?.endsWith('.md')) {
+      console.log('[reportMiddleware] 检测到 .md 文件:', content.path);
+
+      if (onReportReady) {
+        await onReportReady(content.path);
+      }
+      return true;
+    }
+  } catch {
+    // 解析失败，忽略
+  }
+  return false;
+}
+
 /**
  * 创建报告中间件
- * 在 afterModel 中检测 artifact_save 工具调用结果
- * 在 afterAgent 中触发报告显示回调
+ * 在 wrapToolCall 中检测 artifact_save 工具调用结果，立即触发报告
+ * 同时在 afterModel 和 afterAgent 中保留检测逻辑作为备选
  */
 export function createReportMiddleware(options = {}) {
   const { onReportReady } = options;
@@ -25,7 +48,20 @@ export function createReportMiddleware(options = {}) {
     name: 'reportMiddleware',
     stateSchema: reportStateSchema,
 
-    // 模型调用后，检测工具调用结果
+    // 工具调用包装器：在 artifact_save 完成时立即检测
+    wrapToolCall: async (request, handler) => {
+      const toolName = request.tool?.name ?? request.toolCall?.name;
+      const result = await handler(request);
+
+      // 检测 artifact_save 工具返回的 .md 文件
+      if (toolName === 'artifact_save') {
+        await detectAndTriggerReport(result, onReportReady);
+      }
+
+      return result;
+    },
+
+    // 模型调用后，检测工具调用结果（备选方案）
     afterModel: (state) => {
       const messages = state.messages;
       if (!messages || messages.length === 0) return undefined;
@@ -41,10 +77,10 @@ export function createReportMiddleware(options = {}) {
 
             // 检测是否是 artifact_save 写入的 .md 文件
             if (content.success && content.path?.endsWith('.md')) {
-              console.log('[reportMiddleware] 检测到 .md 文件:', content.path);
+              console.log('[reportMiddleware] afterModel 检测到 .md 文件:', content.path);
               return { lastWrittenMdFile: content.path };
             }
-          } catch (e) {
+          } catch {
             // 解析失败，忽略
           }
         }
@@ -52,22 +88,12 @@ export function createReportMiddleware(options = {}) {
       return undefined;
     },
 
-    // Agent 执行完成后
+    // Agent 执行完成后（streamEvents 模式下可能不被调用）
     afterAgent: async (state) => {
       const mdFile = state.lastWrittenMdFile;
 
       if (mdFile) {
         console.log('[reportMiddleware] afterAgent: 准备显示报告:', mdFile);
-
-        // 调用回调通知外部
-        if (onReportReady) {
-          try {
-            await onReportReady(mdFile);
-          } catch (e) {
-            console.error('[reportMiddleware] onReportReady 失败:', e.message);
-          }
-        }
-
         return { reportReady: true };
       }
 

package/src/agent/middleware/subagent.js

@@ -0,0 +1,233 @@
+/**
+ * DeepSpider - 自定义子代理中间件
+ * 复刻 deepagents 内置的 createSubAgentMiddleware，增加 context 结构化传递
+ *
+ * 与内置版本的唯一区别：task tool schema 新增 context 字段（z.record(z.string(), z.string()).optional()），
+ * LLM 按需填写 key-value 对，子代理收到的 HumanMessage 中 context 以 <context> 块拼接在 description 之后。
+ */
+
+import { createMiddleware, createAgent, tool, humanInTheLoopMiddleware } from 'langchain';
+import { HumanMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';
+import { getCurrentTaskInput, Command } from '@langchain/langgraph';
+import { TASK_SYSTEM_PROMPT } from 'deepagents';
+import { z } from 'zod';
+
+// 子代理 state 中需要排除的 key（与 deepagents 内部一致）
+const EXCLUDED_STATE_KEYS = [
+  'messages',
+  'todos',
+  'structuredResponse',
+  'skillsMetadata',
+  'memoryContents',
+];
+
+/**
+ * 过滤 state，排除不应传递给子代理的 key
+ */
+function filterStateForSubagent(state) {
+  const filtered = {};
+  for (const [key, value] of Object.entries(state)) {
+    if (!EXCLUDED_STATE_KEYS.includes(key)) filtered[key] = value;
+  }
+  return filtered;
+}
+
+/**
+ * 构造 Command 返回，将子代理结果的 state 更新 + 最后一条消息作为 ToolMessage 返回
+ */
+const TRUST_SIGNAL = `\n\n---\n⚠️ 子代理已完成任务。请直接使用子代理输出的文件和结论，不要重复执行 artifact_load / artifact_glob / ls 等文件读取操作来检查子代理已保存的文件。如果需要对生成的代码做端到端验证，那是你的职责，请正常执行。`;
+
+function returnCommandWithStateUpdate(result, toolCallId) {
+  const stateUpdate = filterStateForSubagent(result);
+  const messages = result.messages;
+  const lastMessage = messages?.[messages.length - 1];
+  const content = (lastMessage?.content || 'Task completed') + TRUST_SIGNAL;
+  return new Command({
+    update: {
+      ...stateUpdate,
+      messages: [new ToolMessage({
+        content,
+        tool_call_id: toolCallId,
+        name: 'task',
+      })],
+    },
+  });
+}
+
+/**
+ * 生成 task tool 的 description（复刻 deepagents 内部的 getTaskToolDescription）
+ */
+function getTaskToolDescription(subagentDescriptions) {
+  return `
+Launch an ephemeral subagent to handle complex, multi-step independent tasks with isolated context windows.
+
+Available agent types and the tools they have access to:
+${subagentDescriptions.join('\n')}
+
+When using the Task tool, you must specify a subagent_type parameter to select which agent type to use.
+
+## Usage notes:
+1. Launch multiple agents concurrently whenever possible, to maximize performance; to do that, use a single message with multiple tool uses
+2. When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.
+3. Each agent invocation is stateless. You will not be able to send additional messages to the agent, nor will the agent be able to communicate with you outside of its final report. Therefore, your prompt should contain a highly detailed task description for the agent to perform autonomously and you should specify exactly what information the agent should return back to you in its final and only message to you.
+4. The agent's outputs should generally be trusted
+5. Clearly tell the agent whether you expect it to create content, perform analysis, or just do research (search, file reads, web fetches, etc.), since it is not aware of the user's intent
+6. If the agent description mentions that it should be used proactively, then you should try your best to use it without the user having to ask for it first. Use your judgement.
+
+## context 参数
+委托子代理时，使用 context 参数传递结构化上下文（key-value 对），如站点标识、请求 ID、目标参数名等。context 会注入到子代理的初始消息中，确保关键信息不丢失。
+  `.trim();
+}
+
+/**
+ * 编译子代理：遍历 subagents 数组，用 createAgent 编译为可运行实例
+ */
+function getSubagents(options) {
+  const {
+    defaultModel,
+    defaultTools,
+    defaultMiddleware,
+    generalPurposeMiddleware: gpMiddleware,
+    defaultInterruptOn,
+    subagents,
+    generalPurposeAgent,
+  } = options;
+
+  const defaultSubagentMiddleware = defaultMiddleware || [];
+  const generalPurposeMiddlewareBase = gpMiddleware || defaultSubagentMiddleware;
+  const agents = {};
+  const descriptions = [];
+
+  // 通用子代理（DeepSpider 默认不启用，但保留能力）
+  if (generalPurposeAgent) {
+    const generalPurposeMiddleware = [...generalPurposeMiddlewareBase];
+    if (defaultInterruptOn) generalPurposeMiddleware.push(humanInTheLoopMiddleware({ interruptOn: defaultInterruptOn }));
+    agents['general-purpose'] = createAgent({
+      model: defaultModel,
+      systemPrompt: 'In order to complete the objective that the user asks of you, you have access to a number of standard tools.',
+      tools: defaultTools,
+      middleware: generalPurposeMiddleware,
+      name: 'general-purpose',
+    });
+    descriptions.push('- general-purpose: General-purpose agent for researching complex questions, searching for files and content, and executing multi-step tasks.');
+  }
+
+  // 自定义子代理
+  for (const agentParams of subagents) {
+    descriptions.push(`- ${agentParams.name}: ${agentParams.description}`);
+
+    if ('runnable' in agentParams) {
+      // CompiledSubAgent — 已编译，直接使用
+      agents[agentParams.name] = agentParams.runnable;
+    } else {
+      const middleware = agentParams.middleware
+        ? [...defaultSubagentMiddleware, ...agentParams.middleware]
+        : [...defaultSubagentMiddleware];
+      const interruptOn = agentParams.interruptOn || defaultInterruptOn;
+      if (interruptOn) middleware.push(humanInTheLoopMiddleware({ interruptOn }));
+
+      agents[agentParams.name] = createAgent({
+        model: agentParams.model ?? defaultModel,
+        systemPrompt: agentParams.systemPrompt,
+        tools: agentParams.tools ?? defaultTools,
+        middleware,
+        name: agentParams.name,
+      });
+    }
+  }
+
+  return { agents, descriptions };
+}
+
+/**
+ * 创建增强版 task tool：schema 增加 context 字段
+ */
+function createEnhancedTaskTool(options) {
+  const { agents: subagentGraphs, descriptions: subagentDescriptions } = getSubagents(options);
+  const availableTypes = Object.keys(subagentGraphs).join(', ');
+
+  return tool(
+    async (input, config) => {
+      const { description, subagent_type, context } = input;
+
+      if (!(subagent_type in subagentGraphs)) {
+        const allowedTypes = Object.keys(subagentGraphs).map((k) => `\`${k}\``).join(', ');
+        throw new Error(`Error: invoked agent of type ${subagent_type}, the only allowed types are ${allowedTypes}`);
+      }
+
+      // 构造子代理的初始消息：description + context 块
+      let content = description;
+      if (context && Object.keys(context).length > 0) {
+        content += `\n\n<context>\n${JSON.stringify(context)}\n</context>`;
+      }
+
+      const subagent = subagentGraphs[subagent_type];
+      const subagentState = filterStateForSubagent(getCurrentTaskInput());
+      subagentState.messages = [new HumanMessage({ content })];
+
+      const result = await subagent.invoke(subagentState, config);
+      if (!config.toolCall?.id) throw new Error('Tool call ID is required for subagent invocation');
+      return returnCommandWithStateUpdate(result, config.toolCall.id);
+    },
+    {
+      name: 'task',
+      description: getTaskToolDescription(subagentDescriptions),
+      schema: z.object({
+        description: z.string().describe('The task to execute with the selected agent'),
+        subagent_type: z.string().describe(`Name of the agent to use. Available: ${availableTypes}`),
+        context: z.record(z.string(), z.string()).optional().describe('Structured key-value context to pass to the subagent (e.g. site, requestId, targetParam)'),
+      }),
+    },
+  );
+}
+
+/**
+ * 创建自定义子代理中间件
+ * 替换 deepagents 内置的 createSubAgentMiddleware，增加 context 结构化传递
+ *
+ * @param {Object} options
+ * @param {LanguageModelLike} options.defaultModel - LLM 实例
+ * @param {StructuredTool[]} options.defaultTools - 默认工具集
+ * @param {SubAgent[]} options.subagents - 子代理配置数组
+ * @param {AgentMiddleware[]} options.defaultMiddleware - 子代理默认中间件
+ * @param {boolean} [options.generalPurposeAgent=false] - 是否创建通用子代理
+ * @param {Object} [options.defaultInterruptOn] - HITL 配置
+ */
+export function createCustomSubAgentMiddleware(options) {
+  const {
+    defaultModel,
+    defaultTools = [],
+    subagents = [],
+    defaultMiddleware = null,
+    generalPurposeMiddleware = null,
+    generalPurposeAgent = false,
+    defaultInterruptOn = null,
+  } = options;
+
+  const taskToolOptions = {
+    defaultModel,
+    defaultTools,
+    subagents,
+    defaultMiddleware,
+    generalPurposeMiddleware,
+    generalPurposeAgent,
+    defaultInterruptOn,
+  };
+
+  const enhancedTaskTool = createEnhancedTaskTool(taskToolOptions);
+
+  // context 使用说明，拼接到 TASK_SYSTEM_PROMPT 末尾
+  const contextGuide = `\n\n委托子代理时，使用 context 参数传递结构化上下文（key-value 对），如站点标识、请求 ID、目标参数名等。context 会注入到子代理的初始消息中，确保关键信息不丢失。`;
+  const fullSystemPrompt = TASK_SYSTEM_PROMPT + contextGuide;
+
+  return createMiddleware({
+    name: 'subAgentMiddleware',
+    tools: [enhancedTaskTool],
+    wrapModelCall: async (request, handler) => {
+      return handler({
+        ...request,
+        systemMessage: request.systemMessage.concat(new SystemMessage({ content: fullSystemPrompt })),
+      });
+    },
+  });
+}

package/src/agent/middleware/toolGuard.js

@@ -0,0 +1,77 @@
+/**
+ * DeepSpider - 工具连续失败检测中间件
+ * 检测同一工具连续失败（超时、错误），在 ToolMessage 中追加警告引导 LLM 换策略
+ */
+
+import { createMiddleware } from 'langchain';
+
+// 默认配置
+const DEFAULTS = {
+  maxConsecutiveFailures: 3,  // 连续失败 N 次后触发强警告
+  warnAfter: 2,              // 连续失败 N 次后开始追加提示
+  resetOnSuccess: true,       // 成功时重置计数
+};
+
+/**
+ * 判断 ToolMessage 是否表示失败
+ */
+function isToolFailure(result) {
+  // ToolMessage.status === 'error' (toolRetryMiddleware 设置)
+  if (result?.status === 'error') return true;
+
+  // 工具返回的 JSON 中 success === false
+  const content = typeof result?.content === 'string' ? result.content : '';
+  if (!content.startsWith('{')) return false;
+  try {
+    const parsed = JSON.parse(content);
+    return parsed.success === false;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * 创建工具连续失败检测中间件
+ */
+export function createToolGuardMiddleware(options = {}) {
+  const config = { ...DEFAULTS, ...options };
+
+  // toolName → { count, lastArgs }
+  const failureTracker = new Map();
+
+  return createMiddleware({
+    name: 'toolGuardMiddleware',
+
+    wrapToolCall: async (request, handler) => {
+      const toolName = request.tool?.name ?? request.toolCall?.name;
+      const result = await handler(request);
+
+      if (!toolName) return result;
+
+      if (isToolFailure(result)) {
+        const tracker = failureTracker.get(toolName) || { count: 0 };
+        tracker.count++;
+        failureTracker.set(toolName, tracker);
+
+        // 追加警告到 ToolMessage content
+        if (tracker.count >= config.maxConsecutiveFailures) {
+          const warning = `\n\n🚫 工具 ${toolName} 已连续失败 ${tracker.count} 次。请停止使用该工具重试相同逻辑，必须换用其他工具或策略。`;
+          if (typeof result.content === 'string') {
+            result.content += warning;
+          }
+        } else if (tracker.count >= config.warnAfter) {
+          const warning = `\n\n⚠️ 工具 ${toolName} 已连续失败 ${tracker.count} 次（上限 ${config.maxConsecutiveFailures}）。如果继续失败将被限制使用，建议考虑替代方案。`;
+          if (typeof result.content === 'string') {
+            result.content += warning;
+          }
+        }
+      } else if (config.resetOnSuccess) {
+        failureTracker.delete(toolName);
+      }
+
+      return result;
+    },
+  });
+}
+
+export default createToolGuardMiddleware;