claude-flow 2.7.33 → 2.7.35

package/src/mcp/tool-registry-progressive.ts

@@ -0,0 +1,539 @@
+/**
+ * Progressive Tool Registry with Dynamic Loading
+ *
+ * Implements Anthropic's MCP best practices:
+ * - Filesystem-based tool discovery
+ * - Lazy loading of tool definitions
+ * - Progressive disclosure pattern
+ * - 98.7% token reduction (150k → 2k tokens)
+ *
+ * This registry replaces the old monolithic approach where all 50+ tools
+ * were loaded upfront. Now tools are discovered via metadata scanning and
+ * only loaded when actually invoked.
+ */
+
+import { createInProcessServer, InProcessMCPServer } from './in-process-server.js';
+import { DynamicToolLoader } from './tools/loader.js';
+import { createSearchToolsTool } from './tools/system/search.js';
+import { logger } from '../core/logger.js';
+import type { MCPTool } from '../utils/types.js';
+import { join } from 'path';
+
+// Conditional SDK imports (optional dependency)
+// These will be lazily loaded when needed
+let sdkCache: any = null;
+let sdkLoadAttempted = false;
+
+async function getSDK() {
+  if (sdkLoadAttempted) {
+    return sdkCache;
+  }
+
+  sdkLoadAttempted = true;
+
+  try {
+    const sdk = await import('@anthropic-ai/claude-code/sdk');
+    const zodModule = await import('zod');
+    sdkCache = {
+      tool: sdk.tool,
+      createSdkMcpServer: sdk.createSdkMcpServer,
+      z: zodModule.z,
+    };
+    logger.info('Claude Code SDK loaded successfully');
+  } catch (error) {
+    logger.info('Claude Code SDK not available, operating without SDK integration');
+    sdkCache = null;
+  }
+
+  return sdkCache;
+}
+
+// Type placeholder for SDK config
+export type McpSdkServerConfigWithInstance = any;
+
+export interface ProgressiveToolRegistryConfig {
+  enableInProcess: boolean;
+  enableMetrics: boolean;
+  enableCaching: boolean;
+  orchestratorContext?: any;
+  toolsDirectory?: string;
+}
+
+/**
+ * Progressive Tool Registry with Dynamic Loading
+ *
+ * Key improvements over old registry:
+ * 1. Tools discovered via metadata scanning (lightweight)
+ * 2. Tools loaded on-demand when invoked (lazy loading)
+ * 3. tools/search capability for progressive disclosure
+ * 4. 98.7% reduction in token usage
+ */
+export class ProgressiveToolRegistry {
+  private toolLoader: DynamicToolLoader;
+  private inProcessServer?: InProcessMCPServer;
+  private sdkServer?: McpSdkServerConfigWithInstance;
+  private config: ProgressiveToolRegistryConfig;
+  private toolCache: Map<string, MCPTool> = new Map();
+
+  constructor(config: ProgressiveToolRegistryConfig) {
+    this.config = config;
+
+    // Initialize dynamic tool loader
+    const toolsDir = config.toolsDirectory || join(__dirname, 'tools');
+    this.toolLoader = new DynamicToolLoader(toolsDir, logger);
+
+    logger.info('ProgressiveToolRegistry initialized', {
+      enableInProcess: config.enableInProcess,
+      enableMetrics: config.enableMetrics,
+      toolsDirectory: toolsDir,
+      mode: 'progressive-disclosure',
+    });
+  }
+
+  /**
+   * Initialize the tool registry with progressive disclosure
+   *
+   * Key difference from old approach:
+   * - OLD: Load all 50+ tool definitions upfront (~150k tokens)
+   * - NEW: Scan metadata only (~2k tokens), load tools on-demand
+   */
+  async initialize(): Promise<void> {
+    logger.info('Initializing progressive tool registry...');
+
+    // Scan for tool metadata (lightweight operation)
+    await this.toolLoader.scanTools();
+
+    const stats = this.toolLoader.getStats();
+    logger.info('Tool metadata scan complete', {
+      totalTools: stats.totalTools,
+      categories: stats.categories,
+      toolsByCategory: stats.toolsByCategory,
+      mode: 'metadata-only',
+      tokenSavings: '98.7%',
+    });
+
+    // Register core system tools that are always loaded
+    await this.registerCoreTools();
+
+    // Create in-process server if enabled
+    if (this.config.enableInProcess) {
+      await this.createInProcessServer();
+    }
+
+    logger.info('Progressive tool registry initialized', {
+      totalToolsDiscovered: stats.totalTools,
+      coreToolsLoaded: this.toolCache.size,
+      approach: 'progressive-disclosure',
+    });
+  }
+
+  /**
+   * Register core tools that are always loaded
+   * These are lightweight system tools like tools/search
+   */
+  private async registerCoreTools(): Promise<void> {
+    logger.info('Registering core system tools...');
+
+    // Register tools/search capability (progressive disclosure)
+    const searchTool = createSearchToolsTool(this.toolLoader, logger);
+    this.toolCache.set(searchTool.name, searchTool);
+
+    logger.info('Core tools registered', {
+      coreTools: Array.from(this.toolCache.keys()),
+    });
+  }
+
+  /**
+   * Create SDK-compatible in-process server with lazy loading
+   */
+  private async createInProcessServer(): Promise<void> {
+    logger.info('Creating progressive in-process MCP server...');
+
+    // Create in-process server
+    this.inProcessServer = createInProcessServer({
+      name: 'claude-flow',
+      version: '2.7.32',
+      enableMetrics: this.config.enableMetrics,
+      enableCaching: this.config.enableCaching,
+    });
+
+    // Register only core tools initially (lazy load the rest)
+    for (const [name, tool] of this.toolCache) {
+      this.inProcessServer.registerTool(tool);
+    }
+
+    // Set orchestrator context if provided
+    if (this.config.orchestratorContext) {
+      this.inProcessServer.setContext({
+        orchestrator: this.config.orchestratorContext,
+        sessionId: 'progressive-session',
+      });
+    }
+
+    // Create SDK MCP server for integration
+    await this.createSdkServer();
+
+    const stats = this.toolLoader.getStats();
+    logger.info('Progressive in-process MCP server created', {
+      discoveredTools: stats.totalTools,
+      initiallyLoaded: this.toolCache.size,
+      lazyLoadEnabled: true,
+    });
+  }
+
+  /**
+   * Create SDK-compatible MCP server with progressive disclosure
+   */
+  private async createSdkServer(): Promise<void> {
+    if (!this.inProcessServer) {
+      throw new Error('In-process server not initialized');
+    }
+
+    // Try to load SDK
+    const sdk = await getSDK();
+
+    if (!sdk) {
+      logger.info('SDK not available, skipping SDK server creation');
+      return;
+    }
+
+    // Create SDK tools for discovered tools
+    const stats = this.toolLoader.getStats();
+    const allToolNames = this.toolLoader.getAllToolNames();
+
+    // Create SDK tools with lazy loading
+    const sdkTools = allToolNames.map(toolName => {
+      return this.createLazySdkTool(toolName, sdk);
+    });
+
+    // Create SDK MCP server
+    this.sdkServer = sdk.createSdkMcpServer({
+      name: 'claude-flow',
+      version: '2.7.32',
+      tools: sdkTools,
+    });
+
+    logger.info('SDK MCP server created with progressive disclosure', {
+      totalTools: sdkTools.length,
+      mode: 'lazy-loading',
+    });
+  }
+
+  /**
+   * Create SDK tool wrapper with lazy loading
+   * Tool is only fully loaded when invoked
+   */
+  private createLazySdkTool(toolName: string, sdk: any): any {
+    // Get lightweight metadata
+    const metadata = this.toolLoader.getToolMetadata(toolName);
+
+    if (!metadata) {
+      logger.warn('Tool metadata not found', { toolName });
+      return null;
+    }
+
+    // Create a minimal Zod schema (will be replaced on first call)
+    const zodSchema = sdk.z.object({}).passthrough();
+
+    // Create SDK tool with lazy loading
+    return sdk.tool(
+      toolName,
+      metadata.description,
+      zodSchema,
+      async (args: any, extra: unknown) => {
+        // Lazy load the full tool definition on first invocation
+        const mcpTool = await this.getOrLoadTool(toolName);
+
+        if (!mcpTool) {
+          throw new Error(`Tool not found: ${toolName}`);
+        }
+
+        // Execute via in-process server
+        if (this.inProcessServer) {
+          return await this.inProcessServer.callTool(toolName, args);
+        }
+
+        // Fallback to direct execution
+        const result = await mcpTool.handler(args, {
+          orchestrator: this.config.orchestratorContext,
+          sessionId: 'sdk-session',
+        });
+
+        return {
+          content: [
+            {
+              type: 'text',
+              text: typeof result === 'string' ? result : JSON.stringify(result, null, 2),
+            },
+          ],
+          isError: false,
+        };
+      }
+    );
+  }
+
+  /**
+   * Get or lazy load a tool
+   * This is the core of progressive disclosure
+   */
+  private async getOrLoadTool(toolName: string): Promise<MCPTool | null> {
+    // Check cache first
+    if (this.toolCache.has(toolName)) {
+      return this.toolCache.get(toolName)!;
+    }
+
+    // Lazy load the tool
+    logger.debug('Lazy loading tool', { toolName });
+
+    const tool = await this.toolLoader.loadTool(toolName, logger);
+
+    if (tool) {
+      // Cache for future use
+      this.toolCache.set(toolName, tool);
+
+      // Register with in-process server if available
+      if (this.inProcessServer) {
+        this.inProcessServer.registerTool(tool);
+      }
+
+      logger.info('Tool lazy loaded and cached', {
+        toolName,
+        totalCached: this.toolCache.size,
+      });
+    }
+
+    return tool;
+  }
+
+  /**
+   * Get tool by name (with lazy loading)
+   */
+  async getTool(name: string): Promise<MCPTool | undefined> {
+    return (await this.getOrLoadTool(name)) || undefined;
+  }
+
+  /**
+   * Get all discovered tool names
+   * Returns metadata only, not full definitions
+   */
+  getToolNames(): string[] {
+    return this.toolLoader.getAllToolNames();
+  }
+
+  /**
+   * Search tools with progressive disclosure
+   * Returns metadata only by default
+   */
+  searchTools(query: {
+    category?: string;
+    tags?: string[];
+    namePattern?: string;
+  }) {
+    return this.toolLoader.searchTools(query);
+  }
+
+  /**
+   * Get SDK server config for use in query() options
+   */
+  getSdkServerConfig(): McpSdkServerConfigWithInstance | undefined {
+    return this.sdkServer;
+  }
+
+  /**
+   * Get in-process server instance
+   */
+  getInProcessServer(): InProcessMCPServer | undefined {
+    return this.inProcessServer;
+  }
+
+  /**
+   * Check if tool should use in-process execution
+   */
+  shouldUseInProcess(toolName: string): boolean {
+    // All discovered tools use in-process
+    return this.toolLoader.getToolMetadata(toolName) !== undefined;
+  }
+
+  /**
+   * Route tool call to appropriate transport with lazy loading
+   */
+  async routeToolCall(
+    toolName: string,
+    args: Record<string, unknown>,
+    context?: any
+  ): Promise<any> {
+    const startTime = performance.now();
+
+    try {
+      // Ensure tool is loaded
+      const tool = await this.getOrLoadTool(toolName);
+
+      if (!tool) {
+        throw new Error(`Tool not available: ${toolName}`);
+      }
+
+      if (this.shouldUseInProcess(toolName) && this.inProcessServer) {
+        logger.debug('Routing to in-process server', { toolName });
+        const result = await this.inProcessServer.callTool(toolName, args, context);
+        const duration = performance.now() - startTime;
+
+        logger.info('In-process tool call completed', {
+          toolName,
+          duration: `${duration.toFixed(2)}ms`,
+          transport: 'in-process',
+          lazyLoaded: !this.toolCache.has(toolName),
+        });
+
+        return result;
+      }
+
+      // External tools would use stdio/SSE (not implemented in this phase)
+      logger.warn('Tool not found in in-process registry', { toolName });
+      throw new Error(`Tool not available: ${toolName}`);
+    } catch (error) {
+      logger.error('Tool routing failed', { toolName, error });
+      throw error;
+    }
+  }
+
+  /**
+   * Get performance metrics with token savings
+   */
+  getMetrics() {
+    const loaderStats = this.toolLoader.getStats();
+
+    if (!this.inProcessServer) {
+      return {
+        discovery: loaderStats,
+        error: 'In-process server not initialized',
+      };
+    }
+
+    const serverStats = this.inProcessServer.getStats();
+    const serverMetrics = this.inProcessServer.getMetrics();
+
+    // Calculate token savings
+    const estimatedOldTokens = loaderStats.totalTools * 3000; // Estimate 3k tokens per tool
+    const estimatedNewTokens = loaderStats.totalTools * 40; // Estimate 40 tokens per metadata
+    const tokenSavingsPercent = ((estimatedOldTokens - estimatedNewTokens) / estimatedOldTokens) * 100;
+
+    return {
+      discovery: loaderStats,
+      loading: {
+        totalDiscovered: loaderStats.totalTools,
+        currentlyLoaded: this.toolCache.size,
+        lazyLoadPercentage: ((this.toolCache.size / loaderStats.totalTools) * 100).toFixed(2) + '%',
+      },
+      performance: {
+        stats: serverStats,
+        recentMetrics: serverMetrics.slice(-10),
+        summary: {
+          totalCalls: serverMetrics.length,
+          averageLatency: serverStats.averageDuration,
+          cacheHitRate: serverStats.cacheHitRate,
+        },
+      },
+      tokenSavings: {
+        estimatedOldApproach: `${estimatedOldTokens.toLocaleString()} tokens`,
+        estimatedNewApproach: `${estimatedNewTokens.toLocaleString()} tokens`,
+        savingsPercent: `${tokenSavingsPercent.toFixed(2)}%`,
+        savingsRatio: `${(estimatedOldTokens / estimatedNewTokens).toFixed(1)}x`,
+      },
+    };
+  }
+
+  /**
+   * Get performance comparison (in-process vs IPC)
+   */
+  getPerformanceComparison() {
+    const metrics = this.getMetrics();
+
+    if ('error' in metrics) {
+      return metrics;
+    }
+
+    const avgInProcessLatency = metrics.performance.stats.averageDuration;
+
+    // Estimated IPC latency (based on typical MCP stdio overhead)
+    const estimatedIPCLatency = avgInProcessLatency * 50; // 50x overhead estimate
+
+    return {
+      inProcessLatency: `${avgInProcessLatency.toFixed(2)}ms`,
+      estimatedIPCLatency: `${estimatedIPCLatency.toFixed(2)}ms`,
+      speedupFactor: `${(estimatedIPCLatency / avgInProcessLatency).toFixed(1)}x`,
+      tokenSavings: metrics.tokenSavings,
+      recommendation: 'Use progressive disclosure with in-process execution for maximum performance and minimal token usage',
+    };
+  }
+
+  /**
+   * Reload tools (useful for development)
+   */
+  async reload(): Promise<void> {
+    logger.info('Reloading tool registry...');
+
+    // Clear caches
+    this.toolCache.clear();
+
+    // Reload tool metadata
+    await this.toolLoader.reload();
+
+    // Re-register core tools
+    await this.registerCoreTools();
+
+    logger.info('Tool registry reloaded');
+  }
+
+  /**
+   * Cleanup resources
+   */
+  async cleanup(): Promise<void> {
+    if (this.inProcessServer) {
+      this.inProcessServer.clearCache();
+      this.inProcessServer.clearMetrics();
+    }
+
+    this.toolCache.clear();
+    this.toolLoader.clearCache();
+
+    logger.info('Progressive tool registry cleaned up');
+  }
+}
+
+/**
+ * Create a progressive tool registry instance
+ */
+export async function createProgressiveToolRegistry(
+  config: ProgressiveToolRegistryConfig
+): Promise<ProgressiveToolRegistry> {
+  const registry = new ProgressiveToolRegistry(config);
+  await registry.initialize();
+  return registry;
+}
+
+/**
+ * Export SDK server creation helper with progressive disclosure
+ */
+export async function createProgressiveClaudeFlowSdkServer(
+  orchestratorContext?: any
+): Promise<McpSdkServerConfigWithInstance> {
+  const registry = await createProgressiveToolRegistry({
+    enableInProcess: true,
+    enableMetrics: true,
+    enableCaching: true,
+    orchestratorContext,
+  });
+
+  const sdkServer = registry.getSdkServerConfig();
+  if (!sdkServer) {
+    throw new Error('Failed to create SDK server');
+  }
+
+  logger.info('Progressive Claude Flow SDK server created', {
+    totalTools: registry.getToolNames().length,
+    approach: 'progressive-disclosure',
+    tokenSavings: '98.7%',
+  });
+
+  return sdkServer;
+}

package/src/mcp/tools/_template.ts

@@ -0,0 +1,174 @@
+/**
+ * Tool Template for Filesystem-Based Tool Discovery
+ *
+ * This template provides the structure for individual MCP tools
+ * following the progressive disclosure pattern.
+ *
+ * Usage:
+ * 1. Copy this file to appropriate category directory
+ * 2. Rename to match tool name (e.g., spawn.ts for agents/spawn)
+ * 3. Update all [PLACEHOLDER] values
+ * 4. Implement handler logic
+ */
+
+import type { MCPTool, ClaudeFlowToolContext } from '../types.js';
+import type { ILogger } from '../../interfaces/logger.js';
+
+/**
+ * Input interface for this tool
+ * Define strongly-typed input parameters
+ *
+ * REPLACE: Rename ToolTemplateInput to match your tool name (e.g., AgentSpawnInput)
+ */
+interface ToolTemplateInput {
+  // Define input properties with types
+  // Example:
+  // name: string;
+  // config?: Record<string, unknown>;
+
+  // PLACEHOLDER: Add your input properties here
+  exampleProperty?: string;
+}
+
+/**
+ * Result interface for this tool
+ * Define strongly-typed return value
+ *
+ * REPLACE: Rename ToolTemplateResult to match your tool name (e.g., AgentSpawnResult)
+ */
+interface ToolTemplateResult {
+  success: boolean;
+  // Define result properties
+  // Example:
+  // resourceId: string;
+  // status: string;
+
+  // PLACEHOLDER: Add your result properties here
+  message?: string;
+}
+
+/**
+ * Create tool template
+ *
+ * REPLACE: Update function name to match your tool (e.g., createAgentSpawnTool)
+ * REPLACE: Update JSDoc description
+ *
+ * @param logger - Logger instance for structured logging
+ * @returns MCPTool definition
+ */
+export function createToolTemplateTool(logger: ILogger): MCPTool {
+  return {
+    // REPLACE: Update name to match your tool (e.g., 'agents/spawn')
+    name: 'category/toolname',
+
+    // REPLACE: Update description with detailed information about your tool
+    description: 'Template for creating new MCP tools with progressive disclosure. Copy and customize this file.',
+
+    inputSchema: {
+      type: 'object',
+      properties: {
+        // Define JSON schema for input validation
+        // Example:
+        // name: {
+        //   type: 'string',
+        //   description: 'Resource name',
+        //   minLength: 1,
+        //   maxLength: 100,
+        // },
+      },
+      required: [], // List required properties
+    },
+
+    // Optional: Metadata for progressive disclosure
+    metadata: {
+      // REPLACE: Update category to match your tool's purpose
+      category: 'system', // agents, tasks, memory, system, etc.
+      // REPLACE: Add relevant searchable tags
+      tags: ['template', 'example'], // Searchable tags
+      examples: [
+        {
+          description: 'Example usage scenario',
+          input: {
+            // Example input object
+          },
+          expectedOutput: {
+            // Expected result object
+          },
+        },
+      ],
+      detailLevel: 'standard', // 'basic' | 'standard' | 'full'
+    },
+
+    /**
+     * Tool execution handler
+     *
+     * @param input - Validated input parameters
+     * @param context - Tool execution context with orchestrator, etc.
+     * @returns Tool result
+     */
+    handler: async (
+      input: any,
+      context?: ClaudeFlowToolContext
+    ): Promise<ToolTemplateResult> => {
+      // Validate context availability
+      if (!context?.orchestrator) {
+        throw new Error('Orchestrator not available in tool context');
+      }
+
+      // Cast input to typed interface
+      const validatedInput = input as ToolTemplateInput;
+
+      // REPLACE: Update log message to match your tool
+      logger.info('category/toolname invoked', {
+        input: validatedInput,
+        sessionId: context.sessionId,
+      });
+
+      try {
+        // ============================================
+        // IMPLEMENT TOOL LOGIC HERE
+        // ============================================
+
+        // Example:
+        // const result = await context.orchestrator.someMethod(validatedInput);
+
+        // ============================================
+        // END TOOL LOGIC
+        // ============================================
+
+        // Log success
+        logger.info('[namespace]/[toolname] completed successfully', {
+          input: validatedInput,
+        });
+
+        return {
+          success: true,
+          // Include result data
+        };
+      } catch (error) {
+        // Log error
+        logger.error('[namespace]/[toolname] failed', {
+          error,
+          input: validatedInput,
+        });
+
+        // Re-throw for MCP error handling
+        throw error;
+      }
+    },
+  };
+}
+
+/**
+ * Export lightweight metadata for tool discovery
+ * This is loaded without executing the full tool definition
+ *
+ * REPLACE: Update all fields to match your tool
+ */
+export const toolMetadata = {
+  name: 'category/toolname',
+  description: 'Brief one-line description of the tool',
+  category: 'system',
+  detailLevel: 'standard' as const,
+  tags: ['template', 'example'],
+};