@hailer/mcp 0.0.1

package/dist/client/adaptive-documentation-bot.js

@@ -0,0 +1,475 @@
+"use strict";
+/**
+ * Adaptive Documentation Bot
+ *
+ * Monitors MCP logs for tool call errors and automatically improves
+ * tool descriptions and skills based on actual LLM usage patterns.
+ *
+ * Key Features:
+ * - Detects when LLMs make mistakes calling tools
+ * - Uses LLM to analyze what went wrong
+ * - Automatically updates tool descriptions
+ * - Creates/updates skill documents with common mistakes
+ * - Learns and improves over time
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AdaptiveDocumentationBot = void 0;
+const logger_1 = require("../lib/logger");
+const log_parser_1 = require("./log-parser");
+const description_updater_1 = require("./description-updater");
+const skill_generator_1 = require("./skill-generator");
+const simple_llm_caller_1 = require("./simple-llm-caller");
+const logger = (0, logger_1.createLogger)({ component: 'AdaptiveDocumentationBot' });
+class AdaptiveDocumentationBot {
+    logParser;
+    descriptionUpdater;
+    skillGenerator;
+    llmCaller;
+    config;
+    errorPatterns = new Map();
+    monitoringInterval;
+    isMonitoring = false;
+    constructor(apiKey, provider = 'anthropic', config = {}) {
+        this.llmCaller = new simple_llm_caller_1.SimpleLLMCaller(provider, apiKey);
+        this.config = {
+            enabled: config.enabled ?? true,
+            autoUpdate: config.autoUpdate ?? true,
+            updateInterval: config.updateInterval ?? 60000, // 1 minute
+            minErrorCount: config.minErrorCount ?? 3,
+            skillGeneration: config.skillGeneration ?? true,
+            logPath: config.logPath
+        };
+        this.logParser = new log_parser_1.LogParser(this.config.logPath);
+        this.descriptionUpdater = new description_updater_1.DescriptionUpdater();
+        this.skillGenerator = new skill_generator_1.SkillGenerator();
+        logger.info('Adaptive Documentation Bot initialized', {
+            autoUpdate: this.config.autoUpdate,
+            updateInterval: this.config.updateInterval,
+            minErrorCount: this.config.minErrorCount
+        });
+    }
+    /**
+     * Start monitoring logs for errors
+     */
+    startMonitoring() {
+        if (this.isMonitoring) {
+            logger.warn('Already monitoring logs');
+            return;
+        }
+        this.isMonitoring = true;
+        logger.info('🔍 Started monitoring MCP logs for tool call errors');
+        // Run immediately
+        this.monitorAndImprove().catch(error => {
+            logger.error('Error in monitoring loop', error);
+        });
+        // Then run on interval (as backup)
+        this.monitoringInterval = setInterval(() => {
+            this.monitorAndImprove().catch(error => {
+                logger.error('Error in monitoring loop', error);
+            });
+        }, this.config.updateInterval);
+    }
+    /**
+     * Analyze logs immediately after a conversation ends
+     * Called by MCP Client after each bot conversation
+     */
+    async analyzeAfterConversation() {
+        if (!this.isMonitoring) {
+            return;
+        }
+        logger.debug('🔍 Analyzing logs after conversation completion');
+        // Wait 500ms for logger to finish writing
+        // This prevents race condition where we try to read while logger is mid-write
+        await new Promise(resolve => setTimeout(resolve, 500));
+        try {
+            await this.monitorAndImprove();
+        }
+        catch (error) {
+            logger.error('Failed to analyze after conversation', error);
+        }
+    }
+    /**
+     * Report error directly (not from logs)
+     * Use this to capture errors from active conversations
+     */
+    reportError(error) {
+        if (!this.isMonitoring) {
+            return;
+        }
+        const key = this.getPatternKey(error);
+        const existing = this.errorPatterns.get(key);
+        if (existing) {
+            existing.count++;
+            existing.lastSeen = error.timestamp;
+            existing.examples.push(error);
+            logger.debug('Updated error pattern', {
+                pattern: key,
+                count: existing.count
+            });
+        }
+        else {
+            this.errorPatterns.set(key, {
+                pattern: key,
+                description: this.generatePatternDescription(error),
+                count: 1,
+                lastSeen: error.timestamp,
+                firstSeen: error.timestamp,
+                examples: [error]
+            });
+            logger.debug('New error pattern detected', {
+                pattern: key,
+                toolName: error.toolName
+            });
+        }
+        // Check if we should analyze this error immediately
+        const pattern = this.errorPatterns.get(key);
+        if (pattern && pattern.count >= this.config.minErrorCount && !pattern.suggestedFix) {
+            logger.info('Error threshold reached, analyzing immediately', {
+                toolName: error.toolName,
+                count: pattern.count
+            });
+            // Analyze asynchronously
+            this.analyzeAndImprove(pattern).catch(err => {
+                logger.error('Failed to analyze error immediately', err);
+            });
+        }
+    }
+    /**
+     * Stop monitoring logs
+     */
+    stopMonitoring() {
+        if (this.monitoringInterval) {
+            clearInterval(this.monitoringInterval);
+            this.monitoringInterval = undefined;
+        }
+        this.isMonitoring = false;
+        logger.info('Stopped monitoring MCP logs');
+    }
+    /**
+     * Main monitoring loop: parse errors and apply improvements
+     */
+    async monitorAndImprove() {
+        try {
+            // 1. Parse logs for new errors
+            const errors = await this.logParser.parseToolCallErrors();
+            if (errors.length === 0) {
+                return;
+            }
+            logger.info('📊 Detected tool call errors', {
+                count: errors.length,
+                uniqueTools: new Set(errors.map(e => e.toolName)).size
+            });
+            // 2. Group errors by tool and pattern
+            this.updateErrorPatterns(errors);
+            // 3. Analyze and improve tools that meet minimum error threshold
+            for (const pattern of this.errorPatterns.values()) {
+                if (pattern.count >= this.config.minErrorCount && !pattern.suggestedFix) {
+                    await this.analyzeAndImprove(pattern);
+                }
+            }
+        }
+        catch (error) {
+            logger.error('Failed to monitor and improve', error);
+        }
+    }
+    /**
+     * Update error patterns with new errors
+     */
+    updateErrorPatterns(errors) {
+        for (const error of errors) {
+            const key = this.getPatternKey(error);
+            const existing = this.errorPatterns.get(key);
+            if (existing) {
+                existing.count++;
+                existing.lastSeen = error.timestamp;
+                existing.examples.push(error);
+            }
+            else {
+                this.errorPatterns.set(key, {
+                    pattern: key,
+                    description: this.generatePatternDescription(error),
+                    count: 1,
+                    lastSeen: error.timestamp,
+                    firstSeen: error.timestamp,
+                    examples: [error]
+                });
+            }
+        }
+    }
+    /**
+     * Analyze error pattern and apply improvements
+     */
+    async analyzeAndImprove(pattern) {
+        try {
+            logger.info('🔍 Analyzing error pattern', {
+                pattern: pattern.pattern,
+                count: pattern.count,
+                tool: pattern.examples[0].toolName
+            });
+            // 1. Analyze with LLM
+            const analysis = await this.analyzeWithLLM(pattern);
+            if (!analysis.isDescriptionIssue && !analysis.needsSkill) {
+                logger.debug('No action needed for this error pattern', {
+                    pattern: pattern.pattern
+                });
+                pattern.suggestedFix = 'no-action-needed';
+                return;
+            }
+            // 2. Apply improvements based on analysis
+            if (this.config.autoUpdate) {
+                await this.applyImprovements(pattern, analysis);
+            }
+            else {
+                logger.info('Auto-update disabled, would improve:', {
+                    tool: pattern.examples[0].toolName,
+                    analysis
+                });
+            }
+            // Mark as processed
+            pattern.suggestedFix = 'applied';
+        }
+        catch (error) {
+            logger.error('Failed to analyze and improve pattern', error, {
+                pattern: pattern.pattern
+            });
+        }
+    }
+    /**
+     * Analyze error pattern using LLM
+     */
+    async analyzeWithLLM(pattern) {
+        const example = pattern.examples[0];
+        const prompt = `You are analyzing MCP tool call errors to improve tool descriptions for an internal chat bot.
+
+**IMPORTANT CONTEXT**: This chat bot CANNOT use skills (it only has READ + WRITE tools, no get_skill capability).
+All improvements must be in the tool description itself, not in external skill documents.
+
+**Tool**: ${example.toolName}
+**Error Message**: ${example.errorMessage}
+**Error Count**: ${pattern.count} times
+**Example Attempted Call**:
+\`\`\`json
+${JSON.stringify(example.attemptedCall, null, 2)}
+\`\`\`
+
+Analyze this error and respond in JSON format:
+{
+  "isDescriptionIssue": boolean (true if the tool description is unclear - ALWAYS true since bot can't use skills),
+  "needsSkill": boolean (ALWAYS false - bot cannot access skills, only external Claude Code users can),
+  "problem": "string (what was misunderstood in the current description)",
+  "suggestion": "string (how to improve the description INLINE with examples and clear requirements)",
+  "priority": "low" | "medium" | "high" | "critical",
+  "affectedParameter": "string (if specific parameter caused issue)",
+  "incorrectUsage": {...} (the wrong way LLM used it),
+  "correctUsage": {...} (the correct way to use it)
+}
+
+Focus on:
+1. What did the LLM misunderstand from the current description?
+2. Which required parameters are not clearly marked as REQUIRED?
+3. What format requirements are missing or unclear?
+4. Add inline examples directly in the description (NOT in skills)
+5. Use ⚠️ REQUIRED or ✅ EXAMPLE markers in the description
+
+Return ONLY valid JSON, no other text.`;
+        try {
+            const response = await this.llmCaller.generate(prompt, 'You are a technical documentation expert. Analyze errors and suggest improvements.');
+            // Extract JSON from response
+            const jsonMatch = response.match(/\{[\s\S]*\}/);
+            if (!jsonMatch) {
+                throw new Error('LLM response did not contain JSON');
+            }
+            const analysis = JSON.parse(jsonMatch[0]);
+            logger.debug('LLM analysis complete', { analysis });
+            return analysis;
+        }
+        catch (error) {
+            logger.error('Failed to analyze with LLM', error);
+            // Return conservative default
+            return {
+                isDescriptionIssue: true,
+                needsSkill: pattern.count >= 5,
+                problem: 'Analysis failed - assuming description issue',
+                suggestion: 'Add clearer examples and emphasize critical requirements',
+                priority: 'medium'
+            };
+        }
+    }
+    /**
+     * Apply improvements based on analysis
+     */
+    async applyImprovements(pattern, analysis) {
+        const toolName = pattern.examples[0].toolName;
+        // 1. Update description if needed
+        if (analysis.isDescriptionIssue) {
+            const improved = await this.generateImprovedDescription(toolName, pattern, analysis);
+            if (improved) {
+                await this.descriptionUpdater.updateToolDescription(toolName, improved, analysis.problem, pattern.examples[0]);
+            }
+        }
+        // 2. Update parameter description if specific parameter issue
+        if (analysis.affectedParameter) {
+            const paramImproved = await this.generateImprovedParameterDescription(toolName, analysis.affectedParameter, analysis);
+            if (paramImproved) {
+                await this.descriptionUpdater.updateParameterDescription(toolName, analysis.affectedParameter, paramImproved, analysis.problem);
+            }
+        }
+        // 3. Create/update skill if needed
+        if (this.config.skillGeneration && analysis.needsSkill) {
+            await this.skillGenerator.updateOrCreateSkill(toolName, pattern, analysis.incorrectUsage || pattern.examples[0].attemptedCall, analysis.correctUsage || {});
+            // Update skill summary
+            await this.skillGenerator.updateSkillSummary(toolName);
+        }
+        logger.info('✅ Improvements applied', {
+            tool: toolName,
+            descriptionUpdated: analysis.isDescriptionIssue,
+            parameterUpdated: !!analysis.affectedParameter,
+            skillUpdated: analysis.needsSkill && this.config.skillGeneration
+        });
+    }
+    /**
+     * Generate improved description using LLM
+     */
+    async generateImprovedDescription(toolName, pattern, analysis) {
+        // Get current description (would need to read from tool files)
+        // For now, generate based on analysis
+        const prompt = `Generate an improved tool description for "${toolName}" for a chat bot that CANNOT load external skills.
+
+**Problem**: ${analysis.problem}
+**Suggestion**: ${analysis.suggestion}
+**Error Count**: ${pattern.count} times
+**Incorrect Usage (what the bot did WRONG)**: ${JSON.stringify(analysis.incorrectUsage)}
+**Correct Usage (what it SHOULD do)**: ${JSON.stringify(analysis.correctUsage)}
+
+CRITICAL Requirements:
+1. Show BOTH the wrong and right way - use ✅ CORRECT: and ❌ WRONG: labels
+2. Include the ACTUAL incorrect JSON that was attempted
+3. Be EXTREMELY explicit about parameter types (string vs array, required vs optional)
+4. Use ⚠️ CRITICAL: for the most important requirement
+5. Keep description under 400 characters for readability
+
+Example format that SHOWS the contrast:
+"📋 [Brief description]. ⚠️ CRITICAL: workflowId must be STRING (NOT array!), phaseId is REQUIRED. ✅ CORRECT: {"workflowId": "abc123", "phaseId": "xyz"} ❌ WRONG: {"workflowId": ["abc123"]} (causes validation error)"
+
+Return ONLY the improved description text, no JSON or extra formatting.`;
+        try {
+            const improved = await this.llmCaller.generate(prompt, 'You are a technical writer. Write clear, concise tool descriptions.');
+            return improved.trim();
+        }
+        catch (error) {
+            logger.error('Failed to generate improved description', error);
+            return null;
+        }
+    }
+    /**
+     * Generate improved parameter description using LLM
+     */
+    async generateImprovedParameterDescription(toolName, parameterName, analysis) {
+        const prompt = `Generate an improved parameter description for "${parameterName}" in tool "${toolName}".
+
+**Problem**: ${analysis.problem}
+**Incorrect Usage**:
+\`\`\`json
+${JSON.stringify(analysis.incorrectUsage, null, 2)}
+\`\`\`
+
+**Correct Usage**:
+\`\`\`json
+${JSON.stringify(analysis.correctUsage, null, 2)}
+\`\`\`
+
+Requirements:
+- Be specific about format/type requirements
+- Use ⚠️ CRITICAL or ✅ CORRECT for emphasis
+- Include example format if helpful
+- Keep it under 100 words
+
+Return ONLY the improved description text.`;
+        try {
+            const improved = await this.llmCaller.generate(prompt, 'You are a technical writer. Write clear parameter descriptions.');
+            return improved.trim();
+        }
+        catch (error) {
+            logger.error('Failed to generate improved parameter description', error);
+            return null;
+        }
+    }
+    /**
+     * Generate pattern key for grouping similar errors
+     */
+    getPatternKey(error) {
+        // Group by tool name and error message pattern
+        return `${error.toolName}:${error.errorMessage}`;
+    }
+    /**
+     * Generate human-readable description for error pattern
+     */
+    generatePatternDescription(error) {
+        const message = error.errorMessage.toLowerCase();
+        if (message.includes('required') || message.includes('missing')) {
+            return 'Missing required parameter';
+        }
+        else if (message.includes('validation') || message.includes('invalid')) {
+            return 'Parameter validation failed';
+        }
+        else if (message.includes('format') || message.includes('type')) {
+            return 'Incorrect parameter format';
+        }
+        else {
+            return 'Tool call error';
+        }
+    }
+    /**
+     * Get statistics about adaptive bot performance
+     */
+    getStats() {
+        return {
+            monitoring: this.isMonitoring,
+            errorPatterns: this.errorPatterns.size,
+            improvements: this.descriptionUpdater.getStats(),
+            skills: this.skillGenerator.getStats()
+        };
+    }
+    /**
+     * Get error patterns for a specific tool
+     */
+    getToolErrorPatterns(toolName) {
+        const patterns = [];
+        for (const pattern of this.errorPatterns.values()) {
+            if (pattern.examples[0].toolName === toolName) {
+                patterns.push(pattern);
+            }
+        }
+        return patterns;
+    }
+    /**
+     * Test mode - parse errors and show what would be improved (without applying changes)
+     */
+    async testRun() {
+        const errors = await this.logParser.parseToolCallErrors();
+        this.updateErrorPatterns(errors);
+        const suggestions = [];
+        for (const pattern of this.errorPatterns.values()) {
+            if (pattern.count >= this.config.minErrorCount && !pattern.suggestedFix) {
+                const toolName = pattern.examples[0].toolName;
+                // Check if existing skill exists
+                const existingSkill = await this.skillGenerator['findExistingSkill'](toolName);
+                suggestions.push({
+                    toolName,
+                    errorCount: pattern.count,
+                    errorMessage: pattern.examples[0].errorMessage,
+                    existingSkill: existingSkill?.name,
+                    wouldCreateSkill: !existingSkill,
+                    wouldUpdateSkill: !!existingSkill
+                });
+            }
+        }
+        return {
+            errors,
+            patterns: Array.from(this.errorPatterns.values()),
+            suggestions
+        };
+    }
+}
+exports.AdaptiveDocumentationBot = AdaptiveDocumentationBot;
+//# sourceMappingURL=adaptive-documentation-bot.js.map

package/dist/client/adaptive-documentation-types.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Types for Adaptive Documentation Bot
+ *
+ * Monitors MCP logs and automatically improves tool descriptions and skills
+ * based on actual LLM usage errors.
+ */
+export interface ToolCallError {
+    timestamp: number;
+    toolName: string;
+    errorMessage: string;
+    attemptedCall: any;
+    actualParameters?: any;
+    context?: {
+        conversationId?: string;
+        userId?: string;
+        provider?: string;
+        model?: string;
+    };
+}
+export interface ErrorPattern {
+    pattern: string;
+    description: string;
+    count: number;
+    lastSeen: number;
+    firstSeen: number;
+    examples: ToolCallError[];
+    rootCause?: string;
+    suggestedFix?: string;
+}
+export interface ErrorAnalysis {
+    isDescriptionIssue: boolean;
+    needsSkill: boolean;
+    problem: string;
+    suggestion: string;
+    priority: 'low' | 'medium' | 'high' | 'critical';
+    affectedParameter?: string;
+    incorrectUsage?: any;
+    correctUsage?: any;
+}
+export interface Improvement {
+    timestamp: number;
+    toolName: string;
+    type: 'description' | 'schema' | 'skill';
+    oldValue: string;
+    newValue: string;
+    triggeredBy: ToolCallError;
+    reason: string;
+    applied: boolean;
+}
+export interface SkillMistake {
+    title: string;
+    frequency: number;
+    lastSeen: number;
+    wrongExample: any;
+    correctExample: any;
+    explanation: string;
+}
+export interface AdaptiveBotConfig {
+    enabled: boolean;
+    autoUpdate: boolean;
+    updateInterval: number;
+    minErrorCount: number;
+    skillGeneration: boolean;
+    logPath?: string;
+}
+//# sourceMappingURL=adaptive-documentation-types.d.ts.map

package/dist/client/adaptive-documentation-types.js

@@ -0,0 +1,9 @@
+"use strict";
+/**
+ * Types for Adaptive Documentation Bot
+ *
+ * Monitors MCP logs and automatically improves tool descriptions and skills
+ * based on actual LLM usage errors.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=adaptive-documentation-types.js.map

package/dist/client/agent-activity-bot.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Agent Activity Bot
+ *
+ * Monitors agent activity and posts summaries to discussions.
+ * Can respond to queries about agent performance and tool usage.
+ */
+import { AgentTracker } from './agent-tracker';
+export interface AgentActivityBotConfig {
+    enabled: boolean;
+    postSummaryAfterConversation: boolean;
+    minDurationThreshold?: number;
+}
+export declare class AgentActivityBot {
+    private agentTracker;
+    private config;
+    private lastPostedConversation;
+    constructor(agentTracker: AgentTracker, config: AgentActivityBotConfig);
+    /**
+     * Generate a summary message for the last conversation in a discussion
+     */
+    generateSummary(requestId: string, discussionId: string, duration: number, toolsCalled: string[]): string;
+    /**
+     * Format a concise summary message
+     */
+    private formatSummaryMessage;
+    /**
+     * Generate detailed agent activity report
+     */
+    generateDetailedReport(discussionId?: string): string;
+    /**
+     * Check if message is asking for agent activity details
+     */
+    isActivityQuery(messageContent: string): boolean;
+    /**
+     * Should post summary after this conversation?
+     */
+    shouldPostSummary(discussionId: string, duration: number): boolean;
+    /**
+     * Mark that we posted a summary for this discussion
+     */
+    markPosted(discussionId: string): void;
+    /**
+     * Get agent tracker instance
+     */
+    getAgentTracker(): AgentTracker;
+}
+/**
+ * Factory function to create AgentActivityBot
+ */
+export declare function createAgentActivityBot(agentTracker: AgentTracker, config: AgentActivityBotConfig): AgentActivityBot;
+//# sourceMappingURL=agent-activity-bot.d.ts.map