@hailer/mcp 0.0.1

package/.claude/skills/skill-testing/references/testing-guide.md

@@ -0,0 +1,603 @@
+# Comprehensive Testing Guide
+
+Step-by-step guide for testing skills at all levels: unit, integration, and end-to-end.
+
+## Testing Workflow
+
+```
+┌──────────────────────────────────────────────────────┐
+│ 1. Create Skill                                      │
+│    - Write SKILL.md                                  │
+│    - Add references                                  │
+│    - Define keywords                                 │
+└──────────────────────────────────────────────────────┘
+                        ↓
+┌──────────────────────────────────────────────────────┐
+│ 2. Unit Tests (Isolated)                            │
+│    - Test file structure                             │
+│    - Test content validity                           │
+│    - Test loading mechanism                          │
+└──────────────────────────────────────────────────────┘
+                        ↓
+┌──────────────────────────────────────────────────────┐
+│ 3. Integration Tests (Skill System)                 │
+│    - Test with SkillLoader                           │
+│    - Test with SkillManager                          │
+│    - Test keyword matching                           │
+│    - Test caching behavior                           │
+└──────────────────────────────────────────────────────┘
+                        ↓
+┌──────────────────────────────────────────────────────┐
+│ 4. E2E Tests (Full Agent)                           │
+│    - Test with real messages                         │
+│    - Test with LLM providers                         │
+│    - Test prompt enhancement                         │
+│    - Validate agent responses                        │
+└──────────────────────────────────────────────────────┘
+                        ↓
+┌──────────────────────────────────────────────────────┐
+│ 5. Performance Tests                                 │
+│    - Measure load times                              │
+│    - Measure cache efficiency                        │
+│    - Test under load                                 │
+└──────────────────────────────────────────────────────┘
+                        ↓
+┌──────────────────────────────────────────────────────┐
+│ 6. Quality Review                                    │
+│    - Manual content review                           │
+│    - Test with real users                            │
+│    - Gather feedback                                 │
+└──────────────────────────────────────────────────────┘
+```
+
+## Level 1: Unit Tests
+
+### Testing File Structure
+
+```typescript
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+describe('Skill Structure', () => {
+  const skillPath = path.join(__dirname, '../../.claude/skills/my-skill');
+
+  it('should have SKILL.md in root', async () => {
+    const skillMd = path.join(skillPath, 'SKILL.md');
+    const exists = await fs.stat(skillMd)
+      .then(() => true)
+      .catch(() => false);
+    expect(exists).toBe(true);
+  });
+
+  it('should have references directory (optional)', async () => {
+    const refsPath = path.join(skillPath, 'references');
+    const exists = await fs.stat(refsPath)
+      .then(() => true)
+      .catch(() => false);
+    // Optional, so just log if missing
+    if (!exists) {
+      console.log('Note: No references directory found');
+    }
+  });
+
+  it('should contain valid markdown', async () => {
+    const skillMd = path.join(skillPath, 'SKILL.md');
+    const content = await fs.readFile(skillMd, 'utf-8');
+
+    // Basic markdown validation
+    expect(content.length).toBeGreaterThan(100);
+    expect(content).toMatch(/^#\s+/m); // Has at least one heading
+  });
+});
+```
+
+### Testing Content Quality
+
+```typescript
+describe('Skill Content Quality', () => {
+  let skillContent: string;
+
+  beforeAll(async () => {
+    const skillMd = path.join(__dirname, '../../.claude/skills/my-skill/SKILL.md');
+    skillContent = await fs.readFile(skillMd, 'utf-8');
+  });
+
+  it('should have clear title', () => {
+    const titleMatch = skillContent.match(/^#\s+(.+)$/m);
+    expect(titleMatch).toBeTruthy();
+    expect(titleMatch![1].length).toBeGreaterThan(3);
+  });
+
+  it('should have overview section', () => {
+    expect(skillContent.toLowerCase()).toMatch(/overview|introduction|about/);
+  });
+
+  it('should contain examples', () => {
+    expect(skillContent).toMatch(/```/); // Has code blocks
+  });
+
+  it('should not be too short', () => {
+    expect(skillContent.length).toBeGreaterThan(500);
+  });
+
+  it('should not be excessively long', () => {
+    expect(skillContent.length).toBeLessThan(100000); // 100KB
+  });
+
+  it('should have actionable content', () => {
+    const actionWords = ['how to', 'step', 'example', 'guide', 'use'];
+    const hasActionWords = actionWords.some(word =>
+      skillContent.toLowerCase().includes(word)
+    );
+    expect(hasActionWords).toBe(true);
+  });
+});
+```
+
+### Testing Reference Files
+
+```typescript
+describe('Skill References', () => {
+  const refsPath = path.join(__dirname, '../../.claude/skills/my-skill/references');
+
+  it('should load all reference files', async () => {
+    const files = await fs.readdir(refsPath).catch(() => []);
+    const mdFiles = files.filter(f => f.endsWith('.md'));
+
+    for (const file of mdFiles) {
+      const content = await fs.readFile(path.join(refsPath, file), 'utf-8');
+      expect(content.length).toBeGreaterThan(0);
+      console.log(`  ✅ Loaded ${file} (${content.length} chars)`);
+    }
+  });
+});
+```
+
+## Level 2: Integration Tests
+
+### Testing with SkillLoader
+
+```typescript
+import { SkillLoader } from '../src/client/skill-loader';
+import * as path from 'path';
+
+describe('SkillLoader Integration', () => {
+  let loader: SkillLoader;
+
+  beforeAll(() => {
+    const skillsPath = path.join(__dirname, '../.claude/skills');
+    loader = new SkillLoader(skillsPath);
+  });
+
+  afterEach(() => {
+    loader.clearCache();
+  });
+
+  it('should load skill successfully', async () => {
+    const skill = await loader.load('my-skill');
+
+    expect(skill.name).toBe('my-skill');
+    expect(skill.content).toBeTruthy();
+    expect(skill.description).toBeTruthy();
+    expect(skill.loadedAt).toBeGreaterThan(0);
+  });
+
+  it('should include references in content', async () => {
+    const skill = await loader.load('my-skill');
+
+    // References are appended to main content
+    // If references exist, content should be larger
+    expect(skill.content.length).toBeGreaterThan(500);
+  });
+
+  it('should cache skills after first load', async () => {
+    // First load
+    const start1 = performance.now();
+    await loader.load('my-skill');
+    const firstLoadTime = performance.now() - start1;
+
+    // Second load (cached)
+    const start2 = performance.now();
+    await loader.load('my-skill');
+    const cachedLoadTime = performance.now() - start2;
+
+    console.log(`First load: ${firstLoadTime.toFixed(2)}ms`);
+    console.log(`Cached load: ${cachedLoadTime.toFixed(2)}ms`);
+
+    expect(cachedLoadTime).toBeLessThan(firstLoadTime / 5);
+  });
+
+  it('should handle missing skill gracefully', async () => {
+    await expect(
+      loader.load('non-existent-skill')
+    ).rejects.toThrow(/Failed to load skill/);
+  });
+
+  it('should clear cache correctly', async () => {
+    await loader.load('my-skill');
+    loader.clearCache('my-skill');
+
+    // Next load should be from disk again
+    const start = performance.now();
+    await loader.load('my-skill');
+    const loadTime = performance.now() - start;
+
+    // Should take longer than cached load
+    expect(loadTime).toBeGreaterThan(1);
+  });
+});
+```
+
+### Testing with SkillManager
+
+```typescript
+import { SkillManager } from '../src/client/skill-manager';
+import { SkillLoader } from '../src/client/skill-loader';
+
+describe('SkillManager Integration', () => {
+  let manager: SkillManager;
+  let loader: SkillLoader;
+
+  beforeAll(() => {
+    const skillsPath = path.join(__dirname, '../.claude/skills');
+    loader = new SkillLoader(skillsPath);
+    manager = new SkillManager(loader);
+  });
+
+  describe('Keyword Matching', () => {
+    it('should match skill by primary keywords', async () => {
+      const testCases = [
+        {
+          message: 'How do I create a workflow?',
+          expectedSkill: 'mcp-tools',
+          minConfidence: 0.5
+        },
+        {
+          message: 'List all activities in the workspace',
+          expectedSkill: 'hailer-api',
+          minConfidence: 0.5
+        },
+        {
+          message: 'Build a new agent system',
+          expectedSkill: 'agent-building',
+          minConfidence: 0.5
+        },
+      ];
+
+      for (const { message, expectedSkill, minConfidence } of testCases) {
+        const guidance = await manager.analyzeRequest(message);
+
+        expect(guidance.skills).toContain(expectedSkill);
+        expect(guidance.confidence).toBeGreaterThanOrEqual(minConfidence);
+
+        console.log(`  ✅ "${message}" → ${expectedSkill} (${(guidance.confidence * 100).toFixed(0)}%)`);
+      }
+    });
+
+    it('should handle messages with no skill match', async () => {
+      const guidance = await manager.analyzeRequest('Hello world xyz123');
+
+      expect(guidance.skills).toHaveLength(0);
+      expect(guidance.confidence).toBeLessThan(0.5);
+      expect(guidance.guidance).toContain('General assistance');
+    });
+
+    it('should return skill content', async () => {
+      const guidance = await manager.analyzeRequest('create workflow');
+
+      expect(guidance.skillContent).toBeTruthy();
+      expect(guidance.skillContent!.length).toBeGreaterThan(1000);
+    });
+
+    it('should recommend relevant tools', async () => {
+      const guidance = await manager.analyzeRequest('workflow schema');
+
+      expect(guidance.recommendedTools).toBeTruthy();
+      expect(guidance.recommendedTools.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('Multiple Keyword Matches', () => {
+    it('should prioritize highest scoring skill', async () => {
+      // Message with multiple potential matches
+      const guidance = await manager.analyzeRequest(
+        'workflow activity field schema'
+      );
+
+      // Should pick one skill, not multiple
+      expect(guidance.skills.length).toBeGreaterThan(0);
+      expect(guidance.skills.length).toBeLessThanOrEqual(2);
+    });
+  });
+
+  describe('Edge Cases', () => {
+    it('should handle empty message', async () => {
+      const guidance = await manager.analyzeRequest('');
+      expect(guidance.skills).toHaveLength(0);
+    });
+
+    it('should handle very long message', async () => {
+      const longMessage = 'workflow '.repeat(100);
+      const guidance = await manager.analyzeRequest(longMessage);
+      expect(guidance.skills.length).toBeGreaterThan(0);
+    });
+
+    it('should handle special characters', async () => {
+      const guidance = await manager.analyzeRequest('workflow !@#$%^&*()');
+      expect(guidance.skills.length).toBeGreaterThan(0);
+    });
+  });
+});
+```
+
+## Level 3: End-to-End Tests
+
+### Testing with Mock LLM Provider
+
+```typescript
+import { McpClient } from '../src/client/mcp-client';
+import { ChatMessage } from '../src/client/types';
+
+describe('E2E: Skill System with Agent', () => {
+  let mockClient: McpClient;
+
+  beforeAll(() => {
+    // Setup mock MCP client with test config
+    mockClient = new McpClient({
+      enabled: true,
+      mcpServerUrl: 'http://localhost:3030/api/mcp',
+      mcpServerApiKey: 'test-key',
+      providers: [], // No actual providers needed for this test
+      mcpAgentIds: ['test-agent-id'],
+      botConfigs: [],
+      enableDirectMessages: true
+    });
+  });
+
+  it('should enhance prompt with skill content', async () => {
+    // This test verifies the integration works but doesn't call LLM
+    // You'd need to mock the provider or use a test double
+
+    const message: ChatMessage = {
+      id: 'test-msg-1',
+      content: 'How do I create a workflow schema?',
+      timestamp: Date.now(),
+      discussionId: 'test-discussion',
+      userId: 'test-user',
+      userName: 'Test User',
+      workspaceId: 'test-workspace',
+      mentionedOrDirectMessagedBotId: 'test-agent-id'
+    };
+
+    // In a real test, you'd mock the provider and verify
+    // that it receives the skill content in the system prompt
+    // For now, just verify the system initializes
+    expect(mockClient).toBeTruthy();
+  });
+});
+```
+
+### Testing Skill Impact on Responses
+
+```typescript
+describe('E2E: Skill Impact', () => {
+  // These tests require a real or mocked LLM provider
+  // They verify that skills actually improve agent responses
+
+  it('should provide better responses with skill loaded', async () => {
+    // Compare response quality with and without skill
+    // This is typically done manually or with LLM-as-judge
+
+    const messageWithSkill = 'explain workflow schemas';
+    // Response should include detailed workflow schema info
+
+    const messageWithoutSkill = 'explain xyz123abc';
+    // Response should be generic
+
+    // Manual verification or LLM-as-judge scoring
+  });
+});
+```
+
+## Level 4: Performance Tests
+
+### Load Time Benchmarks
+
+```typescript
+describe('Performance: Skill Loading', () => {
+  let loader: SkillLoader;
+
+  beforeAll(() => {
+    const skillsPath = path.join(__dirname, '../.claude/skills');
+    loader = new SkillLoader(skillsPath);
+  });
+
+  it('should load within performance budget', async () => {
+    const iterations = 10;
+    const times: number[] = [];
+
+    for (let i = 0; i < iterations; i++) {
+      loader.clearCache();
+      const start = performance.now();
+      await loader.load('my-skill');
+      times.push(performance.now() - start);
+    }
+
+    const avgTime = times.reduce((a, b) => a + b) / times.length;
+    const maxTime = Math.max(...times);
+
+    console.log(`Average load time: ${avgTime.toFixed(2)}ms`);
+    console.log(`Max load time: ${maxTime.toFixed(2)}ms`);
+
+    expect(avgTime).toBeLessThan(100); // Should average < 100ms
+    expect(maxTime).toBeLessThan(150); // Should never exceed 150ms
+  });
+
+  it('should cache effectively', async () => {
+    await loader.load('my-skill');
+
+    const iterations = 100;
+    const start = performance.now();
+
+    for (let i = 0; i < iterations; i++) {
+      await loader.load('my-skill');
+    }
+
+    const totalTime = performance.now() - start;
+    const avgTime = totalTime / iterations;
+
+    console.log(`Cached load average: ${avgTime.toFixed(2)}ms`);
+
+    expect(avgTime).toBeLessThan(10); // Cached loads should be < 10ms
+  });
+});
+```
+
+### Memory Usage
+
+```typescript
+describe('Performance: Memory Usage', () => {
+  it('should not leak memory', async () => {
+    const loader = new SkillLoader(skillsPath);
+
+    const initialMemory = process.memoryUsage().heapUsed;
+
+    // Load and clear many times
+    for (let i = 0; i < 100; i++) {
+      await loader.load('my-skill');
+      loader.clearCache();
+    }
+
+    // Force GC if available
+    if (global.gc) {
+      global.gc();
+    }
+
+    const finalMemory = process.memoryUsage().heapUsed;
+    const growth = finalMemory - initialMemory;
+    const growthMB = growth / 1024 / 1024;
+
+    console.log(`Memory growth: ${growthMB.toFixed(2)} MB`);
+
+    // Should not grow significantly
+    expect(growthMB).toBeLessThan(10);
+  });
+});
+```
+
+## Level 5: Quality Review
+
+### Manual Testing Checklist
+
+```markdown
+## Manual Testing for {Skill Name}
+
+### Content Quality
+- [ ] Read through entire skill content
+- [ ] Verify all examples are accurate
+- [ ] Check for typos and grammar
+- [ ] Ensure logical flow and structure
+- [ ] Verify technical accuracy
+
+### Usability
+- [ ] Test with 5 different user queries
+- [ ] Verify skill triggers appropriately
+- [ ] Check that content is helpful to LLM
+- [ ] Ensure examples are clear
+- [ ] Validate recommended tools
+
+### Integration
+- [ ] Test in development environment
+- [ ] Verify works with OpenAI provider
+- [ ] Verify works with Anthropic provider
+- [ ] Check log messages are clear
+- [ ] Ensure no errors in production
+
+### Performance
+- [ ] Measure first load time
+- [ ] Measure cached load time
+- [ ] Check skill size is reasonable
+- [ ] Verify caching works
+- [ ] No performance regressions
+```
+
+### User Acceptance Testing
+
+```typescript
+// Script to run UAT with real users
+async function runUserAcceptanceTest() {
+  console.log('🧑‍🤝‍🧑 User Acceptance Testing\n');
+
+  const testQueries = [
+    'How do I work with workflow schemas?',
+    'Show me examples of using the API',
+    'Help me build an agent',
+    // Add more based on skill purpose
+  ];
+
+  console.log('Test these queries with the agent:');
+  testQueries.forEach((q, i) => {
+    console.log(`${i + 1}. ${q}`);
+  });
+
+  console.log('\nFor each query, verify:');
+  console.log('  - Skill loads correctly');
+  console.log('  - Response is helpful');
+  console.log('  - Information is accurate');
+  console.log('  - Examples are clear');
+  console.log('  - User is satisfied\n');
+}
+```
+
+## Continuous Improvement
+
+### Monitoring in Production
+
+```typescript
+// Add to skill-loader.ts for production monitoring
+private logSkillMetrics(skillName: string, loadTime: number, fromCache: boolean) {
+  this.logger.info('Skill loaded', {
+    skillName,
+    loadTime,
+    fromCache,
+    contentSize: this.cache.get(skillName)?.content.length || 0
+  });
+
+  // Send to monitoring service
+  if (process.env.NODE_ENV === 'production') {
+    metrics.timing('skill.load.time', loadTime, { skill: skillName, cached: fromCache });
+  }
+}
+```
+
+### Feedback Collection
+
+```typescript
+// Collect feedback on skill effectiveness
+interface SkillFeedback {
+  skillName: string;
+  userQuery: string;
+  wasHelpful: boolean;
+  comments?: string;
+  timestamp: number;
+}
+
+function logSkillFeedback(feedback: SkillFeedback) {
+  // Store feedback for analysis
+  console.log('Skill Feedback:', feedback);
+  // Send to analytics service
+}
+```
+
+## Summary
+
+Follow this testing workflow for every skill:
+
+1. **Unit Tests**: Structure and content validation
+2. **Integration Tests**: Works with skill system components
+3. **E2E Tests**: Works in full agent context
+4. **Performance Tests**: Meets load time and memory budgets
+5. **Manual Review**: Content quality and usability
+6. **UAT**: Real users validate effectiveness
+7. **Production Monitoring**: Track performance and issues
+8. **Continuous Improvement**: Iterate based on feedback