@hailer/mcp 0.0.5 → 0.1.0

package/.claude/skills/skill-testing/references/test-patterns.md

@@ -1,705 +0,0 @@
-# Reusable Test Patterns
-
-Copy-paste templates and patterns for common skill testing scenarios.
-
-## Quick Test Script Template
-
-Save as `test-{skill-name}.ts`:
-
-```typescript
-/**
- * Quick validation test for {skill-name}
- * Run: npx tsx test-{skill-name}.ts
- */
-
-import { SkillLoader } from './src/client/skill-loader';
-import { SkillManager } from './src/client/skill-manager';
-import * as path from 'path';
-import * as fs from 'fs/promises';
-
-const SKILL_NAME = 'my-skill';
-const TEST_QUERIES = [
-  'primary keyword test',
-  'secondary keyword test',
-  'edge case test',
-];
-
-async function testSkill() {
-  console.log(`\n🧪 Testing Skill: ${SKILL_NAME}\n`);
-
-  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
-  const skillPath = path.join(skillsPath, SKILL_NAME);
-  const loader = new SkillLoader(skillsPath);
-  const manager = new SkillManager(loader);
-
-  let passed = 0;
-  let failed = 0;
-
-  // Test 1: File Structure
-  console.log('📁 Test 1: File Structure');
-  try {
-    const skillMd = await fs.readFile(path.join(skillPath, 'SKILL.md'), 'utf-8');
-    console.log(`  ✅ SKILL.md exists (${skillMd.length} chars)`);
-    passed++;
-  } catch (error) {
-    console.log(`  ❌ SKILL.md not found`);
-    failed++;
-  }
-
-  // Test 2: Load Skill
-  console.log('\n📚 Test 2: Skill Loading');
-  try {
-    const start = Date.now();
-    const skill = await loader.load(SKILL_NAME);
-    const loadTime = Date.now() - start;
-
-    console.log(`  ✅ Loaded successfully in ${loadTime}ms`);
-    console.log(`  ✅ Content size: ${(skill.content.length / 1024).toFixed(1)} KB`);
-    console.log(`  ✅ Description: ${skill.description.substring(0, 60)}...`);
-    passed += 3;
-  } catch (error) {
-    console.log(`  ❌ Failed to load: ${error}`);
-    failed++;
-  }
-
-  // Test 3: Caching
-  console.log('\n⚡ Test 3: Caching');
-  try {
-    const start = Date.now();
-    await loader.load(SKILL_NAME);
-    const cacheTime = Date.now() - start;
-
-    if (cacheTime < 10) {
-      console.log(`  ✅ Cached load: ${cacheTime}ms (fast!)`);
-      passed++;
-    } else {
-      console.log(`  ⚠️  Cached load: ${cacheTime}ms (should be < 10ms)`);
-      failed++;
-    }
-  } catch (error) {
-    console.log(`  ❌ Cache test failed: ${error}`);
-    failed++;
-  }
-
-  // Test 4: Keyword Matching
-  console.log('\n🔍 Test 4: Keyword Matching');
-  for (const query of TEST_QUERIES) {
-    try {
-      const guidance = await manager.analyzeRequest(query);
-
-      if (guidance.skills.includes(SKILL_NAME)) {
-        console.log(`  ✅ "${query}" → Matched (${(guidance.confidence * 100).toFixed(0)}%)`);
-        passed++;
-      } else {
-        console.log(`  ❌ "${query}" → No match`);
-        failed++;
-      }
-    } catch (error) {
-      console.log(`  ❌ "${query}" → Error: ${error}`);
-      failed++;
-    }
-  }
-
-  // Test 5: Content Quality
-  console.log('\n📝 Test 5: Content Quality');
-  try {
-    const skill = await loader.load(SKILL_NAME);
-    const content = skill.content;
-
-    const checks = [
-      { name: 'Has headings', test: () => content.includes('#'), },
-      { name: 'Has examples', test: () => content.includes('```') },
-      { name: 'Not too short', test: () => content.length > 500 },
-      { name: 'Not too long', test: () => content.length < 100000 },
-      { name: 'Has structure', test: () => content.includes('##') },
-    ];
-
-    for (const check of checks) {
-      if (check.test()) {
-        console.log(`  ✅ ${check.name}`);
-        passed++;
-      } else {
-        console.log(`  ❌ ${check.name}`);
-        failed++;
-      }
-    }
-  } catch (error) {
-    console.log(`  ❌ Quality checks failed: ${error}`);
-    failed++;
-  }
-
-  // Summary
-  console.log('\n' + '='.repeat(50));
-  console.log(`Results: ${passed} passed, ${failed} failed`);
-  console.log('='.repeat(50) + '\n');
-
-  if (failed === 0) {
-    console.log(`✅ ${SKILL_NAME} is ready to use!\n`);
-    process.exit(0);
-  } else {
-    console.log(`❌ ${SKILL_NAME} needs fixes before deployment\n`);
-    process.exit(1);
-  }
-}
-
-testSkill().catch((error) => {
-  console.error('\n❌ Test crashed:', error);
-  process.exit(1);
-});
-```
-
-## Pattern: Test All Skills
-
-Save as `test-all-skills.ts`:
-
-```typescript
-/**
- * Test all skills in the system
- * Run: npx tsx test-all-skills.ts
- */
-
-import { SkillLoader } from './src/client/skill-loader';
-import { SkillManager } from './src/client/skill-manager';
-import * as path from 'path';
-
-interface SkillTestResult {
-  skillName: string;
-  passed: boolean;
-  loadTime: number;
-  cacheTime: number;
-  sizeKB: number;
-  errors: string[];
-}
-
-async function testSkill(
-  skillName: string,
-  loader: SkillLoader,
-  manager: SkillManager
-): Promise<SkillTestResult> {
-  const result: SkillTestResult = {
-    skillName,
-    passed: true,
-    loadTime: 0,
-    cacheTime: 0,
-    sizeKB: 0,
-    errors: [],
-  };
-
-  try {
-    // Test loading
-    loader.clearCache(skillName);
-    const start1 = Date.now();
-    const skill = await loader.load(skillName);
-    result.loadTime = Date.now() - start1;
-    result.sizeKB = skill.content.length / 1024;
-
-    // Test caching
-    const start2 = Date.now();
-    await loader.load(skillName);
-    result.cacheTime = Date.now() - start2;
-
-    // Validations
-    if (result.loadTime > 150) {
-      result.errors.push(`Load time too high: ${result.loadTime}ms`);
-      result.passed = false;
-    }
-
-    if (result.cacheTime > 10) {
-      result.errors.push(`Cache time too high: ${result.cacheTime}ms`);
-      result.passed = false;
-    }
-
-    if (result.sizeKB > 100) {
-      result.errors.push(`Size too large: ${result.sizeKB.toFixed(1)} KB`);
-      result.passed = false;
-    }
-
-    if (skill.content.length < 500) {
-      result.errors.push('Content too short');
-      result.passed = false;
-    }
-  } catch (error) {
-    result.passed = false;
-    result.errors.push(String(error));
-  }
-
-  return result;
-}
-
-async function testAllSkills() {
-  console.log('🧪 Testing All Skills\n');
-
-  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
-  const loader = new SkillLoader(skillsPath);
-  const manager = new SkillManager(loader);
-
-  const allSkills = await loader.getAvailableSkills();
-  console.log(`Found ${allSkills.length} skills to test\n`);
-
-  const results: SkillTestResult[] = [];
-
-  for (const skillName of allSkills) {
-    process.stdout.write(`Testing ${skillName}... `);
-    const result = await testSkill(skillName, loader, manager);
-    results.push(result);
-
-    if (result.passed) {
-      console.log('✅');
-    } else {
-      console.log('❌');
-      result.errors.forEach(err => console.log(`  - ${err}`));
-    }
-  }
-
-  // Summary
-  console.log('\n' + '='.repeat(70));
-  console.log('Summary:');
-  console.log('='.repeat(70));
-
-  const passed = results.filter(r => r.passed).length;
-  const failed = results.filter(r => !r.passed).length;
-
-  console.log(`\nSkills: ${passed} passed, ${failed} failed\n`);
-
-  console.log('Performance Metrics:');
-  console.log('────────────────────────────────────────────────────────────────────');
-  console.log('Skill Name'.padEnd(25) + 'Load'.padEnd(15) + 'Cache'.padEnd(15) + 'Size');
-  console.log('────────────────────────────────────────────────────────────────────');
-
-  for (const result of results) {
-    const status = result.passed ? '✅' : '❌';
-    const name = `${status} ${result.skillName}`.padEnd(25);
-    const load = `${result.loadTime}ms`.padEnd(15);
-    const cache = `${result.cacheTime}ms`.padEnd(15);
-    const size = `${result.sizeKB.toFixed(1)} KB`;
-
-    console.log(name + load + cache + size);
-  }
-
-  console.log('────────────────────────────────────────────────────────────────────\n');
-
-  // Exit with error if any failed
-  process.exit(failed > 0 ? 1 : 0);
-}
-
-testAllSkills().catch(console.error);
-```
-
-## Pattern: Keyword Matching Tests
-
-Save as `test-keywords.ts`:
-
-```typescript
-/**
- * Test keyword matching accuracy
- * Run: npx tsx test-keywords.ts
- */
-
-import { SkillManager } from './src/client/skill-manager';
-import { SkillLoader } from './src/client/skill-loader';
-import * as path from 'path';
-
-interface KeywordTest {
-  query: string;
-  expectedSkill: string;
-  minConfidence?: number;
-}
-
-const KEYWORD_TESTS: KeywordTest[] = [
-  // MCP Tools
-  { query: 'create a workflow', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
-  { query: 'list workflow phases', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
-  { query: 'update workflow field', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
-  { query: 'show me the schema', expectedSkill: 'mcp-tools', minConfidence: 0.5 },
-
-  // Hailer API
-  { query: 'list all activities', expectedSkill: 'hailer-api', minConfidence: 0.6 },
-  { query: 'filter activities by date', expectedSkill: 'hailer-api', minConfidence: 0.6 },
-  { query: 'search for tasks', expectedSkill: 'hailer-api', minConfidence: 0.5 },
-  { query: 'show activity details', expectedSkill: 'hailer-api', minConfidence: 0.5 },
-
-  // Agent Building
-  { query: 'how do I build an agent?', expectedSkill: 'agent-building', minConfidence: 0.5 },
-  { query: 'create skill system', expectedSkill: 'agent-building', minConfidence: 0.5 },
-  { query: 'multi-agent architecture', expectedSkill: 'agent-building', minConfidence: 0.5 },
-
-  // Skill Testing
-  { query: 'how do I test skills?', expectedSkill: 'skill-testing', minConfidence: 0.5 },
-  { query: 'validate skill quality', expectedSkill: 'skill-testing', minConfidence: 0.5 },
-  { query: 'skill testing patterns', expectedSkill: 'skill-testing', minConfidence: 0.6 },
-];
-
-async function testKeywordMatching() {
-  console.log('🔍 Testing Keyword Matching\n');
-
-  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
-  const loader = new SkillLoader(skillsPath);
-  const manager = new SkillManager(loader);
-
-  let passed = 0;
-  let failed = 0;
-
-  for (const test of KEYWORD_TESTS) {
-    process.stdout.write(`Testing: "${test.query.substring(0, 40)}"... `);
-
-    try {
-      const guidance = await manager.analyzeRequest(test.query);
-      const matchedSkill = guidance.skills[0];
-      const confidence = guidance.confidence;
-      const minConf = test.minConfidence || 0.5;
-
-      const skillMatches = matchedSkill === test.expectedSkill;
-      const confEnough = confidence >= minConf;
-
-      if (skillMatches && confEnough) {
-        console.log(`✅ ${test.expectedSkill} (${(confidence * 100).toFixed(0)}%)`);
-        passed++;
-      } else if (skillMatches && !confEnough) {
-        console.log(`⚠️  ${test.expectedSkill} but low confidence (${(confidence * 100).toFixed(0)}%)`);
-        failed++;
-      } else {
-        console.log(`❌ Got ${matchedSkill || 'none'}, expected ${test.expectedSkill}`);
-        failed++;
-      }
-    } catch (error) {
-      console.log(`❌ Error: ${error}`);
-      failed++;
-    }
-  }
-
-  // Summary
-  console.log('\n' + '='.repeat(50));
-  const total = passed + failed;
-  const accuracy = (passed / total * 100).toFixed(1);
-  console.log(`Results: ${passed}/${total} passed (${accuracy}% accuracy)`);
-  console.log('='.repeat(50) + '\n');
-
-  process.exit(failed > 0 ? 1 : 0);
-}
-
-testKeywordMatching().catch(console.error);
-```
-
-## Pattern: Performance Benchmarks
-
-Save as `benchmark-skills.ts`:
-
-```typescript
-/**
- * Benchmark skill system performance
- * Run: npx tsx benchmark-skills.ts
- */
-
-import { SkillLoader } from './src/client/skill-loader';
-import { SkillManager } from './src/client/skill-manager';
-import * as path from 'path';
-
-interface BenchmarkResult {
-  operation: string;
-  iterations: number;
-  totalTime: number;
-  avgTime: number;
-  minTime: number;
-  maxTime: number;
-}
-
-async function benchmark(
-  name: string,
-  fn: () => Promise<void>,
-  iterations: number
-): Promise<BenchmarkResult> {
-  const times: number[] = [];
-
-  // Warmup
-  await fn();
-
-  // Benchmark
-  for (let i = 0; i < iterations; i++) {
-    const start = performance.now();
-    await fn();
-    times.push(performance.now() - start);
-  }
-
-  const totalTime = times.reduce((a, b) => a + b, 0);
-  const avgTime = totalTime / iterations;
-  const minTime = Math.min(...times);
-  const maxTime = Math.max(...times);
-
-  return {
-    operation: name,
-    iterations,
-    totalTime,
-    avgTime,
-    minTime,
-    maxTime,
-  };
-}
-
-function printResult(result: BenchmarkResult) {
-  console.log(`\n${result.operation}`);
-  console.log('─'.repeat(50));
-  console.log(`Iterations: ${result.iterations}`);
-  console.log(`Total time: ${result.totalTime.toFixed(2)}ms`);
-  console.log(`Average:    ${result.avgTime.toFixed(2)}ms`);
-  console.log(`Min:        ${result.minTime.toFixed(2)}ms`);
-  console.log(`Max:        ${result.maxTime.toFixed(2)}ms`);
-}
-
-async function runBenchmarks() {
-  console.log('⚡ Skill System Performance Benchmarks\n');
-
-  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
-  const loader = new SkillLoader(skillsPath);
-  const manager = new SkillManager(loader);
-
-  // Benchmark 1: First load (uncached)
-  const firstLoadResult = await benchmark(
-    'First Load (from disk)',
-    async () => {
-      loader.clearCache('mcp-tools');
-      await loader.load('mcp-tools');
-    },
-    10
-  );
-  printResult(firstLoadResult);
-
-  // Benchmark 2: Cached load
-  await loader.load('mcp-tools'); // Ensure cached
-  const cachedLoadResult = await benchmark(
-    'Cached Load (from memory)',
-    async () => {
-      await loader.load('mcp-tools');
-    },
-    100
-  );
-  printResult(cachedLoadResult);
-
-  // Benchmark 3: Keyword analysis
-  const analysisResult = await benchmark(
-    'Keyword Analysis',
-    async () => {
-      await manager.analyzeRequest('create a workflow with phases');
-    },
-    100
-  );
-  printResult(analysisResult);
-
-  // Benchmark 4: Full flow (analysis + load)
-  const fullFlowResult = await benchmark(
-    'Full Flow (analysis + load)',
-    async () => {
-      const guidance = await manager.analyzeRequest('create workflow');
-      // Skill is already loaded during analysis
-    },
-    100
-  );
-  printResult(fullFlowResult);
-
-  // Summary
-  console.log('\n' + '='.repeat(50));
-  console.log('Performance Summary:');
-  console.log('='.repeat(50));
-  console.log(`First load:     ${firstLoadResult.avgTime.toFixed(2)}ms avg (target: < 100ms)`);
-  console.log(`Cached load:    ${cachedLoadResult.avgTime.toFixed(2)}ms avg (target: < 10ms)`);
-  console.log(`Analysis:       ${analysisResult.avgTime.toFixed(2)}ms avg (target: < 5ms)`);
-  console.log(`Full flow:      ${fullFlowResult.avgTime.toFixed(2)}ms avg (target: < 50ms)`);
-  console.log('='.repeat(50) + '\n');
-
-  // Check if targets are met
-  const targets = [
-    { name: 'First load', actual: firstLoadResult.avgTime, target: 100 },
-    { name: 'Cached load', actual: cachedLoadResult.avgTime, target: 10 },
-    { name: 'Analysis', actual: analysisResult.avgTime, target: 5 },
-    { name: 'Full flow', actual: fullFlowResult.avgTime, target: 50 },
-  ];
-
-  const allPassed = targets.every(t => t.actual <= t.target);
-
-  if (allPassed) {
-    console.log('✅ All performance targets met!\n');
-    process.exit(0);
-  } else {
-    console.log('❌ Some performance targets not met:\n');
-    targets.forEach(t => {
-      if (t.actual > t.target) {
-        console.log(`  ⚠️  ${t.name}: ${t.actual.toFixed(2)}ms (target: ${t.target}ms)`);
-      }
-    });
-    console.log();
-    process.exit(1);
-  }
-}
-
-runBenchmarks().catch(console.error);
-```
-
-## Pattern: Integration with Jest
-
-If using Jest for testing, here's the pattern:
-
-```typescript
-// skills.test.ts
-import { SkillLoader } from '../src/client/skill-loader';
-import { SkillManager } from '../src/client/skill-manager';
-import * as path from 'path';
-
-describe('Skill System', () => {
-  let loader: SkillLoader;
-  let manager: SkillManager;
-
-  beforeAll(() => {
-    const skillsPath = path.join(__dirname, '../../.claude/skills');
-    loader = new SkillLoader(skillsPath);
-    manager = new SkillManager(loader);
-  });
-
-  afterEach(() => {
-    loader.clearCache();
-  });
-
-  describe('SkillLoader', () => {
-    it('should load skill successfully', async () => {
-      const skill = await loader.load('mcp-tools');
-      expect(skill.name).toBe('mcp-tools');
-      expect(skill.content.length).toBeGreaterThan(0);
-    });
-
-    it('should cache skills', async () => {
-      await loader.load('mcp-tools');
-      const start = performance.now();
-      await loader.load('mcp-tools');
-      const time = performance.now() - start;
-      expect(time).toBeLessThan(10);
-    });
-
-    it('should handle errors', async () => {
-      await expect(loader.load('invalid-skill')).rejects.toThrow();
-    });
-  });
-
-  describe('SkillManager', () => {
-    it('should match keywords', async () => {
-      const guidance = await manager.analyzeRequest('create workflow');
-      expect(guidance.skills).toContain('mcp-tools');
-    });
-
-    it('should return default for non-matching queries', async () => {
-      const guidance = await manager.analyzeRequest('xyz123random');
-      expect(guidance.skills).toHaveLength(0);
-      expect(guidance.confidence).toBeLessThan(0.5);
-    });
-  });
-});
-```
-
-## Pattern: CI/CD Integration
-
-```yaml
-# .github/workflows/test-skills.yml
-name: Test Skills
-
-on:
-  push:
-    paths:
-      - '.claude/skills/**'
-      - 'server/src/client/skill-*.ts'
-  pull_request:
-    paths:
-      - '.claude/skills/**'
-      - 'server/src/client/skill-*.ts'
-
-jobs:
-  test-skills:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '18'
-
-      - name: Install dependencies
-        working-directory: ./server
-        run: npm install
-
-      - name: Run skill tests
-        working-directory: ./server
-        run: npx tsx test-all-skills.ts
-
-      - name: Run keyword tests
-        working-directory: ./server
-        run: npx tsx test-keywords.ts
-
-      - name: Run performance benchmarks
-        working-directory: ./server
-        run: npx tsx benchmark-skills.ts
-```
-
-## Pattern: Watch Mode for Development
-
-```typescript
-/**
- * Watch mode for skill development
- * Run: npx tsx watch-skills.ts
- *
- * Automatically re-tests skills when files change
- */
-
-import * as chokidar from 'chokidar';
-import { exec } from 'child_process';
-import * as path from 'path';
-
-const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
-
-console.log('👀 Watching skills directory for changes...\n');
-
-const watcher = chokidar.watch(skillsPath, {
-  ignored: /(^|[\/\\])\../, // ignore dotfiles
-  persistent: true,
-});
-
-let testTimeout: NodeJS.Timeout;
-
-function runTests() {
-  console.log('\n🧪 Running tests...\n');
-  exec('npx tsx test-all-skills.ts', (error, stdout, stderr) => {
-    if (stdout) console.log(stdout);
-    if (stderr) console.error(stderr);
-    console.log('\n👀 Watching for changes...\n');
-  });
-}
-
-watcher.on('change', (filePath) => {
-  console.log(`\n📝 Changed: ${path.relative(skillsPath, filePath)}`);
-
-  // Debounce: wait 500ms before running tests
-  clearTimeout(testTimeout);
-  testTimeout = setTimeout(runTests, 500);
-});
-
-watcher.on('add', (filePath) => {
-  console.log(`\n➕ Added: ${path.relative(skillsPath, filePath)}`);
-  clearTimeout(testTimeout);
-  testTimeout = setTimeout(runTests, 500);
-});
-
-// Initial test run
-runTests();
-```
-
-## Usage Tips
-
-1. **Start with the quick test script** - Fastest way to validate a new skill
-2. **Use test-all-skills** - Before committing changes to the skill system
-3. **Run keyword tests** - When adding new skills or modifying keyword mappings
-4. **Benchmark regularly** - Catch performance regressions early
-5. **Use watch mode** - During active skill development for instant feedback
-
-## Next Steps
-
-After using these patterns:
-1. Customize tests for your specific skills
-2. Add skill-specific validation logic
-3. Integrate with your CI/CD pipeline
-4. Set up monitoring in production
-5. Collect feedback and iterate