@hailer/mcp 0.0.1

package/.claude/skills/skill-testing/references/test-patterns.md

@@ -0,0 +1,705 @@
+# Reusable Test Patterns
+
+Copy-paste templates and patterns for common skill testing scenarios.
+
+## Quick Test Script Template
+
+Save as `test-{skill-name}.ts`:
+
+```typescript
+/**
+ * Quick validation test for {skill-name}
+ * Run: npx tsx test-{skill-name}.ts
+ */
+
+import { SkillLoader } from './src/client/skill-loader';
+import { SkillManager } from './src/client/skill-manager';
+import * as path from 'path';
+import * as fs from 'fs/promises';
+
+const SKILL_NAME = 'my-skill';
+const TEST_QUERIES = [
+  'primary keyword test',
+  'secondary keyword test',
+  'edge case test',
+];
+
+async function testSkill() {
+  console.log(`\n🧪 Testing Skill: ${SKILL_NAME}\n`);
+
+  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
+  const skillPath = path.join(skillsPath, SKILL_NAME);
+  const loader = new SkillLoader(skillsPath);
+  const manager = new SkillManager(loader);
+
+  let passed = 0;
+  let failed = 0;
+
+  // Test 1: File Structure
+  console.log('📁 Test 1: File Structure');
+  try {
+    const skillMd = await fs.readFile(path.join(skillPath, 'SKILL.md'), 'utf-8');
+    console.log(`  ✅ SKILL.md exists (${skillMd.length} chars)`);
+    passed++;
+  } catch (error) {
+    console.log(`  ❌ SKILL.md not found`);
+    failed++;
+  }
+
+  // Test 2: Load Skill
+  console.log('\n📚 Test 2: Skill Loading');
+  try {
+    const start = Date.now();
+    const skill = await loader.load(SKILL_NAME);
+    const loadTime = Date.now() - start;
+
+    console.log(`  ✅ Loaded successfully in ${loadTime}ms`);
+    console.log(`  ✅ Content size: ${(skill.content.length / 1024).toFixed(1)} KB`);
+    console.log(`  ✅ Description: ${skill.description.substring(0, 60)}...`);
+    passed += 3;
+  } catch (error) {
+    console.log(`  ❌ Failed to load: ${error}`);
+    failed++;
+  }
+
+  // Test 3: Caching
+  console.log('\n⚡ Test 3: Caching');
+  try {
+    const start = Date.now();
+    await loader.load(SKILL_NAME);
+    const cacheTime = Date.now() - start;
+
+    if (cacheTime < 10) {
+      console.log(`  ✅ Cached load: ${cacheTime}ms (fast!)`);
+      passed++;
+    } else {
+      console.log(`  ⚠️  Cached load: ${cacheTime}ms (should be < 10ms)`);
+      failed++;
+    }
+  } catch (error) {
+    console.log(`  ❌ Cache test failed: ${error}`);
+    failed++;
+  }
+
+  // Test 4: Keyword Matching
+  console.log('\n🔍 Test 4: Keyword Matching');
+  for (const query of TEST_QUERIES) {
+    try {
+      const guidance = await manager.analyzeRequest(query);
+
+      if (guidance.skills.includes(SKILL_NAME)) {
+        console.log(`  ✅ "${query}" → Matched (${(guidance.confidence * 100).toFixed(0)}%)`);
+        passed++;
+      } else {
+        console.log(`  ❌ "${query}" → No match`);
+        failed++;
+      }
+    } catch (error) {
+      console.log(`  ❌ "${query}" → Error: ${error}`);
+      failed++;
+    }
+  }
+
+  // Test 5: Content Quality
+  console.log('\n📝 Test 5: Content Quality');
+  try {
+    const skill = await loader.load(SKILL_NAME);
+    const content = skill.content;
+
+    const checks = [
+      { name: 'Has headings', test: () => content.includes('#'), },
+      { name: 'Has examples', test: () => content.includes('```') },
+      { name: 'Not too short', test: () => content.length > 500 },
+      { name: 'Not too long', test: () => content.length < 100000 },
+      { name: 'Has structure', test: () => content.includes('##') },
+    ];
+
+    for (const check of checks) {
+      if (check.test()) {
+        console.log(`  ✅ ${check.name}`);
+        passed++;
+      } else {
+        console.log(`  ❌ ${check.name}`);
+        failed++;
+      }
+    }
+  } catch (error) {
+    console.log(`  ❌ Quality checks failed: ${error}`);
+    failed++;
+  }
+
+  // Summary
+  console.log('\n' + '='.repeat(50));
+  console.log(`Results: ${passed} passed, ${failed} failed`);
+  console.log('='.repeat(50) + '\n');
+
+  if (failed === 0) {
+    console.log(`✅ ${SKILL_NAME} is ready to use!\n`);
+    process.exit(0);
+  } else {
+    console.log(`❌ ${SKILL_NAME} needs fixes before deployment\n`);
+    process.exit(1);
+  }
+}
+
+testSkill().catch((error) => {
+  console.error('\n❌ Test crashed:', error);
+  process.exit(1);
+});
+```
+
+## Pattern: Test All Skills
+
+Save as `test-all-skills.ts`:
+
+```typescript
+/**
+ * Test all skills in the system
+ * Run: npx tsx test-all-skills.ts
+ */
+
+import { SkillLoader } from './src/client/skill-loader';
+import { SkillManager } from './src/client/skill-manager';
+import * as path from 'path';
+
+interface SkillTestResult {
+  skillName: string;
+  passed: boolean;
+  loadTime: number;
+  cacheTime: number;
+  sizeKB: number;
+  errors: string[];
+}
+
+async function testSkill(
+  skillName: string,
+  loader: SkillLoader,
+  manager: SkillManager
+): Promise<SkillTestResult> {
+  const result: SkillTestResult = {
+    skillName,
+    passed: true,
+    loadTime: 0,
+    cacheTime: 0,
+    sizeKB: 0,
+    errors: [],
+  };
+
+  try {
+    // Test loading
+    loader.clearCache(skillName);
+    const start1 = Date.now();
+    const skill = await loader.load(skillName);
+    result.loadTime = Date.now() - start1;
+    result.sizeKB = skill.content.length / 1024;
+
+    // Test caching
+    const start2 = Date.now();
+    await loader.load(skillName);
+    result.cacheTime = Date.now() - start2;
+
+    // Validations
+    if (result.loadTime > 150) {
+      result.errors.push(`Load time too high: ${result.loadTime}ms`);
+      result.passed = false;
+    }
+
+    if (result.cacheTime > 10) {
+      result.errors.push(`Cache time too high: ${result.cacheTime}ms`);
+      result.passed = false;
+    }
+
+    if (result.sizeKB > 100) {
+      result.errors.push(`Size too large: ${result.sizeKB.toFixed(1)} KB`);
+      result.passed = false;
+    }
+
+    if (skill.content.length < 500) {
+      result.errors.push('Content too short');
+      result.passed = false;
+    }
+  } catch (error) {
+    result.passed = false;
+    result.errors.push(String(error));
+  }
+
+  return result;
+}
+
+async function testAllSkills() {
+  console.log('🧪 Testing All Skills\n');
+
+  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
+  const loader = new SkillLoader(skillsPath);
+  const manager = new SkillManager(loader);
+
+  const allSkills = await loader.getAvailableSkills();
+  console.log(`Found ${allSkills.length} skills to test\n`);
+
+  const results: SkillTestResult[] = [];
+
+  for (const skillName of allSkills) {
+    process.stdout.write(`Testing ${skillName}... `);
+    const result = await testSkill(skillName, loader, manager);
+    results.push(result);
+
+    if (result.passed) {
+      console.log('✅');
+    } else {
+      console.log('❌');
+      result.errors.forEach(err => console.log(`  - ${err}`));
+    }
+  }
+
+  // Summary
+  console.log('\n' + '='.repeat(70));
+  console.log('Summary:');
+  console.log('='.repeat(70));
+
+  const passed = results.filter(r => r.passed).length;
+  const failed = results.filter(r => !r.passed).length;
+
+  console.log(`\nSkills: ${passed} passed, ${failed} failed\n`);
+
+  console.log('Performance Metrics:');
+  console.log('────────────────────────────────────────────────────────────────────');
+  console.log('Skill Name'.padEnd(25) + 'Load'.padEnd(15) + 'Cache'.padEnd(15) + 'Size');
+  console.log('────────────────────────────────────────────────────────────────────');
+
+  for (const result of results) {
+    const status = result.passed ? '✅' : '❌';
+    const name = `${status} ${result.skillName}`.padEnd(25);
+    const load = `${result.loadTime}ms`.padEnd(15);
+    const cache = `${result.cacheTime}ms`.padEnd(15);
+    const size = `${result.sizeKB.toFixed(1)} KB`;
+
+    console.log(name + load + cache + size);
+  }
+
+  console.log('────────────────────────────────────────────────────────────────────\n');
+
+  // Exit with error if any failed
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+testAllSkills().catch(console.error);
+```
+
+## Pattern: Keyword Matching Tests
+
+Save as `test-keywords.ts`:
+
+```typescript
+/**
+ * Test keyword matching accuracy
+ * Run: npx tsx test-keywords.ts
+ */
+
+import { SkillManager } from './src/client/skill-manager';
+import { SkillLoader } from './src/client/skill-loader';
+import * as path from 'path';
+
+interface KeywordTest {
+  query: string;
+  expectedSkill: string;
+  minConfidence?: number;
+}
+
+const KEYWORD_TESTS: KeywordTest[] = [
+  // MCP Tools
+  { query: 'create a workflow', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
+  { query: 'list workflow phases', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
+  { query: 'update workflow field', expectedSkill: 'mcp-tools', minConfidence: 0.6 },
+  { query: 'show me the schema', expectedSkill: 'mcp-tools', minConfidence: 0.5 },
+
+  // Hailer API
+  { query: 'list all activities', expectedSkill: 'hailer-api', minConfidence: 0.6 },
+  { query: 'filter activities by date', expectedSkill: 'hailer-api', minConfidence: 0.6 },
+  { query: 'search for tasks', expectedSkill: 'hailer-api', minConfidence: 0.5 },
+  { query: 'show activity details', expectedSkill: 'hailer-api', minConfidence: 0.5 },
+
+  // Agent Building
+  { query: 'how do I build an agent?', expectedSkill: 'agent-building', minConfidence: 0.5 },
+  { query: 'create skill system', expectedSkill: 'agent-building', minConfidence: 0.5 },
+  { query: 'multi-agent architecture', expectedSkill: 'agent-building', minConfidence: 0.5 },
+
+  // Skill Testing
+  { query: 'how do I test skills?', expectedSkill: 'skill-testing', minConfidence: 0.5 },
+  { query: 'validate skill quality', expectedSkill: 'skill-testing', minConfidence: 0.5 },
+  { query: 'skill testing patterns', expectedSkill: 'skill-testing', minConfidence: 0.6 },
+];
+
+async function testKeywordMatching() {
+  console.log('🔍 Testing Keyword Matching\n');
+
+  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
+  const loader = new SkillLoader(skillsPath);
+  const manager = new SkillManager(loader);
+
+  let passed = 0;
+  let failed = 0;
+
+  for (const test of KEYWORD_TESTS) {
+    process.stdout.write(`Testing: "${test.query.substring(0, 40)}"... `);
+
+    try {
+      const guidance = await manager.analyzeRequest(test.query);
+      const matchedSkill = guidance.skills[0];
+      const confidence = guidance.confidence;
+      const minConf = test.minConfidence || 0.5;
+
+      const skillMatches = matchedSkill === test.expectedSkill;
+      const confEnough = confidence >= minConf;
+
+      if (skillMatches && confEnough) {
+        console.log(`✅ ${test.expectedSkill} (${(confidence * 100).toFixed(0)}%)`);
+        passed++;
+      } else if (skillMatches && !confEnough) {
+        console.log(`⚠️  ${test.expectedSkill} but low confidence (${(confidence * 100).toFixed(0)}%)`);
+        failed++;
+      } else {
+        console.log(`❌ Got ${matchedSkill || 'none'}, expected ${test.expectedSkill}`);
+        failed++;
+      }
+    } catch (error) {
+      console.log(`❌ Error: ${error}`);
+      failed++;
+    }
+  }
+
+  // Summary
+  console.log('\n' + '='.repeat(50));
+  const total = passed + failed;
+  const accuracy = (passed / total * 100).toFixed(1);
+  console.log(`Results: ${passed}/${total} passed (${accuracy}% accuracy)`);
+  console.log('='.repeat(50) + '\n');
+
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+testKeywordMatching().catch(console.error);
+```
+
+## Pattern: Performance Benchmarks
+
+Save as `benchmark-skills.ts`:
+
+```typescript
+/**
+ * Benchmark skill system performance
+ * Run: npx tsx benchmark-skills.ts
+ */
+
+import { SkillLoader } from './src/client/skill-loader';
+import { SkillManager } from './src/client/skill-manager';
+import * as path from 'path';
+
+interface BenchmarkResult {
+  operation: string;
+  iterations: number;
+  totalTime: number;
+  avgTime: number;
+  minTime: number;
+  maxTime: number;
+}
+
+async function benchmark(
+  name: string,
+  fn: () => Promise<void>,
+  iterations: number
+): Promise<BenchmarkResult> {
+  const times: number[] = [];
+
+  // Warmup
+  await fn();
+
+  // Benchmark
+  for (let i = 0; i < iterations; i++) {
+    const start = performance.now();
+    await fn();
+    times.push(performance.now() - start);
+  }
+
+  const totalTime = times.reduce((a, b) => a + b, 0);
+  const avgTime = totalTime / iterations;
+  const minTime = Math.min(...times);
+  const maxTime = Math.max(...times);
+
+  return {
+    operation: name,
+    iterations,
+    totalTime,
+    avgTime,
+    minTime,
+    maxTime,
+  };
+}
+
+function printResult(result: BenchmarkResult) {
+  console.log(`\n${result.operation}`);
+  console.log('─'.repeat(50));
+  console.log(`Iterations: ${result.iterations}`);
+  console.log(`Total time: ${result.totalTime.toFixed(2)}ms`);
+  console.log(`Average:    ${result.avgTime.toFixed(2)}ms`);
+  console.log(`Min:        ${result.minTime.toFixed(2)}ms`);
+  console.log(`Max:        ${result.maxTime.toFixed(2)}ms`);
+}
+
+async function runBenchmarks() {
+  console.log('⚡ Skill System Performance Benchmarks\n');
+
+  const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
+  const loader = new SkillLoader(skillsPath);
+  const manager = new SkillManager(loader);
+
+  // Benchmark 1: First load (uncached)
+  const firstLoadResult = await benchmark(
+    'First Load (from disk)',
+    async () => {
+      loader.clearCache('mcp-tools');
+      await loader.load('mcp-tools');
+    },
+    10
+  );
+  printResult(firstLoadResult);
+
+  // Benchmark 2: Cached load
+  await loader.load('mcp-tools'); // Ensure cached
+  const cachedLoadResult = await benchmark(
+    'Cached Load (from memory)',
+    async () => {
+      await loader.load('mcp-tools');
+    },
+    100
+  );
+  printResult(cachedLoadResult);
+
+  // Benchmark 3: Keyword analysis
+  const analysisResult = await benchmark(
+    'Keyword Analysis',
+    async () => {
+      await manager.analyzeRequest('create a workflow with phases');
+    },
+    100
+  );
+  printResult(analysisResult);
+
+  // Benchmark 4: Full flow (analysis + load)
+  const fullFlowResult = await benchmark(
+    'Full Flow (analysis + load)',
+    async () => {
+      const guidance = await manager.analyzeRequest('create workflow');
+      // Skill is already loaded during analysis
+    },
+    100
+  );
+  printResult(fullFlowResult);
+
+  // Summary
+  console.log('\n' + '='.repeat(50));
+  console.log('Performance Summary:');
+  console.log('='.repeat(50));
+  console.log(`First load:     ${firstLoadResult.avgTime.toFixed(2)}ms avg (target: < 100ms)`);
+  console.log(`Cached load:    ${cachedLoadResult.avgTime.toFixed(2)}ms avg (target: < 10ms)`);
+  console.log(`Analysis:       ${analysisResult.avgTime.toFixed(2)}ms avg (target: < 5ms)`);
+  console.log(`Full flow:      ${fullFlowResult.avgTime.toFixed(2)}ms avg (target: < 50ms)`);
+  console.log('='.repeat(50) + '\n');
+
+  // Check if targets are met
+  const targets = [
+    { name: 'First load', actual: firstLoadResult.avgTime, target: 100 },
+    { name: 'Cached load', actual: cachedLoadResult.avgTime, target: 10 },
+    { name: 'Analysis', actual: analysisResult.avgTime, target: 5 },
+    { name: 'Full flow', actual: fullFlowResult.avgTime, target: 50 },
+  ];
+
+  const allPassed = targets.every(t => t.actual <= t.target);
+
+  if (allPassed) {
+    console.log('✅ All performance targets met!\n');
+    process.exit(0);
+  } else {
+    console.log('❌ Some performance targets not met:\n');
+    targets.forEach(t => {
+      if (t.actual > t.target) {
+        console.log(`  ⚠️  ${t.name}: ${t.actual.toFixed(2)}ms (target: ${t.target}ms)`);
+      }
+    });
+    console.log();
+    process.exit(1);
+  }
+}
+
+runBenchmarks().catch(console.error);
+```
+
+## Pattern: Integration with Jest
+
+If using Jest for testing, here's the pattern:
+
+```typescript
+// skills.test.ts
+import { SkillLoader } from '../src/client/skill-loader';
+import { SkillManager } from '../src/client/skill-manager';
+import * as path from 'path';
+
+describe('Skill System', () => {
+  let loader: SkillLoader;
+  let manager: SkillManager;
+
+  beforeAll(() => {
+    const skillsPath = path.join(__dirname, '../../.claude/skills');
+    loader = new SkillLoader(skillsPath);
+    manager = new SkillManager(loader);
+  });
+
+  afterEach(() => {
+    loader.clearCache();
+  });
+
+  describe('SkillLoader', () => {
+    it('should load skill successfully', async () => {
+      const skill = await loader.load('mcp-tools');
+      expect(skill.name).toBe('mcp-tools');
+      expect(skill.content.length).toBeGreaterThan(0);
+    });
+
+    it('should cache skills', async () => {
+      await loader.load('mcp-tools');
+      const start = performance.now();
+      await loader.load('mcp-tools');
+      const time = performance.now() - start;
+      expect(time).toBeLessThan(10);
+    });
+
+    it('should handle errors', async () => {
+      await expect(loader.load('invalid-skill')).rejects.toThrow();
+    });
+  });
+
+  describe('SkillManager', () => {
+    it('should match keywords', async () => {
+      const guidance = await manager.analyzeRequest('create workflow');
+      expect(guidance.skills).toContain('mcp-tools');
+    });
+
+    it('should return default for non-matching queries', async () => {
+      const guidance = await manager.analyzeRequest('xyz123random');
+      expect(guidance.skills).toHaveLength(0);
+      expect(guidance.confidence).toBeLessThan(0.5);
+    });
+  });
+});
+```
+
+## Pattern: CI/CD Integration
+
+```yaml
+# .github/workflows/test-skills.yml
+name: Test Skills
+
+on:
+  push:
+    paths:
+      - '.claude/skills/**'
+      - 'server/src/client/skill-*.ts'
+  pull_request:
+    paths:
+      - '.claude/skills/**'
+      - 'server/src/client/skill-*.ts'
+
+jobs:
+  test-skills:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+
+      - name: Install dependencies
+        working-directory: ./server
+        run: npm install
+
+      - name: Run skill tests
+        working-directory: ./server
+        run: npx tsx test-all-skills.ts
+
+      - name: Run keyword tests
+        working-directory: ./server
+        run: npx tsx test-keywords.ts
+
+      - name: Run performance benchmarks
+        working-directory: ./server
+        run: npx tsx benchmark-skills.ts
+```
+
+## Pattern: Watch Mode for Development
+
+```typescript
+/**
+ * Watch mode for skill development
+ * Run: npx tsx watch-skills.ts
+ *
+ * Automatically re-tests skills when files change
+ */
+
+import * as chokidar from 'chokidar';
+import { exec } from 'child_process';
+import * as path from 'path';
+
+const skillsPath = path.join(process.cwd(), '..', '.claude', 'skills');
+
+console.log('👀 Watching skills directory for changes...\n');
+
+const watcher = chokidar.watch(skillsPath, {
+  ignored: /(^|[\/\\])\../, // ignore dotfiles
+  persistent: true,
+});
+
+let testTimeout: NodeJS.Timeout;
+
+function runTests() {
+  console.log('\n🧪 Running tests...\n');
+  exec('npx tsx test-all-skills.ts', (error, stdout, stderr) => {
+    if (stdout) console.log(stdout);
+    if (stderr) console.error(stderr);
+    console.log('\n👀 Watching for changes...\n');
+  });
+}
+
+watcher.on('change', (filePath) => {
+  console.log(`\n📝 Changed: ${path.relative(skillsPath, filePath)}`);
+
+  // Debounce: wait 500ms before running tests
+  clearTimeout(testTimeout);
+  testTimeout = setTimeout(runTests, 500);
+});
+
+watcher.on('add', (filePath) => {
+  console.log(`\n➕ Added: ${path.relative(skillsPath, filePath)}`);
+  clearTimeout(testTimeout);
+  testTimeout = setTimeout(runTests, 500);
+});
+
+// Initial test run
+runTests();
+```
+
+## Usage Tips
+
+1. **Start with the quick test script** - Fastest way to validate a new skill
+2. **Use test-all-skills** - Before committing changes to the skill system
+3. **Run keyword tests** - When adding new skills or modifying keyword mappings
+4. **Benchmark regularly** - Catch performance regressions early
+5. **Use watch mode** - During active skill development for instant feedback
+
+## Next Steps
+
+After using these patterns:
+1. Customize tests for your specific skills
+2. Add skill-specific validation logic
+3. Integrate with your CI/CD pipeline
+4. Set up monitoring in production
+5. Collect feedback and iterate