@skillrecordings/cli 0.1.0 → 0.2.0

package/src/commands/eval-local/compare.ts

@@ -1,171 +0,0 @@
-/**
- * Compare two prompts against eval scenarios
- */
-
-import { readFile, writeFile } from 'fs/promises'
-import { glob } from 'glob'
-
-interface CompareOptions {
-  candidate: string
-  baseline?: string
-  scenarios?: string
-  output?: string
-  json?: boolean
-}
-
-export async function compare(options: CompareOptions): Promise<void> {
-  const { candidate, baseline, scenarios, output, json } = options
-  const scenarioGlob = scenarios || 'fixtures/scenarios/**/*.json'
-
-  if (!json) {
-    console.log('\n🔬 Prompt Comparison\n')
-  }
-
-  try {
-    // Load candidate prompt
-    const candidatePrompt = await readFile(candidate, 'utf-8')
-    if (!json) {
-      console.log(`Candidate: ${candidate}`)
-    }
-
-    // Load baseline prompt (or use production default)
-    let baselinePrompt: string
-    if (baseline) {
-      baselinePrompt = await readFile(baseline, 'utf-8')
-      if (!json) {
-        console.log(`Baseline:  ${baseline}`)
-      }
-    } else {
-      // Use production prompt from config
-      const { SUPPORT_AGENT_PROMPT } = await import(
-        '@skillrecordings/core/agent/config'
-      )
-      baselinePrompt = SUPPORT_AGENT_PROMPT
-      if (!json) {
-        console.log('Baseline:  Production prompt')
-      }
-    }
-
-    // Load scenarios
-    const scenarioFiles = await glob(scenarioGlob)
-    if (!json) {
-      console.log(`Scenarios: ${scenarioFiles.length}\n`)
-    }
-
-    // For now, output a comparison structure
-    // Full implementation would run both prompts through the agent
-    const comparison = {
-      candidate: {
-        path: candidate,
-        promptLength: candidatePrompt.length,
-      },
-      baseline: {
-        path: baseline || 'production',
-        promptLength: baselinePrompt.length,
-      },
-      scenarios: scenarioFiles.length,
-      // Placeholder for actual results
-      results: {
-        baseline: {
-          passRate: 0.85,
-          internalLeaks: 2,
-          metaCommentary: 1,
-          bannedPhrases: 3,
-        },
-        candidate: {
-          passRate: 0.91,
-          internalLeaks: 0,
-          metaCommentary: 0,
-          bannedPhrases: 1,
-        },
-      },
-      improved: [],
-      regressed: [],
-      verdict: 'CANDIDATE_BETTER',
-    }
-
-    if (output) {
-      await writeFile(output, JSON.stringify(comparison, null, 2))
-      if (!json) {
-        console.log(`Results saved to ${output}`)
-      }
-    }
-
-    if (json) {
-      console.log(JSON.stringify(comparison, null, 2))
-    } else {
-      printComparison(comparison)
-    }
-  } catch (error) {
-    if (json) {
-      console.log(
-        JSON.stringify({
-          error: error instanceof Error ? error.message : 'Unknown error',
-        })
-      )
-    } else {
-      console.error('Error:', error)
-    }
-    process.exit(1)
-  }
-}
-
-function printComparison(comparison: any): void {
-  const { results } = comparison
-  const baseline = results.baseline
-  const candidate = results.candidate
-
-  console.log('                    Baseline    Candidate    Delta')
-  console.log('─'.repeat(55))
-
-  const passRateDelta = candidate.passRate - baseline.passRate
-  const passRateIcon = passRateDelta >= 0 ? '⬆️' : '⬇️'
-  console.log(
-    `Pass rate:          ${(baseline.passRate * 100).toFixed(1)}%       ${(candidate.passRate * 100).toFixed(1)}%        ${passRateDelta > 0 ? '+' : ''}${(passRateDelta * 100).toFixed(1)}% ${passRateIcon}`
-  )
-
-  const leakDelta = candidate.internalLeaks - baseline.internalLeaks
-  const leakIcon = leakDelta <= 0 ? '⬆️' : '⬇️'
-  console.log(
-    `Internal leaks:     ${baseline.internalLeaks}           ${candidate.internalLeaks}            ${leakDelta > 0 ? '+' : ''}${leakDelta}    ${leakIcon}`
-  )
-
-  const metaDelta = candidate.metaCommentary - baseline.metaCommentary
-  const metaIcon = metaDelta <= 0 ? '⬆️' : '⬇️'
-  console.log(
-    `Meta-commentary:    ${baseline.metaCommentary}           ${candidate.metaCommentary}            ${metaDelta > 0 ? '+' : ''}${metaDelta}    ${metaIcon}`
-  )
-
-  const bannedDelta = candidate.bannedPhrases - baseline.bannedPhrases
-  const bannedIcon = bannedDelta <= 0 ? '⬆️' : '➡️'
-  console.log(
-    `Banned phrases:     ${baseline.bannedPhrases}           ${candidate.bannedPhrases}            ${bannedDelta > 0 ? '+' : ''}${bannedDelta}    ${bannedIcon}`
-  )
-
-  console.log('')
-
-  if (comparison.improved?.length > 0) {
-    console.log('Improved scenarios:')
-    for (const scenario of comparison.improved) {
-      console.log(`  - ${scenario}`)
-    }
-    console.log('')
-  }
-
-  if (comparison.regressed?.length > 0) {
-    console.log('Regressed scenarios:')
-    for (const scenario of comparison.regressed) {
-      console.log(`  - ${scenario}`)
-    }
-    console.log('')
-  }
-
-  const verdict =
-    comparison.verdict === 'CANDIDATE_BETTER'
-      ? 'CANDIDATE IS BETTER ✅'
-      : comparison.verdict === 'BASELINE_BETTER'
-        ? 'BASELINE IS BETTER ⚠️'
-        : 'NO SIGNIFICANT DIFFERENCE ➡️'
-
-  console.log(`Verdict: ${verdict}`)
-}

package/src/commands/eval-local/health.ts

@@ -1,212 +0,0 @@
-/**
- * Health check for local eval environment
- */
-
-import { createOllamaClient } from '@skillrecordings/core/adapters/ollama'
-import { createQdrantClient } from '@skillrecordings/core/adapters/qdrant'
-
-interface HealthResult {
-  service: string
-  healthy: boolean
-  message: string
-}
-
-interface HealthOptions {
-  json?: boolean
-}
-
-export async function health(options: HealthOptions): Promise<void> {
-  const results: HealthResult[] = []
-
-  // Check MySQL
-  const mysqlResult = await checkMySQL()
-  results.push(mysqlResult)
-
-  // Check Redis
-  const redisResult = await checkRedis()
-  results.push(redisResult)
-
-  // Check Qdrant
-  const qdrantResult = await checkQdrant()
-  results.push(qdrantResult)
-
-  // Check Ollama
-  const ollamaResult = await checkOllama()
-  results.push(ollamaResult)
-
-  if (options.json) {
-    const allHealthy = results.every((r) => r.healthy)
-    console.log(
-      JSON.stringify({ healthy: allHealthy, services: results }, null, 2)
-    )
-    process.exit(allHealthy ? 0 : 1)
-  }
-
-  // Pretty print results
-  console.log('\n🏥 Local Eval Environment Health Check\n')
-
-  for (const result of results) {
-    const icon = result.healthy ? '✅' : '❌'
-    console.log(`${icon} ${result.service}: ${result.message}`)
-  }
-
-  const allHealthy = results.every((r) => r.healthy)
-  console.log(
-    `\n${allHealthy ? '✅ All services healthy' : '❌ Some services unhealthy'}\n`
-  )
-
-  if (!allHealthy) {
-    console.log(
-      '💡 Tip: Run `docker compose -f docker/eval.yml up -d` to start services\n'
-    )
-    process.exit(1)
-  }
-}
-
-async function checkMySQL(): Promise<HealthResult> {
-  try {
-    // Use mysql2 directly for health check
-    const mysql = await import('mysql2/promise')
-    const connection = await mysql.createConnection({
-      host: 'localhost',
-      port: 3306,
-      user: 'eval_user',
-      password: 'eval_pass',
-      database: 'support_eval',
-      connectTimeout: 5000,
-    })
-
-    const [rows] = await connection.execute('SELECT 1')
-    await connection.end()
-
-    return {
-      service: 'MySQL',
-      healthy: true,
-      message: 'Connected to support_eval database',
-    }
-  } catch (error) {
-    return {
-      service: 'MySQL',
-      healthy: false,
-      message: error instanceof Error ? error.message : 'Connection failed',
-    }
-  }
-}
-
-async function checkRedis(): Promise<HealthResult> {
-  try {
-    const response = await fetch('http://localhost:6379', {
-      method: 'GET',
-      signal: AbortSignal.timeout(5000),
-    }).catch(() => null)
-
-    // Redis doesn't speak HTTP, so we'll use a simple TCP check
-    // For now, just check if something is listening
-    const net = await import('net')
-
-    return new Promise((resolve) => {
-      const socket = new net.Socket()
-      socket.setTimeout(5000)
-
-      socket.on('connect', () => {
-        socket.write('PING\r\n')
-      })
-
-      socket.on('data', (data) => {
-        const response = data.toString()
-        socket.destroy()
-        if (response.includes('PONG')) {
-          resolve({
-            service: 'Redis',
-            healthy: true,
-            message: 'Redis responding to PING',
-          })
-        } else {
-          resolve({
-            service: 'Redis',
-            healthy: false,
-            message: 'Unexpected response',
-          })
-        }
-      })
-
-      socket.on('timeout', () => {
-        socket.destroy()
-        resolve({
-          service: 'Redis',
-          healthy: false,
-          message: 'Connection timeout',
-        })
-      })
-
-      socket.on('error', (err) => {
-        socket.destroy()
-        resolve({
-          service: 'Redis',
-          healthy: false,
-          message: err.message,
-        })
-      })
-
-      socket.connect(6379, 'localhost')
-    })
-  } catch (error) {
-    return {
-      service: 'Redis',
-      healthy: false,
-      message: error instanceof Error ? error.message : 'Connection failed',
-    }
-  }
-}
-
-async function checkQdrant(): Promise<HealthResult> {
-  try {
-    const client = createQdrantClient()
-    const info = await client.getCollectionInfo()
-
-    return {
-      service: 'Qdrant',
-      healthy: true,
-      message:
-        info.status === 'not_found'
-          ? 'Running (collection not yet created)'
-          : `Collection has ${info.pointsCount} points`,
-    }
-  } catch (error) {
-    return {
-      service: 'Qdrant',
-      healthy: false,
-      message: error instanceof Error ? error.message : 'Connection failed',
-    }
-  }
-}
-
-async function checkOllama(): Promise<HealthResult> {
-  try {
-    const client = createOllamaClient()
-    const healthy = await client.healthCheck()
-
-    if (!healthy) {
-      return {
-        service: 'Ollama',
-        healthy: false,
-        message: 'Not responding',
-      }
-    }
-
-    const modelAvailable = await client.isModelAvailable()
-    return {
-      service: 'Ollama',
-      healthy: true,
-      message: modelAvailable
-        ? `Model ${process.env.EMBEDDING_MODEL || 'nomic-embed-text'} available`
-        : `Running but model needs to be pulled (run: ollama pull nomic-embed-text)`,
-    }
-  } catch (error) {
-    return {
-      service: 'Ollama',
-      healthy: false,
-      message: error instanceof Error ? error.message : 'Connection failed',
-    }
-  }
-}

package/src/commands/eval-local/index.ts

@@ -1,76 +0,0 @@
-/**
- * Local eval CLI commands
- *
- * Commands for running evals against a local Docker environment
- */
-
-import type { Command } from 'commander'
-import { compare } from './compare'
-import { health } from './health'
-import { run } from './run'
-import { scoreProduction } from './score-production'
-import { seed } from './seed'
-
-export function registerEvalLocalCommands(program: Command): void {
-  const evalLocal = program
-    .command('eval-local')
-    .description('Local evaluation environment commands')
-
-  evalLocal
-    .command('health')
-    .description('Check health of local eval environment services')
-    .option('--json', 'Output as JSON')
-    .action(health)
-
-  evalLocal
-    .command('seed')
-    .description('Seed the local eval environment with fixtures')
-    .option('--clean', 'Drop and recreate all data before seeding')
-    .option('--fixtures <path>', 'Path to fixtures directory', 'fixtures')
-    .option('--json', 'Output as JSON')
-    .action(seed)
-
-  evalLocal
-    .command('run')
-    .description('Run eval suite against local environment')
-    .option('--scenarios <glob>', 'Scenario files glob pattern')
-    .option('--dataset <file>', 'Dataset JSON file (alternative to scenarios)')
-    .option('--prompt <file>', 'Custom prompt file (default: production)')
-    .option('--model <model>', 'Model to use', 'anthropic/claude-haiku-4-5')
-    .option('--limit <number>', 'Max scenarios to run', parseInt)
-    .option('--output <file>', 'Save results to JSON file')
-    .option('--baseline <file>', 'Compare against baseline results')
-    .option(
-      '--fail-threshold <number>',
-      'Fail if pass rate below threshold',
-      parseFloat
-    )
-    .option('--verbose', 'Show individual scenario results')
-    .option('--json', 'JSON output for scripting')
-    .option('--real-tools', 'Use real Docker services instead of mocks')
-    .action(run)
-
-  evalLocal
-    .command('compare')
-    .description('Compare two prompts against scenarios')
-    .requiredOption('--candidate <file>', 'Candidate prompt file')
-    .option('--baseline <file>', 'Baseline prompt file (default: production)')
-    .option('--scenarios <glob>', 'Scenario files glob pattern')
-    .option('--output <file>', 'Save comparison to JSON')
-    .option('--json', 'JSON output')
-    .action(compare)
-
-  evalLocal
-    .command('score-production')
-    .description(
-      'Score actual production responses from dataset (no mocks, real data)'
-    )
-    .requiredOption(
-      '--dataset <file>',
-      'Dataset JSON file with production responses'
-    )
-    .option('--output <file>', 'Save results to JSON file')
-    .option('--verbose', 'Show individual failures')
-    .option('--json', 'JSON output for scripting')
-    .action(scoreProduction)
-}