@skillrecordings/cli 0.1.0

package/src/commands/eval.test.ts

@@ -0,0 +1,253 @@
+import * as fs from 'node:fs/promises'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { runEval } from './eval'
+
+// Mock process.exit to prevent test termination
+const mockExit = vi.spyOn(process, 'exit').mockImplementation((code) => {
+  throw new Error(`process.exit(${code})`)
+})
+
+// Mock fs module
+vi.mock('node:fs/promises')
+
+// Mock core evals module
+vi.mock('@skillrecordings/core/evals/routing', () => ({
+  evalRouting: vi.fn(),
+}))
+
+describe('eval command', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockExit.mockClear()
+  })
+
+  afterEach(() => {
+    mockExit.mockClear()
+  })
+
+  it('should require dataset path', async () => {
+    const consoleErrorSpy = vi.spyOn(console, 'error')
+
+    await expect(runEval('routing', undefined)).rejects.toThrow(
+      'process.exit(1)'
+    )
+
+    expect(consoleErrorSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dataset path is required')
+    )
+  })
+
+  it('should fail if dataset file does not exist', async () => {
+    const consoleErrorSpy = vi.spyOn(console, 'error')
+    vi.mocked(fs.access).mockRejectedValue(new Error('File not found'))
+
+    await expect(
+      runEval('routing', '/path/to/nonexistent.json')
+    ).rejects.toThrow('process.exit(1)')
+
+    expect(consoleErrorSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dataset file not found')
+    )
+  })
+
+  it('should fail if dataset is invalid JSON', async () => {
+    const consoleErrorSpy = vi.spyOn(console, 'error')
+    vi.mocked(fs.access).mockResolvedValue(undefined)
+    vi.mocked(fs.readFile).mockResolvedValue('invalid json')
+
+    await expect(runEval('routing', '/path/to/invalid.json')).rejects.toThrow(
+      'process.exit(1)'
+    )
+
+    expect(consoleErrorSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Invalid JSON')
+    )
+  })
+
+  it('should print pretty results table by default', async () => {
+    const consoleSpy = vi.spyOn(console, 'log')
+    const mockDataset = [
+      {
+        message: 'Test message',
+        expectedCategory: 'needs_response',
+        expectedRoute: 'classifier' as const,
+      },
+    ]
+    const mockReport = {
+      precision: 0.92,
+      recall: 0.95,
+      fpRate: 0.03,
+      fnRate: 0.02,
+      byCategory: {
+        needs_response: {
+          tp: 10,
+          fp: 1,
+          fn: 1,
+          tn: 5,
+          precision: 0.95,
+          recall: 0.93,
+          f1: 0.94,
+          count: 17,
+        },
+      },
+      cost: {
+        tokens: 5000,
+        estimatedUsd: 0.00125,
+      },
+      latency: {
+        p50: 120,
+        p95: 250,
+        p99: 350,
+      },
+      passed: true,
+    }
+
+    vi.mocked(fs.access).mockResolvedValue(undefined)
+    vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(mockDataset))
+
+    const { evalRouting } = await import('@skillrecordings/core/evals/routing')
+    vi.mocked(evalRouting).mockResolvedValue(mockReport)
+
+    await expect(runEval('routing', '/path/to/dataset.json')).rejects.toThrow(
+      'process.exit(0)'
+    )
+
+    const output = consoleSpy.mock.calls.flat().join('\n')
+    expect(output).toContain('Precision')
+    expect(output).toContain('92.0%')
+    expect(output).toContain('Recall')
+    expect(output).toContain('95.0%')
+    expect(output).toContain('Latency')
+    expect(output).toContain('120ms')
+  })
+
+  it('should output JSON when --json flag is used', async () => {
+    const consoleSpy = vi.spyOn(console, 'log')
+    const mockDataset = [
+      {
+        message: 'Test message',
+        expectedCategory: 'needs_response',
+        expectedRoute: 'classifier' as const,
+      },
+    ]
+    const mockReport = {
+      precision: 0.92,
+      recall: 0.95,
+      fpRate: 0.03,
+      fnRate: 0.02,
+      byCategory: {},
+      cost: {
+        tokens: 5000,
+        estimatedUsd: 0.00125,
+      },
+      latency: {
+        p50: 120,
+        p95: 250,
+        p99: 350,
+      },
+      passed: true,
+    }
+
+    vi.mocked(fs.access).mockResolvedValue(undefined)
+    vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(mockDataset))
+
+    const { evalRouting } = await import('@skillrecordings/core/evals/routing')
+    vi.mocked(evalRouting).mockResolvedValue(mockReport)
+
+    await expect(
+      runEval('routing', '/path/to/dataset.json', { json: true })
+    ).rejects.toThrow('process.exit(0)')
+
+    const output = consoleSpy.mock.calls.flat().join('\n')
+    const parsed = JSON.parse(output)
+
+    expect(parsed.precision).toBe(0.92)
+    expect(parsed.recall).toBe(0.95)
+    expect(parsed.passed).toBe(true)
+  })
+
+  it('should exit with code 1 when gates fail', async () => {
+    const mockDataset = [
+      {
+        message: 'Test message',
+        expectedCategory: 'needs_response',
+        expectedRoute: 'classifier' as const,
+      },
+    ]
+    const mockReport = {
+      precision: 0.85, // Below threshold
+      recall: 0.88, // Below threshold
+      fpRate: 0.05,
+      fnRate: 0.04,
+      byCategory: {},
+      cost: {
+        tokens: 5000,
+        estimatedUsd: 0.00125,
+      },
+      latency: {
+        p50: 120,
+        p95: 250,
+        p99: 350,
+      },
+      passed: false,
+    }
+
+    vi.mocked(fs.access).mockResolvedValue(undefined)
+    vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(mockDataset))
+
+    const { evalRouting } = await import('@skillrecordings/core/evals/routing')
+    vi.mocked(evalRouting).mockResolvedValue(mockReport)
+
+    await expect(
+      runEval('routing', '/path/to/dataset.json', {
+        gates: { minPrecision: 0.92, minRecall: 0.95 },
+      })
+    ).rejects.toThrow('process.exit(1)')
+  })
+
+  it('should accept custom gates', async () => {
+    const mockDataset = [
+      {
+        message: 'Test message',
+        expectedCategory: 'needs_response',
+        expectedRoute: 'classifier' as const,
+      },
+    ]
+    const mockReport = {
+      precision: 0.92,
+      recall: 0.95,
+      fpRate: 0.03,
+      fnRate: 0.02,
+      byCategory: {},
+      cost: {
+        tokens: 5000,
+        estimatedUsd: 0.00125,
+      },
+      latency: {
+        p50: 120,
+        p95: 250,
+        p99: 350,
+      },
+      passed: true,
+    }
+
+    vi.mocked(fs.access).mockResolvedValue(undefined)
+    vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify(mockDataset))
+
+    const { evalRouting } = await import('@skillrecordings/core/evals/routing')
+    vi.mocked(evalRouting).mockResolvedValue(mockReport)
+
+    // Test with custom gates
+    await expect(
+      runEval('routing', '/path/to/dataset.json', {
+        gates: { minPrecision: 0.9, minRecall: 0.93, maxFpRate: 0.05 },
+      })
+    ).rejects.toThrow('process.exit(0)')
+
+    expect(evalRouting).toHaveBeenCalledWith(mockDataset, {
+      minPrecision: 0.9,
+      minRecall: 0.93,
+      maxFpRate: 0.05,
+    })
+  })
+})

package/src/commands/eval.ts

@@ -0,0 +1,108 @@
+import { access, readFile } from 'node:fs/promises'
+import type {
+  EvalDatapoint,
+  EvalGates,
+  EvalReport,
+} from '@skillrecordings/core/evals/routing'
+
+export interface EvalOptions {
+  json?: boolean
+  gates?: EvalGates
+}
+
+/**
+ * Run evals against a dataset
+ *
+ * Usage: skill eval routing --dataset path/to/dataset.json [--gates strict|relaxed] [--json]
+ *
+ * @param evalType - Type of eval to run (currently only 'routing' supported)
+ * @param datasetPath - Path to JSON dataset file
+ * @param options - Command options
+ */
+export async function runEval(
+  evalType: string,
+  datasetPath: string | undefined,
+  options: EvalOptions = {}
+): Promise<void> {
+  const { json = false, gates } = options
+
+  // Validate inputs
+  if (!datasetPath) {
+    console.error(
+      'Error: Dataset path is required. Usage: skill eval routing --dataset <path>'
+    )
+    process.exit(1)
+  }
+
+  // Check if file exists
+  try {
+    await access(datasetPath)
+  } catch {
+    console.error(`Error: Dataset file not found: ${datasetPath}`)
+    process.exit(1)
+  }
+
+  // Read and parse dataset
+  let dataset: EvalDatapoint[]
+  try {
+    const content = await readFile(datasetPath, 'utf-8')
+    dataset = JSON.parse(content)
+  } catch (error) {
+    console.error(
+      `Error: Invalid JSON in dataset file: ${error instanceof Error ? error.message : 'Unknown error'}`
+    )
+    process.exit(1)
+  }
+
+  // Import evalRouting (only when needed to avoid circular deps)
+  const { evalRouting } = await import('@skillrecordings/core/evals/routing')
+
+  // Run eval with optional gates
+  const report: EvalReport = await evalRouting(dataset, gates)
+
+  // Output results
+  if (json) {
+    console.log(JSON.stringify(report, null, 2))
+  } else {
+    printPrettyResults(report)
+  }
+
+  // Exit with appropriate code
+  process.exit(report.passed ? 0 : 1)
+}
+
+/**
+ * Print pretty-formatted results table
+ */
+function printPrettyResults(report: EvalReport): void {
+  console.log('\n📊 Evaluation Results\n')
+  console.log('Overall Metrics:')
+  console.log(`  Precision:           ${(report.precision * 100).toFixed(1)}%`)
+  console.log(`  Recall:              ${(report.recall * 100).toFixed(1)}%`)
+  console.log(`  False Positive Rate: ${(report.fpRate * 100).toFixed(1)}%`)
+  console.log(`  False Negative Rate: ${(report.fnRate * 100).toFixed(1)}%`)
+
+  console.log('\nPerformance:')
+  console.log(`  Latency (p50):       ${report.latency.p50.toFixed(0)}ms`)
+  console.log(`  Latency (p95):       ${report.latency.p95.toFixed(0)}ms`)
+  console.log(`  Latency (p99):       ${report.latency.p99.toFixed(0)}ms`)
+  console.log(`  Total Tokens:        ${report.cost.tokens.toLocaleString()}`)
+  console.log(`  Estimated Cost:      $${report.cost.estimatedUsd.toFixed(4)}`)
+
+  if (Object.keys(report.byCategory).length > 0) {
+    console.log('\nCategory Breakdown:')
+    for (const [category, metrics] of Object.entries(report.byCategory)) {
+      console.log(`\n  ${category}:`)
+      console.log(`    Precision: ${(metrics.precision * 100).toFixed(1)}%`)
+      console.log(`    Recall:    ${(metrics.recall * 100).toFixed(1)}%`)
+      console.log(`    F1:        ${(metrics.f1 * 100).toFixed(1)}%`)
+      console.log(`    Count:     ${metrics.count}`)
+    }
+  }
+
+  console.log(`\n${report.passed ? '✅ PASSED' : '❌ FAILED'}`)
+
+  if (!report.passed) {
+    console.log('\nOne or more metrics fell below threshold gates.')
+  }
+}