@skillrecordings/cli 0.1.0

package/src/commands/memory/index.ts

@@ -0,0 +1,97 @@
+import type { Command } from 'commander'
+import { find } from './find'
+import { get } from './get'
+import { stale, stats } from './stats'
+import { store } from './store'
+import { deleteMemory, downvote, upvote, validate } from './vote'
+
+/**
+ * Register memory commands with Commander
+ */
+export function registerMemoryCommands(program: Command): void {
+  const memory = program
+    .command('memory')
+    .description('Manage semantic memory for agent learning')
+
+  memory
+    .command('store')
+    .description('Store a new memory')
+    .argument('<content>', 'Memory content to store')
+    .option('--tags <tags>', 'Comma-separated tags')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--app <app>', 'App slug to associate with memory')
+    .option('--json', 'Output as JSON')
+    .action(store)
+
+  memory
+    .command('find')
+    .description('Search memories by semantic similarity')
+    .argument('<query>', 'Search query text')
+    .option('--limit <number>', 'Max results (1-100, default: 10)')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--app <app>', 'Filter by app slug')
+    .option(
+      '--min-confidence <confidence>',
+      'Minimum confidence threshold (0-1, default: 0.5)'
+    )
+    .option('--json', 'Output as JSON')
+    .action(find)
+
+  memory
+    .command('get')
+    .description('Get a specific memory by ID')
+    .argument('<id>', 'Memory ID')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--json', 'Output as JSON')
+    .action(get)
+
+  memory
+    .command('validate')
+    .description('Validate a memory (resets decay clock)')
+    .argument('<id>', 'Memory ID')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--json', 'Output as JSON')
+    .action(validate)
+
+  memory
+    .command('upvote')
+    .description('Upvote a memory')
+    .argument('<id>', 'Memory ID')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--reason <reason>', 'Optional reason for upvote')
+    .option('--json', 'Output as JSON')
+    .action(upvote)
+
+  memory
+    .command('downvote')
+    .description('Downvote a memory')
+    .argument('<id>', 'Memory ID')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--reason <reason>', 'Optional reason for downvote')
+    .option('--json', 'Output as JSON')
+    .action(downvote)
+
+  memory
+    .command('delete')
+    .description('Delete a memory')
+    .argument('<id>', 'Memory ID')
+    .option('--collection <collection>', 'Collection name (default: learnings)')
+    .option('--json', 'Output as JSON')
+    .action(deleteMemory)
+
+  memory
+    .command('stats')
+    .description('Display memory statistics')
+    .option('--collection <collection>', 'Filter by collection')
+    .option('--app <app>', 'Filter by app slug')
+    .option('--json', 'Output as JSON')
+    .action(stats)
+
+  memory
+    .command('stale')
+    .description('List stale memories needing validation')
+    .option('--collection <collection>', 'Filter by collection')
+    .option('--threshold <threshold>', 'Confidence threshold (default: 0.25)')
+    .option('--json', 'Output as JSON')
+    .action(stale)
+}

package/src/commands/memory/stats.ts

@@ -0,0 +1,163 @@
+import { calculateConfidence } from '@skillrecordings/memory/decay'
+import { VotingService } from '@skillrecordings/memory/voting'
+
+/**
+ * Display memory statistics
+ */
+export async function stats(options: {
+  collection?: string
+  app?: string
+  json?: boolean
+}): Promise<void> {
+  try {
+    const statsResult = await VotingService.stats(options.collection)
+
+    if (options.json) {
+      console.log(JSON.stringify(statsResult, null, 2))
+      return
+    }
+
+    // Format output
+    const collections = Object.keys(statsResult)
+
+    if (collections.length === 0) {
+      console.log('No memories found')
+      return
+    }
+
+    console.log('\nMemory Statistics')
+    console.log('─'.repeat(60))
+
+    for (const collection of collections) {
+      const stats = statsResult[collection]
+      if (!stats) continue
+
+      console.log(`\n${collection}:`)
+      console.log(`  Total memories: ${stats.count}`)
+      console.log(
+        `  Avg confidence: ${(stats.avg_confidence * 100).toFixed(1)}%`
+      )
+      console.log(`  Upvotes: ${stats.total_upvotes}`)
+      console.log(`  Downvotes: ${stats.total_downvotes}`)
+      console.log(`  Citations: ${stats.total_citations}`)
+      console.log(
+        `  Avg success rate: ${(stats.avg_success_rate * 100).toFixed(1)}%`
+      )
+    }
+
+    console.log('')
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}
+
+/**
+ * List stale memories (low confidence, needing validation)
+ */
+export async function stale(options: {
+  collection?: string
+  threshold?: number
+  json?: boolean
+}): Promise<void> {
+  try {
+    const threshold = options.threshold ?? 0.25
+
+    // Get all collections or specified one
+    const collections = options.collection
+      ? [options.collection]
+      : await VotingService._listCollections()
+
+    const staleMemories: Array<{
+      id: string
+      collection: string
+      confidence: number
+      age_days: number
+      content_preview: string
+    }> = []
+
+    // Check each collection for stale memories
+    for (const collection of collections) {
+      const memories = await VotingService._fetchAllMemories(collection)
+
+      for (const memory of memories) {
+        const confidence = calculateConfidence(memory)
+
+        if (confidence < threshold) {
+          const createdAt = new Date(memory.metadata.created_at)
+          const lastValidatedAt = memory.metadata.last_validated_at
+            ? new Date(memory.metadata.last_validated_at)
+            : undefined
+          const referenceDate = lastValidatedAt || createdAt
+          const ageDays =
+            (Date.now() - referenceDate.getTime()) / (24 * 60 * 60 * 1000)
+
+          // Truncate content for preview
+          const contentPreview =
+            memory.content.length > 60
+              ? memory.content.slice(0, 57) + '...'
+              : memory.content
+
+          staleMemories.push({
+            id: memory.id,
+            collection,
+            confidence,
+            age_days: ageDays,
+            content_preview: contentPreview,
+          })
+        }
+      }
+    }
+
+    if (options.json) {
+      console.log(JSON.stringify(staleMemories, null, 2))
+      return
+    }
+
+    if (staleMemories.length === 0) {
+      console.log(
+        `No stale memories found (threshold: ${(threshold * 100).toFixed(0)}%)`
+      )
+      return
+    }
+
+    console.log(
+      `\nStale Memories (confidence < ${(threshold * 100).toFixed(0)}%)`
+    )
+    console.log('─'.repeat(80))
+
+    for (const mem of staleMemories) {
+      console.log(`\n${mem.id} [${mem.collection}]`)
+      console.log(`  Confidence: ${(mem.confidence * 100).toFixed(1)}%`)
+      console.log(`  Age: ${mem.age_days.toFixed(1)} days`)
+      console.log(`  Preview: ${mem.content_preview}`)
+    }
+
+    console.log('')
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}

package/src/commands/memory/store.ts

@@ -0,0 +1,49 @@
+import { MemoryService } from '@skillrecordings/memory/memory'
+
+/**
+ * Store a new memory with optional tags and collection
+ */
+export async function store(
+  content: string,
+  options: {
+    tags?: string
+    collection?: string
+    app?: string
+    json?: boolean
+  }
+): Promise<void> {
+  try {
+    const memory = await MemoryService.store(content, {
+      collection: options.collection || 'learnings',
+      source: 'human',
+      app_slug: options.app,
+      tags: options.tags?.split(',').map((t) => t.trim()) ?? [],
+    })
+
+    if (options.json) {
+      console.log(JSON.stringify(memory, null, 2))
+    } else {
+      console.log(`✓ Stored memory: ${memory.id}`)
+      if (memory.metadata.tags && memory.metadata.tags.length > 0) {
+        console.log(`  Tags: ${memory.metadata.tags.join(', ')}`)
+      }
+      if (memory.metadata.app_slug) {
+        console.log(`  App: ${memory.metadata.app_slug}`)
+      }
+    }
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}

package/src/commands/memory/vote.ts

@@ -0,0 +1,159 @@
+import { MemoryService } from '@skillrecordings/memory/memory'
+import { VotingService } from '@skillrecordings/memory/voting'
+
+/**
+ * Validate a memory (resets decay clock)
+ */
+export async function validate(
+  id: string,
+  options: {
+    collection?: string
+    json?: boolean
+  }
+): Promise<void> {
+  try {
+    const collection = options.collection || 'learnings'
+    await MemoryService.validate(id, collection)
+
+    if (options.json) {
+      console.log(JSON.stringify({ success: true, id }, null, 2))
+    } else {
+      console.log(`✓ Validated memory: ${id}`)
+      console.log('  Decay clock has been reset')
+    }
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}
+
+/**
+ * Upvote a memory
+ */
+export async function upvote(
+  id: string,
+  options: {
+    collection?: string
+    reason?: string
+    json?: boolean
+  }
+): Promise<void> {
+  try {
+    const collection = options.collection || 'learnings'
+    await VotingService.vote(id, collection, 'upvote')
+
+    if (options.json) {
+      console.log(
+        JSON.stringify({ success: true, id, vote: 'upvote' }, null, 2)
+      )
+    } else {
+      console.log(`✓ Upvoted memory: ${id}`)
+      if (options.reason) {
+        console.log(`  Reason: ${options.reason}`)
+      }
+    }
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}
+
+/**
+ * Downvote a memory
+ */
+export async function downvote(
+  id: string,
+  options: {
+    collection?: string
+    reason?: string
+    json?: boolean
+  }
+): Promise<void> {
+  try {
+    const collection = options.collection || 'learnings'
+    await VotingService.vote(id, collection, 'downvote')
+
+    if (options.json) {
+      console.log(
+        JSON.stringify({ success: true, id, vote: 'downvote' }, null, 2)
+      )
+    } else {
+      console.log(`✓ Downvoted memory: ${id}`)
+      if (options.reason) {
+        console.log(`  Reason: ${options.reason}`)
+      }
+    }
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}
+
+/**
+ * Delete a memory
+ */
+export async function deleteMemory(
+  id: string,
+  options: {
+    collection?: string
+    json?: boolean
+  }
+): Promise<void> {
+  try {
+    const collection = options.collection || 'learnings'
+    await MemoryService.delete(id, collection)
+
+    if (options.json) {
+      console.log(JSON.stringify({ success: true, id }, null, 2))
+    } else {
+      console.log(`✓ Deleted memory: ${id}`)
+    }
+  } catch (error) {
+    if (options.json) {
+      console.error(
+        JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        })
+      )
+    } else {
+      console.error(
+        'Error:',
+        error instanceof Error ? error.message : 'Unknown error'
+      )
+    }
+    process.exit(1)
+  }
+}

package/src/commands/pipeline.ts

@@ -0,0 +1,127 @@
+/**
+ * Pipeline CLI commands
+ * 
+ * Commands for running and evaluating the pipeline steps.
+ */
+
+import type { Command } from 'commander'
+import { runClassifyEval, buildClassifyDataset } from '@skillrecordings/core/pipeline/evals/classify.eval'
+import { runValidateEval, buildValidateDatasetFromProduction } from '@skillrecordings/core/pipeline/evals/validate.eval'
+import { runE2EEval } from '@skillrecordings/core/pipeline/evals/e2e.eval'
+
+export function registerPipelineCommands(program: Command): void {
+  const pipeline = program
+    .command('pipeline')
+    .description('Pipeline step commands and evals')
+
+  // -------------------------------------------------------------------------
+  // Classify eval
+  // -------------------------------------------------------------------------
+  pipeline
+    .command('eval-classify')
+    .description('Run classifier evaluation')
+    .requiredOption('--dataset <file>', 'Path to labeled scenarios JSON')
+    .option('--output <file>', 'Save results to JSON')
+    .option('--verbose', 'Show individual failures')
+    .option('--json', 'JSON output')
+    .option('--force-llm', 'Skip fast path, always use LLM')
+    .option('--model <model>', 'Model for LLM classification', 'anthropic/claude-haiku-4-5')
+    .action(async (opts) => {
+      await runClassifyEval(opts)
+    })
+
+  pipeline
+    .command('build-classify-dataset')
+    .description('Build classify eval dataset from production data')
+    .requiredOption('--production <file>', 'Production dataset JSON')
+    .requiredOption('--output <file>', 'Output scenarios JSON')
+    .action(async (opts) => {
+      await buildClassifyDataset(opts.production, opts.output)
+    })
+
+  // -------------------------------------------------------------------------
+  // Validate eval
+  // -------------------------------------------------------------------------
+  pipeline
+    .command('eval-validate')
+    .description('Run validator evaluation')
+    .option('--dataset <file>', 'Path to scenarios JSON (uses built-in if not provided)')
+    .option('--output <file>', 'Save results to JSON')
+    .option('--verbose', 'Show individual failures')
+    .option('--json', 'JSON output')
+    .action(async (opts) => {
+      await runValidateEval(opts)
+    })
+
+  pipeline
+    .command('build-validate-dataset')
+    .description('Build validate eval dataset from production failures')
+    .requiredOption('--production <file>', 'Production baseline results JSON')
+    .requiredOption('--output <file>', 'Output scenarios JSON')
+    .action(async (opts) => {
+      await buildValidateDatasetFromProduction(opts.production, opts.output)
+    })
+
+  // -------------------------------------------------------------------------
+  // E2E eval
+  // -------------------------------------------------------------------------
+  pipeline
+    .command('eval-e2e')
+    .description('Run end-to-end pipeline evaluation')
+    .requiredOption('--dataset <file>', 'Production dataset JSON')
+    .option('--output <file>', 'Save results to JSON')
+    .option('--verbose', 'Show individual failures')
+    .option('--json', 'JSON output')
+    .option('--limit <number>', 'Max scenarios to run', parseInt)
+    .option('--model <model>', 'Model for LLM steps', 'anthropic/claude-haiku-4-5')
+    .action(async (opts) => {
+      await runE2EEval(opts)
+    })
+
+  // -------------------------------------------------------------------------
+  // Run pipeline
+  // -------------------------------------------------------------------------
+  pipeline
+    .command('run')
+    .description('Run pipeline on a single message')
+    .requiredOption('--subject <text>', 'Message subject')
+    .requiredOption('--body <text>', 'Message body')
+    .option('--app <id>', 'App ID', 'total-typescript')
+    .option('--dry-run', 'Don\'t actually send', true)
+    .option('--json', 'JSON output')
+    .action(async (opts) => {
+      const { runPipeline } = await import('@skillrecordings/core/pipeline')
+      
+      const result = await runPipeline({
+        message: {
+          subject: opts.subject,
+          body: opts.body,
+        },
+        appConfig: {
+          appId: opts.app,
+          instructorConfigured: false,
+          autoSendEnabled: false,
+        },
+        dryRun: opts.dryRun,
+      })
+
+      if (opts.json) {
+        console.log(JSON.stringify(result, null, 2))
+      } else {
+        console.log(`\n📬 Pipeline Result\n`)
+        console.log(`Action: ${result.action}`)
+        if (result.response) {
+          console.log(`\nResponse:\n${result.response}`)
+        }
+        console.log(`\nSteps:`)
+        for (const step of result.steps) {
+          const icon = step.success ? '✅' : '❌'
+          console.log(`  ${icon} ${step.step} (${step.durationMs}ms)`)
+          if (step.error) {
+            console.log(`     Error: ${step.error}`)
+          }
+        }
+        console.log(`\nTotal: ${result.totalDurationMs}ms`)
+      }
+    })
+}