@skillrecordings/cli 0.1.0 → 0.2.0

package/src/commands/build-dataset.ts

@@ -1,311 +0,0 @@
-/**
- * Build eval dataset from real agent responses
- *
- * Pulls trigger messages and agent responses from Front + DB
- * to create labeled datasets for eval improvement.
- *
- * Usage:
- *   skill dataset build --since 2025-01-01 --output /tmp/dataset.json
- *   skill dataset build --app total-typescript --labeled-only
- */
-
-import { writeFileSync } from 'fs'
-import { createInstrumentedFrontClient } from '@skillrecordings/core/front/instrumented-client'
-import {
-  ActionsTable,
-  AppsTable,
-  ConversationsTable,
-  and,
-  desc,
-  eq,
-  getDb,
-  gte,
-} from '@skillrecordings/database'
-import { type Message } from '@skillrecordings/front-sdk'
-import type { Command } from 'commander'
-
-interface EvalDataPoint {
-  id: string
-  app: string
-  conversationId: string
-  customerEmail: string
-  triggerMessage: {
-    subject: string
-    body: string
-    timestamp: number
-  }
-  agentResponse: {
-    text: string
-    category: string
-    timestamp: string
-  }
-  label?: 'good' | 'bad'
-  labeledBy?: string
-  conversationHistory?: Array<{
-    direction: 'in' | 'out'
-    body: string
-    timestamp: number
-    author?: string
-  }>
-}
-
-/**
- * Build eval dataset from responses
- */
-async function buildDataset(options: {
-  app?: string
-  since?: string
-  output?: string
-  labeledOnly?: boolean
-  limit?: number
-  includeHistory?: boolean
-}): Promise<void> {
-  const db = getDb()
-  const limit = options.limit || 100
-
-  // Get Front token
-  const frontToken = process.env.FRONT_API_TOKEN
-  if (!frontToken) {
-    console.error('Error: FRONT_API_TOKEN environment variable required')
-    console.error('Set it or source the env file: source apps/front/.env.local')
-    process.exit(1)
-  }
-
-  const front = createInstrumentedFrontClient({ apiToken: frontToken })
-
-  try {
-    // Build query conditions
-    const conditions = [eq(ActionsTable.type, 'draft-response')]
-
-    if (options.app) {
-      const appResults = await db
-        .select()
-        .from(AppsTable)
-        .where(eq(AppsTable.slug, options.app))
-        .limit(1)
-
-      const foundApp = appResults[0]
-      if (!foundApp) {
-        console.error(`App not found: ${options.app}`)
-        process.exit(1)
-      }
-      conditions.push(eq(ActionsTable.app_id, foundApp.id))
-    }
-
-    if (options.since) {
-      const sinceDate = new Date(options.since)
-      conditions.push(gte(ActionsTable.created_at, sinceDate))
-    }
-
-    // Query actions
-    const results = await db
-      .select({
-        action: ActionsTable,
-        app: AppsTable,
-        conversation: ConversationsTable,
-      })
-      .from(ActionsTable)
-      .leftJoin(AppsTable, eq(ActionsTable.app_id, AppsTable.id))
-      .leftJoin(
-        ConversationsTable,
-        eq(
-          ActionsTable.conversation_id,
-          ConversationsTable.front_conversation_id
-        )
-      )
-      .where(and(...conditions))
-      .orderBy(desc(ActionsTable.created_at))
-      .limit(limit)
-
-    console.log(`Found ${results.length} responses, fetching context...`)
-
-    const dataset: EvalDataPoint[] = []
-    let processed = 0
-
-    for (const r of results) {
-      processed++
-      process.stdout.write(`\rProcessing ${processed}/${results.length}...`)
-
-      const params = r.action.parameters as {
-        response?: string
-        category?: string
-      }
-
-      // Skip if no response
-      if (!params.response) continue
-
-      // Determine label
-      let label: 'good' | 'bad' | undefined
-      let labeledBy: string | undefined
-
-      if (r.action.approved_by) {
-        label = 'good'
-        labeledBy = r.action.approved_by
-      } else if (r.action.rejected_by) {
-        label = 'bad'
-        labeledBy = r.action.rejected_by
-      }
-
-      // Skip unlabeled if labeledOnly
-      if (options.labeledOnly && !label) continue
-
-      // Fetch conversation from Front
-      if (!r.action.conversation_id) continue
-
-      let triggerMessage: EvalDataPoint['triggerMessage'] | undefined
-      let conversationHistory: EvalDataPoint['conversationHistory'] | undefined
-
-      try {
-        const messageList = (await front.conversations.listMessages(
-          r.action.conversation_id
-        )) as { _results?: Message[] }
-        const messages = messageList._results ?? []
-
-        // Find trigger message (most recent inbound before draft)
-        const draftTime = r.action.created_at?.getTime() ?? Date.now()
-        const inboundBefore = messages
-          .filter((m) => m.is_inbound && m.created_at * 1000 < draftTime)
-          .sort((a, b) => b.created_at - a.created_at)
-
-        const trigger = inboundBefore[0]
-        if (trigger) {
-          triggerMessage = {
-            subject: trigger.subject ?? '',
-            body:
-              trigger.text ??
-              trigger.body
-                ?.replace(/<[^>]*>/g, ' ')
-                .replace(/\s+/g, ' ')
-                .trim() ??
-              '',
-            timestamp: trigger.created_at,
-          }
-        }
-
-        // Include history if requested
-        if (options.includeHistory) {
-          conversationHistory = messages.map((m) => ({
-            direction: m.is_inbound ? ('in' as const) : ('out' as const),
-            body:
-              m.text ??
-              m.body
-                ?.replace(/<[^>]*>/g, ' ')
-                .replace(/\s+/g, ' ')
-                .trim() ??
-              '',
-            timestamp: m.created_at,
-            author: m.author?.email,
-          }))
-        }
-      } catch (err) {
-        // Skip if can't fetch context
-        continue
-      }
-
-      // Skip if no trigger message
-      if (!triggerMessage) continue
-
-      dataset.push({
-        id: r.action.id,
-        app: r.app?.slug ?? 'unknown',
-        conversationId: r.action.conversation_id,
-        customerEmail: r.conversation?.customer_email ?? 'unknown',
-        triggerMessage,
-        agentResponse: {
-          text: params.response,
-          category: params.category ?? 'unknown',
-          timestamp: r.action.created_at?.toISOString() ?? '',
-        },
-        label,
-        labeledBy,
-        conversationHistory,
-      })
-    }
-
-    console.log(`\n\nBuilt dataset with ${dataset.length} eval points`)
-    console.log(`  Labeled: ${dataset.filter((d) => d.label).length}`)
-    console.log(`  Good: ${dataset.filter((d) => d.label === 'good').length}`)
-    console.log(`  Bad: ${dataset.filter((d) => d.label === 'bad').length}`)
-    console.log(`  Unlabeled: ${dataset.filter((d) => !d.label).length}`)
-
-    const outputJson = JSON.stringify(dataset, null, 2)
-
-    if (options.output) {
-      writeFileSync(options.output, outputJson, 'utf-8')
-      console.log(`\nSaved to ${options.output}`)
-    } else {
-      console.log('\n' + outputJson)
-    }
-  } catch (error) {
-    console.error(
-      '\nError:',
-      error instanceof Error ? error.message : 'Unknown error'
-    )
-    process.exit(1)
-  }
-}
-
-/**
- * Convert dataset to evalite format
- */
-async function toEvalite(options: {
-  input: string
-  output?: string
-}): Promise<void> {
-  const { readFileSync } = await import('fs')
-
-  const data = JSON.parse(
-    readFileSync(options.input, 'utf-8')
-  ) as EvalDataPoint[]
-
-  const evaliteData = data.map((d) => ({
-    input: d.triggerMessage.body,
-    output: d.agentResponse.text,
-    expected: d.label === 'good' ? d.agentResponse.text : '',
-    metadata: {
-      id: d.id,
-      app: d.app,
-      category: d.agentResponse.category,
-      label: d.label,
-      subject: d.triggerMessage.subject,
-    },
-  }))
-
-  const outputJson = JSON.stringify(evaliteData, null, 2)
-
-  if (options.output) {
-    writeFileSync(options.output, outputJson, 'utf-8')
-    console.log(
-      `Converted ${evaliteData.length} points to evalite format: ${options.output}`
-    )
-  } else {
-    console.log(outputJson)
-  }
-}
-
-/**
- * Register dataset commands
- */
-export function registerDatasetCommands(program: Command): void {
-  const dataset = program
-    .command('dataset')
-    .description('Build and manage eval datasets')
-
-  dataset
-    .command('build')
-    .description('Build eval dataset from agent responses')
-    .option('-a, --app <slug>', 'Filter by app slug')
-    .option('-s, --since <date>', 'Filter responses since date (YYYY-MM-DD)')
-    .option('-o, --output <file>', 'Output file path')
-    .option('-l, --limit <n>', 'Max responses to process', parseInt)
-    .option('--labeled-only', 'Only include labeled responses')
-    .option('--include-history', 'Include full conversation history')
-    .action(buildDataset)
-
-  dataset
-    .command('to-evalite')
-    .description('Convert dataset to evalite format')
-    .requiredOption('-i, --input <file>', 'Input dataset JSON file')
-    .option('-o, --output <file>', 'Output file path')
-    .action(toEvalite)
-}

package/src/commands/db-status.ts

@@ -1,47 +0,0 @@
-import { program } from 'commander'
-import {
-  ConversationsTable,
-  database,
-  desc,
-  sql,
-} from '@skillrecordings/database'
-
-export const registerDbStatusCommand = (prog: typeof program) => {
-  prog
-    .command('db-status')
-    .description('Check database status and conversation counts')
-    .action(async () => {
-      try {
-        // Count by status
-        const statusCounts = await database
-          .select({
-            status: ConversationsTable.status,
-            count: sql<number>`COUNT(*)`.as('count'),
-          })
-          .from(ConversationsTable)
-          .groupBy(ConversationsTable.status)
-
-        console.log('Conversation counts by status:')
-        for (const row of statusCounts) {
-          console.log(`  ${row.status}: ${row.count}`)
-        }
-
-        // Get recent conversations
-        const recent = await database
-          .select()
-          .from(ConversationsTable)
-          .orderBy(desc(ConversationsTable.updated_at))
-          .limit(5)
-
-        console.log('\nRecent conversations:')
-        for (const c of recent) {
-          console.log(`  ${c.front_conversation_id}: ${c.status} (${c.updated_at})`)
-        }
-
-        process.exit(0)
-      } catch (error) {
-        console.error('Error:', error)
-        process.exit(1)
-      }
-    })
-}

package/src/commands/deploys.ts

@@ -1,219 +0,0 @@
-import { execSync } from 'child_process'
-import { Command } from 'commander'
-
-const VERCEL_SCOPE = 'skillrecordings'
-
-// Map of app names to their Vercel project names
-const APPS: Record<string, { vercel: string; description: string }> = {
-  front: {
-    vercel: 'skill-support-agent-front',
-    description: 'Front webhook handler (main support pipeline)',
-  },
-  slack: {
-    vercel: 'skill-support-agent-slack',
-    description: 'Slack interactions (approvals, notifications)',
-  },
-}
-
-function stripAnsi(str: string): string {
-  // eslint-disable-next-line no-control-regex
-  return str.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '').replace(/\x1B\].*?\x07/g, '')
-}
-
-function runVercel(args: string): string {
-  try {
-    // Capture both stdout and stderr (Vercel CLI writes table to stderr)
-    const raw = execSync(`vercel ${args} --scope ${VERCEL_SCOPE} --yes 2>&1`, {
-      encoding: 'utf-8',
-      timeout: 30000,
-      stdio: ['pipe', 'pipe', 'pipe'],
-      env: { ...process.env, FORCE_COLOR: '0', NO_COLOR: '1' },
-    }).trim()
-    return stripAnsi(raw)
-  } catch (err: any) {
-    const out = err.stdout?.trim() || err.stderr?.trim() || err.message
-    return stripAnsi(out)
-  }
-}
-
-function listApps() {
-  console.log('\n📦 Support Platform Apps\n')
-  for (const [name, app] of Object.entries(APPS)) {
-    console.log(
-      `  ${name.padEnd(10)} ${app.vercel.padEnd(35)} ${app.description}`
-    )
-  }
-  console.log()
-}
-
-function resolveApp(
-  nameOrAll: string | undefined
-): [string, { vercel: string; description: string }][] {
-  if (!nameOrAll || nameOrAll === 'all') {
-    return Object.entries(APPS)
-  }
-  const app = APPS[nameOrAll]
-  if (!app) {
-    console.error(`❌ Unknown app: ${nameOrAll}`)
-    console.error(`   Available: ${Object.keys(APPS).join(', ')}, all`)
-    process.exit(1)
-  }
-  return [[nameOrAll, app]]
-}
-
-async function deploysStatus(
-  appName: string | undefined,
-  options: { limit?: string; json?: boolean }
-) {
-  const apps = resolveApp(appName)
-  const limit = parseInt(options.limit || '5')
-
-  for (const [name, app] of apps) {
-    console.log(`\n🚀 ${name} (${app.vercel})`)
-    console.log(`   ${app.description}\n`)
-
-    const output = runVercel(`ls ${app.vercel}`)
-    const lines = output.split('\n')
-    // Deploy lines contain https:// URLs
-    const deployLines = lines.filter((l) => l.includes('https://'))
-
-    if (deployLines.length === 0) {
-      console.log('   No recent deployments found')
-    } else {
-      // Filter production-only for cleaner output, show all if few deploys
-      const prodLines = deployLines.filter((l) => l.includes('Production'))
-      const showLines = (
-        prodLines.length >= limit ? prodLines : deployLines
-      ).slice(0, limit)
-
-      for (const line of showLines) {
-        const trimmed = line.trim()
-        // Extract known tokens from the line
-        const hasReady = trimmed.includes('Ready')
-        const hasError = trimmed.includes('Error')
-        const hasCanceled = trimmed.includes('Canceled')
-        const isProd = trimmed.includes('Production')
-        const isPreview = trimmed.includes('Preview')
-        const prefix = hasError
-          ? '❌'
-          : hasCanceled
-            ? '⚪'
-            : isProd
-              ? '🟢'
-              : '🔵'
-
-        // Pull age (first token) and environment
-        const age = trimmed.split(/\s+/)[0] || ''
-        const status = hasError
-          ? 'Error'
-          : hasCanceled
-            ? 'Canceled'
-            : hasReady
-              ? 'Ready'
-              : 'Unknown'
-        const env = isProd ? 'Production' : isPreview ? 'Preview' : ''
-
-        // Pull duration (pattern like "30s", "17s")
-        const durMatch = trimmed.match(/(\d+s)/)
-        const duration = durMatch ? durMatch[1] : ''
-
-        console.log(
-          `   ${prefix} ${age.padEnd(6)} ${status.padEnd(10)} ${env.padEnd(12)} ${duration}`
-        )
-      }
-    }
-  }
-  console.log()
-}
-
-async function deploysLogs(appName: string, options: { lines?: string }) {
-  const apps = resolveApp(appName)
-  if (apps.length > 1) {
-    console.error('❌ Specify a single app for logs (front or slack)')
-    process.exit(1)
-  }
-
-  const [name, app] = apps[0]!
-  console.log(`\n📋 Recent logs for ${name} (${app.vercel})\n`)
-
-  // Get latest production deployment URL
-  const lsOutput = runVercel(`ls ${app.vercel} --limit 5`)
-  const prodLine = lsOutput
-    .split('\n')
-    .find((l) => l.includes('Production') && l.includes('Ready'))
-  const urlMatch = prodLine?.match(/https:\/\/\S+/)
-
-  if (!urlMatch) {
-    console.error('   Could not find latest production deployment')
-    process.exit(1)
-  }
-
-  const url = urlMatch[0]
-  console.log(`   Deployment: ${url}\n`)
-
-  try {
-    const logsOutput = execSync(
-      `vercel logs ${url} --scope ${VERCEL_SCOPE} --output short 2>&1 | tail -${options.lines || '30'}`,
-      { encoding: 'utf-8', timeout: 30000 }
-    )
-    console.log(logsOutput)
-  } catch (err: any) {
-    console.log(err.stdout || err.message)
-  }
-}
-
-async function deploysInspect(appName: string) {
-  const apps = resolveApp(appName)
-  if (apps.length > 1) {
-    console.error('❌ Specify a single app for inspect (front or slack)')
-    process.exit(1)
-  }
-
-  const [name, app] = apps[0]!
-
-  // Get latest production deployment URL
-  const lsOutput = runVercel(`ls ${app.vercel} --limit 3`)
-  const prodLine = lsOutput
-    .split('\n')
-    .find((l) => l.includes('Production') && l.includes('Ready'))
-  const urlMatch = prodLine?.match(/https:\/\/\S+/)
-
-  if (!urlMatch) {
-    console.error('   Could not find latest production deployment')
-    process.exit(1)
-  }
-
-  console.log(`\n🔍 Inspecting ${name} latest production deploy\n`)
-  const output = runVercel(`inspect ${urlMatch[0]}`)
-  console.log(output)
-}
-
-export function registerDeployCommands(program: Command) {
-  const deploys = program
-    .command('deploys')
-    .description('Check Vercel deployment status for support platform apps')
-
-  deploys
-    .command('status')
-    .description('Show recent deployments for one or all apps')
-    .argument('[app]', 'App name (front, slack, all)', 'all')
-    .option('-n, --limit <number>', 'Number of deploys to show', '5')
-    .option('--json', 'JSON output')
-    .action(deploysStatus)
-
-  deploys
-    .command('logs')
-    .description('Show recent logs for an app')
-    .argument('<app>', 'App name (front, slack)')
-    .option('-n, --lines <number>', 'Number of log lines', '30')
-    .action(deploysLogs)
-
-  deploys
-    .command('inspect')
-    .description('Inspect latest production deployment')
-    .argument('<app>', 'App name (front, slack)')
-    .action(deploysInspect)
-
-  // Default: `deploys` with no subcommand = status all
-  deploys.action(() => deploysStatus('all', { limit: '3' }))
-}