@indemn/cli 1.5.0 → 1.7.0

package/src/cli/eval.ts

@@ -3,7 +3,10 @@ import * as fs from 'fs';
 import ora from 'ora';
 import chalk from 'chalk';
 import { IndemnClient } from '../sdk/client.js';
-import { EvalSDK } from '../sdk/eval.js';
+import { EvalSDK, gateVerdict, type GateOptions } from '../sdk/eval.js';
+import type { EvaluationTriggerInput } from '../sdk/types.js';
+import { CliError, EXIT } from '../core/runtime.js';
+import { confirmOrExit } from './link.js';
 import { RubricsSDK } from '../sdk/rubrics.js';
 import { TestSetsSDK } from '../sdk/test-sets.js';
 import { exportEvalMarkdown, exportEvalReport } from '../report/export.js';
@@ -15,63 +18,105 @@ export function registerEvalCommands(program: Command): void {
   eval_
     .command('run')
     .argument('<agent-id>', 'Agent ID')
-    .description('Run an evaluation')
-    .requiredOption('--test-set <id>', 'Test set ID')
+    .description('Run an evaluation (costs model calls; asks first). With a threshold it becomes a CI gate')
+    .requiredOption('--test-set <id>', 'Test set ID (must belong to this agent)')
     .option('--rubric <id>', 'Rubric ID')
     .option('--concurrency <n>', 'Concurrency level')
     .option('--wait', 'Wait for evaluation to complete')
+    .option('--timeout <seconds>', 'With --wait: give up after this long', '1800')
     .option('--eval-model <model>', 'Model for evaluation judging')
-    .option('--keep', 'Keep evaluation data after completion')
     .option('--limit <n>', 'Limit number of items to evaluate')
     .option('--component-scope <scope>', 'Component scope filter')
     .option('--component-ids <ids>', 'Comma-separated component IDs')
-    .option('--conversation-ids <ids>', 'Comma-separated conversation IDs')
-    .option('--mode <mode>', 'Evaluation mode (e.g. "transcript")')
+    .option('--conversation-ids <ids>', 'Comma-separated conversation IDs (transcript mode)')
+    .option('--date-from <date>', 'Grade real conversations from this date (transcript mode)')
+    .option('--date-to <date>', 'Up to this date, exclusive (needs --date-from)')
+    .option('--mode <mode>', 'Evaluation mode: "transcript" grades real conversations')
+    .option('--webhook-url <url>', 'Called when the run finishes')
+    .option('--min-pass-rate <rate>', 'CI gate: exit 1 when passed/total is below this (0..1). Implies --wait')
+    .option('--max-p95-ms <ms>', 'CI gate: exit 1 when p95 latency is above this. Implies --wait')
+    .option('--error-policy <policy>', 'CI gate: block (a failed run fails the gate) or warn', 'block')
+    .option('--fail-on-error', 'CI gate: also exit 1 when any case had an infrastructure error. Implies --wait')
+    .option('--allow-other-agent', 'Run a test set written for another agent')
+    .option('--yes', 'Do not ask for confirmation')
+    .option('--json', 'Print the final run as JSON')
     .action(async (agentId: string, options) => {
       try {
         const client = new IndemnClient();
         const sdk = new EvalSDK(client);
-        const spinner = ora('Triggering evaluation...').start();
+        await sdk.assertAgentInOrg(agentId);
 
-        const input: Record<string, unknown> = {
-          bot_id: agentId,
-          test_set_id: options.testSet,
-        };
+        const testSet = await new TestSetsSDK(client).getTestSet(options.testSet);
+        const owner = (testSet as { agent_id?: string }).agent_id;
+        if (owner && owner !== agentId && !options.allowOtherAgent) {
+          throw new CliError(`Test set ${options.testSet} was written for agent ${owner}, not ${agentId}. Pass --allow-other-agent to run it anyway.`, EXIT.NEEDS_INPUT);
+        }
+
+        const input: EvaluationTriggerInput = { bot_id: agentId, test_set_id: options.testSet };
         if (options.rubric) input.rubric_id = options.rubric;
         if (options.concurrency) input.concurrency = parseInt(options.concurrency, 10);
+        // Voice simulations share call rooms; the dashboard runs them one at a time.
+        if (((testSet as { items?: Array<{ type?: string }> }).items ?? []).some((i) => i.type === 'voice_simulation')) input.concurrency = 1;
         if (options.evalModel) input.eval_model = options.evalModel;
-        if (options.keep !== undefined) input.keep = options.keep;
         if (options.limit) input.limit = parseInt(options.limit, 10);
         if (options.componentScope) input.component_scope = options.componentScope;
         if (options.componentIds) input.component_ids = options.componentIds.split(',');
         if (options.conversationIds) input.conversation_ids = options.conversationIds.split(',');
-        if (options.mode) input.mode = options.mode;
+        if (options.webhookUrl) input.webhook_url = options.webhookUrl;
+        if (Boolean(options.dateFrom) !== Boolean(options.dateTo)) {
+          throw new CliError('--date-from and --date-to go together.', EXIT.NEEDS_INPUT);
+        }
+        if (options.dateFrom) {
+          input.date_from = options.dateFrom;
+          input.date_to = options.dateTo;
+          if (options.mode && options.mode !== 'transcript') throw new CliError('A date range only works with --mode transcript.', EXIT.NEEDS_INPUT);
+          // The service ignores dates outside transcript mode and would run the test set instead.
+          input.mode = 'transcript';
+        } else if (options.mode) {
+          input.mode = options.mode;
+        }
+
+        const items = (testSet as { items?: unknown[] }).items?.length ?? 0;
+        const count = input.mode === 'transcript' ? 'real conversations' : `${input.limit ? Math.min(input.limit, items) : items} item(s)`;
+        await confirmOrExit(`Run an evaluation of agent ${agentId} on ${count} with concurrency ${input.concurrency ?? 1}? It uses paid model calls and cannot be cancelled.`, Boolean(options.yes));
+
+        const gate: GateOptions = {
+          minPassRate: options.minPassRate !== undefined ? Number(options.minPassRate) : undefined,
+          maxP95Ms: options.maxP95Ms !== undefined ? Number(options.maxP95Ms) : undefined,
+          errorPolicy: options.errorPolicy === 'warn' ? 'warn' : 'block',
+          failOnError: Boolean(options.failOnError),
+        };
+        const gating = gate.minPassRate !== undefined || gate.maxP95Ms !== undefined || gate.failOnError;
 
-        const result = await sdk.triggerEvaluation(input as any);
+        const spinner = ora('Triggering evaluation...').start();
+        const result = await sdk.triggerEvaluation(input);
         spinner.stop();
+        if (!options.json) {
+          printSuccess(`Evaluation started: ${result.run_id}`);
+          if (result.total) console.log(`  Total items: ${result.total}`);
+        }
+        if (!options.wait && !gating) return;
 
-        printSuccess(`Evaluation started: ${result.run_id}`);
-        console.log(`  Status: ${result.status}`);
-        if (result.total) console.log(`  Total items: ${result.total}`);
-
-        if (options.wait) {
-          const pollSpinner = ora('Waiting for evaluation to complete...').start();
-
-          try {
-            const run = await sdk.pollRunUntilComplete(result.run_id);
-            pollSpinner.stop();
-
-            printSuccess(`Evaluation completed: ${run.status}`);
-            if (run.passed !== undefined) console.log(`  Passed: ${run.passed}`);
-            if (run.failed !== undefined) console.log(`  Failed: ${run.failed}`);
-            if (run.criteria_passed !== undefined && run.criteria_total !== undefined) {
-              console.log(`  Criteria: ${run.criteria_passed}/${run.criteria_total}`);
-            }
-          } catch (err) {
-            pollSpinner.fail('Evaluation did not complete in time');
-            throw err;
-          }
+        const pollSpinner = options.json ? null : ora('Waiting for evaluation to complete...').start();
+        const run = await sdk.pollRunUntilComplete(result.run_id, 10_000, Number(options.timeout) * 1000).catch((err) => {
+          pollSpinner?.fail('Evaluation did not complete in time');
+          throw err;
+        });
+        pollSpinner?.stop();
+        const verdict = gateVerdict(run, gate);
+        if (options.json) {
+          printJson({ run, gate: gating ? verdict : undefined });
+        } else {
+          printSuccess(`Evaluation ${run.status}: ${run.passed ?? 0}/${run.total ?? 0} passed (${(verdict.passRate * 100).toFixed(1)}%)`);
+          if (run.errors) console.log(chalk.yellow(`  Infrastructure errors: ${run.errors}`));
+          if (run.latency?.p95_ms !== undefined) console.log(`  p95 latency: ${run.latency.p95_ms} ms`);
+          if (run.criteria_passed !== undefined && run.criteria_total !== undefined) console.log(`  Criteria: ${run.criteria_passed}/${run.criteria_total}`);
+        }
+        if (gating && !verdict.pass) {
+          console.error(chalk.red(`Gate failed: ${verdict.reasons.join('; ')}`));
+          process.exit(1);
         }
+        if (gating && !options.json) printSuccess('Gate passed.');
       } catch (err) {
         printError(err);
         process.exit(1);
@@ -80,8 +125,8 @@ export function registerEvalCommands(program: Command): void {
 
   eval_
     .command('list')
-    .description('List evaluation runs')
-    .option('--agent-id <id>', 'Filter by agent ID')
+    .description('List evaluation runs of an agent')
+    .requiredOption('--agent-id <id>', 'Agent ID (runs of other orgs are never listed)')
     .option('--json', 'Output as JSON')
     .option('--limit <n>', 'Limit number of results')
     .option('--page <n>', 'Page number (use with --limit)')
@@ -91,6 +136,7 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new EvalSDK(client);
         const spinner = ora('Fetching evaluation runs...').start();
 
+        await sdk.assertAgentInOrg(options.agentId);
         let runs = await sdk.listRuns(options.agentId);
         spinner.stop();
 
@@ -139,7 +185,7 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new EvalSDK(client);
         const spinner = ora('Fetching run status...').start();
 
-        const run = await sdk.getRunStatus(runId);
+        const run = await sdk.getRunInOrg(runId);
         spinner.stop();
 
         if (options.json) {
@@ -179,6 +225,7 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new EvalSDK(client);
         const spinner = ora('Fetching results...').start();
 
+        await sdk.getRunInOrg(runId);
         const results = await sdk.getRunResults(runId);
         spinner.stop();
 
@@ -211,6 +258,7 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new EvalSDK(client);
         const spinner = ora('Fetching bot context...').start();
 
+        await sdk.assertAgentInOrg(agentId);
         const context = await sdk.getBotContext(agentId);
         spinner.stop();
 
@@ -253,6 +301,7 @@ export function registerEvalCommands(program: Command): void {
         const client = new IndemnClient();
         const spinner = ora('Fetching evaluation data...').start();
 
+        await new EvalSDK(client).getRunInOrg(runId);
         const result = await exportEvalMarkdown(runId, options.output, client);
 
         spinner.stop();
@@ -266,16 +315,32 @@ export function registerEvalCommands(program: Command): void {
   eval_
     .command('report')
     .argument('<run-id>', 'Evaluation run ID')
-    .description('Generate branded evaluation report PDF')
+    .description('Branded evaluation report PDF (made here, or by the evaluations service with --server)')
     .option('--output <path>', 'Output file path')
+    .option('--server', 'Ask the evaluations service to render it (completed runs only)')
+    .option('--force', 'With --server: render again instead of reusing the last report')
     .action(async (runId: string, options) => {
       try {
         const client = new IndemnClient();
-        const spinner = ora('Fetching evaluation data...').start();
-
-        spinner.text = 'Generating PDF...';
+        const sdk = new EvalSDK(client);
+        await sdk.getRunInOrg(runId);
+        if (options.server) {
+          const spinner = ora('Rendering on the evaluations service...').start();
+          const { download_url, report_id } = await sdk.exportReport(runId, Boolean(options.force));
+          spinner.stop();
+          if (options.output) {
+            const res = await fetch(download_url);
+            if (!res.ok) throw new Error(`Download failed (${res.status}).`);
+            fs.writeFileSync(options.output, Buffer.from(await res.arrayBuffer()));
+            printSuccess(`Report saved to ${options.output}`);
+          } else {
+            console.log(download_url);
+            console.log(chalk.dim(`Link valid for 15 minutes. New link later: indemn eval report-link ${report_id}`));
+          }
+          return;
+        }
+        const spinner = ora('Generating PDF...').start();
         const result = await exportEvalReport(runId, options.output, client);
-
         spinner.stop();
         printSuccess(`Report saved to ${result.file_path}`);
       } catch (err) {
@@ -284,13 +349,146 @@ export function registerEvalCommands(program: Command): void {
       }
     });
 
+  eval_
+    .command('report-link')
+    .argument('<report-id>', 'Report id printed by `eval report --server`')
+    .description('A fresh download link for a server-rendered report')
+    .action(async (reportId: string) => {
+      try { console.log((await new EvalSDK(new IndemnClient()).downloadReport(reportId)).download_url); } catch (err) { printError(err); process.exit(1); }
+    });
+
+  eval_
+    .command('trends')
+    .requiredOption('--agent <id>', 'Agent ID')
+    .option('--test-set <id>', 'Only runs of this test set')
+    .option('--limit <n>', 'Runs to include', '50')
+    .option('--json', 'Output as JSON')
+    .description('Pass rate and latency over past runs, with a drift warning when the pass rate dropped')
+    .action(async (options) => {
+      try {
+        const sdk = new EvalSDK(new IndemnClient());
+        await sdk.assertAgentInOrg(options.agent);
+        const res = await sdk.trends(options.agent, options.testSet, Number(options.limit)) as { series: Array<Record<string, unknown>>; drift: Record<string, unknown> | null };
+        if (options.json) { printJson(res); return; }
+        console.log(formatTable(['Run', 'When', 'Pass rate', 'Total', 'p95 ms'], res.series.map((r) => [
+          String(r.run_id), String(r.created_at ?? ''), `${(Number(r.pass_rate ?? 0) * 100).toFixed(1)}%`, String(r.total ?? ''), String(r.latency_p95_ms ?? ''),
+        ])));
+        if (res.drift) console.log(chalk.yellow(`Drift: pass rate fell from ${res.drift.from} to ${res.drift.to} in run ${res.drift.run_id}.`));
+      } catch (err) { printError(err); process.exit(1); }
+    });
+
+  eval_
+    .command('compare')
+    .argument('<run-a>', 'First run')
+    .argument('<run-b>', 'Second run')
+    .description('Pass rate and p95 latency difference between two runs')
+    .action(async (a: string, b: string) => {
+      try {
+        const sdk = new EvalSDK(new IndemnClient());
+        await sdk.getRunInOrg(a);
+        await sdk.getRunInOrg(b);
+        printJson(await sdk.compareRuns(a, b));
+      } catch (err) { printError(err); process.exit(1); }
+    });
+
+  eval_
+    .command('metrics')
+    .description('Voice quality metrics a test item may require (names for expected.voice.required_metrics)')
+    .action(async () => {
+      try { printJson(await new EvalSDK(new IndemnClient()).qualityMetrics()); } catch (err) { printError(err); process.exit(1); }
+    });
+
+  eval_
+    .command('transcript')
+    .requiredOption('--agent <id>', 'Agent ID (needs a stored evaluator config)')
+    .requiredOption('--baseline <file>', 'JSON list of { sender: "user"|"bot", text }')
+    .requiredOption('--replay <file>', 'JSON list of { sender: "user"|"bot", text }')
+    .option('--yes', 'Do not ask for confirmation')
+    .description('Grade a replayed conversation against a baseline one')
+    .action(async (options) => {
+      try {
+        const sdk = new EvalSDK(new IndemnClient());
+        await sdk.assertAgentInOrg(options.agent);
+        await confirmOrExit('Grade the transcript? It uses paid model calls.', Boolean(options.yes));
+        printJson(await sdk.evaluateTranscript({
+          agent_id: options.agent,
+          baseline_transcript: JSON.parse(fs.readFileSync(options.baseline, 'utf8')),
+          replay_transcript: JSON.parse(fs.readFileSync(options.replay, 'utf8')),
+        }));
+      } catch (err) { printError(err); process.exit(1); }
+    });
+
+  const prod = eval_.command('prod-calls').description('Ingested production voice calls, and turning them into regression tests');
+  prod.command('list')
+    .requiredOption('--agent <id>', 'Agent ID')
+    .option('--search <text>', 'Only calls whose transcript contains this')
+    .option('--limit <n>', 'At most this many', '50')
+    .action(async (options) => {
+      try {
+        const sdk = new EvalSDK(new IndemnClient());
+        await sdk.assertAgentInOrg(options.agent);
+        printJson(await sdk.prodCalls(options.agent, { search: options.search, limit: Number(options.limit) }));
+      } catch (err) { printError(err); process.exit(1); }
+    });
+  prod.command('promote')
+    .argument('<session-id>', 'Call session id')
+    .option('--yes', 'Do not ask for confirmation')
+    .description('Add the call as a test item to the agent\'s "Regressions (review)" test set')
+    .action(async (sessionId: string, options) => {
+      try {
+        const sdk = new EvalSDK(new IndemnClient());
+        const call = await sdk.prodCall(sessionId);
+        if (call.bot_id) await sdk.assertAgentInOrg(String(call.bot_id));
+        await confirmOrExit(`Add call ${sessionId} to the "Regressions (review)" test set?`, Boolean(options.yes));
+        printJson(await sdk.promoteProdCall(sessionId));
+      } catch (err) { printError(err); process.exit(1); }
+    });
+
+  const datasets = eval_.command('datasets').description('Low-level datasets the evaluation trigger creates');
+  datasets.command('list').requiredOption('--agent <id>', 'Agent ID').action(async (options) => {
+    try {
+      const sdk = new EvalSDK(new IndemnClient());
+      await sdk.assertAgentInOrg(options.agent);
+      printJson(await sdk.listDatasets(options.agent));
+    } catch (err) { printError(err); process.exit(1); }
+  });
+  datasets.command('cases').argument('<dataset-id>').description('Test cases of a dataset').action(async (id: string) => {
+    try {
+      const sdk = new EvalSDK(new IndemnClient());
+      const ds = await sdk.getDataset(id);
+      if (ds.agent_id) await sdk.assertAgentInOrg(String(ds.agent_id));
+      printJson(await sdk.listTestCases(id));
+    } catch (err) { printError(err); process.exit(1); }
+  });
+
+  const evaluators = eval_.command('evaluators').description('Evaluator types and the stored evaluator configs of an agent');
+  evaluators.command('types').action(async () => {
+    try { printJson(await new EvalSDK(new IndemnClient()).evaluatorTypes()); } catch (err) { printError(err); process.exit(1); }
+  });
+  evaluators.command('list').requiredOption('--agent <id>', 'Agent ID').action(async (options) => {
+    try {
+      const sdk = new EvalSDK(new IndemnClient());
+      await sdk.assertAgentInOrg(options.agent);
+      printJson(await sdk.listEvaluatorConfigs(options.agent));
+    } catch (err) { printError(err); process.exit(1); }
+  });
+  evaluators.command('delete').argument('<config-id>').option('--yes', 'Do not ask for confirmation').action(async (id: string, options) => {
+    try {
+      const sdk = new EvalSDK(new IndemnClient());
+      const cfg = await sdk.getEvaluatorConfig(id);
+      if (cfg.agent_id) await sdk.assertAgentInOrg(String(cfg.agent_id));
+      await confirmOrExit(`Delete evaluator config ${id}?`, Boolean(options.yes));
+      printJson(await sdk.deleteEvaluatorConfig(id));
+    } catch (err) { printError(err); process.exit(1); }
+  });
+
   // Rubric commands — registered on program, not nested under eval
   const rubric = program.command('rubric').description('Manage evaluation rubrics');
 
   rubric
     .command('list')
-    .description('List rubrics')
-    .option('--agent-id <id>', 'Filter by agent ID')
+    .description('List rubrics of an agent')
+    .requiredOption('--agent-id <id>', 'Agent ID (rubrics of other orgs are never listed)')
     .option('--json', 'Output as JSON')
     .option('--limit <n>', 'Limit number of results')
     .option('--page <n>', 'Page number (use with --limit)')
@@ -300,6 +498,7 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new RubricsSDK(client);
         const spinner = ora('Fetching rubrics...').start();
 
+        await new EvalSDK(client).assertAgentInOrg(options.agentId);
         let rubrics = await sdk.listRubrics(options.agentId);
         spinner.stop();
 
@@ -473,13 +672,10 @@ export function registerEvalCommands(program: Command): void {
     .command('delete')
     .argument('<id>', 'Rubric ID')
     .description('Delete a rubric')
-    .action(async (id: string) => {
+    .option('--yes', 'Do not ask for confirmation')
+    .action(async (id: string, options) => {
       try {
-        const confirmed = await confirm(`Delete rubric ${id}? This cannot be undone.`);
-        if (!confirmed) {
-          console.log('Cancelled.');
-          return;
-        }
+        await confirmOrExit(`Delete rubric ${id}? This cannot be undone.`, Boolean(options.yes));
 
         const client = new IndemnClient();
         const sdk = new RubricsSDK(client);
@@ -537,8 +733,8 @@ export function registerEvalCommands(program: Command): void {
 
   testset
     .command('list')
-    .description('List test sets')
-    .option('--agent-id <id>', 'Filter by agent ID')
+    .description('List test sets of an agent')
+    .requiredOption('--agent-id <id>', 'Agent ID (test sets of other orgs are never listed)')
     .option('--json', 'Output as JSON')
     .option('--limit <n>', 'Limit number of results')
     .option('--page <n>', 'Page number (use with --limit)')
@@ -548,17 +744,8 @@ export function registerEvalCommands(program: Command): void {
         const sdk = new TestSetsSDK(client);
         const spinner = ora('Fetching test sets...').start();
 
-        let sets: any[];
-        try {
-          sets = await sdk.listTestSets(options.agentId);
-        } catch (err: any) {
-          spinner.stop();
-          if (!options.agentId && err?.message?.includes('500')) {
-            printError('The server returned an error listing all test sets. Try filtering by agent: indemn testset list --agent-id <id>');
-            process.exit(1);
-          }
-          throw err;
-        }
+        await new EvalSDK(client).assertAgentInOrg(options.agentId);
+        let sets: any[] = await sdk.listTestSets(options.agentId);
         spinner.stop();
 
         if (options.page && options.limit) {
@@ -731,13 +918,10 @@ export function registerEvalCommands(program: Command): void {
     .command('delete')
     .argument('<id>', 'Test set ID')
     .description('Delete a test set')
-    .action(async (id: string) => {
+    .option('--yes', 'Do not ask for confirmation')
+    .action(async (id: string, options) => {
       try {
-        const confirmed = await confirm(`Delete test set ${id}? This cannot be undone.`);
-        if (!confirmed) {
-          console.log('Cancelled.');
-          return;
-        }
+        await confirmOrExit(`Delete test set ${id}? This cannot be undone.`, Boolean(options.yes));
 
         const client = new IndemnClient();
         const sdk = new TestSetsSDK(client);