@indemn/cli 1.6.0 → 1.7.0

package/dist/cli/eval.js

@@ -2,45 +2,60 @@ import * as fs from 'fs';
 import ora from 'ora';
 import chalk from 'chalk';
 import { IndemnClient } from '../sdk/client.js';
-import { EvalSDK } from '../sdk/eval.js';
+import { EvalSDK, gateVerdict } from '../sdk/eval.js';
+import { CliError, EXIT } from '../core/runtime.js';
+import { confirmOrExit } from './link.js';
 import { RubricsSDK } from '../sdk/rubrics.js';
 import { TestSetsSDK } from '../sdk/test-sets.js';
 import { exportEvalMarkdown, exportEvalReport } from '../report/export.js';
-import { formatTable, printJson, printError, printSuccess, confirm } from './utils/output.js';
+import { formatTable, printJson, printError, printSuccess } from './utils/output.js';
 export function registerEvalCommands(program) {
     const eval_ = program.command('eval').description('Run and manage evaluations');
     eval_
         .command('run')
         .argument('<agent-id>', 'Agent ID')
-        .description('Run an evaluation')
-        .requiredOption('--test-set <id>', 'Test set ID')
+        .description('Run an evaluation (costs model calls; asks first). With a threshold it becomes a CI gate')
+        .requiredOption('--test-set <id>', 'Test set ID (must belong to this agent)')
         .option('--rubric <id>', 'Rubric ID')
         .option('--concurrency <n>', 'Concurrency level')
         .option('--wait', 'Wait for evaluation to complete')
+        .option('--timeout <seconds>', 'With --wait: give up after this long', '1800')
         .option('--eval-model <model>', 'Model for evaluation judging')
-        .option('--keep', 'Keep evaluation data after completion')
         .option('--limit <n>', 'Limit number of items to evaluate')
         .option('--component-scope <scope>', 'Component scope filter')
         .option('--component-ids <ids>', 'Comma-separated component IDs')
-        .option('--conversation-ids <ids>', 'Comma-separated conversation IDs')
-        .option('--mode <mode>', 'Evaluation mode (e.g. "transcript")')
+        .option('--conversation-ids <ids>', 'Comma-separated conversation IDs (transcript mode)')
+        .option('--date-from <date>', 'Grade real conversations from this date (transcript mode)')
+        .option('--date-to <date>', 'Up to this date, exclusive (needs --date-from)')
+        .option('--mode <mode>', 'Evaluation mode: "transcript" grades real conversations')
+        .option('--webhook-url <url>', 'Called when the run finishes')
+        .option('--min-pass-rate <rate>', 'CI gate: exit 1 when passed/total is below this (0..1). Implies --wait')
+        .option('--max-p95-ms <ms>', 'CI gate: exit 1 when p95 latency is above this. Implies --wait')
+        .option('--error-policy <policy>', 'CI gate: block (a failed run fails the gate) or warn', 'block')
+        .option('--fail-on-error', 'CI gate: also exit 1 when any case had an infrastructure error. Implies --wait')
+        .option('--allow-other-agent', 'Run a test set written for another agent')
+        .option('--yes', 'Do not ask for confirmation')
+        .option('--json', 'Print the final run as JSON')
         .action(async (agentId, options) => {
         try {
             const client = new IndemnClient();
             const sdk = new EvalSDK(client);
-            const spinner = ora('Triggering evaluation...').start();
-            const input = {
-                bot_id: agentId,
-                test_set_id: options.testSet,
-            };
+            await sdk.assertAgentInOrg(agentId);
+            const testSet = await new TestSetsSDK(client).getTestSet(options.testSet);
+            const owner = testSet.agent_id;
+            if (owner && owner !== agentId && !options.allowOtherAgent) {
+                throw new CliError(`Test set ${options.testSet} was written for agent ${owner}, not ${agentId}. Pass --allow-other-agent to run it anyway.`, EXIT.NEEDS_INPUT);
+            }
+            const input = { bot_id: agentId, test_set_id: options.testSet };
             if (options.rubric)
                 input.rubric_id = options.rubric;
             if (options.concurrency)
                 input.concurrency = parseInt(options.concurrency, 10);
+            // Voice simulations share call rooms; the dashboard runs them one at a time.
+            if ((testSet.items ?? []).some((i) => i.type === 'voice_simulation'))
+                input.concurrency = 1;
             if (options.evalModel)
                 input.eval_model = options.evalModel;
-            if (options.keep !== undefined)
-                input.keep = options.keep;
             if (options.limit)
                 input.limit = parseInt(options.limit, 10);
             if (options.componentScope)
@@ -49,33 +64,67 @@ export function registerEvalCommands(program) {
                 input.component_ids = options.componentIds.split(',');
             if (options.conversationIds)
                 input.conversation_ids = options.conversationIds.split(',');
-            if (options.mode)
+            if (options.webhookUrl)
+                input.webhook_url = options.webhookUrl;
+            if (Boolean(options.dateFrom) !== Boolean(options.dateTo)) {
+                throw new CliError('--date-from and --date-to go together.', EXIT.NEEDS_INPUT);
+            }
+            if (options.dateFrom) {
+                input.date_from = options.dateFrom;
+                input.date_to = options.dateTo;
+                if (options.mode && options.mode !== 'transcript')
+                    throw new CliError('A date range only works with --mode transcript.', EXIT.NEEDS_INPUT);
+                // The service ignores dates outside transcript mode and would run the test set instead.
+                input.mode = 'transcript';
+            }
+            else if (options.mode) {
                 input.mode = options.mode;
+            }
+            const items = testSet.items?.length ?? 0;
+            const count = input.mode === 'transcript' ? 'real conversations' : `${input.limit ? Math.min(input.limit, items) : items} item(s)`;
+            await confirmOrExit(`Run an evaluation of agent ${agentId} on ${count} with concurrency ${input.concurrency ?? 1}? It uses paid model calls and cannot be cancelled.`, Boolean(options.yes));
+            const gate = {
+                minPassRate: options.minPassRate !== undefined ? Number(options.minPassRate) : undefined,
+                maxP95Ms: options.maxP95Ms !== undefined ? Number(options.maxP95Ms) : undefined,
+                errorPolicy: options.errorPolicy === 'warn' ? 'warn' : 'block',
+                failOnError: Boolean(options.failOnError),
+            };
+            const gating = gate.minPassRate !== undefined || gate.maxP95Ms !== undefined || gate.failOnError;
+            const spinner = ora('Triggering evaluation...').start();
             const result = await sdk.triggerEvaluation(input);
             spinner.stop();
-            printSuccess(`Evaluation started: ${result.run_id}`);
-            console.log(`  Status: ${result.status}`);
-            if (result.total)
-                console.log(`  Total items: ${result.total}`);
-            if (options.wait) {
-                const pollSpinner = ora('Waiting for evaluation to complete...').start();
-                try {
-                    const run = await sdk.pollRunUntilComplete(result.run_id);
-                    pollSpinner.stop();
-                    printSuccess(`Evaluation completed: ${run.status}`);
-                    if (run.passed !== undefined)
-                        console.log(`  Passed: ${run.passed}`);
-                    if (run.failed !== undefined)
-                        console.log(`  Failed: ${run.failed}`);
-                    if (run.criteria_passed !== undefined && run.criteria_total !== undefined) {
-                        console.log(`  Criteria: ${run.criteria_passed}/${run.criteria_total}`);
-                    }
-                }
-                catch (err) {
-                    pollSpinner.fail('Evaluation did not complete in time');
-                    throw err;
-                }
+            if (!options.json) {
+                printSuccess(`Evaluation started: ${result.run_id}`);
+                if (result.total)
+                    console.log(`  Total items: ${result.total}`);
             }
+            if (!options.wait && !gating)
+                return;
+            const pollSpinner = options.json ? null : ora('Waiting for evaluation to complete...').start();
+            const run = await sdk.pollRunUntilComplete(result.run_id, 10_000, Number(options.timeout) * 1000).catch((err) => {
+                pollSpinner?.fail('Evaluation did not complete in time');
+                throw err;
+            });
+            pollSpinner?.stop();
+            const verdict = gateVerdict(run, gate);
+            if (options.json) {
+                printJson({ run, gate: gating ? verdict : undefined });
+            }
+            else {
+                printSuccess(`Evaluation ${run.status}: ${run.passed ?? 0}/${run.total ?? 0} passed (${(verdict.passRate * 100).toFixed(1)}%)`);
+                if (run.errors)
+                    console.log(chalk.yellow(`  Infrastructure errors: ${run.errors}`));
+                if (run.latency?.p95_ms !== undefined)
+                    console.log(`  p95 latency: ${run.latency.p95_ms} ms`);
+                if (run.criteria_passed !== undefined && run.criteria_total !== undefined)
+                    console.log(`  Criteria: ${run.criteria_passed}/${run.criteria_total}`);
+            }
+            if (gating && !verdict.pass) {
+                console.error(chalk.red(`Gate failed: ${verdict.reasons.join('; ')}`));
+                process.exit(1);
+            }
+            if (gating && !options.json)
+                printSuccess('Gate passed.');
         }
         catch (err) {
             printError(err);
@@ -84,8 +133,8 @@ export function registerEvalCommands(program) {
     });
     eval_
         .command('list')
-        .description('List evaluation runs')
-        .option('--agent-id <id>', 'Filter by agent ID')
+        .description('List evaluation runs of an agent')
+        .requiredOption('--agent-id <id>', 'Agent ID (runs of other orgs are never listed)')
         .option('--json', 'Output as JSON')
         .option('--limit <n>', 'Limit number of results')
         .option('--page <n>', 'Page number (use with --limit)')
@@ -94,6 +143,7 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new EvalSDK(client);
             const spinner = ora('Fetching evaluation runs...').start();
+            await sdk.assertAgentInOrg(options.agentId);
             let runs = await sdk.listRuns(options.agentId);
             spinner.stop();
             if (options.page && options.limit) {
@@ -137,7 +187,7 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new EvalSDK(client);
             const spinner = ora('Fetching run status...').start();
-            const run = await sdk.getRunStatus(runId);
+            const run = await sdk.getRunInOrg(runId);
             spinner.stop();
             if (options.json) {
                 printJson(run);
@@ -181,6 +231,7 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new EvalSDK(client);
             const spinner = ora('Fetching results...').start();
+            await sdk.getRunInOrg(runId);
             const results = await sdk.getRunResults(runId);
             spinner.stop();
             if (options.json) {
@@ -209,6 +260,7 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new EvalSDK(client);
             const spinner = ora('Fetching bot context...').start();
+            await sdk.assertAgentInOrg(agentId);
             const context = await sdk.getBotContext(agentId);
             spinner.stop();
             if (options.json) {
@@ -248,6 +300,7 @@ export function registerEvalCommands(program) {
         try {
             const client = new IndemnClient();
             const spinner = ora('Fetching evaluation data...').start();
+            await new EvalSDK(client).getRunInOrg(runId);
             const result = await exportEvalMarkdown(runId, options.output, client);
             spinner.stop();
             printSuccess(`Exported to ${result.file_path}`);
@@ -260,13 +313,33 @@ export function registerEvalCommands(program) {
     eval_
         .command('report')
         .argument('<run-id>', 'Evaluation run ID')
-        .description('Generate branded evaluation report PDF')
+        .description('Branded evaluation report PDF (made here, or by the evaluations service with --server)')
         .option('--output <path>', 'Output file path')
+        .option('--server', 'Ask the evaluations service to render it (completed runs only)')
+        .option('--force', 'With --server: render again instead of reusing the last report')
         .action(async (runId, options) => {
         try {
             const client = new IndemnClient();
-            const spinner = ora('Fetching evaluation data...').start();
-            spinner.text = 'Generating PDF...';
+            const sdk = new EvalSDK(client);
+            await sdk.getRunInOrg(runId);
+            if (options.server) {
+                const spinner = ora('Rendering on the evaluations service...').start();
+                const { download_url, report_id } = await sdk.exportReport(runId, Boolean(options.force));
+                spinner.stop();
+                if (options.output) {
+                    const res = await fetch(download_url);
+                    if (!res.ok)
+                        throw new Error(`Download failed (${res.status}).`);
+                    fs.writeFileSync(options.output, Buffer.from(await res.arrayBuffer()));
+                    printSuccess(`Report saved to ${options.output}`);
+                }
+                else {
+                    console.log(download_url);
+                    console.log(chalk.dim(`Link valid for 15 minutes. New link later: indemn eval report-link ${report_id}`));
+                }
+                return;
+            }
+            const spinner = ora('Generating PDF...').start();
             const result = await exportEvalReport(runId, options.output, client);
             spinner.stop();
             printSuccess(`Report saved to ${result.file_path}`);
@@ -276,12 +349,198 @@ export function registerEvalCommands(program) {
             process.exit(1);
         }
     });
+    eval_
+        .command('report-link')
+        .argument('<report-id>', 'Report id printed by `eval report --server`')
+        .description('A fresh download link for a server-rendered report')
+        .action(async (reportId) => {
+        try {
+            console.log((await new EvalSDK(new IndemnClient()).downloadReport(reportId)).download_url);
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    eval_
+        .command('trends')
+        .requiredOption('--agent <id>', 'Agent ID')
+        .option('--test-set <id>', 'Only runs of this test set')
+        .option('--limit <n>', 'Runs to include', '50')
+        .option('--json', 'Output as JSON')
+        .description('Pass rate and latency over past runs, with a drift warning when the pass rate dropped')
+        .action(async (options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.assertAgentInOrg(options.agent);
+            const res = await sdk.trends(options.agent, options.testSet, Number(options.limit));
+            if (options.json) {
+                printJson(res);
+                return;
+            }
+            console.log(formatTable(['Run', 'When', 'Pass rate', 'Total', 'p95 ms'], res.series.map((r) => [
+                String(r.run_id), String(r.created_at ?? ''), `${(Number(r.pass_rate ?? 0) * 100).toFixed(1)}%`, String(r.total ?? ''), String(r.latency_p95_ms ?? ''),
+            ])));
+            if (res.drift)
+                console.log(chalk.yellow(`Drift: pass rate fell from ${res.drift.from} to ${res.drift.to} in run ${res.drift.run_id}.`));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    eval_
+        .command('compare')
+        .argument('<run-a>', 'First run')
+        .argument('<run-b>', 'Second run')
+        .description('Pass rate and p95 latency difference between two runs')
+        .action(async (a, b) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.getRunInOrg(a);
+            await sdk.getRunInOrg(b);
+            printJson(await sdk.compareRuns(a, b));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    eval_
+        .command('metrics')
+        .description('Voice quality metrics a test item may require (names for expected.voice.required_metrics)')
+        .action(async () => {
+        try {
+            printJson(await new EvalSDK(new IndemnClient()).qualityMetrics());
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    eval_
+        .command('transcript')
+        .requiredOption('--agent <id>', 'Agent ID (needs a stored evaluator config)')
+        .requiredOption('--baseline <file>', 'JSON list of { sender: "user"|"bot", text }')
+        .requiredOption('--replay <file>', 'JSON list of { sender: "user"|"bot", text }')
+        .option('--yes', 'Do not ask for confirmation')
+        .description('Grade a replayed conversation against a baseline one')
+        .action(async (options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.assertAgentInOrg(options.agent);
+            await confirmOrExit('Grade the transcript? It uses paid model calls.', Boolean(options.yes));
+            printJson(await sdk.evaluateTranscript({
+                agent_id: options.agent,
+                baseline_transcript: JSON.parse(fs.readFileSync(options.baseline, 'utf8')),
+                replay_transcript: JSON.parse(fs.readFileSync(options.replay, 'utf8')),
+            }));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    const prod = eval_.command('prod-calls').description('Ingested production voice calls, and turning them into regression tests');
+    prod.command('list')
+        .requiredOption('--agent <id>', 'Agent ID')
+        .option('--search <text>', 'Only calls whose transcript contains this')
+        .option('--limit <n>', 'At most this many', '50')
+        .action(async (options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.assertAgentInOrg(options.agent);
+            printJson(await sdk.prodCalls(options.agent, { search: options.search, limit: Number(options.limit) }));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    prod.command('promote')
+        .argument('<session-id>', 'Call session id')
+        .option('--yes', 'Do not ask for confirmation')
+        .description('Add the call as a test item to the agent\'s "Regressions (review)" test set')
+        .action(async (sessionId, options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            const call = await sdk.prodCall(sessionId);
+            if (call.bot_id)
+                await sdk.assertAgentInOrg(String(call.bot_id));
+            await confirmOrExit(`Add call ${sessionId} to the "Regressions (review)" test set?`, Boolean(options.yes));
+            printJson(await sdk.promoteProdCall(sessionId));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    const datasets = eval_.command('datasets').description('Low-level datasets the evaluation trigger creates');
+    datasets.command('list').requiredOption('--agent <id>', 'Agent ID').action(async (options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.assertAgentInOrg(options.agent);
+            printJson(await sdk.listDatasets(options.agent));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    datasets.command('cases').argument('<dataset-id>').description('Test cases of a dataset').action(async (id) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            const ds = await sdk.getDataset(id);
+            if (ds.agent_id)
+                await sdk.assertAgentInOrg(String(ds.agent_id));
+            printJson(await sdk.listTestCases(id));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    const evaluators = eval_.command('evaluators').description('Evaluator types and the stored evaluator configs of an agent');
+    evaluators.command('types').action(async () => {
+        try {
+            printJson(await new EvalSDK(new IndemnClient()).evaluatorTypes());
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    evaluators.command('list').requiredOption('--agent <id>', 'Agent ID').action(async (options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            await sdk.assertAgentInOrg(options.agent);
+            printJson(await sdk.listEvaluatorConfigs(options.agent));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
+    evaluators.command('delete').argument('<config-id>').option('--yes', 'Do not ask for confirmation').action(async (id, options) => {
+        try {
+            const sdk = new EvalSDK(new IndemnClient());
+            const cfg = await sdk.getEvaluatorConfig(id);
+            if (cfg.agent_id)
+                await sdk.assertAgentInOrg(String(cfg.agent_id));
+            await confirmOrExit(`Delete evaluator config ${id}?`, Boolean(options.yes));
+            printJson(await sdk.deleteEvaluatorConfig(id));
+        }
+        catch (err) {
+            printError(err);
+            process.exit(1);
+        }
+    });
     // Rubric commands — registered on program, not nested under eval
     const rubric = program.command('rubric').description('Manage evaluation rubrics');
     rubric
         .command('list')
-        .description('List rubrics')
-        .option('--agent-id <id>', 'Filter by agent ID')
+        .description('List rubrics of an agent')
+        .requiredOption('--agent-id <id>', 'Agent ID (rubrics of other orgs are never listed)')
         .option('--json', 'Output as JSON')
         .option('--limit <n>', 'Limit number of results')
         .option('--page <n>', 'Page number (use with --limit)')
@@ -290,6 +549,7 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new RubricsSDK(client);
             const spinner = ora('Fetching rubrics...').start();
+            await new EvalSDK(client).assertAgentInOrg(options.agentId);
             let rubrics = await sdk.listRubrics(options.agentId);
             spinner.stop();
             if (options.page && options.limit) {
@@ -457,13 +717,10 @@ export function registerEvalCommands(program) {
         .command('delete')
         .argument('<id>', 'Rubric ID')
         .description('Delete a rubric')
-        .action(async (id) => {
+        .option('--yes', 'Do not ask for confirmation')
+        .action(async (id, options) => {
         try {
-            const confirmed = await confirm(`Delete rubric ${id}? This cannot be undone.`);
-            if (!confirmed) {
-                console.log('Cancelled.');
-                return;
-            }
+            await confirmOrExit(`Delete rubric ${id}? This cannot be undone.`, Boolean(options.yes));
             const client = new IndemnClient();
             const sdk = new RubricsSDK(client);
             const spinner = ora('Deleting rubric...').start();
@@ -512,8 +769,8 @@ export function registerEvalCommands(program) {
     const testset = program.command('testset').description('Manage evaluation test sets');
     testset
         .command('list')
-        .description('List test sets')
-        .option('--agent-id <id>', 'Filter by agent ID')
+        .description('List test sets of an agent')
+        .requiredOption('--agent-id <id>', 'Agent ID (test sets of other orgs are never listed)')
         .option('--json', 'Output as JSON')
         .option('--limit <n>', 'Limit number of results')
         .option('--page <n>', 'Page number (use with --limit)')
@@ -522,18 +779,8 @@ export function registerEvalCommands(program) {
             const client = new IndemnClient();
             const sdk = new TestSetsSDK(client);
             const spinner = ora('Fetching test sets...').start();
-            let sets;
-            try {
-                sets = await sdk.listTestSets(options.agentId);
-            }
-            catch (err) {
-                spinner.stop();
-                if (!options.agentId && err?.message?.includes('500')) {
-                    printError('The server returned an error listing all test sets. Try filtering by agent: indemn testset list --agent-id <id>');
-                    process.exit(1);
-                }
-                throw err;
-            }
+            await new EvalSDK(client).assertAgentInOrg(options.agentId);
+            let sets = await sdk.listTestSets(options.agentId);
             spinner.stop();
             if (options.page && options.limit) {
                 const page = parseInt(options.page, 10);
@@ -700,13 +947,10 @@ export function registerEvalCommands(program) {
         .command('delete')
         .argument('<id>', 'Test set ID')
         .description('Delete a test set')
-        .action(async (id) => {
+        .option('--yes', 'Do not ask for confirmation')
+        .action(async (id, options) => {
         try {
-            const confirmed = await confirm(`Delete test set ${id}? This cannot be undone.`);
-            if (!confirmed) {
-                console.log('Cancelled.');
-                return;
-            }
+            await confirmOrExit(`Delete test set ${id}? This cannot be undone.`, Boolean(options.yes));
             const client = new IndemnClient();
             const sdk = new TestSetsSDK(client);
             const spinner = ora('Deleting test set...').start();