@indemn/cli 1.0.0

package/src/cli/eval.ts

@@ -0,0 +1,750 @@
+import { Command } from 'commander';
+import * as fs from 'fs';
+import ora from 'ora';
+import chalk from 'chalk';
+import { IndemnClient } from '../sdk/client.js';
+import { EvalSDK } from '../sdk/eval.js';
+import { RubricsSDK } from '../sdk/rubrics.js';
+import { TestSetsSDK } from '../sdk/test-sets.js';
+import { formatTable, printJson, printError, printSuccess, confirm } from './utils/output.js';
+
+export function registerEvalCommands(program: Command): void {
+  const eval_ = program.command('eval').description('Run and manage evaluations');
+
+  eval_
+    .command('run')
+    .argument('<agent-id>', 'Agent ID')
+    .description('Run an evaluation')
+    .requiredOption('--test-set <id>', 'Test set ID')
+    .option('--rubric <id>', 'Rubric ID')
+    .option('--concurrency <n>', 'Concurrency level')
+    .option('--wait', 'Wait for evaluation to complete')
+    .option('--eval-model <model>', 'Model for evaluation judging')
+    .option('--keep', 'Keep evaluation data after completion')
+    .option('--limit <n>', 'Limit number of items to evaluate')
+    .option('--component-scope <scope>', 'Component scope filter')
+    .option('--component-ids <ids>', 'Comma-separated component IDs')
+    .option('--conversation-ids <ids>', 'Comma-separated conversation IDs')
+    .option('--mode <mode>', 'Evaluation mode (e.g. "transcript")')
+    .action(async (agentId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new EvalSDK(client);
+        const spinner = ora('Triggering evaluation...').start();
+
+        const input: Record<string, unknown> = {
+          bot_id: agentId,
+          test_set_id: options.testSet,
+        };
+        if (options.rubric) input.rubric_id = options.rubric;
+        if (options.concurrency) input.concurrency = parseInt(options.concurrency, 10);
+        if (options.evalModel) input.eval_model = options.evalModel;
+        if (options.keep !== undefined) input.keep = options.keep;
+        if (options.limit) input.limit = parseInt(options.limit, 10);
+        if (options.componentScope) input.component_scope = options.componentScope;
+        if (options.componentIds) input.component_ids = options.componentIds.split(',');
+        if (options.conversationIds) input.conversation_ids = options.conversationIds.split(',');
+        if (options.mode) input.mode = options.mode;
+
+        const result = await sdk.triggerEvaluation(input as any);
+        spinner.stop();
+
+        printSuccess(`Evaluation started: ${result.run_id}`);
+        console.log(`  Status: ${result.status}`);
+        if (result.total) console.log(`  Total items: ${result.total}`);
+
+        if (options.wait) {
+          const pollSpinner = ora('Waiting for evaluation to complete...').start();
+
+          try {
+            const run = await sdk.pollRunUntilComplete(result.run_id);
+            pollSpinner.stop();
+
+            printSuccess(`Evaluation completed: ${run.status}`);
+            if (run.passed !== undefined) console.log(`  Passed: ${run.passed}`);
+            if (run.failed !== undefined) console.log(`  Failed: ${run.failed}`);
+            if (run.criteria_passed !== undefined && run.criteria_total !== undefined) {
+              console.log(`  Criteria: ${run.criteria_passed}/${run.criteria_total}`);
+            }
+          } catch (err) {
+            pollSpinner.fail('Evaluation did not complete in time');
+            throw err;
+          }
+        }
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  eval_
+    .command('list')
+    .description('List evaluation runs')
+    .option('--agent-id <id>', 'Filter by agent ID')
+    .option('--json', 'Output as JSON')
+    .option('--limit <n>', 'Limit number of results')
+    .option('--page <n>', 'Page number (use with --limit)')
+    .action(async (options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new EvalSDK(client);
+        const spinner = ora('Fetching evaluation runs...').start();
+
+        let runs = await sdk.listRuns(options.agentId);
+        spinner.stop();
+
+        if (options.page && options.limit) {
+          const page = parseInt(options.page, 10);
+          const limit = parseInt(options.limit, 10);
+          runs = runs.slice((page - 1) * limit, page * limit);
+        } else if (options.limit) {
+          runs = runs.slice(0, parseInt(options.limit, 10));
+        }
+
+        if (options.json) {
+          printJson(runs);
+          return;
+        }
+
+        if (runs.length === 0) {
+          console.log('No evaluation runs found.');
+          return;
+        }
+
+        const rows = runs.map((r) => [
+          r.run_id || '',
+          r.status || '',
+          r.agent_id || '',
+          r.passed !== undefined ? String(r.passed) : '',
+          r.failed !== undefined ? String(r.failed) : '',
+          r.started_at ? new Date(r.started_at).toLocaleDateString() : '',
+        ]);
+
+        console.log(formatTable(['Run ID', 'Status', 'Agent', 'Passed', 'Failed', 'Started'], rows));
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  eval_
+    .command('status')
+    .argument('<run-id>', 'Evaluation run ID')
+    .description('Get evaluation run status')
+    .option('--json', 'Output as JSON')
+    .action(async (runId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new EvalSDK(client);
+        const spinner = ora('Fetching run status...').start();
+
+        const run = await sdk.getRunStatus(runId);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(run);
+          return;
+        }
+
+        const statusColor = run.status === 'completed' ? chalk.green : run.status === 'failed' ? chalk.red : chalk.yellow;
+
+        console.log(`Run ID:   ${run.run_id}`);
+        console.log(`Status:   ${statusColor(run.status)}`);
+        if (run.agent_id) console.log(`Agent:    ${run.agent_id}`);
+        if (run.total !== undefined) console.log(`Total:    ${run.total}`);
+        if (run.completed !== undefined) console.log(`Done:     ${run.completed}`);
+        if (run.passed !== undefined) console.log(`Passed:   ${run.passed}`);
+        if (run.failed !== undefined) console.log(`Failed:   ${run.failed}`);
+        if (run.criteria_passed !== undefined && run.criteria_total !== undefined) {
+          console.log(`Criteria: ${run.criteria_passed}/${run.criteria_total}`);
+        }
+        if (run.started_at) console.log(`Started:  ${run.started_at}`);
+        if (run.completed_at) console.log(`Finished: ${run.completed_at}`);
+        if (run.error) console.log(`Error:    ${chalk.red(run.error)}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  eval_
+    .command('results')
+    .argument('<run-id>', 'Evaluation run ID')
+    .description('Get evaluation results')
+    .option('--json', 'Output as JSON')
+    .action(async (runId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new EvalSDK(client);
+        const spinner = ora('Fetching results...').start();
+
+        const results = await sdk.getRunResults(runId);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(results);
+          return;
+        }
+
+        if (results.length === 0) {
+          console.log('No results available.');
+          return;
+        }
+
+        // Print results as formatted JSON since structure varies
+        printJson(results);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  eval_
+    .command('bot-context')
+    .argument('<agent-id>', 'Agent ID')
+    .description('Get bot context used for evaluations')
+    .option('--json', 'Output as JSON')
+    .action(async (agentId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new EvalSDK(client);
+        const spinner = ora('Fetching bot context...').start();
+
+        const context = await sdk.getBotContext(agentId);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(context);
+          return;
+        }
+
+        if (context.system_prompt) {
+          console.log(chalk.bold('System Prompt:'));
+          console.log(context.system_prompt);
+          console.log();
+        }
+        if (context.tools && Array.isArray(context.tools)) {
+          console.log(chalk.bold(`Tools (${context.tools.length}):`));
+          for (const tool of context.tools) {
+            console.log(`  - ${tool.name || tool._id}: ${tool.description || ''}`);
+          }
+          console.log();
+        }
+        if (context.knowledge_bases && Array.isArray(context.knowledge_bases)) {
+          console.log(chalk.bold(`Knowledge Bases (${context.knowledge_bases.length}):`));
+          for (const kb of context.knowledge_bases) {
+            console.log(`  - ${kb.name || kb._id}: ${kb.type || ''}`);
+          }
+        }
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  // Rubric commands — registered on program, not nested under eval
+  const rubric = program.command('rubric').description('Manage evaluation rubrics');
+
+  rubric
+    .command('list')
+    .description('List rubrics')
+    .option('--agent-id <id>', 'Filter by agent ID')
+    .option('--json', 'Output as JSON')
+    .option('--limit <n>', 'Limit number of results')
+    .option('--page <n>', 'Page number (use with --limit)')
+    .action(async (options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Fetching rubrics...').start();
+
+        let rubrics = await sdk.listRubrics(options.agentId);
+        spinner.stop();
+
+        if (options.page && options.limit) {
+          const page = parseInt(options.page, 10);
+          const limit = parseInt(options.limit, 10);
+          rubrics = rubrics.slice((page - 1) * limit, page * limit);
+        } else if (options.limit) {
+          rubrics = rubrics.slice(0, parseInt(options.limit, 10));
+        }
+
+        if (options.json) {
+          printJson(rubrics);
+          return;
+        }
+
+        if (rubrics.length === 0) {
+          console.log('No rubrics found.');
+          return;
+        }
+
+        const rows = rubrics.map((r) => [
+          r._id || r.rubric_id || '',
+          r.name,
+          String(r.version || 1),
+          String(r.rules?.length || 0),
+          r.updatedAt ? new Date(r.updatedAt).toLocaleDateString() : '',
+        ]);
+
+        console.log(formatTable(['ID', 'Name', 'Version', 'Rules', 'Updated'], rows));
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  rubric
+    .command('get')
+    .argument('<id>', 'Rubric ID')
+    .description('Get rubric details')
+    .option('--json', 'Output as JSON')
+    .option('--revision <n>', 'Get a specific version/revision')
+    .action(async (id: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Fetching rubric...').start();
+
+        const r = options.revision
+          ? await sdk.getVersion(id, parseInt(options.revision, 10))
+          : await sdk.getRubric(id);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(r);
+          return;
+        }
+
+        console.log(`ID:          ${r._id || r.rubric_id}`);
+        console.log(`Name:        ${r.name}`);
+        console.log(`Description: ${r.description || '(none)'}`);
+        console.log(`Version:     ${r.version || 1}`);
+        console.log(`Rules:       ${r.rules?.length || 0}`);
+        if (r.rules?.length) {
+          console.log();
+          for (const rule of r.rules) {
+            console.log(`  [${rule.severity}] ${rule.name}: ${rule.description}`);
+          }
+        }
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  rubric
+    .command('create')
+    .description('Create a rubric')
+    .option('--file <path>', 'Path to JSON file with rubric definition')
+    .option('--name <name>', 'Rubric name')
+    .option('--agent-id <id>', 'Agent ID')
+    .action(async (options) => {
+      try {
+        let input: Record<string, unknown>;
+
+        if (options.file) {
+          const filePath = options.file as string;
+          if (!fs.existsSync(filePath)) {
+            printError(`File not found: ${filePath}`);
+            process.exit(1);
+          }
+          const content = fs.readFileSync(filePath, 'utf-8');
+          try {
+            input = JSON.parse(content);
+          } catch {
+            printError('File must contain valid JSON.');
+            process.exit(1);
+          }
+          // Merge CLI options into file content (CLI flags override file values)
+          if (options.name) input.name = options.name;
+          if (options.agentId) input.agent_id = options.agentId;
+        } else if (options.name) {
+          input = { name: options.name, rules: [] };
+          if (options.agentId) input.agent_id = options.agentId;
+        } else {
+          printError('Provide --file with rubric JSON or --name for an empty rubric.');
+          process.exit(1);
+        }
+
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Creating rubric...').start();
+
+        const rubricResult = await sdk.createRubric(input as any);
+        spinner.stop();
+
+        printSuccess(`Rubric created: ${rubricResult._id || rubricResult.rubric_id}`);
+        console.log(`  Name:  ${rubricResult.name}`);
+        console.log(`  Rules: ${rubricResult.rules?.length || 0}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  rubric
+    .command('update')
+    .argument('<id>', 'Rubric ID')
+    .description('Update a rubric')
+    .option('--file <path>', 'Path to JSON file with rubric updates')
+    .option('--name <name>', 'New rubric name')
+    .action(async (id: string, options) => {
+      try {
+        let input: Record<string, unknown>;
+
+        if (options.file) {
+          const filePath = options.file as string;
+          if (!fs.existsSync(filePath)) {
+            printError(`File not found: ${filePath}`);
+            process.exit(1);
+          }
+          const content = fs.readFileSync(filePath, 'utf-8');
+          try {
+            input = JSON.parse(content);
+          } catch {
+            printError('File must contain valid JSON.');
+            process.exit(1);
+          }
+        } else if (options.name) {
+          input = { name: options.name };
+        } else {
+          printError('Provide --file with rubric JSON or --name.');
+          process.exit(1);
+        }
+
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Updating rubric...').start();
+
+        await sdk.updateRubric(id, input as any);
+        spinner.stop();
+
+        printSuccess(`Rubric ${id} updated.`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  rubric
+    .command('delete')
+    .argument('<id>', 'Rubric ID')
+    .description('Delete a rubric')
+    .action(async (id: string) => {
+      try {
+        const confirmed = await confirm(`Delete rubric ${id}? This cannot be undone.`);
+        if (!confirmed) {
+          console.log('Cancelled.');
+          return;
+        }
+
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Deleting rubric...').start();
+
+        await sdk.deleteRubric(id);
+        spinner.stop();
+
+        printSuccess(`Rubric ${id} deleted.`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  rubric
+    .command('versions')
+    .argument('<id>', 'Rubric ID')
+    .description('List rubric versions')
+    .option('--json', 'Output as JSON')
+    .action(async (id: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new RubricsSDK(client);
+        const spinner = ora('Fetching versions...').start();
+
+        const versions = await sdk.listVersions(id);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(versions);
+          return;
+        }
+
+        if (versions.length === 0) {
+          console.log('No versions found.');
+          return;
+        }
+
+        const rows = versions.map((v) => [
+          String(v.version || ''),
+          String(v.rules?.length || 0),
+          v.updatedAt ? new Date(v.updatedAt).toLocaleDateString() : '',
+        ]);
+
+        console.log(formatTable(['Version', 'Rules', 'Updated'], rows));
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  // Test set commands — registered on program, not nested under eval
+  const testset = program.command('testset').description('Manage evaluation test sets');
+
+  testset
+    .command('list')
+    .description('List test sets')
+    .option('--agent-id <id>', 'Filter by agent ID')
+    .option('--json', 'Output as JSON')
+    .option('--limit <n>', 'Limit number of results')
+    .option('--page <n>', 'Page number (use with --limit)')
+    .action(async (options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Fetching test sets...').start();
+
+        let sets: any[];
+        try {
+          sets = await sdk.listTestSets(options.agentId);
+        } catch (err: any) {
+          spinner.stop();
+          if (!options.agentId && err?.message?.includes('500')) {
+            printError('The server returned an error listing all test sets. Try filtering by agent: indemn testset list --agent-id <id>');
+            process.exit(1);
+          }
+          throw err;
+        }
+        spinner.stop();
+
+        if (options.page && options.limit) {
+          const page = parseInt(options.page, 10);
+          const limit = parseInt(options.limit, 10);
+          sets = sets.slice((page - 1) * limit, page * limit);
+        } else if (options.limit) {
+          sets = sets.slice(0, parseInt(options.limit, 10));
+        }
+
+        if (options.json) {
+          printJson(sets);
+          return;
+        }
+
+        if (sets.length === 0) {
+          console.log('No test sets found.');
+          return;
+        }
+
+        const rows = sets.map((s) => [
+          s._id || s.test_set_id || '',
+          s.name,
+          String(s.version || 1),
+          String(s.items?.length || 0),
+          s.updatedAt ? new Date(s.updatedAt).toLocaleDateString() : '',
+        ]);
+
+        console.log(formatTable(['ID', 'Name', 'Version', 'Items', 'Updated'], rows));
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  testset
+    .command('get')
+    .argument('<id>', 'Test set ID')
+    .description('Get test set details')
+    .option('--json', 'Output as JSON')
+    .option('--revision <n>', 'Get a specific version/revision')
+    .action(async (id: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Fetching test set...').start();
+
+        const ts = options.revision
+          ? await sdk.getVersion(id, parseInt(options.revision, 10))
+          : await sdk.getTestSet(id);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(ts);
+          return;
+        }
+
+        console.log(`ID:          ${ts._id || ts.test_set_id}`);
+        console.log(`Name:        ${ts.name}`);
+        console.log(`Description: ${ts.description || '(none)'}`);
+        console.log(`Version:     ${ts.version || 1}`);
+        console.log(`Items:       ${ts.items?.length || 0}`);
+        if (ts.items?.length) {
+          console.log();
+          for (const item of ts.items) {
+            console.log(`  [${item.type}] ${item.name}: ${item.inputs.message || item.inputs.initial_message || ''}`);
+          }
+        }
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  testset
+    .command('create')
+    .description('Create a test set')
+    .option('--file <path>', 'Path to JSON file with test set definition')
+    .option('--name <name>', 'Test set name')
+    .option('--agent-id <id>', 'Agent ID')
+    .action(async (options) => {
+      try {
+        let input: Record<string, unknown>;
+
+        if (options.file) {
+          const filePath = options.file as string;
+          if (!fs.existsSync(filePath)) {
+            printError(`File not found: ${filePath}`);
+            process.exit(1);
+          }
+          const content = fs.readFileSync(filePath, 'utf-8');
+          try {
+            input = JSON.parse(content);
+          } catch {
+            printError('File must contain valid JSON.');
+            process.exit(1);
+          }
+          // Merge CLI options into file content (CLI flags override file values)
+          if (options.name) input.name = options.name;
+          if (options.agentId) input.agent_id = options.agentId;
+        } else if (options.name) {
+          input = { name: options.name, items: [] };
+          if (options.agentId) input.agent_id = options.agentId;
+        } else {
+          printError('Provide --file with test set JSON or --name for an empty test set.');
+          process.exit(1);
+        }
+
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Creating test set...').start();
+
+        const testSetResult = await sdk.createTestSet(input as any);
+        spinner.stop();
+
+        printSuccess(`Test set created: ${testSetResult._id || testSetResult.test_set_id}`);
+        console.log(`  Name:  ${testSetResult.name}`);
+        console.log(`  Items: ${testSetResult.items?.length || 0}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  testset
+    .command('update')
+    .argument('<id>', 'Test set ID')
+    .description('Update a test set')
+    .option('--file <path>', 'Path to JSON file with test set updates')
+    .option('--name <name>', 'New test set name')
+    .action(async (id: string, options) => {
+      try {
+        let input: Record<string, unknown>;
+
+        if (options.file) {
+          const filePath = options.file as string;
+          if (!fs.existsSync(filePath)) {
+            printError(`File not found: ${filePath}`);
+            process.exit(1);
+          }
+          const content = fs.readFileSync(filePath, 'utf-8');
+          try {
+            input = JSON.parse(content);
+          } catch {
+            printError('File must contain valid JSON.');
+            process.exit(1);
+          }
+        } else if (options.name) {
+          input = { name: options.name };
+        } else {
+          printError('Provide --file with test set JSON or --name.');
+          process.exit(1);
+        }
+
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Updating test set...').start();
+
+        await sdk.updateTestSet(id, input as any);
+        spinner.stop();
+
+        printSuccess(`Test set ${id} updated.`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  testset
+    .command('delete')
+    .argument('<id>', 'Test set ID')
+    .description('Delete a test set')
+    .action(async (id: string) => {
+      try {
+        const confirmed = await confirm(`Delete test set ${id}? This cannot be undone.`);
+        if (!confirmed) {
+          console.log('Cancelled.');
+          return;
+        }
+
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Deleting test set...').start();
+
+        await sdk.deleteTestSet(id);
+        spinner.stop();
+
+        printSuccess(`Test set ${id} deleted.`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  testset
+    .command('versions')
+    .argument('<id>', 'Test set ID')
+    .description('List test set versions')
+    .option('--json', 'Output as JSON')
+    .action(async (id: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const sdk = new TestSetsSDK(client);
+        const spinner = ora('Fetching versions...').start();
+
+        const versions = await sdk.listVersions(id);
+        spinner.stop();
+
+        if (options.json) {
+          printJson(versions);
+          return;
+        }
+
+        if (versions.length === 0) {
+          console.log('No versions found.');
+          return;
+        }
+
+        const rows = versions.map((v) => [
+          String(v.version || ''),
+          String(v.items?.length || 0),
+          v.updatedAt ? new Date(v.updatedAt).toLocaleDateString() : '',
+        ]);
+
+        console.log(formatTable(['Version', 'Items', 'Updated'], rows));
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+}