@artemiskit/cli 0.1.2

package/src/commands/compare.ts

@@ -0,0 +1,104 @@
+/**
+ * Compare command - Compare two test runs
+ */
+
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import { Command } from 'commander';
+import { loadConfig } from '../config/loader';
+import { createStorage } from '../utils/storage';
+
+interface CompareOptions {
+  threshold?: number;
+  config?: string;
+}
+
+export function compareCommand(): Command {
+  const cmd = new Command('compare');
+
+  cmd
+    .description('Compare two test runs')
+    .argument('<baseline>', 'Baseline run ID')
+    .argument('<current>', 'Current run ID')
+    .option('--threshold <number>', 'Regression threshold (0-1)', '0.05')
+    .option('--config <path>', 'Path to config file')
+    .action(async (baselineId: string, currentId: string, options: CompareOptions) => {
+      try {
+        const config = await loadConfig(options.config);
+        const storage = createStorage({ fileConfig: config });
+
+        if (!storage.compare) {
+          console.error(chalk.red('Storage adapter does not support comparison'));
+          process.exit(1);
+        }
+
+        console.log(chalk.bold('Comparing runs...'));
+        console.log();
+
+        const comparison = await storage.compare(baselineId, currentId);
+        const { baseline, current, delta } = comparison;
+
+        // Summary table
+        const summaryTable = new Table({
+          head: [
+            chalk.bold('Metric'),
+            chalk.bold('Baseline'),
+            chalk.bold('Current'),
+            chalk.bold('Delta'),
+          ],
+          style: { head: [], border: [] },
+        });
+
+        const formatDelta = (value: number, inverse = false) => {
+          const improved = inverse ? value < 0 : value > 0;
+          const color = improved ? chalk.green : value === 0 ? chalk.dim : chalk.red;
+          const sign = value > 0 ? '+' : '';
+          return color(`${sign}${value.toFixed(2)}`);
+        };
+
+        summaryTable.push(
+          [
+            'Success Rate',
+            `${(baseline.metrics.success_rate * 100).toFixed(1)}%`,
+            `${(current.metrics.success_rate * 100).toFixed(1)}%`,
+            `${formatDelta(delta.successRate * 100)}%`,
+          ],
+          [
+            'Median Latency',
+            `${baseline.metrics.median_latency_ms}ms`,
+            `${current.metrics.median_latency_ms}ms`,
+            `${formatDelta(delta.latency, true)}ms`,
+          ],
+          [
+            'Total Tokens',
+            baseline.metrics.total_tokens.toLocaleString(),
+            current.metrics.total_tokens.toLocaleString(),
+            formatDelta(delta.tokens, true),
+          ]
+        );
+
+        console.log(summaryTable.toString());
+        console.log();
+
+        // Check for regression
+        const threshold = Number.parseFloat(String(options.threshold)) || 0.05;
+        const hasRegression = delta.successRate < -threshold;
+
+        if (hasRegression) {
+          console.log(
+            chalk.red('⚠ Regression detected!'),
+            `Success rate dropped by ${Math.abs(delta.successRate * 100).toFixed(1)}%`,
+            `(threshold: ${threshold * 100}%)`
+          );
+          process.exit(1);
+        } else {
+          console.log(chalk.green('✓ No regression detected'));
+        }
+      } catch (error) {
+        console.error(chalk.red('Error:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  return cmd;
+}

package/src/commands/history.ts

@@ -0,0 +1,80 @@
+/**
+ * History command - View run history
+ */
+
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import { Command } from 'commander';
+import { loadConfig } from '../config/loader';
+import { createStorage } from '../utils/storage';
+
+interface HistoryOptions {
+  project?: string;
+  scenario?: string;
+  limit?: number;
+  config?: string;
+}
+
+export function historyCommand(): Command {
+  const cmd = new Command('history');
+
+  cmd
+    .description('View run history')
+    .option('-p, --project <project>', 'Filter by project')
+    .option('-s, --scenario <scenario>', 'Filter by scenario')
+    .option('-l, --limit <number>', 'Limit number of results', '20')
+    .option('--config <path>', 'Path to config file')
+    .action(async (options: HistoryOptions) => {
+      try {
+        const config = await loadConfig(options.config);
+        const storage = createStorage({ fileConfig: config });
+        const limit = Number.parseInt(String(options.limit)) || 20;
+
+        const runs = await storage.list({
+          project: options.project,
+          scenario: options.scenario,
+          limit,
+        });
+
+        if (runs.length === 0) {
+          console.log(chalk.dim('No runs found.'));
+          return;
+        }
+
+        const table = new Table({
+          head: [
+            chalk.bold('Run ID'),
+            chalk.bold('Scenario'),
+            chalk.bold('Success Rate'),
+            chalk.bold('Date'),
+          ],
+          style: { head: [], border: [] },
+        });
+
+        for (const run of runs) {
+          const successColor =
+            run.successRate >= 0.9
+              ? chalk.green
+              : run.successRate >= 0.7
+                ? chalk.yellow
+                : chalk.red;
+
+          table.push([
+            run.runId,
+            run.scenario,
+            successColor(`${(run.successRate * 100).toFixed(1)}%`),
+            new Date(run.createdAt).toLocaleString(),
+          ]);
+        }
+
+        console.log(table.toString());
+        console.log();
+        console.log(chalk.dim(`Showing ${runs.length} runs`));
+      } catch (error) {
+        console.error(chalk.red('Error:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  return cmd;
+}

package/src/commands/index.ts

@@ -0,0 +1,8 @@
+/**
+ * CLI commands exports
+ */
+
+export { initCommand } from './init';
+export { runCommand } from './run';
+export { compareCommand } from './compare';
+export { historyCommand } from './history';

package/src/commands/init.ts

@@ -0,0 +1,111 @@
+/**
+ * Init command - Initialize ArtemisKit in a project
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import chalk from 'chalk';
+import { Command } from 'commander';
+
+const DEFAULT_CONFIG = `# ArtemisKit Configuration
+project: my-project
+
+# Default provider settings
+provider: openai
+model: gpt-4
+
+# Provider configurations
+providers:
+  openai:
+    apiKey: \${OPENAI_API_KEY}
+    defaultModel: gpt-4
+
+  azure-openai:
+    apiKey: \${AZURE_OPENAI_API_KEY}
+    resourceName: \${AZURE_OPENAI_RESOURCE}
+    deploymentName: \${AZURE_OPENAI_DEPLOYMENT}
+    apiVersion: "2024-02-15-preview"
+
+# Storage configuration
+storage:
+  type: local
+  basePath: ./artemis-runs
+
+# Scenarios directory
+scenariosDir: ./scenarios
+
+# Output settings
+output:
+  format: json
+  dir: ./artemis-output
+`;
+
+const DEFAULT_SCENARIO = `name: Example Scenario
+description: Basic example scenario for testing
+version: "1.0"
+provider: openai
+model: gpt-4
+temperature: 0
+
+cases:
+  - id: greeting
+    name: Simple Greeting
+    prompt: "Say hello in exactly 3 words."
+    expected:
+      type: regex
+      pattern: "^\\\\w+\\\\s+\\\\w+\\\\s+\\\\w+$"
+    tags:
+      - greeting
+      - basic
+
+  - id: math
+    name: Basic Math
+    prompt: "What is 2 + 2? Reply with just the number."
+    expected:
+      type: exact
+      value: "4"
+    tags:
+      - math
+      - basic
+`;
+
+export function initCommand(): Command {
+  const cmd = new Command('init');
+
+  cmd
+    .description('Initialize ArtemisKit in the current directory')
+    .option('-f, --force', 'Overwrite existing configuration')
+    .action(async () => {
+      try {
+        const cwd = process.cwd();
+
+        // Create directories
+        await mkdir(join(cwd, 'scenarios'), { recursive: true });
+        await mkdir(join(cwd, 'artemis-runs'), { recursive: true });
+        await mkdir(join(cwd, 'artemis-output'), { recursive: true });
+
+        // Write config file
+        const configPath = join(cwd, 'artemis.config.yaml');
+        await writeFile(configPath, DEFAULT_CONFIG);
+        console.log(chalk.green('✓'), 'Created artemis.config.yaml');
+
+        // Write example scenario
+        const scenarioPath = join(cwd, 'scenarios', 'example.yaml');
+        await writeFile(scenarioPath, DEFAULT_SCENARIO);
+        console.log(chalk.green('✓'), 'Created scenarios/example.yaml');
+
+        console.log();
+        console.log(chalk.bold('ArtemisKit initialized successfully!'));
+        console.log();
+        console.log('Next steps:');
+        console.log('  1. Configure your API keys in .env or environment variables');
+        console.log('  2. Edit scenarios/example.yaml to add your test cases');
+        console.log('  3. Run tests with: artemiskit run scenarios/example.yaml');
+      } catch (error) {
+        console.error(chalk.red('Error:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  return cmd;
+}