@artemiskit/cli 0.1.4 → 0.1.6

package/src/commands/init.ts

@@ -2,23 +2,25 @@
  * Init command - Initialize ArtemisKit in a project
  */
 
-import { mkdir, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, writeFile, appendFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import chalk from 'chalk';
 import { Command } from 'commander';
+import { createSpinner, icons } from '../ui/index.js';
 
 const DEFAULT_CONFIG = `# ArtemisKit Configuration
 project: my-project
 
 # Default provider settings
 provider: openai
-model: gpt-4
+model: gpt-4o-mini
 
 # Provider configurations
 providers:
   openai:
     apiKey: \${OPENAI_API_KEY}
-    defaultModel: gpt-4
+    defaultModel: gpt-4o-mini
 
   azure-openai:
     apiKey: \${AZURE_OPENAI_API_KEY}
@@ -26,6 +28,10 @@ providers:
     deploymentName: \${AZURE_OPENAI_DEPLOYMENT}
     apiVersion: "2024-02-15-preview"
 
+  anthropic:
+    apiKey: \${ANTHROPIC_API_KEY}
+    defaultModel: claude-sonnet-4-20250514
+
 # Storage configuration
 storage:
   type: local
@@ -44,7 +50,7 @@ const DEFAULT_SCENARIO = `name: Example Scenario
 description: Basic example scenario for testing
 version: "1.0"
 provider: openai
-model: gpt-4
+model: gpt-4o-mini
 temperature: 0
 
 cases:
@@ -69,40 +75,194 @@ cases:
       - basic
 `;
 
+const ENV_KEYS = [
+  '# ArtemisKit Environment Variables',
+  'OPENAI_API_KEY=',
+  'AZURE_OPENAI_API_KEY=',
+  'AZURE_OPENAI_RESOURCE=',
+  'AZURE_OPENAI_DEPLOYMENT=',
+  'AZURE_OPENAI_API_VERSION=',
+  'ANTHROPIC_API_KEY=',
+];
+
+function renderWelcomeBanner(): string {
+  const lines = [
+    '',
+    chalk.cyan('  ╔═══════════════════════════════════════════════════════╗'),
+    chalk.cyan('  ║                                                       ║'),
+    chalk.cyan('  ║') +
+      chalk.bold.white('     🎯 Welcome to ArtemisKit                         ') +
+      chalk.cyan('║'),
+    chalk.cyan('  ║') +
+      chalk.gray('     LLM Testing & Evaluation Toolkit                 ') +
+      chalk.cyan('║'),
+    chalk.cyan('  ║                                                       ║'),
+    chalk.cyan('  ╚═══════════════════════════════════════════════════════╝'),
+    '',
+  ];
+  return lines.join('\n');
+}
+
+function renderSuccessPanel(): string {
+  const lines = [
+    '',
+    chalk.green('  ╭─────────────────────────────────────────────────────────╮'),
+    chalk.green('  │') +
+      chalk.bold.green('  ✓ ArtemisKit initialized successfully!                 ') +
+      chalk.green('│'),
+    chalk.green('  ├─────────────────────────────────────────────────────────┤'),
+    chalk.green('  │                                                         │'),
+    chalk.green('  │') +
+      chalk.white('  Next steps:                                            ') +
+      chalk.green('│'),
+    chalk.green('  │                                                         │'),
+    chalk.green('  │') +
+      chalk.white('  1. Set your API key:                                   ') +
+      chalk.green('│'),
+    chalk.green('  │') +
+      chalk.cyan('     export OPENAI_API_KEY="sk-..."                      ') +
+      chalk.green('│'),
+    chalk.green('  │                                                         │'),
+    chalk.green('  │') +
+      chalk.white('  2. Run your first test:                                ') +
+      chalk.green('│'),
+    chalk.green('  │') +
+      chalk.cyan('     artemiskit run scenarios/example.yaml               ') +
+      chalk.green('│'),
+    chalk.green('  │                                                         │'),
+    chalk.green('  │') +
+      chalk.white('  3. View the docs:                                      ') +
+      chalk.green('│'),
+    chalk.green('  │') +
+      chalk.cyan('     https://artemiskit.vercel.app/docs                  ') +
+      chalk.green('│'),
+    chalk.green('  │                                                         │'),
+    chalk.green('  ╰─────────────────────────────────────────────────────────╯'),
+    '',
+  ];
+  return lines.join('\n');
+}
+
+async function appendEnvKeys(cwd: string): Promise<{ added: string[]; skipped: string[] }> {
+  const envPath = join(cwd, '.env');
+  const added: string[] = [];
+  const skipped: string[] = [];
+
+  let existingContent = '';
+  if (existsSync(envPath)) {
+    existingContent = await readFile(envPath, 'utf-8');
+  }
+
+  const linesToAdd: string[] = [];
+
+  for (const key of ENV_KEYS) {
+    // Skip comments
+    if (key.startsWith('#')) {
+      // Only add comment if we're adding new keys and it's not already there
+      if (!existingContent.includes(key)) {
+        linesToAdd.push(key);
+      }
+      continue;
+    }
+
+    const keyName = key.split('=')[0];
+    // Check if key already exists (with or without value)
+    const keyPattern = new RegExp(`^${keyName}=`, 'm');
+    if (keyPattern.test(existingContent)) {
+      skipped.push(keyName);
+    } else {
+      linesToAdd.push(key);
+      added.push(keyName);
+    }
+  }
+
+  if (linesToAdd.length > 0) {
+    // Add newline before our content if file exists and doesn't end with newline
+    const prefix =
+      existingContent && !existingContent.endsWith('\n') ? '\n\n' : existingContent ? '\n' : '';
+    await appendFile(envPath, prefix + linesToAdd.join('\n') + '\n');
+  }
+
+  return { added, skipped };
+}
+
 export function initCommand(): Command {
   const cmd = new Command('init');
 
   cmd
     .description('Initialize ArtemisKit in the current directory')
     .option('-f, --force', 'Overwrite existing configuration')
-    .action(async () => {
+    .option('--skip-env', 'Skip adding environment variables to .env')
+    .action(async (options: { force?: boolean; skipEnv?: boolean }) => {
+      const spinner = createSpinner();
+
       try {
         const cwd = process.cwd();
 
-        // Create directories
+        // Show welcome banner
+        console.log(renderWelcomeBanner());
+
+        // Step 1: Create directories
+        spinner.start('Creating project structure...');
         await mkdir(join(cwd, 'scenarios'), { recursive: true });
         await mkdir(join(cwd, 'artemis-runs'), { recursive: true });
         await mkdir(join(cwd, 'artemis-output'), { recursive: true });
+        spinner.succeed('Created project structure');
 
-        // Write config file
+        // Step 2: Write config file
         const configPath = join(cwd, 'artemis.config.yaml');
-        await writeFile(configPath, DEFAULT_CONFIG);
-        console.log(chalk.green('✓'), 'Created artemis.config.yaml');
+        const configExists = existsSync(configPath);
 
-        // Write example scenario
+        if (configExists && !options.force) {
+          spinner.info('Config file already exists (use --force to overwrite)');
+        } else {
+          spinner.start('Writing configuration...');
+          await writeFile(configPath, DEFAULT_CONFIG);
+          spinner.succeed(
+            configExists ? 'Overwrote artemis.config.yaml' : 'Created artemis.config.yaml'
+          );
+        }
+
+        // Step 3: Write example scenario
         const scenarioPath = join(cwd, 'scenarios', 'example.yaml');
-        await writeFile(scenarioPath, DEFAULT_SCENARIO);
-        console.log(chalk.green('✓'), 'Created scenarios/example.yaml');
-
-        console.log();
-        console.log(chalk.bold('ArtemisKit initialized successfully!'));
-        console.log();
-        console.log('Next steps:');
-        console.log('  1. Configure your API keys in .env or environment variables');
-        console.log('  2. Edit scenarios/example.yaml to add your test cases');
-        console.log('  3. Run tests with: artemiskit run scenarios/example.yaml');
+        const scenarioExists = existsSync(scenarioPath);
+
+        if (scenarioExists && !options.force) {
+          spinner.info('Example scenario already exists (use --force to overwrite)');
+        } else {
+          spinner.start('Creating example scenario...');
+          await writeFile(scenarioPath, DEFAULT_SCENARIO);
+          spinner.succeed(
+            scenarioExists ? 'Overwrote scenarios/example.yaml' : 'Created scenarios/example.yaml'
+          );
+        }
+
+        // Step 4: Update .env file
+        if (!options.skipEnv) {
+          spinner.start('Updating .env file...');
+          const { added, skipped } = await appendEnvKeys(cwd);
+
+          if (added.length > 0) {
+            spinner.succeed(`Added ${added.length} environment variable(s) to .env`);
+            if (skipped.length > 0) {
+              console.log(
+                chalk.dim(
+                  `  ${icons.info} Skipped ${skipped.length} existing key(s): ${skipped.join(', ')}`
+                )
+              );
+            }
+          } else if (skipped.length > 0) {
+            spinner.info('All environment variables already exist in .env');
+          } else {
+            spinner.succeed('Created .env with environment variables');
+          }
+        }
+
+        // Show success panel
+        console.log(renderSuccessPanel());
       } catch (error) {
-        console.error(chalk.red('Error:'), (error as Error).message);
+        spinner.fail('Error');
+        console.error(chalk.red(`\n${icons.failed} ${(error as Error).message}`));
         process.exit(1);
       }
     });

package/src/commands/redteam.ts

@@ -30,17 +30,25 @@ import {
 } from '@artemiskit/redteam';
 import { generateJSONReport, generateRedTeamHTMLReport } from '@artemiskit/reports';
 import chalk from 'chalk';
-import Table from 'cli-table3';
 import { Command } from 'commander';
 import { nanoid } from 'nanoid';
-import ora from 'ora';
-import { loadConfig } from '../config/loader';
+import { loadConfig } from '../config/loader.js';
+import {
+  createSpinner,
+  renderRedteamSummaryPanel,
+  renderError,
+  renderInfoBox,
+  renderProgressBar,
+  getProviderErrorContext,
+  isTTY,
+  icons,
+} from '../ui/index.js';
 import {
   buildAdapterConfig,
   resolveModelWithSource,
   resolveProviderWithSource,
-} from '../utils/adapter';
-import { createStorage } from '../utils/storage';
+} from '../utils/adapter.js';
+import { createStorage } from '../utils/storage.js';
 
 interface RedteamOptions {
   provider?: string;
@@ -78,7 +86,8 @@ export function redteamCommand(): Command {
       'Custom redaction patterns (regex or built-in: email, phone, credit_card, ssn, api_key)'
     )
     .action(async (scenarioPath: string, options: RedteamOptions) => {
-      const spinner = ora('Loading configuration...').start();
+      const spinner = createSpinner('Loading configuration...');
+      spinner.start();
       const startTime = new Date();
 
       try {
@@ -125,10 +134,22 @@ export function redteamCommand(): Command {
         const mutations = selectMutations(options.mutations);
         const generator = new RedTeamGenerator(mutations);
         const detector = new UnsafeResponseDetector();
+        const count = Number.parseInt(String(options.count)) || 5;
 
+        // Display configuration using info box
+        console.log();
+        const configLines = [
+          `Mutations: ${mutations.map((m) => m.name).join(', ')}`,
+          `Prompts per case: ${count}`,
+          `Total cases: ${scenario.cases.length}`,
+        ];
+        if (options.redact) {
+          configLines.push(
+            `Redaction: enabled${options.redactPatterns ? ` (${options.redactPatterns.join(', ')})` : ''}`
+          );
+        }
+        console.log(renderInfoBox('Red Team Configuration', configLines));
         console.log();
-        console.log(chalk.bold('Red-Team Testing'));
-        console.log(chalk.dim(`Mutations: ${mutations.map((m) => m.name).join(', ')}`));
 
         // Set up redaction if enabled
         let redactionConfig: RedactionConfig | undefined;
@@ -143,20 +164,17 @@ export function redteamCommand(): Command {
             replacement: '[REDACTED]',
           };
           redactor = new Redactor(redactionConfig);
-          console.log(
-            chalk.dim(
-              `Redaction enabled${options.redactPatterns ? ` with patterns: ${options.redactPatterns.join(', ')}` : ' (default patterns)'}`
-            )
-          );
         }
-        console.log();
 
-        const count = Number.parseInt(String(options.count)) || 5;
         const results: RedTeamCaseResult[] = [];
         let promptsRedacted = 0;
         let responsesRedacted = 0;
         let totalRedactions = 0;
 
+        // Calculate total tests for progress
+        const totalTests = scenario.cases.length * count;
+        let completedTests = 0;
+
         // Run mutated tests for each case
         for (const testCase of scenario.cases) {
           console.log(chalk.bold(`Testing case: ${testCase.id}`));
@@ -169,6 +187,18 @@ export function redteamCommand(): Command {
           const mutatedPrompts = generator.generate(originalPrompt, count);
 
           for (const mutated of mutatedPrompts) {
+            completedTests++;
+
+            // Show progress bar in TTY mode
+            if (isTTY) {
+              const progressBar = renderProgressBar(completedTests, totalTests, {
+                width: 15,
+                showPercentage: true,
+                showCount: false,
+              });
+              process.stdout.write(`\r  ${progressBar}  `);
+            }
+
             const requestStart = Date.now();
             try {
               const result = await client.generate({
@@ -180,14 +210,21 @@ export function redteamCommand(): Command {
               const detection = detector.detect(result.text);
 
               const resultStatus: RedTeamStatus = detection.unsafe ? 'unsafe' : 'safe';
-              const statusDisplay = detection.unsafe
+
+              // Clear progress line
+              if (isTTY) {
+                process.stdout.write('\r' + ' '.repeat(60) + '\r');
+              }
+
+              // Display status with appropriate icon
+              const statusIcon = detection.unsafe ? icons.failed : icons.passed;
+              const statusLabel = detection.unsafe
                 ? chalk.red(`UNSAFE (${detection.severity})`)
                 : chalk.green('SAFE');
-
-              console.log(`  ${statusDisplay} [${mutated.mutations.join(', ')}]`);
+              console.log(`  ${statusIcon} ${statusLabel} [${mutated.mutations.join(', ')}]`);
 
               if (detection.unsafe && options.verbose) {
-                console.log(chalk.dim(`    Reasons: ${detection.reasons.join(', ')}`));
+                console.log(chalk.dim(`      Reasons: ${detection.reasons.join(', ')}`));
               }
 
               // Apply redaction if enabled
@@ -228,6 +265,11 @@ export function redteamCommand(): Command {
               const errorMessage = (error as Error).message;
               const isContentFiltered = isProviderContentFilter(errorMessage);
 
+              // Clear progress line
+              if (isTTY) {
+                process.stdout.write('\r' + ' '.repeat(60) + '\r');
+              }
+
               // Apply redaction to prompt even for errors/blocked
               let errorPrompt = mutated.mutated;
               let errorCaseRedaction: CaseRedactionInfo | undefined;
@@ -249,8 +291,11 @@ export function redteamCommand(): Command {
 
               if (isContentFiltered) {
                 console.log(
-                  `  ${chalk.cyan('BLOCKED')} [${mutated.mutations.join(', ')}]: Provider content filter triggered`
+                  `  ${chalk.cyan('⊘')} ${chalk.cyan('BLOCKED')} [${mutated.mutations.join(', ')}]`
                 );
+                if (options.verbose) {
+                  console.log(chalk.dim('      Provider content filter triggered'));
+                }
                 results.push({
                   caseId: testCase.id,
                   mutation: mutated.mutations.join('+'),
@@ -264,8 +309,11 @@ export function redteamCommand(): Command {
                 });
               } else {
                 console.log(
-                  `  ${chalk.yellow('ERROR')} [${mutated.mutations.join(', ')}]: ${errorMessage}`
+                  `  ${icons.warning} ${chalk.yellow('ERROR')} [${mutated.mutations.join(', ')}]`
                 );
+                if (options.verbose) {
+                  console.log(chalk.dim(`      ${errorMessage}`));
+                }
                 results.push({
                   caseId: testCase.id,
                   mutation: mutated.mutations.join('+'),
@@ -335,8 +383,48 @@ export function redteamCommand(): Command {
           redaction: redactionInfo,
         };
 
-        // Display summary
-        displaySummary(metrics, runId);
+        // Display summary using enhanced panel
+        const summaryData = {
+          totalCases: metrics.total_tests,
+          safeResponses: metrics.safe_responses,
+          unsafeResponses: metrics.unsafe_responses,
+          blockedResponses: metrics.blocked_responses,
+          errorResponses: metrics.error_responses,
+          defenseRate: metrics.defense_rate * 100,
+          severityBreakdown: metrics.by_severity,
+        };
+        console.log(renderRedteamSummaryPanel(summaryData));
+
+        // Show run ID and defense rate detail
+        console.log();
+        const testableResults = metrics.total_tests - metrics.error_responses;
+        console.log(
+          chalk.dim(
+            `Run ID: ${runId}  |  Defense Rate: ${(metrics.defense_rate * 100).toFixed(1)}% (${metrics.defended}/${testableResults})`
+          )
+        );
+
+        // Show severity breakdown if there were unsafe responses
+        if (metrics.unsafe_responses > 0) {
+          console.log();
+          console.log(chalk.bold('Severity Breakdown:'));
+          for (const severity of ['critical', 'high', 'medium', 'low'] as const) {
+            if (metrics.by_severity[severity]) {
+              const info = SeverityMapper.getInfo(severity);
+              console.log(`  ${info.label}: ${metrics.by_severity[severity]}`);
+            }
+          }
+
+          console.log();
+          console.log(
+            chalk.red(
+              `${icons.warning} ${metrics.unsafe_responses} potentially unsafe responses detected`
+            )
+          );
+        } else if (testableResults > 0) {
+          console.log();
+          console.log(chalk.green(`${icons.passed} No unsafe responses detected`));
+        }
 
         // Save results if requested
         if (options.save) {
@@ -372,7 +460,13 @@ export function redteamCommand(): Command {
         }
       } catch (error) {
         spinner.fail('Error');
-        console.error(chalk.red('Error:'), (error as Error).message);
+
+        // Display enhanced error message
+        const provider = options.provider || 'unknown';
+        const errorContext = getProviderErrorContext(provider, error as Error);
+        console.log();
+        console.log(renderError(errorContext));
+
         process.exit(1);
       }
     });
@@ -458,54 +552,3 @@ function calculateMetrics(results: RedTeamCaseResult[]): RedTeamMetrics {
     by_severity: bySeverity,
   };
 }
-
-function displaySummary(metrics: RedTeamMetrics, runId: string): void {
-  const table = new Table({
-    head: [chalk.bold('Metric'), chalk.bold('Value')],
-    style: { head: [], border: [] },
-  });
-
-  table.push(
-    ['Run ID', runId],
-    ['Total Tests', metrics.total_tests.toString()],
-    ['Defended', chalk.green(metrics.defended.toString())],
-    [`  ${chalk.dim('Model handled safely')}`, chalk.green(metrics.safe_responses.toString())],
-    [`  ${chalk.dim('Provider blocked')}`, chalk.cyan(metrics.blocked_responses.toString())],
-    [
-      'Unsafe Responses',
-      metrics.unsafe_responses > 0 ? chalk.red(metrics.unsafe_responses.toString()) : '0',
-    ]
-  );
-
-  for (const severity of ['critical', 'high', 'medium', 'low'] as const) {
-    if (metrics.by_severity[severity]) {
-      const info = SeverityMapper.getInfo(severity);
-      table.push([`  ${info.label}`, metrics.by_severity[severity].toString()]);
-    }
-  }
-
-  if (metrics.error_responses > 0) {
-    table.push(['Errors', chalk.yellow(metrics.error_responses.toString())]);
-  }
-
-  console.log(chalk.bold('Summary'));
-  console.log(table.toString());
-
-  // Calculate defense rate (excluding errors from denominator)
-  const testableResults = metrics.total_tests - metrics.error_responses;
-  if (testableResults > 0) {
-    const defenseRate = (metrics.defense_rate * 100).toFixed(1);
-    console.log();
-    console.log(
-      chalk.dim(`Defense Rate: ${defenseRate}% (${metrics.defended}/${testableResults})`)
-    );
-  }
-
-  if (metrics.unsafe_responses > 0) {
-    console.log();
-    console.log(chalk.red(`⚠ ${metrics.unsafe_responses} potentially unsafe responses detected`));
-  } else if (testableResults > 0) {
-    console.log();
-    console.log(chalk.green('✓ No unsafe responses detected'));
-  }
-}

package/src/commands/report.ts

@@ -11,11 +11,10 @@ import {
   generateRedTeamHTMLReport,
   generateStressHTMLReport,
 } from '@artemiskit/reports';
-import chalk from 'chalk';
 import { Command } from 'commander';
-import ora from 'ora';
-import { loadConfig } from '../config/loader';
-import { createStorage } from '../utils/storage';
+import { loadConfig } from '../config/loader.js';
+import { createSpinner, renderError, renderInfoBox, icons } from '../ui/index.js';
+import { createStorage } from '../utils/storage.js';
 
 interface ReportOptions {
   format?: 'html' | 'json' | 'both';
@@ -74,7 +73,8 @@ export function reportCommand(): Command {
     .option('-o, --output <dir>', 'Output directory', './artemis-output')
     .option('--config <path>', 'Path to config file')
     .action(async (runId: string, options: ReportOptions) => {
-      const spinner = ora('Loading run...').start();
+      const spinner = createSpinner('Loading run...');
+      spinner.start();
 
       try {
         const config = await loadConfig(options.config);
@@ -96,7 +96,7 @@ export function reportCommand(): Command {
           const htmlPath = join(outputDir, `${runId}.html`);
           await writeFile(htmlPath, html);
           generatedFiles.push(htmlPath);
-          spinner.succeed(`Generated HTML report: ${htmlPath}`);
+          spinner.succeed(`Generated HTML report`);
         }
 
         if (format === 'json' || format === 'both') {
@@ -105,19 +105,34 @@ export function reportCommand(): Command {
           const jsonPath = join(outputDir, `${runId}.json`);
           await writeFile(jsonPath, json);
           generatedFiles.push(jsonPath);
-          spinner.succeed(`Generated JSON report: ${jsonPath}`);
+          spinner.succeed(`Generated JSON report`);
         }
 
+        // Show success panel
         console.log();
-        console.log(chalk.bold('Report generated successfully!'));
-        console.log();
-        console.log('Files:');
-        for (const file of generatedFiles) {
-          console.log(`  ${chalk.green('•')} ${file}`);
-        }
+        console.log(
+          renderInfoBox('Report Generated', [
+            `Run ID: ${runId}`,
+            `Type: ${manifestType}`,
+            '',
+            'Files:',
+            ...generatedFiles.map((f) => `${icons.passed} ${f}`),
+          ])
+        );
       } catch (error) {
         spinner.fail('Error');
-        console.error(chalk.red('Error:'), (error as Error).message);
+        console.log();
+        console.log(
+          renderError({
+            title: 'Failed to Generate Report',
+            reason: (error as Error).message,
+            suggestions: [
+              'Check that the run ID exists',
+              'Run "artemiskit history" to see available runs',
+              'Verify storage configuration in artemis.config.yaml',
+            ],
+          })
+        );
         process.exit(1);
       }
     });