@artemiskit/cli 0.2.3 → 0.2.4

package/dist/src/cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAYpC,wBAAgB,SAAS,IAAI,OAAO,CAwCnC"}
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAapC,wBAAgB,SAAS,IAAI,OAAO,CAyCnC"}

package/dist/src/commands/redteam.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAAA;;GAEG;AAsCH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoCpC,wBAAgB,cAAc,IAAI,OAAO,CAycxC"}
+{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoCpC,wBAAgB,cAAc,IAAI,OAAO,CAidxC"}

package/dist/src/commands/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiBH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAyiBpC,wBAAgB,UAAU,IAAI,OAAO,CAggBpC"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiBH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAyiBpC,wBAAgB,UAAU,IAAI,OAAO,CAwgBpC"}

package/dist/src/commands/validate.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Validate command - Validate scenarios without running them
+ */
+import { Command } from 'commander';
+export declare function validateCommand(): Command;
+//# sourceMappingURL=validate.d.ts.map

package/dist/src/commands/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../../src/commands/validate.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,wBAAgB,eAAe,IAAI,OAAO,CAkHzC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@artemiskit/cli",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Command-line interface for ArtemisKit LLM evaluation toolkit",
   "type": "module",
   "license": "Apache-2.0",
@@ -45,11 +45,11 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@artemiskit/adapter-openai": "0.1.10",
-    "@artemiskit/adapter-vercel-ai": "0.1.10",
-    "@artemiskit/core": "0.2.3",
-    "@artemiskit/redteam": "0.2.3",
-    "@artemiskit/reports": "0.2.3",
+    "@artemiskit/adapter-openai": "0.1.11",
+    "@artemiskit/adapter-vercel-ai": "0.1.11",
+    "@artemiskit/core": "0.2.4",
+    "@artemiskit/redteam": "0.2.4",
+    "@artemiskit/reports": "0.2.4",
     "chalk": "^5.3.0",
     "cli-table3": "^0.6.3",
     "commander": "^12.0.0",

package/src/cli.ts

@@ -12,6 +12,7 @@ import { redteamCommand } from './commands/redteam';
 import { reportCommand } from './commands/report';
 import { runCommand } from './commands/run';
 import { stressCommand } from './commands/stress';
+import { validateCommand } from './commands/validate';
 import { checkForUpdate, formatUpdateMessage, formatVersionDisplay } from './utils/update-checker';
 
 export function createCLI(): Command {
@@ -46,6 +47,7 @@ export function createCLI(): Command {
 
   program.addCommand(initCommand());
   program.addCommand(runCommand());
+  program.addCommand(validateCommand());
   program.addCommand(baselineCommand());
   program.addCommand(compareCommand());
   program.addCommand(historyCommand());

package/src/commands/redteam.ts

@@ -35,6 +35,7 @@ import {
 import {
   generateJSONReport,
   generateRedTeamHTMLReport,
+  generateRedTeamJUnitReport,
   generateRedTeamMarkdownReport,
 } from '@artemiskit/reports';
 import chalk from 'chalk';
@@ -70,7 +71,7 @@ interface RedteamOptions {
   config?: string;
   redact?: boolean;
   redactPatterns?: string[];
-  export?: 'markdown';
+  export?: 'markdown' | 'junit';
   exportOutput?: string;
 }
 
@@ -97,7 +98,7 @@ export function redteamCommand(): Command {
       '--redact-patterns <patterns...>',
       'Custom redaction patterns (regex or built-in: email, phone, credit_card, ssn, api_key)'
     )
-    .option('--export <format>', 'Export results to format (markdown)')
+    .option('--export <format>', 'Export results to format (markdown or junit)')
     .option('--export-output <dir>', 'Output directory for exports (default: ./artemis-exports)')
     .action(async (scenarioPath: string, options: RedteamOptions) => {
       const spinner = createSpinner('Loading configuration...');
@@ -503,14 +504,22 @@ export function redteamCommand(): Command {
           console.log(chalk.dim(`  JSON: ${jsonPath}`));
         }
 
-        // Export to markdown if requested
-        if (options.export === 'markdown') {
+        // Export if requested
+        if (options.export) {
           const exportDir = options.exportOutput || './artemis-exports';
           await mkdir(exportDir, { recursive: true });
-          const markdown = generateRedTeamMarkdownReport(manifest);
-          const mdPath = join(exportDir, `${runId}.md`);
-          await writeFile(mdPath, markdown);
-          console.log(chalk.dim(`Exported: ${mdPath}`));
+
+          if (options.export === 'markdown') {
+            const markdown = generateRedTeamMarkdownReport(manifest);
+            const mdPath = join(exportDir, `${runId}.md`);
+            await writeFile(mdPath, markdown);
+            console.log(chalk.dim(`Exported: ${mdPath}`));
+          } else if (options.export === 'junit') {
+            const junit = generateRedTeamJUnitReport(manifest);
+            const junitPath = join(exportDir, `${runId}.xml`);
+            await writeFile(junitPath, junit);
+            console.log(chalk.dim(`Exported: ${junitPath}`));
+          }
         }
 
         // Exit with error if there were unsafe responses

package/src/commands/run.ts

@@ -15,7 +15,7 @@ import {
   resolveScenarioPaths,
   runScenario,
 } from '@artemiskit/core';
-import { generateMarkdownReport } from '@artemiskit/reports';
+import { generateJUnitReport, generateMarkdownReport } from '@artemiskit/reports';
 import chalk from 'chalk';
 import { Command } from 'commander';
 import { loadConfig } from '../config/loader.js';
@@ -68,8 +68,8 @@ interface RunOptions {
   threshold?: number;
   /** Budget limit in USD - fail if cost exceeds this */
   budget?: number;
-  /** Export format: markdown */
-  export?: 'markdown';
+  /** Export format: markdown or junit */
+  export?: 'markdown' | 'junit';
   /** Output directory for exports */
   exportOutput?: string;
 }
@@ -607,7 +607,7 @@ export function runCommand(): Command {
     .option('--baseline', 'Compare against baseline and detect regression')
     .option('--threshold <number>', 'Regression threshold (0-1), e.g., 0.05 for 5%', '0.05')
     .option('--budget <amount>', 'Maximum budget in USD - fail if estimated cost exceeds this')
-    .option('--export <format>', 'Export format: markdown')
+    .option('--export <format>', 'Export format: markdown or junit (for CI integration)')
     .option('--export-output <dir>', 'Output directory for exports (default: ./artemis-exports)')
     .action(async (scenarioPath: string | undefined, options: RunOptions) => {
       // Determine CI mode: explicit flag, environment variable, or summary format that implies CI
@@ -819,14 +819,22 @@ export function runCommand(): Command {
                 console.log(chalk.dim(`Saved: ${savedPath}`));
               }
 
-              // Export to markdown if requested
-              if (options.export === 'markdown') {
+              // Export if requested
+              if (options.export) {
                 const exportDir = options.exportOutput || './artemis-exports';
                 await mkdir(exportDir, { recursive: true });
-                const markdown = generateMarkdownReport(result.manifest);
-                const mdPath = join(exportDir, `${result.manifest.run_id}.md`);
-                await writeFile(mdPath, markdown);
-                console.log(chalk.dim(`Exported: ${mdPath}`));
+
+                if (options.export === 'markdown') {
+                  const markdown = generateMarkdownReport(result.manifest);
+                  const mdPath = join(exportDir, `${result.manifest.run_id}.md`);
+                  await writeFile(mdPath, markdown);
+                  console.log(chalk.dim(`Exported: ${mdPath}`));
+                } else if (options.export === 'junit') {
+                  const junit = generateJUnitReport(result.manifest);
+                  const junitPath = join(exportDir, `${result.manifest.run_id}.xml`);
+                  await writeFile(junitPath, junit);
+                  console.log(chalk.dim(`Exported: ${junitPath}`));
+                }
               }
             } catch (error) {
               // Record failed scenario

package/src/commands/validate.ts

@@ -0,0 +1,254 @@
+/**
+ * Validate command - Validate scenarios without running them
+ */
+
+import { readdirSync, statSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { basename, join, resolve } from 'node:path';
+import { ScenarioValidator, type ValidationResult, type ValidationSummary } from '@artemiskit/core';
+import { generateValidationJUnitReport } from '@artemiskit/reports';
+import { Glob } from 'bun';
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { icons } from '../ui/index.js';
+
+interface ValidateOptions {
+  json?: boolean;
+  strict?: boolean;
+  quiet?: boolean;
+  export?: 'junit';
+  exportOutput?: string;
+}
+
+export function validateCommand(): Command {
+  const cmd = new Command('validate');
+
+  cmd
+    .description('Validate scenario files without running them')
+    .argument('<path>', 'Path to scenario file, directory, or glob pattern')
+    .option('--json', 'Output as JSON')
+    .option('--strict', 'Treat warnings as errors')
+    .option('-q, --quiet', 'Only output errors (no success messages)')
+    .option('--export <format>', 'Export results to format (junit for CI integration)')
+    .option('--export-output <dir>', 'Output directory for exports (default: ./artemis-exports)')
+    .action(async (pathArg: string, options: ValidateOptions) => {
+      const validator = new ScenarioValidator({ strict: options.strict });
+
+      // Resolve files to validate
+      const files = resolveFiles(pathArg);
+
+      if (files.length === 0) {
+        if (options.json) {
+          console.log(
+            JSON.stringify(
+              {
+                valid: false,
+                error: `No scenario files found matching: ${pathArg}`,
+                results: [],
+                summary: { total: 0, passed: 0, failed: 0, withWarnings: 0 },
+              },
+              null,
+              2
+            )
+          );
+        } else {
+          console.log(chalk.red(`${icons.failed} No scenario files found matching: ${pathArg}`));
+        }
+        process.exit(2);
+      }
+
+      // Validate all files
+      const results: ValidationResult[] = [];
+
+      if (!options.json && !options.quiet) {
+        console.log(chalk.bold('Validating scenarios...\n'));
+      }
+
+      for (const file of files) {
+        const result = validator.validate(file);
+        results.push(result);
+
+        // In strict mode, warnings become errors
+        if (options.strict && result.warnings.length > 0) {
+          result.valid = false;
+          result.errors.push(
+            ...result.warnings.map((w: ValidationResult['warnings'][0]) => ({
+              ...w,
+              severity: 'error' as const,
+            }))
+          );
+        }
+
+        if (!options.json) {
+          printFileResult(result, options);
+        }
+      }
+
+      // Calculate summary
+      const summary: ValidationSummary = {
+        total: results.length,
+        passed: results.filter((r) => r.valid && r.warnings.length === 0).length,
+        failed: results.filter((r) => !r.valid).length,
+        withWarnings: results.filter((r) => r.valid && r.warnings.length > 0).length,
+      };
+
+      // Output results
+      if (options.json) {
+        console.log(
+          JSON.stringify(
+            {
+              valid: summary.failed === 0,
+              results: results.map((r) => ({
+                file: r.file,
+                valid: r.valid,
+                errors: r.errors,
+                warnings: r.warnings,
+              })),
+              summary,
+            },
+            null,
+            2
+          )
+        );
+      } else if (!options.quiet) {
+        console.log();
+        printSummary(summary, options.strict);
+      }
+
+      // Export to JUnit if requested
+      if (options.export === 'junit') {
+        const exportDir = options.exportOutput || './artemis-exports';
+        await mkdir(exportDir, { recursive: true });
+        const junit = generateValidationJUnitReport(results);
+        const junitPath = join(exportDir, `validation-${Date.now()}.xml`);
+        await writeFile(junitPath, junit);
+        if (!options.quiet) {
+          console.log(chalk.dim(`Exported: ${junitPath}`));
+        }
+      }
+
+      // Exit with appropriate code
+      if (summary.failed > 0) {
+        process.exit(1);
+      }
+    });
+
+  return cmd;
+}
+
+/**
+ * Resolve files from path argument (file, directory, or glob)
+ */
+function resolveFiles(pathArg: string): string[] {
+  const resolved = resolve(pathArg);
+
+  try {
+    const stat = statSync(resolved);
+
+    if (stat.isFile()) {
+      // Single file
+      return [resolved];
+    }
+
+    if (stat.isDirectory()) {
+      // Directory - find all yaml files recursively
+      return findYamlFiles(resolved);
+    }
+  } catch {
+    // Path doesn't exist as file/directory - try as glob
+  }
+
+  // Try as glob pattern using Bun's Glob
+  const glob = new Glob(pathArg);
+  const matches: string[] = [];
+  for (const file of glob.scanSync({ absolute: true, onlyFiles: true })) {
+    if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+      matches.push(file);
+    }
+  }
+
+  return matches;
+}
+
+/**
+ * Find all YAML files in a directory recursively
+ */
+function findYamlFiles(dir: string): string[] {
+  const files: string[] = [];
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      files.push(...findYamlFiles(fullPath));
+    } else if (entry.isFile() && (entry.name.endsWith('.yaml') || entry.name.endsWith('.yml'))) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Print result for a single file
+ */
+function printFileResult(result: ValidationResult, options: ValidateOptions): void {
+  const fileName = basename(result.file);
+
+  if (result.valid && result.warnings.length === 0) {
+    if (!options.quiet) {
+      console.log(`${icons.passed} ${chalk.green(fileName)}`);
+    }
+  } else if (result.valid && result.warnings.length > 0) {
+    console.log(`${icons.warning} ${chalk.yellow(fileName)}`);
+    for (const warning of result.warnings) {
+      const location = warning.column
+        ? `Line ${warning.line}:${warning.column}`
+        : `Line ${warning.line}`;
+      console.log(chalk.yellow(`    ${location}: ${warning.message}`));
+      if (warning.suggestion) {
+        console.log(chalk.dim(`    Suggestion: ${warning.suggestion}`));
+      }
+    }
+  } else {
+    console.log(`${icons.failed} ${chalk.red(fileName)}`);
+    for (const error of result.errors) {
+      const location = error.column ? `Line ${error.line}:${error.column}` : `Line ${error.line}`;
+      console.log(chalk.red(`    ${location}: ${error.message}`));
+      if (error.suggestion) {
+        console.log(chalk.dim(`    Suggestion: ${error.suggestion}`));
+      }
+    }
+    for (const warning of result.warnings) {
+      const location = warning.column
+        ? `Line ${warning.line}:${warning.column}`
+        : `Line ${warning.line}`;
+      console.log(chalk.yellow(`    ${location}: ${warning.message}`));
+    }
+  }
+}
+
+/**
+ * Print validation summary
+ */
+function printSummary(summary: ValidationSummary, strict?: boolean): void {
+  const parts: string[] = [];
+
+  if (summary.passed > 0) {
+    parts.push(chalk.green(`${summary.passed} passed`));
+  }
+  if (summary.failed > 0) {
+    parts.push(chalk.red(`${summary.failed} failed`));
+  }
+  if (summary.withWarnings > 0 && !strict) {
+    parts.push(chalk.yellow(`${summary.withWarnings} with warnings`));
+  }
+
+  const statusIcon = summary.failed > 0 ? icons.failed : icons.passed;
+  const statusColor = summary.failed > 0 ? chalk.red : chalk.green;
+
+  console.log(statusColor(`${statusIcon} ${parts.join(', ')}`));
+  console.log(chalk.dim(`${summary.total} scenario(s) validated`));
+}