@artemiskit/cli 0.1.2

package/src/commands/stress.ts

@@ -0,0 +1,501 @@
+/**
+ * Stress command - Run load/stress tests against an LLM
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import {
+  type ManifestRedactionInfo,
+  type RedactionConfig,
+  Redactor,
+  type StressManifest,
+  type StressMetrics,
+  type StressRequestResult,
+  createAdapter,
+  getGitInfo,
+  parseScenarioFile,
+} from '@artemiskit/core';
+import { generateJSONReport, generateStressHTMLReport } from '@artemiskit/reports';
+import chalk from 'chalk';
+import Table from 'cli-table3';
+import { Command } from 'commander';
+import { nanoid } from 'nanoid';
+import ora from 'ora';
+import { loadConfig } from '../config/loader';
+import {
+  buildAdapterConfig,
+  resolveModelWithSource,
+  resolveProviderWithSource,
+} from '../utils/adapter';
+import { createStorage } from '../utils/storage';
+
+interface StressOptions {
+  provider?: string;
+  model?: string;
+  concurrency?: number;
+  requests?: number;
+  duration?: number;
+  rampUp?: number;
+  save?: boolean;
+  output?: string;
+  verbose?: boolean;
+  config?: string;
+  redact?: boolean;
+  redactPatterns?: string[];
+}
+
+export function stressCommand(): Command {
+  const cmd = new Command('stress');
+
+  cmd
+    .description('Run load/stress tests against an LLM')
+    .argument('<scenario>', 'Path to scenario YAML file')
+    .option('-p, --provider <provider>', 'Provider to use')
+    .option('-m, --model <model>', 'Model to use')
+    .option('-c, --concurrency <number>', 'Number of concurrent requests', '10')
+    .option('-n, --requests <number>', 'Total number of requests to make')
+    .option('-d, --duration <seconds>', 'Duration to run the test in seconds', '30')
+    .option('--ramp-up <seconds>', 'Ramp-up time in seconds', '5')
+    .option('--save', 'Save results to storage')
+    .option('-o, --output <dir>', 'Output directory for reports')
+    .option('-v, --verbose', 'Verbose output')
+    .option('--config <path>', 'Path to config file')
+    .option('--redact', 'Enable PII/sensitive data redaction in results')
+    .option(
+      '--redact-patterns <patterns...>',
+      'Custom redaction patterns (regex or built-in: email, phone, credit_card, ssn, api_key)'
+    )
+    .action(async (scenarioPath: string, options: StressOptions) => {
+      const spinner = ora('Loading configuration...').start();
+      const startTime = new Date();
+
+      try {
+        // Load config file if present
+        const config = await loadConfig(options.config);
+        if (config) {
+          spinner.succeed('Loaded config file');
+        } else {
+          spinner.info('No config file found, using defaults');
+        }
+
+        // Parse scenario
+        spinner.start('Loading scenario...');
+        const scenario = await parseScenarioFile(scenarioPath);
+        spinner.succeed(`Loaded scenario: ${scenario.name}`);
+
+        // Resolve provider and model with precedence and source tracking:
+        // CLI > Scenario > Config > Default
+        const { provider, source: providerSource } = resolveProviderWithSource(
+          options.provider,
+          scenario.provider,
+          config?.provider
+        );
+        const { model, source: modelSource } = resolveModelWithSource(
+          options.model,
+          scenario.model,
+          config?.model
+        );
+
+        // Build adapter config with full precedence chain and source tracking
+        spinner.start(`Connecting to ${provider}...`);
+        const { adapterConfig, resolvedConfig } = buildAdapterConfig({
+          provider,
+          model,
+          providerSource,
+          modelSource,
+          scenarioConfig: scenario.providerConfig,
+          fileConfig: config,
+        });
+        const client = await createAdapter(adapterConfig);
+        spinner.succeed(`Connected to ${provider}`);
+
+        // Configuration
+        const concurrency = Number.parseInt(String(options.concurrency)) || 10;
+        const durationSec = Number.parseInt(String(options.duration)) || 30;
+        const rampUpSec = Number.parseInt(String(options.rampUp)) || 5;
+        const maxRequests = options.requests
+          ? Number.parseInt(String(options.requests))
+          : undefined;
+
+        // Set up redaction if enabled
+        let redactionConfig: RedactionConfig | undefined;
+        let redactor: Redactor | undefined;
+        if (options.redact) {
+          redactionConfig = {
+            enabled: true,
+            patterns: options.redactPatterns,
+            redactPrompts: true,
+            redactResponses: true,
+            redactMetadata: false,
+            replacement: '[REDACTED]',
+          };
+          redactor = new Redactor(redactionConfig);
+        }
+
+        console.log();
+        console.log(chalk.bold('Stress Test Configuration'));
+        console.log(chalk.dim(`Concurrency: ${concurrency}`));
+        console.log(chalk.dim(`Duration: ${durationSec}s`));
+        console.log(chalk.dim(`Ramp-up: ${rampUpSec}s`));
+        if (maxRequests) {
+          console.log(chalk.dim(`Max requests: ${maxRequests}`));
+        }
+        if (options.redact) {
+          console.log(
+            chalk.dim(
+              `Redaction: enabled${options.redactPatterns ? ` (${options.redactPatterns.join(', ')})` : ' (default patterns)'}`
+            )
+          );
+        }
+        console.log();
+
+        // Get test prompts from scenario cases
+        const prompts = scenario.cases.map((c) =>
+          typeof c.prompt === 'string' ? c.prompt : c.prompt.map((m) => m.content).join('\n')
+        );
+
+        if (prompts.length === 0) {
+          throw new Error('No test cases found in scenario');
+        }
+
+        // Run stress test
+        spinner.start('Running stress test...');
+        const results = await runStressTest({
+          client,
+          model,
+          prompts,
+          concurrency,
+          durationMs: durationSec * 1000,
+          rampUpMs: rampUpSec * 1000,
+          maxRequests,
+          temperature: scenario.temperature,
+          onProgress: (completed, active) => {
+            spinner.text = `Running stress test... ${completed} completed, ${active} active`;
+          },
+          verbose: options.verbose,
+        });
+
+        spinner.succeed('Stress test completed');
+        const endTime = new Date();
+        console.log();
+
+        // Calculate stats
+        const metrics = calculateMetrics(results, endTime.getTime() - startTime.getTime());
+
+        // Build redaction metadata if enabled
+        let redactionInfo: ManifestRedactionInfo | undefined;
+        if (redactor && redactionConfig?.enabled) {
+          redactionInfo = {
+            enabled: true,
+            patternsUsed: redactor.patternNames,
+            replacement: redactor.replacement,
+            summary: {
+              promptsRedacted: 0, // Stress test doesn't track individual prompts
+              responsesRedacted: 0,
+              totalRedactions: 0,
+            },
+          };
+        }
+
+        // Build manifest
+        const runId = `st_${nanoid(12)}`;
+        const manifest: StressManifest = {
+          version: '1.0',
+          type: 'stress',
+          run_id: runId,
+          project: config?.project || process.env.ARTEMIS_PROJECT || 'default',
+          start_time: startTime.toISOString(),
+          end_time: endTime.toISOString(),
+          duration_ms: endTime.getTime() - startTime.getTime(),
+          config: {
+            scenario: basename(scenarioPath, '.yaml'),
+            provider,
+            model: resolvedConfig.model,
+            concurrency,
+            duration_seconds: durationSec,
+            ramp_up_seconds: rampUpSec,
+            max_requests: maxRequests,
+          },
+          resolved_config: resolvedConfig,
+          metrics,
+          git: await getGitInfo(),
+          provenance: {
+            run_by: process.env.USER || process.env.USERNAME || 'unknown',
+          },
+          // Sample results (keep only a sample to avoid huge files)
+          sample_results: sampleResults(results, 100),
+          environment: {
+            node_version: process.version,
+            platform: process.platform,
+            arch: process.arch,
+          },
+          redaction: redactionInfo,
+        };
+
+        // Display stats
+        displayStats(metrics, runId);
+
+        // Display latency histogram if verbose
+        if (options.verbose) {
+          displayHistogram(results);
+        }
+
+        // Save results if requested
+        if (options.save) {
+          spinner.start('Saving results...');
+          const storage = createStorage({ fileConfig: config });
+          const path = await storage.save(manifest);
+          spinner.succeed(`Results saved: ${path}`);
+        }
+
+        // Generate reports if output directory specified
+        if (options.output) {
+          spinner.start('Generating reports...');
+          await mkdir(options.output, { recursive: true });
+
+          // HTML report
+          const html = generateStressHTMLReport(manifest);
+          const htmlPath = join(options.output, `${runId}.html`);
+          await writeFile(htmlPath, html);
+
+          // JSON report
+          const json = generateJSONReport(manifest);
+          const jsonPath = join(options.output, `${runId}.json`);
+          await writeFile(jsonPath, json);
+
+          spinner.succeed(`Reports generated: ${options.output}`);
+          console.log(chalk.dim(`  HTML: ${htmlPath}`));
+          console.log(chalk.dim(`  JSON: ${jsonPath}`));
+        }
+      } catch (error) {
+        spinner.fail('Error');
+        console.error(chalk.red('Error:'), (error as Error).message);
+        process.exit(1);
+      }
+    });
+
+  return cmd;
+}
+
+interface StressTestOptions {
+  client: {
+    generate: (req: { prompt: string; model?: string; temperature?: number }) => Promise<{
+      text: string;
+    }>;
+  };
+  model?: string;
+  prompts: string[];
+  concurrency: number;
+  durationMs: number;
+  rampUpMs: number;
+  maxRequests?: number;
+  temperature?: number;
+  onProgress?: (completed: number, active: number) => void;
+  verbose?: boolean;
+}
+
+async function runStressTest(options: StressTestOptions): Promise<StressRequestResult[]> {
+  const {
+    client,
+    model,
+    prompts,
+    concurrency,
+    durationMs,
+    rampUpMs,
+    maxRequests,
+    temperature,
+    onProgress,
+  } = options;
+
+  const results: StressRequestResult[] = [];
+  const startTime = Date.now();
+  const endTime = startTime + durationMs;
+  let completed = 0;
+  let active = 0;
+  let promptIndex = 0;
+
+  const makeRequest = async (): Promise<void> => {
+    const prompt = prompts[promptIndex % prompts.length];
+    promptIndex++;
+
+    const requestStart = Date.now();
+    active++;
+
+    try {
+      await client.generate({
+        prompt,
+        model,
+        temperature,
+      });
+
+      results.push({
+        success: true,
+        latencyMs: Date.now() - requestStart,
+        timestamp: requestStart,
+      });
+    } catch (error) {
+      results.push({
+        success: false,
+        latencyMs: Date.now() - requestStart,
+        error: (error as Error).message,
+        timestamp: requestStart,
+      });
+    } finally {
+      active--;
+      completed++;
+      onProgress?.(completed, active);
+    }
+  };
+
+  // Calculate target concurrency based on ramp-up
+  const getTargetConcurrency = (elapsed: number): number => {
+    if (elapsed >= rampUpMs) return concurrency;
+    return Math.ceil((elapsed / rampUpMs) * concurrency);
+  };
+
+  // Main loop
+  const promises: Promise<void>[] = [];
+
+  while (Date.now() < endTime) {
+    if (maxRequests && completed >= maxRequests) break;
+
+    const elapsed = Date.now() - startTime;
+    const targetConcurrency = getTargetConcurrency(elapsed);
+
+    // Launch new requests if below target
+    while (active < targetConcurrency && Date.now() < endTime) {
+      if (maxRequests && completed + active >= maxRequests) break;
+      promises.push(makeRequest());
+    }
+
+    // Small delay to prevent tight loop
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+
+  // Wait for all pending requests
+  await Promise.all(promises);
+
+  return results;
+}
+
+function calculateMetrics(results: StressRequestResult[], durationMs: number): StressMetrics {
+  const successful = results.filter((r) => r.success);
+  const latencies = successful.map((r) => r.latencyMs).sort((a, b) => a - b);
+
+  const totalRequests = results.length;
+  const successfulRequests = successful.length;
+  const failedRequests = totalRequests - successfulRequests;
+
+  const minLatency = latencies[0] || 0;
+  const maxLatency = latencies[latencies.length - 1] || 0;
+  const avgLatency =
+    latencies.length > 0 ? latencies.reduce((sum, l) => sum + l, 0) / latencies.length : 0;
+
+  const requestsPerSecond = durationMs > 0 ? (totalRequests / durationMs) * 1000 : 0;
+  const successRate = totalRequests > 0 ? successfulRequests / totalRequests : 0;
+
+  return {
+    total_requests: totalRequests,
+    successful_requests: successfulRequests,
+    failed_requests: failedRequests,
+    success_rate: successRate,
+    requests_per_second: requestsPerSecond,
+    min_latency_ms: minLatency,
+    max_latency_ms: maxLatency,
+    avg_latency_ms: Math.round(avgLatency),
+    p50_latency_ms: percentile(latencies, 50),
+    p90_latency_ms: percentile(latencies, 90),
+    p95_latency_ms: percentile(latencies, 95),
+    p99_latency_ms: percentile(latencies, 99),
+  };
+}
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  const index = Math.ceil((p / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, Math.min(index, sorted.length - 1))];
+}
+
+function sampleResults(results: StressRequestResult[], maxSamples: number): StressRequestResult[] {
+  if (results.length <= maxSamples) return results;
+
+  // Sample evenly across the results
+  const step = Math.floor(results.length / maxSamples);
+  const sampled: StressRequestResult[] = [];
+  for (let i = 0; i < results.length && sampled.length < maxSamples; i += step) {
+    sampled.push(results[i]);
+  }
+  return sampled;
+}
+
+function displayStats(metrics: StressMetrics, runId: string): void {
+  const table = new Table({
+    head: [chalk.bold('Metric'), chalk.bold('Value')],
+    style: { head: [], border: [] },
+  });
+
+  table.push(
+    ['Run ID', runId],
+    ['Total Requests', metrics.total_requests.toString()],
+    ['Successful', chalk.green(metrics.successful_requests.toString())],
+    ['Failed', metrics.failed_requests > 0 ? chalk.red(metrics.failed_requests.toString()) : '0'],
+    ['', ''],
+    ['Requests/sec', metrics.requests_per_second.toFixed(2)],
+    ['', ''],
+    ['Min Latency', `${metrics.min_latency_ms}ms`],
+    ['Max Latency', `${metrics.max_latency_ms}ms`],
+    ['Avg Latency', `${metrics.avg_latency_ms}ms`],
+    ['p50 Latency', `${metrics.p50_latency_ms}ms`],
+    ['p90 Latency', `${metrics.p90_latency_ms}ms`],
+    ['p95 Latency', `${metrics.p95_latency_ms}ms`],
+    ['p99 Latency', `${metrics.p99_latency_ms}ms`]
+  );
+
+  console.log(chalk.bold('Results'));
+  console.log(table.toString());
+
+  // Success rate
+  const successRate = metrics.success_rate * 100;
+
+  console.log();
+  if (successRate >= 99) {
+    console.log(chalk.green(`✓ Success rate: ${successRate.toFixed(2)}%`));
+  } else if (successRate >= 95) {
+    console.log(chalk.yellow(`⚠ Success rate: ${successRate.toFixed(2)}%`));
+  } else {
+    console.log(chalk.red(`✗ Success rate: ${successRate.toFixed(2)}%`));
+  }
+}
+
+function displayHistogram(results: StressRequestResult[]): void {
+  const successful = results.filter((r) => r.success);
+  if (successful.length === 0) return;
+
+  const latencies = successful.map((r) => r.latencyMs);
+  const maxLatency = Math.max(...latencies);
+  const bucketSize = Math.ceil(maxLatency / 10);
+  const buckets = new Array(10).fill(0);
+
+  for (const latency of latencies) {
+    const bucket = Math.min(Math.floor(latency / bucketSize), 9);
+    buckets[bucket]++;
+  }
+
+  const maxCount = Math.max(...buckets);
+
+  console.log();
+  console.log(chalk.bold('Latency Distribution'));
+  console.log();
+
+  for (let i = 0; i < 10; i++) {
+    const rangeStart = i * bucketSize;
+    const rangeEnd = (i + 1) * bucketSize;
+    const count = buckets[i];
+    const barLength = maxCount > 0 ? Math.round((count / maxCount) * 30) : 0;
+    const bar = '█'.repeat(barLength);
+
+    console.log(
+      `${chalk.dim(`${rangeStart.toString().padStart(5)}-${rangeEnd.toString().padStart(5)}ms`)} │ ${chalk.cyan(bar)} ${count}`
+    );
+  }
+}

package/src/config/index.ts

@@ -0,0 +1,6 @@
+/**
+ * Config module exports
+ */
+
+export * from './schema';
+export * from './loader';

package/src/config/loader.ts

@@ -0,0 +1,112 @@
+/**
+ * Configuration file loader
+ */
+
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { type ArtemisConfig, ArtemisConfigSchema } from './schema';
+
+const CONFIG_FILENAMES = ['artemis.config.yaml', 'artemis.config.yml', 'artemis.yaml'];
+
+/**
+ * Find and load the configuration file
+ */
+export async function loadConfig(configPath?: string): Promise<ArtemisConfig | null> {
+  const path = configPath || findConfigFile();
+
+  if (!path) {
+    return null;
+  }
+
+  try {
+    const content = await readFile(path, 'utf-8');
+    const raw = parseYaml(content);
+
+    // Expand environment variables
+    const expanded = expandEnvVars(raw);
+
+    const result = ArtemisConfigSchema.safeParse(expanded);
+
+    if (!result.success) {
+      const issues = result.error.issues
+        .map((i) => `  - ${i.path.join('.')}: ${i.message}`)
+        .join('\n');
+      throw new Error(`Invalid config file ${path}:\n${issues}`);
+    }
+
+    return result.data;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return null;
+    }
+    throw error;
+  }
+}
+
+/**
+ * Find config file in current directory or parents
+ */
+function findConfigFile(): string | null {
+  let dir = process.cwd();
+  const root = resolve('/');
+
+  while (dir !== root) {
+    for (const filename of CONFIG_FILENAMES) {
+      const path = join(dir, filename);
+      if (existsSync(path)) {
+        return path;
+      }
+    }
+    dir = resolve(dir, '..');
+  }
+
+  return null;
+}
+
+/**
+ * Expand environment variables in config values
+ * Supports ${VAR} and ${VAR:-default} syntax
+ */
+function expandEnvVars(obj: unknown): unknown {
+  if (typeof obj === 'string') {
+    return obj.replace(/\$\{([^}]+)\}/g, (_, expr) => {
+      const [varName, defaultValue] = expr.split(':-');
+      return process.env[varName] || defaultValue || '';
+    });
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(expandEnvVars);
+  }
+
+  if (obj && typeof obj === 'object') {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = expandEnvVars(value);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
+/**
+ * Get a merged config with CLI options taking precedence
+ */
+export function mergeConfig(
+  fileConfig: ArtemisConfig | null,
+  cliOptions: Partial<ArtemisConfig>
+): ArtemisConfig {
+  const defaults: ArtemisConfig = {
+    project: 'default',
+    scenariosDir: './scenarios',
+  };
+
+  return {
+    ...defaults,
+    ...fileConfig,
+    ...Object.fromEntries(Object.entries(cliOptions).filter(([_, v]) => v !== undefined)),
+  } as ArtemisConfig;
+}

package/src/config/schema.ts

@@ -0,0 +1,56 @@
+/**
+ * Configuration schema for artemis.config.yaml
+ */
+
+import { z } from 'zod';
+
+const ProviderConfigSchema = z.object({
+  apiKey: z.string().optional(),
+  baseUrl: z.string().optional(),
+  defaultModel: z.string().optional(),
+  timeout: z.number().optional(),
+  maxRetries: z.number().optional(),
+  // OpenAI specific
+  organization: z.string().optional(),
+  // Azure specific
+  resourceName: z.string().optional(),
+  deploymentName: z.string().optional(),
+  apiVersion: z.string().optional(),
+  // Vercel AI specific
+  underlyingProvider: z.enum(['openai', 'azure', 'anthropic', 'google', 'mistral']).optional(),
+});
+
+const StorageConfigSchema = z.object({
+  type: z.enum(['supabase', 'local']).default('local'),
+  url: z.string().optional(),
+  anonKey: z.string().optional(),
+  bucket: z.string().optional(),
+  basePath: z.string().optional(),
+});
+
+const OutputConfigSchema = z.object({
+  format: z.enum(['json', 'html', 'both']).default('json'),
+  dir: z.string().default('./artemis-output'),
+});
+
+const CIConfigSchema = z.object({
+  failOnRegression: z.boolean().default(true),
+  regressionThreshold: z.number().min(0).max(1).default(0.05),
+  baselineStrategy: z.enum(['latest', 'tagged', 'specific']).default('latest'),
+  baselineRunId: z.string().optional(),
+});
+
+export const ArtemisConfigSchema = z.object({
+  project: z.string().default('default'),
+  provider: z.string().optional(),
+  model: z.string().optional(),
+  providers: z.record(ProviderConfigSchema).optional(),
+  storage: StorageConfigSchema.optional(),
+  scenariosDir: z.string().default('./scenarios'),
+  output: OutputConfigSchema.optional(),
+  ci: CIConfigSchema.optional(),
+});
+
+export type ArtemisConfig = z.infer<typeof ArtemisConfigSchema>;
+export type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
+export type StorageConfig = z.infer<typeof StorageConfigSchema>;

package/src/index.ts

@@ -0,0 +1,6 @@
+/**
+ * @artemiskit/cli
+ * Command-line interface for Artemis Agent Reliability Toolkit
+ */
+
+export { createCLI } from './cli';