@curl-runner/cli 1.13.0 → 1.15.0

package/src/executor/profile-executor.ts

@@ -0,0 +1,167 @@
+import type {
+  ExecutionResult,
+  GlobalConfig,
+  ProfileConfig,
+  ProfileResult,
+  RequestConfig,
+} from '../types/config';
+import { CurlBuilder } from '../utils/curl-builder';
+import { Logger } from '../utils/logger';
+import { calculateProfileStats } from '../utils/stats';
+
+export class ProfileExecutor {
+  private logger: Logger;
+  private profileConfig: ProfileConfig;
+
+  constructor(globalConfig: GlobalConfig, profileConfig: ProfileConfig) {
+    this.profileConfig = profileConfig;
+    this.logger = new Logger(globalConfig.output);
+  }
+
+  /**
+   * Execute a single iteration of the request (minimal overhead version).
+   * Skips logging and validation for accurate timing.
+   */
+  private async executeSingleIteration(config: RequestConfig): Promise<ExecutionResult> {
+    const startTime = performance.now();
+    const command = CurlBuilder.buildCommand(config);
+    const result = await CurlBuilder.executeCurl(command);
+
+    if (result.success) {
+      let body = result.body;
+      try {
+        if (
+          result.headers?.['content-type']?.includes('application/json') ||
+          (body && (body.trim().startsWith('{') || body.trim().startsWith('[')))
+        ) {
+          body = JSON.parse(body);
+        }
+      } catch (_e) {
+        // Keep raw body
+      }
+
+      return {
+        request: config,
+        success: true,
+        status: result.status,
+        headers: result.headers,
+        body,
+        metrics: {
+          ...result.metrics,
+          duration: performance.now() - startTime,
+        },
+      };
+    }
+
+    return {
+      request: config,
+      success: false,
+      error: result.error,
+      metrics: {
+        duration: performance.now() - startTime,
+      },
+    };
+  }
+
+  /**
+   * Execute iterations in chunks for controlled concurrency.
+   */
+  private async executeWithConcurrency(
+    config: RequestConfig,
+    iterations: number,
+    concurrency: number,
+  ): Promise<ExecutionResult[]> {
+    const results: ExecutionResult[] = [];
+
+    for (let i = 0; i < iterations; i += concurrency) {
+      const chunkSize = Math.min(concurrency, iterations - i);
+      const chunk = await Promise.all(
+        Array.from({ length: chunkSize }, () => this.executeSingleIteration(config)),
+      );
+      results.push(...chunk);
+    }
+
+    return results;
+  }
+
+  /**
+   * Profile a single request by running it multiple times.
+   */
+  async profileRequest(config: RequestConfig, index = 0): Promise<ProfileResult> {
+    const iterations = this.profileConfig.iterations;
+    const warmup = this.profileConfig.warmup ?? 1;
+    const concurrency = this.profileConfig.concurrency ?? 1;
+    const requestName = config.name || `Request ${index + 1}`;
+
+    this.logger.logProfileStart(requestName, iterations, warmup, concurrency);
+
+    const results =
+      concurrency === 1
+        ? await this.executeSequentially(config, iterations)
+        : await this.executeWithConcurrency(config, iterations, concurrency);
+
+    // Collect timings and count failures
+    const timings: number[] = [];
+    let failures = 0;
+
+    for (const result of results) {
+      if (result.success && result.metrics?.duration !== undefined) {
+        timings.push(result.metrics.duration);
+      } else {
+        failures++;
+        // Use 0 as placeholder for failed requests (excluded from stats)
+        timings.push(0);
+      }
+    }
+
+    // Filter out failed timings (0s) for stats calculation
+    const successfulTimings = timings
+      .map((t, i) => (results[i].success ? t : -1))
+      .filter((t) => t >= 0);
+
+    // Recalculate stats with only successful timings
+    const stats = calculateProfileStats(
+      successfulTimings,
+      Math.min(warmup, successfulTimings.length),
+      failures,
+    );
+
+    return {
+      request: config,
+      stats,
+      iterations: results,
+    };
+  }
+
+  /**
+   * Execute iterations sequentially (default behavior).
+   */
+  private async executeSequentially(
+    config: RequestConfig,
+    iterations: number,
+  ): Promise<ExecutionResult[]> {
+    const results: ExecutionResult[] = [];
+
+    for (let i = 0; i < iterations; i++) {
+      const result = await this.executeSingleIteration(config);
+      results.push(result);
+    }
+
+    return results;
+  }
+
+  /**
+   * Profile multiple requests.
+   */
+  async profileRequests(requests: RequestConfig[]): Promise<ProfileResult[]> {
+    const results: ProfileResult[] = [];
+
+    for (let i = 0; i < requests.length; i++) {
+      const result = await this.profileRequest(requests[i], i);
+      results.push(result);
+      this.logger.logProfileResult(result, this.profileConfig.histogram ?? false);
+    }
+
+    return results;
+  }
+}

package/src/types/config.ts

@@ -257,6 +257,11 @@ export interface RequestConfig {
    * Use `true` to enable with defaults, or provide detailed config.
    */
   snapshot?: SnapshotConfig | boolean;
+  /**
+   * Response diffing configuration for this request.
+   * Use `true` to enable with defaults, or provide detailed config.
+   */
+  diff?: DiffConfig | boolean;
   sourceOutputConfig?: {
     verbose?: boolean;
     showHeaders?: boolean;
@@ -325,11 +330,21 @@ export interface GlobalConfig {
    * Automatically re-runs requests when YAML files change.
    */
   watch?: WatchConfig;
+  /**
+   * Performance profiling mode configuration.
+   * Runs requests multiple times to collect p50/p95/p99 latency stats.
+   */
+  profile?: ProfileConfig;
   /**
    * Snapshot testing configuration.
    * Saves response snapshots and compares future runs against them.
    */
   snapshot?: GlobalSnapshotConfig;
+  /**
+   * Response diffing configuration.
+   * Compare responses between environments or runs to detect API drift.
+   */
+  diff?: GlobalDiffConfig;
   variables?: Record<string, string>;
   output?: {
     verbose?: boolean;
@@ -369,6 +384,8 @@ export interface ExecutionResult {
   };
   /** Snapshot comparison result (if snapshot testing enabled). */
   snapshotResult?: SnapshotCompareResult;
+  /** Diff comparison result (if response diffing enabled). */
+  diffResult?: DiffCompareResult;
   /** Whether this request was skipped due to a `when` condition. */
   skipped?: boolean;
   /** Reason the request was skipped (condition that failed). */
@@ -403,6 +420,65 @@ export interface WatchConfig {
   clear?: boolean;
 }
 
+/**
+ * Configuration for performance profiling mode.
+ * Runs requests multiple times to collect latency statistics.
+ */
+export interface ProfileConfig {
+  /** Number of iterations to run. Default: 10 */
+  iterations: number;
+  /** Number of warmup iterations to exclude from stats. Default: 1 */
+  warmup?: number;
+  /** Number of concurrent iterations. Default: 1 (sequential) */
+  concurrency?: number;
+  /** Show ASCII histogram in output. Default: false */
+  histogram?: boolean;
+  /** Export raw timings to file (JSON or CSV based on extension) */
+  exportFile?: string;
+}
+
+/**
+ * Statistics computed from profile run timings.
+ */
+export interface ProfileStats {
+  /** Total iterations run (excluding warmup) */
+  iterations: number;
+  /** Warmup iterations excluded */
+  warmup: number;
+  /** Minimum latency in ms */
+  min: number;
+  /** Maximum latency in ms */
+  max: number;
+  /** Mean latency in ms */
+  mean: number;
+  /** Median latency in ms (same as p50) */
+  median: number;
+  /** 50th percentile latency in ms */
+  p50: number;
+  /** 95th percentile latency in ms */
+  p95: number;
+  /** 99th percentile latency in ms */
+  p99: number;
+  /** Standard deviation in ms */
+  stdDev: number;
+  /** Number of failed iterations */
+  failures: number;
+  /** Failure rate as percentage */
+  failureRate: number;
+  /** Raw timing values (for export) */
+  timings: number[];
+}
+
+/**
+ * Result of a profiled request execution.
+ */
+export interface ProfileResult {
+  request: RequestConfig;
+  stats: ProfileStats;
+  /** Individual results from each iteration */
+  iterations: ExecutionResult[];
+}
+
 /**
  * Configuration for snapshot testing.
  * Snapshots save response data and compare future runs against them.
@@ -470,3 +546,93 @@ export interface SnapshotCompareResult {
   updated: boolean;
   differences: SnapshotDiff[];
 }
+
+/**
+ * Configuration for response diffing at request level.
+ */
+export interface DiffConfig {
+  /** Enable diffing for this request. */
+  enabled?: boolean;
+  /** Paths to exclude from comparison (e.g., 'body.timestamp'). */
+  exclude?: string[];
+  /** Match rules for dynamic values (path -> '*' or 'regex:pattern'). */
+  match?: Record<string, string>;
+  /** Include timing differences in comparison. Default: false */
+  includeTimings?: boolean;
+}
+
+/**
+ * Global configuration for response diffing.
+ */
+export interface GlobalDiffConfig extends DiffConfig {
+  /** Directory for baseline files. Default: '__baselines__' */
+  dir?: string;
+  /** Label for current run (e.g., 'staging', 'production'). */
+  label?: string;
+  /** Label to compare against. */
+  compareWith?: string;
+  /** Save current run as baseline. */
+  save?: boolean;
+  /** Output format for diff results. Default: 'terminal' */
+  outputFormat?: 'terminal' | 'json' | 'markdown';
+}
+
+/**
+ * Stored baseline data for a single request.
+ */
+export interface Baseline {
+  status?: number;
+  headers?: Record<string, string>;
+  body?: JsonValue;
+  timing?: number;
+  hash: string;
+  capturedAt: string;
+}
+
+/**
+ * Baseline file format.
+ */
+export interface BaselineFile {
+  version: number;
+  label: string;
+  capturedAt: string;
+  baselines: Record<string, Baseline>;
+}
+
+/**
+ * Single difference in response comparison.
+ */
+export interface ResponseDiff {
+  path: string;
+  baseline: unknown;
+  current: unknown;
+  type: 'added' | 'removed' | 'changed' | 'type_mismatch';
+}
+
+/**
+ * Result of comparing a response against a baseline.
+ */
+export interface DiffCompareResult {
+  requestName: string;
+  hasDifferences: boolean;
+  isNewBaseline: boolean;
+  baselineLabel: string;
+  currentLabel: string;
+  differences: ResponseDiff[];
+  timingDiff?: {
+    baseline: number;
+    current: number;
+    changePercent: number;
+  };
+}
+
+/**
+ * Summary of diff comparison across all requests.
+ */
+export interface DiffSummary {
+  totalRequests: number;
+  unchanged: number;
+  changed: number;
+  newBaselines: number;
+  results: DiffCompareResult[];
+}

package/src/utils/logger.ts

@@ -3,8 +3,10 @@ import type {
   ExecutionResult,
   ExecutionSummary,
   GlobalConfig,
+  ProfileResult,
   RequestConfig,
 } from '../types/config';
+import { generateHistogram } from './stats';
 
 interface TreeNode {
   label: string;
@@ -732,4 +734,123 @@ export class Logger {
     );
     console.log();
   }
+
+  logProfileStart(
+    requestName: string,
+    iterations: number,
+    warmup: number,
+    concurrency: number,
+  ): void {
+    if (!this.shouldShowOutput()) {
+      return;
+    }
+
+    console.log();
+    console.log(`${this.color('⚡ PROFILING', 'magenta')} ${this.color(requestName, 'bright')}`);
+    console.log(
+      this.color(
+        `   ${iterations} iterations, ${warmup} warmup, concurrency: ${concurrency}`,
+        'dim',
+      ),
+    );
+  }
+
+  logProfileResult(result: ProfileResult, showHistogram: boolean): void {
+    const { stats, request } = result;
+    const name = request.name || request.url;
+
+    if (this.config.format === 'json') {
+      console.log(
+        JSON.stringify({
+          request: { name, url: request.url, method: request.method || 'GET' },
+          stats: {
+            iterations: stats.iterations,
+            warmup: stats.warmup,
+            failures: stats.failures,
+            failureRate: stats.failureRate,
+            min: stats.min,
+            max: stats.max,
+            mean: stats.mean,
+            median: stats.median,
+            p50: stats.p50,
+            p95: stats.p95,
+            p99: stats.p99,
+            stdDev: stats.stdDev,
+          },
+        }),
+      );
+      return;
+    }
+
+    if (this.config.format === 'raw') {
+      // Raw format: just print the key stats
+      console.log(`${stats.p50}\t${stats.p95}\t${stats.p99}\t${stats.mean}`);
+      return;
+    }
+
+    // Pretty format
+    console.log();
+    const statusIcon = stats.failures === 0 ? this.color('✓', 'green') : this.color('⚠', 'yellow');
+    console.log(`${statusIcon} ${this.color(name, 'bright')}`);
+
+    // Latency stats table
+    console.log(this.color('   ┌─────────────────────────────────────┐', 'dim'));
+    console.log(
+      `   │ ${this.color('p50', 'cyan')}    ${this.formatLatency(stats.p50).padStart(10)} │ ${this.color('min', 'dim')}   ${this.formatLatency(stats.min).padStart(10)} │`,
+    );
+    console.log(
+      `   │ ${this.color('p95', 'yellow')}    ${this.formatLatency(stats.p95).padStart(10)} │ ${this.color('max', 'dim')}   ${this.formatLatency(stats.max).padStart(10)} │`,
+    );
+    console.log(
+      `   │ ${this.color('p99', 'red')}    ${this.formatLatency(stats.p99).padStart(10)} │ ${this.color('mean', 'dim')}  ${this.formatLatency(stats.mean).padStart(10)} │`,
+    );
+    console.log(this.color('   └─────────────────────────────────────┘', 'dim'));
+
+    // Additional stats
+    console.log(
+      this.color(
+        `   σ ${stats.stdDev.toFixed(2)}ms | ${stats.iterations} samples | ${stats.failures} failures (${stats.failureRate}%)`,
+        'dim',
+      ),
+    );
+
+    // Optional histogram
+    if (showHistogram && stats.timings.length > 0) {
+      console.log();
+      console.log(this.color('   Distribution:', 'dim'));
+      const histogramLines = generateHistogram(stats.timings, 8, 30);
+      for (const line of histogramLines) {
+        console.log(`   ${this.color(line, 'dim')}`);
+      }
+    }
+  }
+
+  private formatLatency(ms: number): string {
+    if (ms < 1) {
+      return `${(ms * 1000).toFixed(0)}µs`;
+    }
+    if (ms < 1000) {
+      return `${ms.toFixed(1)}ms`;
+    }
+    return `${(ms / 1000).toFixed(2)}s`;
+  }
+
+  logProfileSummary(results: ProfileResult[]): void {
+    if (!this.shouldShowOutput()) {
+      return;
+    }
+
+    const totalIterations = results.reduce((sum, r) => sum + r.stats.iterations, 0);
+    const totalFailures = results.reduce((sum, r) => sum + r.stats.failures, 0);
+
+    console.log();
+    console.log(this.color('─'.repeat(50), 'dim'));
+    console.log(
+      `${this.color('⚡ Profile Summary:', 'magenta')} ${results.length} request${results.length === 1 ? '' : 's'}, ${totalIterations} total iterations`,
+    );
+
+    if (totalFailures > 0) {
+      console.log(this.color(`   ${totalFailures} total failures`, 'yellow'));
+    }
+  }
 }

package/src/utils/stats.test.ts

@@ -0,0 +1,161 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  calculateMean,
+  calculatePercentile,
+  calculateProfileStats,
+  calculateStdDev,
+  exportToCSV,
+  exportToJSON,
+  generateHistogram,
+} from './stats';
+
+describe('calculatePercentile', () => {
+  test('returns 0 for empty array', () => {
+    expect(calculatePercentile([], 50)).toBe(0);
+  });
+
+  test('returns single value for array of 1', () => {
+    expect(calculatePercentile([100], 50)).toBe(100);
+    expect(calculatePercentile([100], 99)).toBe(100);
+  });
+
+  test('calculates p50 (median) correctly', () => {
+    expect(calculatePercentile([1, 2, 3, 4, 5], 50)).toBe(3);
+    expect(calculatePercentile([1, 2, 3, 4], 50)).toBe(2.5);
+  });
+
+  test('calculates p95 correctly', () => {
+    const values = Array.from({ length: 100 }, (_, i) => i + 1);
+    expect(calculatePercentile(values, 95)).toBeCloseTo(95.05, 1);
+  });
+
+  test('calculates p99 correctly', () => {
+    const values = Array.from({ length: 100 }, (_, i) => i + 1);
+    expect(calculatePercentile(values, 99)).toBeCloseTo(99.01, 1);
+  });
+
+  test('handles unsorted input (requires pre-sorting)', () => {
+    // Note: function expects sorted input
+    const sorted = [10, 20, 30, 40, 50].sort((a, b) => a - b);
+    expect(calculatePercentile(sorted, 50)).toBe(30);
+  });
+});
+
+describe('calculateMean', () => {
+  test('returns 0 for empty array', () => {
+    expect(calculateMean([])).toBe(0);
+  });
+
+  test('calculates mean correctly', () => {
+    expect(calculateMean([1, 2, 3, 4, 5])).toBe(3);
+    expect(calculateMean([10, 20, 30])).toBe(20);
+    expect(calculateMean([100])).toBe(100);
+  });
+});
+
+describe('calculateStdDev', () => {
+  test('returns 0 for empty array', () => {
+    expect(calculateStdDev([], 0)).toBe(0);
+  });
+
+  test('returns 0 for single value', () => {
+    expect(calculateStdDev([100], 100)).toBe(0);
+  });
+
+  test('calculates standard deviation correctly', () => {
+    const values = [2, 4, 4, 4, 5, 5, 7, 9];
+    const mean = calculateMean(values);
+    expect(calculateStdDev(values, mean)).toBeCloseTo(2, 0);
+  });
+});
+
+describe('calculateProfileStats', () => {
+  test('calculates stats correctly with no warmup', () => {
+    const timings = [10, 20, 30, 40, 50];
+    const stats = calculateProfileStats(timings, 0, 0);
+
+    expect(stats.iterations).toBe(5);
+    expect(stats.warmup).toBe(0);
+    expect(stats.min).toBe(10);
+    expect(stats.max).toBe(50);
+    expect(stats.mean).toBe(30);
+    expect(stats.failures).toBe(0);
+    expect(stats.failureRate).toBe(0);
+  });
+
+  test('excludes warmup iterations from stats', () => {
+    const timings = [100, 10, 20, 30, 40]; // First value is warmup outlier
+    const stats = calculateProfileStats(timings, 1, 0);
+
+    expect(stats.iterations).toBe(4);
+    expect(stats.warmup).toBe(1);
+    expect(stats.min).toBe(10);
+    expect(stats.max).toBe(40);
+    expect(stats.mean).toBe(25);
+  });
+
+  test('calculates failure rate correctly', () => {
+    const timings = [10, 20, 30];
+    const stats = calculateProfileStats(timings, 0, 2);
+
+    expect(stats.failures).toBe(2);
+    expect(stats.failureRate).toBeCloseTo(66.67, 1);
+  });
+
+  test('handles empty timings', () => {
+    const stats = calculateProfileStats([], 0, 0);
+
+    expect(stats.iterations).toBe(0);
+    expect(stats.min).toBe(0);
+    expect(stats.max).toBe(0);
+    expect(stats.mean).toBe(0);
+  });
+});
+
+describe('generateHistogram', () => {
+  test('returns "No data" for empty array', () => {
+    const result = generateHistogram([]);
+    expect(result).toEqual(['No data']);
+  });
+
+  test('generates histogram with correct bucket count', () => {
+    const timings = Array.from({ length: 100 }, (_, i) => i);
+    const result = generateHistogram(timings, 5, 20);
+
+    expect(result.length).toBe(5);
+  });
+
+  test('histogram lines contain bucket ranges', () => {
+    const timings = [10, 20, 30, 40, 50];
+    const result = generateHistogram(timings, 2, 10);
+
+    expect(result[0]).toContain('ms -');
+    expect(result[0]).toContain('ms │');
+  });
+});
+
+describe('exportToCSV', () => {
+  test('exports stats to CSV format', () => {
+    const stats = calculateProfileStats([10, 20, 30], 0, 0);
+    const csv = exportToCSV(stats, 'Test Request');
+
+    expect(csv).toContain('iteration,latency_ms');
+    expect(csv).toContain('1,10');
+    expect(csv).toContain('2,20');
+    expect(csv).toContain('3,30');
+  });
+});
+
+describe('exportToJSON', () => {
+  test('exports stats to JSON format', () => {
+    const stats = calculateProfileStats([10, 20, 30], 0, 0);
+    const json = exportToJSON(stats, 'Test Request');
+    const parsed = JSON.parse(json);
+
+    expect(parsed.request).toBe('Test Request');
+    expect(parsed.summary.iterations).toBe(3);
+    expect(parsed.summary.min).toBe(10);
+    expect(parsed.summary.max).toBe(30);
+    expect(parsed.timings).toEqual([10, 20, 30]);
+  });
+});