modestbench 0.3.2 → 0.5.0

package/src/cli/commands/test.ts

@@ -0,0 +1,546 @@
+/**
+ * ModestBench Test Adapter Command
+ *
+ * Run existing test files as benchmarks by capturing test definitions and
+ * executing them through a lightweight benchmark runner.
+ */
+
+import { hostname } from 'node:os';
+import { cpus, freemem, totalmem } from 'node:os';
+import { resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+import type {
+  BenchmarkRun,
+  FileResult,
+  ModestBenchConfig,
+  RunSummary,
+  SuiteResult,
+  TaskResult,
+} from '../../types/core.js';
+import type { CliContext } from '../index.js';
+
+import { AvaAdapter } from '../../adapters/ava-adapter.js';
+import { MochaAdapter } from '../../adapters/mocha-adapter.js';
+import { NodeTestAdapter } from '../../adapters/node-test-adapter.js';
+import {
+  type CapturedSuite,
+  type CapturedTestFile,
+  capturedToBenchmark,
+  type ConvertedBenchmarkSuite,
+  type TestFramework,
+} from '../../adapters/types.js';
+import { ExitCodes } from '../../types/cli.js';
+import { createRunId } from '../../types/core.js';
+import { isError } from '../../utils/type-guards.js';
+
+/**
+ * Get a minimal default config for test runs
+ */
+const getDefaultTestConfig = (
+  iterations: number,
+  warmup: number,
+  verbose: boolean,
+): ModestBenchConfig => ({
+  bail: false,
+  exclude: [],
+  excludeTags: [],
+  iterations,
+  limitBy: 'iterations',
+  metadata: {},
+  pattern: [],
+  quiet: false,
+  reporterConfig: {},
+  reporters: ['human'],
+  tags: [],
+  thresholds: {},
+  time: 1000,
+  timeout: 30000,
+  verbose,
+  warmup,
+});
+
+/**
+ * Default iteration count for test benchmarks
+ */
+const DEFAULT_ITERATIONS = 100;
+
+/**
+ * Default warmup iterations
+ */
+const DEFAULT_WARMUP = 5;
+
+/**
+ * Test command options
+ */
+export interface TestOptions {
+  /** Bail on first failure */
+  bail?: boolean;
+  /** Working directory */
+  cwd?: string;
+  /** Test framework to use */
+  framework: TestFramework;
+  /** Number of iterations per test */
+  iterations?: number;
+  /** Output JSON */
+  json?: boolean;
+  /** Disable color */
+  noColor?: boolean;
+  /** Test file paths or patterns */
+  pattern?: string[];
+  /** Quiet mode */
+  quiet?: boolean;
+  /** Verbose output */
+  verbose?: boolean;
+  /** Warmup iterations */
+  warmup?: number;
+}
+
+/**
+ * Result of benchmarking a single test
+ */
+interface TestBenchmarkResult {
+  /** Error if test failed */
+  error?: Error;
+  /** Number of iterations run */
+  iterations: number;
+  /** Maximum execution time in ms */
+  max: number;
+  /** Mean execution time in ms */
+  mean: number;
+  /** Minimum execution time in ms */
+  min: number;
+  /** Test name */
+  name: string;
+  /** Operations per second */
+  opsPerSecond: number;
+  /** Standard deviation in ms */
+  stdDev: number;
+  /** Total time in ms */
+  totalTime: number;
+}
+
+/**
+ * Handle test command - run test files as benchmarks
+ */
+export const handleTestCommand = async (
+  context: CliContext,
+  options: TestOptions,
+): Promise<number> => {
+  const verbose = options.verbose ?? false;
+  const quiet = options.quiet ?? false;
+  const iterations = options.iterations ?? DEFAULT_ITERATIONS;
+  const warmup = options.warmup ?? DEFAULT_WARMUP;
+
+  // Get the reporter (default to human unless JSON requested)
+  const reporterName = options.json ? 'json' : 'human';
+  const reporter = context.reporterRegistry.get(reporterName);
+
+  try {
+    // Select the appropriate adapter
+    const adapter = selectAdapter(options.framework);
+
+    if (verbose && !quiet) {
+      console.error(`Using ${options.framework} adapter`);
+    }
+
+    // Resolve file paths
+    const cwd = options.cwd ?? process.cwd();
+    const patterns = options.pattern ?? [];
+
+    if (patterns.length === 0) {
+      console.error('Error: At least one test file path is required');
+      return ExitCodes.CONFIG_ERROR;
+    }
+
+    // Build config for the run
+    const config = getDefaultTestConfig(iterations, warmup, verbose);
+
+    // Initialize run tracking
+    const runStartTime = new Date();
+    const fileResults: FileResult[] = [];
+    let hasFailures = false;
+
+    // Build initial run for onStart (will be updated at end)
+    const runId = createRunId(crypto.randomUUID());
+    const initialRun = buildBenchmarkRun(
+      [],
+      config,
+      runStartTime,
+      runStartTime,
+      runId,
+    );
+
+    // Notify reporter of start
+    await reporter?.onStart?.(initialRun);
+
+    for (const pattern of patterns) {
+      const filePath = resolve(cwd, pattern);
+      const fileStartTime = new Date();
+
+      if (verbose && !quiet) {
+        console.error(`\nCapturing tests from: ${filePath}`);
+      }
+
+      await reporter?.onFileStart?.(filePath);
+
+      try {
+        const captured = await adapter.capture(filePath);
+
+        if (verbose && !quiet) {
+          const testCount = countAllTests(captured);
+          console.error(`  Found ${testCount} test(s)`);
+        }
+
+        // Convert to benchmark definition
+        const benchmarkDef = capturedToBenchmark(captured);
+
+        // Run benchmarks and collect suite results
+        const suiteResults: SuiteResult[] = [];
+
+        // Execute benchmarks for each suite
+        for (const [suiteName, suite] of Object.entries(
+          benchmarkDef.suites,
+        ) as Array<[string, ConvertedBenchmarkSuite]>) {
+          const suiteStartTime = new Date();
+          const taskResults: TaskResult[] = [];
+
+          await reporter?.onSuiteStart?.(suiteName);
+
+          // Run setup if present
+          if (suite.setup) {
+            try {
+              await suite.setup();
+            } catch (setupError) {
+              const err = isError(setupError)
+                ? setupError
+                : new Error(String(setupError));
+              await reporter?.onError?.(err);
+              hasFailures = true;
+              if (options.bail) {
+                return ExitCodes.BENCHMARK_FAILURES;
+              }
+              continue;
+            }
+          }
+
+          // Run each benchmark
+          for (const [benchName, bench] of Object.entries(suite.benchmarks)) {
+            await reporter?.onTaskStart?.(benchName);
+
+            const result = await runBenchmark(
+              benchName,
+              bench.fn,
+              iterations,
+              warmup,
+            );
+
+            // Convert to TaskResult format
+            const taskResult = convertToTaskResult(result);
+            taskResults.push(taskResult);
+
+            // Notify reporter
+            await reporter?.onTaskResult?.(taskResult);
+
+            if (result.error) {
+              hasFailures = true;
+              if (options.bail) {
+                // Run teardown before bailing
+                if (suite.teardown) {
+                  try {
+                    await suite.teardown();
+                  } catch {
+                    // Ignore teardown errors when bailing
+                  }
+                }
+                return ExitCodes.BENCHMARK_FAILURES;
+              }
+            }
+          }
+
+          // Run teardown if present
+          if (suite.teardown) {
+            try {
+              await suite.teardown();
+            } catch (teardownError) {
+              const err = isError(teardownError)
+                ? teardownError
+                : new Error(String(teardownError));
+              await reporter?.onError?.(err);
+            }
+          }
+
+          const suiteEndTime = new Date();
+          const suiteResult: SuiteResult = {
+            duration: suiteEndTime.getTime() - suiteStartTime.getTime(),
+            endTime: suiteEndTime,
+            name: suiteName,
+            startTime: suiteStartTime,
+            tasks: taskResults,
+          };
+
+          suiteResults.push(suiteResult);
+          await reporter?.onSuiteEnd?.(suiteResult);
+        }
+
+        const fileEndTime = new Date();
+        const fileResult: FileResult = {
+          duration: fileEndTime.getTime() - fileStartTime.getTime(),
+          endTime: fileEndTime,
+          filePath,
+          startTime: fileStartTime,
+          suites: suiteResults,
+        };
+
+        fileResults.push(fileResult);
+        await reporter?.onFileEnd?.(fileResult);
+      } catch (captureError) {
+        const err = isError(captureError)
+          ? captureError
+          : new Error(String(captureError));
+        await reporter?.onError?.(err);
+        hasFailures = true;
+        if (options.bail) {
+          return ExitCodes.BENCHMARK_FAILURES;
+        }
+      }
+    }
+
+    // Build final run result
+    const runEndTime = new Date();
+    const run = buildBenchmarkRun(
+      fileResults,
+      config,
+      runStartTime,
+      runEndTime,
+      runId,
+    );
+
+    // Notify reporter of end
+    await reporter?.onEnd?.(run);
+
+    return hasFailures ? ExitCodes.BENCHMARK_FAILURES : ExitCodes.SUCCESS;
+  } catch (error) {
+    const err = isError(error) ? error : new Error(String(error));
+    await reporter?.onError?.(err);
+    return ExitCodes.RUNTIME_ERROR;
+  }
+};
+
+/**
+ * Run a benchmark for a single test function
+ */
+const runBenchmark = async (
+  name: string,
+  fn: (...args: any[]) => unknown,
+  iterations: number,
+  warmup: number,
+): Promise<TestBenchmarkResult> => {
+  const times: number[] = [];
+
+  try {
+    // Warmup phase
+    for (let i = 0; i < warmup; i++) {
+      await fn();
+    }
+
+    // Measurement phase
+    for (let i = 0; i < iterations; i++) {
+      const start = performance.now();
+      await fn();
+      const end = performance.now();
+      times.push(end - start);
+    }
+
+    // Calculate statistics
+    const totalTime = times.reduce((a, b) => a + b, 0);
+    const mean = totalTime / times.length;
+    const min = Math.min(...times);
+    const max = Math.max(...times);
+
+    // Standard deviation
+    const squaredDiffs = times.map((t) => Math.pow(t - mean, 2));
+    const avgSquaredDiff =
+      squaredDiffs.reduce((a, b) => a + b, 0) / squaredDiffs.length;
+    const stdDev = Math.sqrt(avgSquaredDiff);
+
+    // Ops per second
+    const opsPerSecond = mean > 0 ? 1000 / mean : 0;
+
+    return {
+      iterations,
+      max,
+      mean,
+      min,
+      name,
+      opsPerSecond,
+      stdDev,
+      totalTime,
+    };
+  } catch (error) {
+    return {
+      error: isError(error) ? error : new Error(String(error)),
+      iterations: 0,
+      max: 0,
+      mean: 0,
+      min: 0,
+      name,
+      opsPerSecond: 0,
+      stdDev: 0,
+      totalTime: 0,
+    };
+  }
+};
+
+/**
+ * Convert internal TestBenchmarkResult to standard TaskResult format
+ *
+ * Note: Times are converted from milliseconds to nanoseconds
+ */
+const convertToTaskResult = (result: TestBenchmarkResult): TaskResult => {
+  const MS_TO_NS = 1_000_000;
+
+  // Calculate coefficient of variation (stdDev/mean × 100)
+  const cv = result.mean > 0 ? (result.stdDev / result.mean) * 100 : 0;
+
+  // Estimate margin of error (using 95% confidence, ~1.96 * stdErr)
+  const stdErr = result.stdDev / Math.sqrt(result.iterations);
+  const marginOfError =
+    result.mean > 0 ? ((1.96 * stdErr) / result.mean) * 100 : 0;
+
+  // Variance in ns² = (stdDev in ms)² × (MS_TO_NS)²
+  // This is equivalent to (stdDev × MS_TO_NS)² since variance = stdDev²
+  const variance = result.stdDev * result.stdDev * MS_TO_NS * MS_TO_NS;
+
+  return {
+    cv,
+    error: result.error,
+    iterations: result.iterations,
+    marginOfError,
+    max: result.max * MS_TO_NS,
+    mean: result.mean * MS_TO_NS,
+    min: result.min * MS_TO_NS,
+    name: result.name,
+    opsPerSecond: result.opsPerSecond,
+    // Percentile approximations: without storing all timing samples, we estimate
+    // p95/p99 as fractions of max. These are NOT true percentiles from the distribution.
+    p95: result.max * MS_TO_NS * 0.95,
+    p99: result.max * MS_TO_NS * 0.99,
+    stdDev: result.stdDev * MS_TO_NS,
+    variance,
+  };
+};
+
+/**
+ * Build the final BenchmarkRun result structure
+ */
+const buildBenchmarkRun = (
+  files: FileResult[],
+  config: ModestBenchConfig,
+  startTime: Date,
+  endTime: Date,
+  runId: ReturnType<typeof createRunId>,
+): BenchmarkRun => {
+  // Collect all task results for summary
+  const allTasks: TaskResult[] = [];
+  for (const file of files) {
+    for (const suite of file.suites) {
+      allTasks.push(...suite.tasks);
+    }
+  }
+
+  // Find fastest and slowest
+  const validTasks = allTasks.filter((t) => !t.error && t.opsPerSecond > 0);
+  const fastest =
+    validTasks.length > 0
+      ? validTasks.reduce((a, b) => (a.opsPerSecond > b.opsPerSecond ? a : b))
+      : null;
+  const slowest =
+    validTasks.length > 0
+      ? validTasks.reduce((a, b) => (a.opsPerSecond < b.opsPerSecond ? a : b))
+      : null;
+
+  // Calculate summary
+  const summary: RunSummary = {
+    failedTasks: allTasks.filter((t) => t.error).length,
+    fastest,
+    overallMean:
+      validTasks.length > 0
+        ? validTasks.reduce((sum, t) => sum + t.mean, 0) / validTasks.length
+        : 0,
+    passedTasks: validTasks.length,
+    slowest,
+    totalFiles: files.length,
+    totalOperations: allTasks.reduce((sum, t) => sum + t.iterations, 0),
+    totalSuites: files.reduce((sum, f) => sum + f.suites.length, 0),
+    totalTasks: allTasks.length,
+  };
+
+  return {
+    config,
+    duration: endTime.getTime() - startTime.getTime(),
+    endTime,
+    environment: {
+      arch: process.arch,
+      availableMemory: freemem(),
+      cpu: {
+        cores: cpus().length,
+        model: cpus()[0]?.model ?? 'unknown',
+        speed: cpus()[0]?.speed ?? 0,
+      },
+      env: {},
+      hostname: hostname(),
+      memory: {
+        free: freemem(),
+        total: totalmem(),
+        used: totalmem() - freemem(),
+      },
+      nodeVersion: process.version,
+      platform: process.platform,
+    },
+    files,
+    id: runId,
+    startTime,
+    summary,
+  };
+};
+
+/**
+ * Select the appropriate adapter for the framework
+ */
+const selectAdapter = (framework: TestFramework) => {
+  switch (framework) {
+    case 'ava':
+      return new AvaAdapter();
+    case 'mocha':
+      return new MochaAdapter();
+    case 'node-test':
+      return new NodeTestAdapter();
+    default: {
+      const _exhaustive: never = framework;
+      throw new Error(`Unknown framework: ${_exhaustive}`);
+    }
+  }
+};
+
+/**
+ * Count all tests in a captured file
+ */
+const countAllTests = (captured: CapturedTestFile): number => {
+  let count = captured.rootTests.length;
+  for (const suite of captured.rootSuites) {
+    count += countTestsInSuite(suite);
+  }
+  return count;
+};
+
+/**
+ * Count tests in a suite recursively
+ */
+const countTestsInSuite = (suite: CapturedSuite): number => {
+  let count = suite.tests.length;
+  for (const child of suite.children) {
+    count += countTestsInSuite(child);
+  }
+  return count;
+};

package/src/cli/index.ts

@@ -24,6 +24,7 @@ import type {
 import { bootstrap } from '../bootstrap.js';
 import {
   ABORT_TIMEOUT,
+  DEFAULT_BENCHMARK_DIR,
   DEFAULT_ENGINE,
   DEFAULT_REPORTER,
   Engines,
@@ -65,6 +66,10 @@ import {
   RUN_COMMAND_DEFAULTS,
   handleRunCommand as runCommand,
 } from './commands/run.js';
+import {
+  handleTestCommand as testCommand,
+  type TestOptions,
+} from './commands/test.js';
 
 /**
  * CLI context with initialized services
@@ -177,7 +182,7 @@ export const main = async (
           yargs
             .positional('pattern', {
               array: true,
-              defaultDescription: '(auto-discovered from bench/ directory)',
+              defaultDescription: `(auto-discovered from ${DEFAULT_BENCHMARK_DIR} directory)`,
               describe:
                 'File paths, directory paths, or glob patterns for benchmark files',
               type: 'string',
@@ -986,6 +991,81 @@ export const main = async (
           process.exitCode = await analyzeCommand(context, options);
         },
       )
+      .command(
+        'test <framework> [files..]',
+        'Run test files as benchmarks (captures tests from Mocha, node:test, or AVA)',
+        (yargs) => {
+          return yargs
+            .positional('framework', {
+              choices: ['mocha', 'node-test', 'ava'] as const,
+              demandOption: true,
+              describe: 'Test framework to use',
+              nargs: 1,
+              type: 'string',
+            })
+            .positional('files', {
+              array: true,
+              describe: 'Test file paths or glob patterns',
+              type: 'string',
+            })
+            .option('iterations', {
+              alias: 'i',
+              default: 100,
+              description: 'Number of iterations per test',
+              type: 'number',
+            })
+            .option('warmup', {
+              alias: 'w',
+              default: 5,
+              description: 'Number of warmup iterations',
+              type: 'number',
+            })
+            .option('bail', {
+              alias: 'b',
+              default: false,
+              description: 'Stop on first failure',
+              type: 'boolean',
+            })
+            .option('quiet', {
+              alias: 'q',
+              default: false,
+              description: 'Minimal output',
+              type: 'boolean',
+            })
+            .example([
+              ['$0 test mocha test/*.spec.js', 'Run Mocha tests as benchmarks'],
+              [
+                '$0 test node-test test/*.test.js',
+                'Run node:test tests as benchmarks',
+              ],
+              [
+                '$0 test ava test/*.js --iterations 500',
+                'Run AVA tests with 500 iterations',
+              ],
+              [
+                '$0 test mocha test/unit.spec.js --json',
+                'Output results as JSON',
+              ],
+            ]);
+        },
+        async (argv) => {
+          const context = await createCliContext(argv, abortController!);
+          const options: TestOptions = {
+            bail: argv.bail,
+            cwd: argv.cwd,
+            framework: argv.framework as TestOptions['framework'],
+            iterations: argv.iterations,
+            json: argv.json,
+            noColor: argv.noColor,
+            pattern: argv.files,
+            quiet: argv.quiet,
+            verbose: argv.verbose,
+            warmup: argv.warmup,
+          };
+          const exitCode = await testCommand(context, options);
+          process.exit(exitCode);
+        },
+      )
       .fail((msg, err, yargs) => {
         if (err) {
           console.error('Error:', err.message);

package/src/constants.ts

@@ -131,3 +131,18 @@ export const ErrorCodes = {
   VALIDATION_TYPE_FAILED: 'ERR_MB_VALIDATION_TYPE_FAILED',
   //#endregion
 } as const;
+
+/**
+ * Default output directory for reporters, history, cache, etc.
+ */
+export const DEFAULT_OUTPUT_DIR = './.modestbench';
+
+/**
+ * Default directory containing `*.bench.*` files
+ */
+export const DEFAULT_BENCHMARK_DIR = 'bench';
+
+/**
+ * URL of the site
+ */
+export const SITE_URL = 'https://modestbench.dev';

package/src/core/engine.ts

@@ -323,6 +323,11 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         file.suites.flatMap((suite: SuiteResult) => suite.tasks),
       );
       const finalTotalTasks = allTasks.length;
+
+      // Count task failures and passed tasks
+      // Note: Suite-level failures (setup errors) are NOT counted here to keep
+      // passedTasks + failedTasks == finalTotalTasks. Suite failures are detected
+      // separately in the CLI for exit code purposes.
       const failedTasks = allTasks.filter((task) => task.error).length;
       const passedTasks = finalTotalTasks - failedTasks;