modestbench 0.7.0 → 0.9.0

package/src/cli/commands/run.ts

@@ -7,14 +7,24 @@
 
 import { resolve } from 'node:path';
 
-import type { BenchmarkRun, ModestBenchConfig } from '../../types/index.js';
+import type {
+  BenchmarkRun,
+  ModestBenchConfig,
+  Reporter,
+  ReporterConfig,
+} from '../../types/index.js';
 import type { CliContext } from '../index.js';
 
 import { ErrorCodes, ExitCodes } from '../../constants.js';
-import { resolveOutputPath } from '../../core/output-path-resolver.js';
+import {
+  generateTimestampedFilename,
+  resolveOutputPath,
+} from '../../core/output-path-resolver.js';
 import {
   type BudgetExceededError,
   InvalidArgumentError,
+  ReporterLoadError,
+  ReporterValidationError,
   UnknownReporterError,
 } from '../../errors/index.js';
 import { CsvReporter } from '../../reporters/csv.js';
@@ -22,6 +32,11 @@ import { HumanReporter } from '../../reporters/human.js';
 import { JsonReporter } from '../../reporters/json.js';
 import { NyanReporter } from '../../reporters/nyan.js';
 import { SimpleReporter } from '../../reporters/simple.js';
+import {
+  isBuiltInReporter,
+  isFilePath,
+  loadReporter,
+} from '../../services/reporter-loader.js';
 import { hasErrorCode, isError } from '../../utils/type-guards.js';
 
 /**
@@ -50,6 +65,7 @@ interface RunOptions {
   excludeTags?: string[] | undefined;
   iterations?: number | undefined;
   json?: boolean | undefined;
+  jsonPretty?: boolean | undefined;
   noColor?: boolean | undefined;
   outputDir?: string | undefined;
   outputFile?: string | undefined;
@@ -106,7 +122,7 @@ export const handleRunCommand = async (
     if (showCliMessages) {
       console.error('Setting up reporters...');
     }
-    const reporters = setupReporters(
+    const reporters = await setupReporters(
       context,
       config,
       verbose,
@@ -115,6 +131,7 @@ export const handleRunCommand = async (
       options.outputDir,
       options.outputFile,
       options.progress,
+      options.jsonPretty,
     );
 
     // Step 3: Discovery phase
@@ -370,19 +387,27 @@ const loadConfiguration = async (context: CliContext, options: RunOptions) => {
 
 /**
  * Setup and configure reporters based on configuration
+ *
+ * Supports built-in reporters, registry-based custom reporters, and external
+ * reporters loaded from file paths or npm packages.
  */
-const setupReporters = (
+const setupReporters = async (
   context: CliContext,
-  config: { outputDir?: string; reporters?: string[] },
+  config: {
+    outputDir?: string;
+    reporterConfig?: ReporterConfig;
+    reporters?: string[];
+  },
   isVerbose: boolean,
   showCliMessages: boolean,
   explicitQuiet: boolean,
   explicitOutputDir?: string,
   explicitOutputFile?: string,
   progressOption?: boolean,
-) => {
+  explicitJsonPretty?: boolean,
+): Promise<Reporter[]> => {
   try {
-    const reporters = [];
+    const reporters: Reporter[] = [];
     // Dedupe requested reporters
     const requestedReporters = [...new Set(config.reporters || ['human'])];
 
@@ -398,67 +423,108 @@ const setupReporters = (
     const builtInReporters = ['human', 'json', 'csv', 'nyan', 'simple'];
 
     for (const reporterName of requestedReporters) {
-      let reporter;
-
-      // Create reporter instances with output path configuration
-      switch (reporterName) {
-        case 'csv': {
-          const outputPath = resolveOutputPath(
-            outputDir,
-            explicitOutputFile,
-            'results.csv',
-          );
-          reporter = new CsvReporter({
-            includeHeaders: true,
-            includeMetadata: true,
-            ...(outputPath ? { outputPath } : {}),
-            quiet: explicitQuiet, // Only applies explicit --quiet flag; CSV output can coexist with progress messages on different streams
-            verbose: isVerbose,
-          });
-          break;
-        }
+      let reporter: Reporter;
+
+      // Check if this is a built-in reporter
+      if (isBuiltInReporter(reporterName)) {
+        // Create reporter instances with output path configuration
+        switch (reporterName) {
+          case 'csv': {
+            const outputPath = resolveOutputPath(
+              outputDir,
+              explicitOutputFile,
+              generateTimestampedFilename('csv'),
+            );
+            reporter = new CsvReporter({
+              includeHeaders: true,
+              includeMetadata: true,
+              ...(outputPath ? { outputPath } : {}),
+              quiet: explicitQuiet,
+              verbose: isVerbose,
+            });
+            break;
+          }
 
-        case 'human':
-          reporter = new HumanReporter({
-            color: true,
-            progress: progressOption ?? true,
-            quiet: explicitQuiet, // Only applies explicit --quiet flag; JSON reporter forcing quiet mode does not affect HumanReporter progress output
-            verbose: isVerbose,
-          });
-          break;
-
-        case 'json': {
-          const outputPath = resolveOutputPath(
-            outputDir,
-            explicitOutputFile,
-            'results.json',
-          );
-          reporter = new JsonReporter({
-            ...(outputPath ? { outputPath } : {}),
-            prettyPrint: true,
-          });
-          break;
-        }
+          case 'human':
+            reporter = new HumanReporter({
+              color: true,
+              progress: progressOption ?? true,
+              quiet: explicitQuiet,
+              verbose: isVerbose,
+            });
+            break;
+
+          case 'json': {
+            const outputPath = resolveOutputPath(
+              outputDir,
+              explicitOutputFile,
+              generateTimestampedFilename('json'),
+            );
+            // Precedence: CLI flag > config file > default (false)
+            const prettyPrint =
+              explicitJsonPretty ??
+              config.reporterConfig?.json?.prettyPrint ??
+              false;
+            reporter = new JsonReporter({
+              ...(outputPath ? { outputPath } : {}),
+              prettyPrint,
+            });
+            break;
+          }
 
-        case 'nyan':
-          reporter = new NyanReporter({
-            color: true,
-            quiet: explicitQuiet,
-          });
-          break;
-
-        case 'simple':
-          reporter = new SimpleReporter({
-            quiet: explicitQuiet,
-            verbose: isVerbose,
-          });
-          break;
-
-        default:
-          // Fall back to registry for custom reporters
-          reporter = context.reporterRegistry.get(reporterName);
-          if (!reporter) {
-            // Combine built-in reporters with registered custom reporters
+          case 'nyan':
+            reporter = new NyanReporter({
+              color: true,
+              quiet: explicitQuiet,
+            });
+            break;
+
+          case 'simple':
+            reporter = new SimpleReporter({
+              quiet: explicitQuiet,
+              verbose: isVerbose,
+            });
+            break;
+
+          default:
+            // TypeScript exhaustiveness check - should never reach here
+            throw new Error(`Unhandled built-in reporter: ${reporterName}`);
+        }
+      } else if (isFilePath(reporterName)) {
+        // External reporter from file path
+        const reporterOptions =
+          (config.reporterConfig?.[reporterName] as Record<string, unknown>) ??
+          {};
+        if (showCliMessages) {
+          console.error(`Loading external reporter: ${reporterName}`);
+        }
+        reporter = await loadReporter(reporterName, reporterOptions);
+      } else {
+        // Try registry first, then npm package
+        const registryReporter = context.reporterRegistry.get(reporterName);
+        if (registryReporter) {
+          reporter = registryReporter;
+        } else {
+          // Try loading as npm package
+          const reporterOptions =
+            (config.reporterConfig?.[reporterName] as Record<
+              string,
+              unknown
+            >) ?? {};
+          if (showCliMessages) {
+            console.error(`Loading reporter package: ${reporterName}`);
+          }
+          try {
+            reporter = await loadReporter(reporterName, reporterOptions);
+          } catch (error) {
+            // If loading fails and it's not a file path, provide helpful error
+            if (
+              error instanceof ReporterLoadError ||
+              error instanceof ReporterValidationError
+            ) {
+              throw error;
+            }
+            // Combine built-in reporters with registered custom reporters for error message
             const registeredReporters = Object.keys(
               context.reporterRegistry.getAll(),
             );
@@ -467,10 +533,11 @@ const setupReporters = (
               ...registeredReporters,
             ];
             throw new UnknownReporterError(
-              `Unknown reporter: ${reporterName}. Available: ${availableReporters.join(', ')}`,
+              `Unknown reporter: ${reporterName}. Available built-in reporters: ${availableReporters.join(', ')}. ` +
+                `For external reporters, use a file path (./my-reporter.js) or npm package name.`,
             );
           }
-          break;
+        }
       }
 
       reporters.push(reporter);
@@ -484,7 +551,11 @@ const setupReporters = (
   } catch (error) {
     // Re-throw our custom errors
     const errorCode = hasErrorCode(error) ? error.code : undefined;
-    if (errorCode === ErrorCodes.REPORTER_UNKNOWN) {
+    if (
+      errorCode === ErrorCodes.REPORTER_UNKNOWN ||
+      errorCode === ErrorCodes.REPORTER_LOAD_FAILED ||
+      errorCode === ErrorCodes.REPORTER_INVALID
+    ) {
       throw error;
     }
     throw new InvalidArgumentError(

package/src/cli/index.ts

@@ -276,6 +276,12 @@ export const main = async (
                 'Benchmark engine: tinybench (default) or accurate (requires --allow-natives-syntax)',
               type: 'string',
             })
+            .option('json-pretty', {
+              defaultDescription: 'false',
+              description:
+                'Pretty-print JSON output (only affects json reporter)',
+              type: 'boolean',
+            })
             .example([
               ['$0 run', 'Run benchmarks in current directory and bench/'],
               ['$0 run benchmarks/', 'Run all benchmarks in a directory'],
@@ -315,6 +321,7 @@ export const main = async (
             excludeTags: argv['exclude-tag'],
             iterations: argv.iterations,
             json: argv.json,
+            jsonPretty: argv['json-pretty'],
             noColor: argv.noColor,
             outputDir: argv.output,
             outputFile: argv['output-file'],
@@ -1133,7 +1140,7 @@ const createCliContext = async (
     engine.registerReporter(
       'json',
       new JsonReporter({
-        prettyPrint: true,
+        prettyPrint: false,
       }),
     );
 

package/src/{core → config}/benchmark-schema.ts

@@ -11,7 +11,7 @@ import { z } from 'zod';
 
 import type { ModestBenchConfig } from '../types/core.js';
 
-import { partialModestBenchConfigSchema } from '../config/schema.js';
+import { partialModestBenchConfigSchema } from './schema.js';
 
 /**
  * Schema for benchmark functions