modestbench 0.6.0 → 0.8.0

package/src/services/reporter-loader.ts

@@ -0,0 +1,323 @@
+/**
+ * ModestBench Reporter Loader
+ *
+ * Service for loading third-party reporter plugins from file paths or npm
+ * packages. Supports multiple export patterns: plain objects, classes, and
+ * factory functions (sync or async).
+ */
+
+import { isAbsolute, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import type { Logger, Reporter, ReporterContext } from '../types/index.js';
+
+import { Reporters } from '../constants.js';
+import {
+  ReporterLoadError,
+  ReporterValidationError,
+} from '../errors/reporter.js';
+import { getPackageVersion } from '../utils/package.js';
+import { reporterUtils } from '../utils/reporter-utils.js';
+
+/**
+ * Current plugin API version
+ *
+ * Increment this when making breaking changes to the plugin API.
+ */
+export const PLUGIN_API_VERSION = 1;
+
+/**
+ * Set of built-in reporter names
+ */
+const BUILT_IN_REPORTERS = new Set(Object.values(Reporters));
+
+/**
+ * Required methods that all reporters must implement
+ */
+const REQUIRED_REPORTER_METHODS = [
+  'onStart',
+  'onEnd',
+  'onError',
+  'onTaskResult',
+] as const;
+
+/**
+ * Default logger implementation using console
+ *
+ * This provides a simple console-based logger for reporter plugins.
+ */
+const defaultLogger: Logger = {
+  debug: (message, ...args) => console.debug(message, ...args),
+  error: (message, ...args) => console.error(message, ...args),
+  info: (message, ...args) => console.info(message, ...args),
+  trace: (message, ...args) => console.trace(message, ...args),
+  warn: (message, ...args) => console.warn(message, ...args),
+};
+
+/**
+ * Create a ReporterContext for passing to plugins
+ *
+ * @param logger - Optional logger to use (defaults to console-based logger)
+ * @returns ReporterContext with version info and utilities
+ */
+export const createReporterContext = (logger?: Logger): ReporterContext => {
+  return {
+    logger: logger ?? defaultLogger,
+    pluginApiVersion: PLUGIN_API_VERSION,
+    utils: reporterUtils,
+    version: getPackageVersion(),
+  };
+};
+
+/**
+ * Get the list of missing required methods from a reporter object
+ *
+ * @param obj - Object to check
+ * @returns Array of missing method names
+ */
+const getMissingMethods = (obj: unknown): string[] => {
+  if (typeof obj !== 'object' || obj === null) {
+    return [...REQUIRED_REPORTER_METHODS];
+  }
+
+  return REQUIRED_REPORTER_METHODS.filter(
+    (method) => typeof (obj as Record<string, unknown>)[method] !== 'function',
+  );
+};
+
+/**
+ * Check if a specifier refers to a built-in reporter
+ *
+ * @param specifier - Reporter name or path
+ * @returns True if the specifier is a built-in reporter name
+ */
+export const isBuiltInReporter = (specifier: string): boolean => {
+  return BUILT_IN_REPORTERS.has(
+    specifier as (typeof Reporters)[keyof typeof Reporters],
+  );
+};
+
+/**
+ * Check if a function is a class constructor
+ *
+ * Uses heuristics to distinguish classes from regular functions:
+ *
+ * - Classes have a non-writable prototype property
+ * - Class syntax produces different toString() output
+ *
+ * @param func - Function to check
+ * @returns True if the function appears to be a class constructor
+ */
+const isClass = (
+  func: unknown,
+): func is new (...args: unknown[]) => unknown => {
+  if (typeof func !== 'function') {
+    return false;
+  }
+
+  // Classes have a non-writable prototype
+  const protoDescriptor = Object.getOwnPropertyDescriptor(func, 'prototype');
+  if (!protoDescriptor || protoDescriptor.writable) {
+    return false;
+  }
+
+  // Check if it uses class syntax (handles both 'class Foo' and 'class{')
+  const funcStr = func.toString();
+  return /^class\b/.test(funcStr);
+};
+
+/**
+ * Check if a specifier looks like a file path
+ *
+ * @param specifier - Reporter name or path
+ * @returns True if the specifier appears to be a file path
+ */
+export const isFilePath = (specifier: string): boolean => {
+  return (
+    specifier.startsWith('.') ||
+    specifier.startsWith('/') ||
+    // isAbsolute handles Windows paths like 'C:\path\to\file.js'
+    isAbsolute(specifier)
+  );
+};
+
+/**
+ * Check if an object implements the Reporter interface
+ *
+ * Validates that all required methods are present and are functions.
+ *
+ * @param obj - Object to validate
+ * @returns True if the object has all required reporter methods
+ */
+const isReporterObject = (obj: unknown): obj is Reporter => {
+  if (typeof obj !== 'object' || obj === null) {
+    return false;
+  }
+
+  return REQUIRED_REPORTER_METHODS.every(
+    (method) => typeof (obj as Record<string, unknown>)[method] === 'function',
+  );
+};
+
+/**
+ * Load a reporter from a file path or npm package name
+ *
+ * Supports multiple export patterns:
+ *
+ * 1. Plain Reporter object (simplest, no options support)
+ * 2. Class constructor (instantiated with options and context)
+ * 3. Factory function (called with options and context, can be async)
+ *
+ * @example
+ *
+ * ```typescript
+ * // Load from file path
+ * const reporter = await loadReporter('./my-reporter.js', {
+ *   verbose: true,
+ * });
+ *
+ * // Load from npm package
+ * const reporter = await loadReporter('@company/custom-reporter', {
+ *   apiKey: 'xxx',
+ * });
+ * ```
+ *
+ * @param specifier - File path (relative or absolute) or npm package name
+ * @param options - Options to pass to the reporter factory/constructor
+ * @param cwd - Current working directory for resolving relative paths
+ * @returns Loaded reporter instance
+ * @throws ReporterLoadError if the module cannot be loaded
+ * @throws ReporterValidationError if the module doesn't implement Reporter
+ */
+export const loadReporter = async (
+  specifier: string,
+  options: Record<string, unknown> = {},
+  cwd: string = process.cwd(),
+): Promise<Reporter> => {
+  const context = createReporterContext();
+  const resolvedSpecifier = resolveSpecifier(specifier, cwd);
+
+  let module: unknown;
+
+  try {
+    module = await import(resolvedSpecifier);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new ReporterLoadError(message, specifier, { cause: error });
+  }
+
+  // Handle ESM/CJS interop - get default export if present
+  const exported = (module as { default?: unknown }).default ?? module;
+
+  // Case 1: Already a Reporter object (plain object export)
+  if (isReporterObject(exported)) {
+    return exported;
+  }
+
+  // Case 2: Class constructor
+  if (isClass(exported)) {
+    let instance: unknown;
+
+    try {
+      instance = new (exported as new (
+        options: Record<string, unknown>,
+        context: ReporterContext,
+      ) => unknown)(options, context);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ReporterLoadError(
+        `Constructor threw error: ${message}`,
+        specifier,
+        { cause: error },
+      );
+    }
+
+    validateReporter(instance, specifier);
+    return instance;
+  }
+
+  // Case 3: Factory function (sync or async)
+  if (typeof exported === 'function') {
+    let result: unknown;
+
+    try {
+      result = await (
+        exported as (
+          options: Record<string, unknown>,
+          context: ReporterContext,
+        ) => Promise<unknown>
+      )(options, context);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ReporterLoadError(
+        `Factory function threw error: ${message}`,
+        specifier,
+        { cause: error },
+      );
+    }
+
+    validateReporter(result, specifier);
+    return result;
+  }
+
+  // None of the above - could be an object with missing methods or invalid type
+  if (typeof exported === 'object' && exported !== null) {
+    // It's an object but missing required methods
+    const missing = getMissingMethods(exported);
+    throw new ReporterValidationError(
+      'Module does not implement Reporter interface.',
+      specifier,
+      missing,
+    );
+  }
+
+  // Completely invalid export type
+  throw new ReporterValidationError(
+    'Module must export a Reporter object, class, or factory function.',
+    specifier,
+  );
+};
+
+/**
+ * Resolve a specifier to an importable URL or module name
+ *
+ * @param specifier - File path or npm package name
+ * @param cwd - Current working directory for resolving relative paths
+ * @returns Resolved module specifier
+ */
+const resolveSpecifier = (specifier: string, cwd: string): string => {
+  if (isFilePath(specifier)) {
+    const absolutePath = resolve(cwd, specifier);
+    return pathToFileURL(absolutePath).href;
+  }
+
+  // npm package name - return as-is for dynamic import
+  return specifier;
+};
+
+/**
+ * Validate that an object implements the Reporter interface
+ *
+ * @param obj - Object to validate
+ * @param specifier - Original specifier for error messages
+ * @throws ReporterValidationError if validation fails
+ */
+/**
+ * Type signature for the validateReporter assertion function
+ */
+type ValidateReporterFn = (
+  obj: unknown,
+  specifier: string,
+) => asserts obj is Reporter;
+
+const validateReporter: ValidateReporterFn = (obj, specifier) => {
+  const missing = getMissingMethods(obj);
+
+  if (missing.length > 0) {
+    throw new ReporterValidationError(
+      'Module does not implement Reporter interface.',
+      specifier,
+      missing,
+    );
+  }
+};

package/src/types/core.ts

@@ -37,6 +37,20 @@ export type {
   BenchmarkTaskInput,
 } from '../core/benchmark-schema.js';
 
+/**
+ * Benchmark file structure after parsing
+ */
+export interface BenchmarkFile {
+  /** Raw file content */
+  readonly content: string;
+  /** Parsed exports from the file */
+  readonly exports: unknown;
+  /** Absolute path to the file */
+  readonly filePath: string;
+  /** File metadata */
+  readonly metadata: FileMetadata;
+}
+
 /**
  * ModestBench Core Types
  *
@@ -52,20 +66,6 @@ export type {
 // Re-export identifier helper functions
 export { createRunId, createTaskId } from '../utils/identifiers.js';
 
-/**
- * Benchmark file structure after parsing
- */
-export interface BenchmarkFile {
-  /** Raw file content */
-  readonly content: string;
-  /** Parsed exports from the file */
-  readonly exports: unknown;
-  /** Absolute path to the file */
-  readonly filePath: string;
-  /** File metadata */
-  readonly metadata: FileMetadata;
-}
-
 /**
  * Represents a complete benchmark run across multiple files
  */
@@ -128,6 +128,8 @@ export interface CpuInfo {
   readonly speed: number;
 }
 
+export type Engine = 'accurate' | 'tinybench';
+
 /**
  * Environment information captured during benchmark execution
  */

package/src/types/index.ts

@@ -5,9 +5,6 @@
  * This file re-exports all types from the individual type modules.
  */
 
-// CLI-specific types
-export * from './cli.js';
-
 // Core data types
 export type * from './core.js';
 // Helper functions from core (value exports)
@@ -16,6 +13,9 @@ export { createRunId, createTaskId } from './core.js';
 // Interface contracts
 export type * from './interfaces.js';
 
+// Plugin types (for third-party reporter authors)
+export type * from './plugin.js';
+
 // Profiler types
 export type * from './profiler.js';
 

package/src/types/plugin.ts

@@ -0,0 +1,197 @@
+/**
+ * ModestBench Plugin Types
+ *
+ * Type definitions for third-party reporter plugins. These types are exported
+ * from the main package for use by plugin authors.
+ */
+
+import type { Reporter } from './interfaces.js';
+import type { Logger } from './utility.js';
+
+/**
+ * Class constructor for reporter plugins
+ *
+ * Plugin authors can export a default class matching this signature. The class
+ * constructor receives options from the config file and a context object with
+ * utilities.
+ *
+ * @example
+ *
+ * ```typescript
+ * import type { Reporter, ReporterContext } from 'modestbench';
+ *
+ * interface MyReporterOptions {
+ *   verbose?: boolean;
+ *   outputFormat?: 'text' | 'markdown';
+ * }
+ *
+ * class MyReporter implements Reporter {
+ *   constructor(
+ *     private options: MyReporterOptions,
+ *     private context: ReporterContext,
+ *   ) {}
+ *
+ *   onStart(run) {
+ *     if (this.options.verbose) console.log('Starting');
+ *   }
+ *   onEnd(run) {
+ *     console.log('Done');
+ *   }
+ *   onError(error) {
+ *     console.error(error);
+ *   }
+ *   onTaskResult(result) {
+ *     console.log(
+ *       `${result.name}: ${this.context.utils.formatDuration(result.mean)}`,
+ *     );
+ *   }
+ * }
+ *
+ * export default MyReporter;
+ * ```
+ *
+ * @typeParam TOptions - The shape of the options object (defaults to
+ *   Record<string, unknown>)
+ */
+export interface ReporterClass<
+  TOptions extends Record<string, unknown> = Record<string, unknown>,
+> {
+  new (options?: TOptions, context?: ReporterContext): Reporter;
+}
+
+/**
+ * Context provided to reporter plugins
+ *
+ * Contains version information and utility functions that plugins can use.
+ */
+export interface ReporterContext {
+  /**
+   * Logger for reporter output
+   *
+   * Use this instead of console.log/console.error to ensure output respects the
+   * user's verbosity settings and uses the correct output streams.
+   */
+  readonly logger: Logger;
+
+  /**
+   * Plugin API version
+   *
+   * Incremented when breaking changes are made to the plugin API. Currently
+   * version 1.
+   */
+  readonly pluginApiVersion: number;
+
+  /**
+   * Utility functions for formatting benchmark data
+   */
+  readonly utils: ReporterUtils;
+
+  /**
+   * ModestBench version
+   *
+   * Plugins can use this to check compatibility.
+   */
+  readonly version: string;
+}
+
+/**
+ * Factory function for creating reporter instances
+ *
+ * Plugin authors can export a default function matching this signature. The
+ * function receives options from the config file and a context object with
+ * utilities. Use the generic parameter to define the shape of your options.
+ *
+ * @example
+ *
+ * ```typescript
+ * import type { ReporterFactory } from 'modestbench';
+ *
+ * interface MyReporterOptions {
+ *   verbose?: boolean;
+ *   outputFormat?: 'text' | 'markdown';
+ * }
+ *
+ * const createReporter: ReporterFactory<MyReporterOptions> = (
+ *   options,
+ *   context,
+ * ) => {
+ *   return {
+ *     onStart(run) {
+ *       if (options.verbose) console.log('Starting');
+ *     },
+ *     onEnd(run) {
+ *       console.log('Done');
+ *     },
+ *     onError(error) {
+ *       console.error(error);
+ *     },
+ *     onTaskResult(result) {
+ *       console.log(
+ *         `${result.name}: ${context.utils.formatDuration(result.mean)}`,
+ *       );
+ *     },
+ *   };
+ * };
+ *
+ * export default createReporter;
+ * ```
+ *
+ * @typeParam TOptions - The shape of the options object (defaults to
+ *   Record<string, unknown>)
+ */
+export type ReporterFactory<
+  TOptions extends Record<string, unknown> = Record<string, unknown>,
+> = (
+  options: TOptions,
+  context: ReporterContext,
+) => Promise<Reporter> | Reporter;
+
+/**
+ * Union type representing all valid reporter plugin exports
+ *
+ * A reporter plugin module can export:
+ *
+ * - A plain Reporter object (simplest form, no options)
+ * - A ReporterClass constructor (instantiated with options)
+ * - A ReporterFactory function (most flexible, supports async)
+ */
+export type ReporterPlugin = Reporter | ReporterClass | ReporterFactory;
+
+/**
+ * Utility functions available to reporter plugins
+ *
+ * These functions help format benchmark data consistently.
+ */
+export interface ReporterUtils {
+  /**
+   * Format bytes in human-readable format
+   *
+   * @param bytes - Number of bytes
+   * @returns Formatted string (e.g., "1.5 GB", "256 MB")
+   */
+  formatBytes(bytes: number): string;
+
+  /**
+   * Format duration in human-readable format
+   *
+   * @param nanoseconds - Duration in nanoseconds
+   * @returns Formatted string (e.g., "1.23ms", "456.78μs")
+   */
+  formatDuration(nanoseconds: number): string;
+
+  /**
+   * Format operations per second
+   *
+   * @param opsPerSecond - Operations per second
+   * @returns Formatted string (e.g., "1.2M ops/sec", "456K ops/sec")
+   */
+  formatOpsPerSecond(opsPerSecond: number): string;
+
+  /**
+   * Format percentage value
+   *
+   * @param value - Percentage value
+   * @returns Formatted string (e.g., "12.34%")
+   */
+  formatPercentage(value: number): string;
+}

package/src/utils/package.ts

@@ -6,8 +6,39 @@
  * @packageDocumentation
  */
 
+import { readFileSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
-import path from 'node:path';
+import path, { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/**
+ * Cached package version, loaded at module initialization
+ *
+ * NOTE: This relies on package.json being at the same relative path from both
+ * src/ and dist/ directories (../../package.json). If the build output
+ * structure changes, this will break.
+ */
+const cachedPackageVersion = (() => {
+  try {
+    const __dirname = dirname(fileURLToPath(import.meta.url));
+    const pkgPath = join(__dirname, '..', '..', 'package.json');
+    const pkgContent = readFileSync(pkgPath, 'utf8');
+    const pkg = JSON.parse(pkgContent) as { version: string };
+    return pkg.version;
+  } catch {
+    // Fallback if package.json cannot be read (shouldn't happen in normal use)
+    return 'unknown';
+  }
+})();
+
+/**
+ * Get the ModestBench package version
+ *
+ * @returns The version string from package.json
+ */
+export const getPackageVersion = (): string => {
+  return cachedPackageVersion;
+};
 
 /**
  * Find the nearest package.json and return its directory

package/src/utils/reporter-utils.ts

@@ -0,0 +1,85 @@
+/**
+ * ModestBench Reporter Utilities
+ *
+ * Formatting functions for benchmark data, exported for use by third-party
+ * reporter plugins.
+ */
+
+import type { ReporterUtils } from '../types/plugin.js';
+
+/**
+ * Format bytes in human-readable format
+ *
+ * @param bytes - Number of bytes
+ * @returns Formatted string (e.g., "1.5 GB", "256 MB", "1.2 KB")
+ */
+export const formatBytes = (bytes: number): string => {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let size = bytes;
+  let unitIndex = 0;
+
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex++;
+  }
+
+  return `${size.toFixed(1)} ${units[unitIndex]}`;
+};
+
+/**
+ * Format duration in human-readable format
+ *
+ * @param nanoseconds - Duration in nanoseconds
+ * @returns Formatted string (e.g., "1.23ms", "456.78μs", "789.00ns")
+ */
+export const formatDuration = (nanoseconds: number): string => {
+  if (nanoseconds < 1000) {
+    return `${nanoseconds.toFixed(2)}ns`;
+  } else if (nanoseconds < 1_000_000) {
+    return `${(nanoseconds / 1000).toFixed(2)}μs`;
+  } else if (nanoseconds < 1_000_000_000) {
+    return `${(nanoseconds / 1_000_000).toFixed(2)}ms`;
+  } else {
+    return `${(nanoseconds / 1_000_000_000).toFixed(2)}s`;
+  }
+};
+
+/**
+ * Format operations per second in human-readable format
+ *
+ * @param opsPerSecond - Operations per second
+ * @returns Formatted string (e.g., "1.2M ops/sec", "456K ops/sec")
+ */
+export const formatOpsPerSecond = (opsPerSecond: number): string => {
+  if (opsPerSecond < 1000) {
+    return `${opsPerSecond.toFixed(2)} ops/sec`;
+  } else if (opsPerSecond < 1_000_000) {
+    return `${(opsPerSecond / 1000).toFixed(2)}K ops/sec`;
+  } else if (opsPerSecond < 1_000_000_000) {
+    return `${(opsPerSecond / 1_000_000).toFixed(2)}M ops/sec`;
+  } else {
+    return `${(opsPerSecond / 1_000_000_000).toFixed(2)}B ops/sec`;
+  }
+};
+
+/**
+ * Format percentage value
+ *
+ * @param value - Percentage value (e.g., 12.345 for 12.345%)
+ * @returns Formatted string (e.g., "12.35%")
+ */
+export const formatPercentage = (value: number): string => {
+  return `${value.toFixed(2)}%`;
+};
+
+/**
+ * Reporter utilities object implementing the ReporterUtils interface
+ *
+ * This object is provided to reporter plugins via the ReporterContext.
+ */
+export const reporterUtils: ReporterUtils = {
+  formatBytes,
+  formatDuration,
+  formatOpsPerSecond,
+  formatPercentage,
+};