modestbench 0.0.3 → 0.2.0

package/src/core/engine.ts

@@ -6,6 +6,8 @@
  * architecture.
  */
 
+import { randomBytes } from 'node:crypto';
+
 import type {
   BenchmarkDefinition,
   BenchmarkEngine,
@@ -15,8 +17,6 @@ import type {
   CiInfo,
   ConfigurationManager,
   EnvironmentInfo,
-  ErrorManager,
-  ExecutionPhase,
   FileLoader,
   FileResult,
   GitInfo,
@@ -33,12 +33,19 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import {
+  BenchmarkExecutionError,
+  FileDiscoveryError,
+  SchemaValidationError,
+  SetupError,
+  StructureValidationError,
+} from '../errors/index.js';
+
 /**
  * Dependencies required by the BenchmarkEngine
  */
 interface EngineDependencies {
   readonly configManager: ConfigurationManager;
-  readonly errorManager: ErrorManager;
   readonly fileLoader: FileLoader;
   readonly historyStorage: HistoryStorage;
   readonly progressManager: ProgressManager;
@@ -55,8 +62,6 @@ interface EngineDependencies {
 export abstract class ModestBenchEngine implements BenchmarkEngine {
   public readonly configManager: ConfigurationManager;
 
-  public readonly errorManager: ErrorManager;
-
   public readonly fileLoader: FileLoader;
 
   public readonly historyStorage: HistoryStorage;
@@ -71,7 +76,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     this.reporterRegistry = dependencies.reporterRegistry;
     this.historyStorage = dependencies.historyStorage;
     this.progressManager = dependencies.progressManager;
-    this.errorManager = dependencies.errorManager;
   }
 
   /**
@@ -86,13 +90,10 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     } catch (error) {
       const discoveryError =
         error instanceof Error ? error : new Error(String(error));
-      this.errorManager.handleError(discoveryError, {
-        metadata: { exclude, pattern },
-        phase: 'discovery',
-        timestamp: new Date(),
-      });
-
-      throw new Error(`File discovery failed: ${discoveryError.message}`);
+      throw new FileDiscoveryError(
+        `File discovery failed: ${discoveryError.message}`,
+        { cause: discoveryError },
+      );
     }
   }
 
@@ -105,11 +106,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     signal?: AbortSignal,
   ): Promise<BenchmarkRun> {
     const startTime = new Date();
-    let currentPhase: ExecutionPhase = 'discovery';
 
     try {
       // 1. Merge configuration with defaults
-      currentPhase = 'discovery';
       const mergedConfig = await this.configManager.load(
         undefined, // No specific config path for now
         config as Record<string, unknown>,
@@ -125,30 +124,18 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         if (mergedConfig.exclude?.length) {
           msg += ` and excluding "${mergedConfig.exclude.join(', ')}"`;
         }
-        const error = new Error(msg);
-        this.errorManager.handleError(error, {
-          phase: currentPhase,
-          timestamp: new Date(),
-        });
-        throw error;
+        throw new FileDiscoveryError(msg);
       }
 
       // 3. Validate files
-      currentPhase = 'validation';
       const validationResult = await this.validate(files);
       if (!validationResult.valid) {
-        const error = new Error(
+        throw new SchemaValidationError(
           `Validation failed: ${validationResult.errors.map((e) => e.message).join(', ')}`,
         );
-        this.errorManager.handleError(error, {
-          phase: currentPhase,
-          timestamp: new Date(),
-        });
-        throw error;
       }
 
       // 4. Initialize progress tracking
-      currentPhase = 'setup';
       const runId = this.generateRunId();
 
       // Pre-calculate total tasks for progress tracking
@@ -236,7 +223,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       await this.callReporters(reporters, 'onStart', initialRun);
 
       // 6. Execute benchmark files
-      currentPhase = 'execution';
       const fileResults: FileResult[] = [];
 
       for (const filePath of files) {
@@ -263,11 +249,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         } catch (error) {
           const fileError =
             error instanceof Error ? error : new Error(String(error));
-          this.errorManager.handleError(fileError, {
-            file: filePath,
-            phase: currentPhase,
-            timestamp: new Date(),
-          });
 
           // Call reporter onError
           await this.callReporters(reporters, 'onError', fileError);
@@ -350,15 +331,23 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       // 9. Return completed run
       return finalRun;
     } catch (error) {
+      // Re-throw our custom errors
+      if (
+        error instanceof FileDiscoveryError ||
+        error instanceof SchemaValidationError ||
+        error instanceof BenchmarkExecutionError
+      ) {
+        throw error;
+      }
+
       const executionError =
         error instanceof Error ? error : new Error(String(error));
-      const handledError = this.errorManager.handleError(executionError, {
-        phase: currentPhase,
-        timestamp: new Date(),
-      });
 
       // Re-throw the original error with more context
-      throw new Error(`Benchmark execution failed: ${handledError.message}`);
+      throw new BenchmarkExecutionError(
+        `Benchmark execution failed: ${executionError.message}`,
+        { cause: executionError },
+      );
     }
   }
 
@@ -423,11 +412,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         } catch (error) {
           const validationError =
             error instanceof Error ? error : new Error(String(error));
-          this.errorManager.handleError(validationError, {
-            file,
-            phase: 'validation',
-            timestamp: new Date(),
-          });
 
           errors.push({
             code: 'FILE_VALIDATION_ERROR',
@@ -516,7 +500,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
 
       if (!benchmarkDef || typeof benchmarkDef !== 'object') {
-        throw new Error(
+        throw new StructureValidationError(
           'Benchmark file must export a default object with suites',
         );
       }
@@ -634,7 +618,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
             error instanceof Error
               ? error
               : new Error(`Setup failed: ${String(error)}`);
-          throw new Error(`Suite setup failed: ${setupError.message}`);
+          throw new SetupError(`Suite setup failed: ${setupError.message}`, {
+            cause: setupError,
+          });
         }
       }
 
@@ -713,9 +699,15 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
 
   /**
    * Generate a unique run ID
+   *
+   * Uses crypto.randomBytes for cryptographically random 7-character IDs.
+   * Format: 7 lowercase alphanumeric characters (e.g., "k3m9x2p")
    */
   private generateRunId(): string {
-    return `run-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
+    // Generate random bytes, convert to hex, then to base36, take first 7 chars
+    const hex = randomBytes(4).toString('hex');
+    const num = parseInt(hex, 16);
+    return num.toString(36).padStart(7, '0').substring(0, 7);
   }
 
   /**

package/src/core/engines/accurate-engine.ts

@@ -26,6 +26,7 @@ import type {
   TaskResult,
 } from '../../types/index.js';
 
+import { StructureValidationError } from '../../errors/index.js';
 import { ModestBenchEngine } from '../engine.js';
 import { calculateStatistics, removeOutliersIQR } from '../stats-utils.js';
 
@@ -59,7 +60,9 @@ export class AccurateEngine extends ModestBenchEngine {
   ): Promise<TaskResult> {
     try {
       if (!taskData.fn || typeof taskData.fn !== 'function') {
-        throw new Error('Benchmark task must have a "fn" function property');
+        throw new StructureValidationError(
+          'Benchmark task must have a "fn" function property',
+        );
       }
 
       // Check for V8 native syntax support

package/src/core/engines/tinybench-engine.ts

@@ -14,6 +14,11 @@ import type {
   TaskResult,
 } from '../../types/index.js';
 
+import {
+  BenchmarkExecutionError,
+  OperationTooFastError,
+  StructureValidationError,
+} from '../../errors/index.js';
 import { ModestBenchEngine } from '../engine.js';
 import { calculateStatistics, removeOutliersIQR } from '../stats-utils.js';
 
@@ -33,7 +38,9 @@ export class TinybenchEngine extends ModestBenchEngine {
   ): Promise<TaskResult> {
     try {
       if (!taskData.fn || typeof taskData.fn !== 'function') {
-        throw new Error('Benchmark task must have a "fn" function property');
+        throw new StructureValidationError(
+          'Benchmark task must have a "fn" function property',
+        );
       }
 
       // Determine effective time and iterations based on limitBy mode
@@ -137,13 +144,13 @@ export class TinybenchEngine extends ModestBenchEngine {
             await minimalBench.run();
           } catch {
             // If still failing, the operation is too fast even for tinybench
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark operation is too fast to measure reliably (execution time < 1ns)`,
             );
           }
           const minimalResults = minimalBench.results[0];
           if (!minimalResults || minimalResults.error) {
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark too fast to measure reliably: ${minimalResults?.error?.message || 'unknown error'}`,
             );
           }
@@ -180,7 +187,7 @@ export class TinybenchEngine extends ModestBenchEngine {
       // Get results
       const results = bench.results[0];
       if (!results) {
-        throw new Error('No benchmark results returned');
+        throw new BenchmarkExecutionError('No benchmark results returned');
       }
 
       // Check if the task was aborted
@@ -251,7 +258,7 @@ export class TinybenchEngine extends ModestBenchEngine {
 
           if (!minimalResults || minimalResults.error) {
             // If retry also fails, just accept it failed
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark operation is too fast to measure reliably`,
             );
           }

package/src/core/output-path-resolver.ts

@@ -0,0 +1,38 @@
+import { isAbsolute, join, resolve } from 'node:path';
+
+/**
+ * Resolves the final output path for a reporter
+ *
+ * @param outputDir - Optional output directory from --output flag
+ * @param outputFile - Optional output filename from --output-file flag
+ * @param defaultFilename - Default filename to use if none specified
+ * @returns Resolved output path, or undefined if no output to file requested
+ */
+export const resolveOutputPath = (
+  outputDir?: string,
+  outputFile?: string,
+  defaultFilename?: string,
+): string | undefined => {
+  // If outputFile is provided
+  if (outputFile) {
+    // If outputFile is absolute, use as-is
+    if (isAbsolute(outputFile)) {
+      return outputFile;
+    }
+
+    // If outputDir specified, join them
+    if (outputDir) {
+      return join(outputDir, outputFile);
+    }
+
+    // Otherwise, resolve relative to cwd
+    return resolve(process.cwd(), outputFile);
+  }
+
+  // Fall back to default behavior
+  if (outputDir && defaultFilename) {
+    return join(outputDir, defaultFilename);
+  }
+
+  return undefined;
+};

package/src/errors/base.ts

@@ -0,0 +1,152 @@
+/**
+ * ModestBench Custom Error System
+ *
+ * Base error classes providing structured error handling with error codes,
+ * documentation URLs, and consistent error display.
+ */
+
+/**
+ * Base URL for error documentation
+ */
+const ERROR_DOC_BASE_URL =
+  'https://boneskull.github.io/modestbench/reference/errors';
+
+/**
+ * Abstract base class for ModestBench aggregate errors
+ *
+ * Extends AggregateError to support multiple errors with ModestBench error
+ * system features.
+ */
+export abstract class ModestBenchAggregateError extends AggregateError {
+  /**
+   * Unique error code for this error type Must be in format:
+   * ERR_MB_CATEGORY_DESCRIPTION
+   */
+  abstract readonly code: string;
+
+  /**
+   * Error name (matches class name)
+   */
+  public override readonly name: string;
+
+  /**
+   * Create a new ModestBench aggregate error
+   *
+   * @param errors - Array of errors that occurred
+   * @param message - Human-readable error message
+   * @param options - Optional Error options (e.g., cause)
+   */
+  constructor(errors: unknown[], message: string, options?: ErrorOptions) {
+    super(errors, message, options);
+    this.name = this.constructor.name;
+  }
+
+  /**
+   * Get the documentation URL for this error
+   *
+   * Returns a URL to the error reference page with an anchor to the specific
+   * error.
+   *
+   * @returns Documentation URL
+   */
+  getDocUrl(): string {
+    // Use the error class name as the anchor (e.g., ConfigValidationError -> #configvalidationerror)
+    const anchor = this.name.toLowerCase();
+    return `${ERROR_DOC_BASE_URL}#${anchor}`;
+  }
+
+  /**
+   * Convert error to string with code, documentation URL, and nested errors
+   *
+   * @returns Formatted error string
+   */
+  override toString(): string {
+    let result = `${this.name} [${this.code}]: ${this.message}\n`;
+    result += `See: ${this.getDocUrl()}\n`;
+
+    if (this.errors.length > 0) {
+      result += `\nContains ${this.errors.length} error(s):\n`;
+      this.errors.forEach((err, index) => {
+        const errMsg = err instanceof Error ? err.message : String(err);
+        result += `  ${index + 1}. ${errMsg}\n`;
+      });
+    }
+
+    return result;
+  }
+}
+
+/**
+ * Abstract base class for all ModestBench errors
+ *
+ * Provides:
+ *
+ * - Unique error codes with ERR_MB_ prefix
+ * - Documentation URL generation
+ * - Consistent error display format
+ * - TypeScript type safety
+ */
+export abstract class ModestBenchError extends Error {
+  /**
+   * Unique error code for this error type Must be in format:
+   * ERR_MB_CATEGORY_DESCRIPTION
+   */
+  abstract readonly code: string;
+
+  /**
+   * Error name (matches class name)
+   */
+  public override readonly name: string;
+
+  /**
+   * Create a new ModestBench error
+   *
+   * @param message - Human-readable error message
+   * @param options - Optional Error options (e.g., cause)
+   */
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = this.constructor.name;
+  }
+
+  /**
+   * Get the documentation URL for this error
+   *
+   * Returns a URL to the error reference page with an anchor to the specific
+   * error.
+   *
+   * @returns Documentation URL
+   */
+  getDocUrl(): string {
+    // Use the error class name as the anchor (e.g., ConfigValidationError -> #configvalidationerror)
+    const anchor = this.name.toLowerCase();
+    return `${ERROR_DOC_BASE_URL}#${anchor}`;
+  }
+
+  /**
+   * Convert error to string with code and documentation URL
+   *
+   * @returns Formatted error string
+   */
+  override toString(): string {
+    return `${this.name} [${this.code}]: ${this.message}\nSee: ${this.getDocUrl()}`;
+  }
+}
+
+/**
+ * Type guard to check if an error is a ModestBench error
+ *
+ * @param error - The error to check
+ * @returns True if the error is a ModestBench error
+ */
+export const isModestBenchError = (
+  error: unknown,
+): error is ModestBenchError => {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    'code' in error &&
+    typeof (error as { code: unknown }).code === 'string' &&
+    (error as { code: string }).code.startsWith('ERR_MB_')
+  );
+};

package/src/errors/cli.ts

@@ -0,0 +1,59 @@
+/**
+ * CLI-related errors
+ *
+ * Errors that occur during command-line interface operations.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Invalid CLI argument
+ *
+ * Thrown when a CLI argument is invalid or cannot be parsed.
+ */
+export class InvalidArgumentError extends ModestBenchError {
+  readonly code = 'ERR_MB_CLI_INVALID_ARGUMENT';
+}
+
+/**
+ * Invalid date format
+ *
+ * Thrown when a date string cannot be parsed into a valid date.
+ */
+export class InvalidDateFormatError extends ModestBenchError {
+  readonly code = 'ERR_MB_CLI_INVALID_DATE_FORMAT';
+}
+
+/**
+ * Unknown error
+ *
+ * Thrown at the CLI boundary to wrap unexpected errors that are not ModestBench
+ * errors. This ensures all errors have proper structure and documentation
+ * links.
+ */
+export class UnknownError extends ModestBenchError {
+  readonly code = 'ERR_MB_UNKNOWN';
+
+  /**
+   * Create a new UnknownError wrapping an unexpected error
+   *
+   * @param message - The error message (typically from the original error)
+   * @param options - Error options with the original error as the cause
+   */
+  constructor(message: string, options: ErrorOptions) {
+    super(message, options);
+  }
+
+  /**
+   * Override toString to show full details of the wrapped error
+   */
+  override toString(): string {
+    let result = super.toString();
+
+    if (this.cause instanceof Error && this.cause.stack) {
+      result += '\n\nOriginal error:\n' + this.cause.stack;
+    }
+
+    return result;
+  }
+}

package/src/errors/configuration.ts

@@ -0,0 +1,45 @@
+/**
+ * Configuration-related errors
+ *
+ * Errors that occur during configuration file loading, parsing, and validation.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Failed to load configuration
+ *
+ * Thrown when configuration loading fails for reasons other than file not found
+ * (e.g., parse errors, module loading errors).
+ */
+export class ConfigLoadError extends ModestBenchError {
+  readonly code = 'ERR_MB_CONFIG_LOAD_FAILED';
+}
+
+/**
+ * Configuration file not found
+ *
+ * Thrown when a specified configuration file cannot be found.
+ */
+export class ConfigNotFoundError extends ModestBenchError {
+  readonly code = 'ERR_MB_CONFIG_NOT_FOUND';
+}
+
+/**
+ * Configuration validation failed
+ *
+ * Thrown when a configuration file or configuration options fail schema
+ * validation.
+ */
+export class ConfigValidationError extends ModestBenchError {
+  readonly code = 'ERR_MB_CONFIG_VALIDATION_FAILED';
+}
+
+/**
+ * Unsupported configuration format
+ *
+ * Thrown when attempting to use an unsupported configuration file format.
+ */
+export class UnsupportedConfigFormatError extends ModestBenchError {
+  readonly code = 'ERR_MB_CONFIG_UNSUPPORTED_FORMAT';
+}

package/src/errors/execution.ts

@@ -0,0 +1,62 @@
+/**
+ * Execution-related errors
+ *
+ * Errors that occur during benchmark execution, setup, and teardown.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Benchmark execution failed
+ *
+ * Thrown when overall benchmark execution fails.
+ */
+export class BenchmarkExecutionError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_BENCHMARK_FAILED';
+}
+
+/**
+ * Operation too fast to measure
+ *
+ * Thrown when a benchmark operation executes so quickly that it cannot be
+ * reliably measured (typically < 1ns per operation).
+ */
+export class OperationTooFastError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_TOO_FAST';
+}
+
+/**
+ * Setup function failed
+ *
+ * Thrown when a benchmark setup function fails.
+ */
+export class SetupError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_SETUP_FAILED';
+}
+
+/**
+ * Task execution failed
+ *
+ * Thrown when a specific benchmark task fails during execution.
+ */
+export class TaskExecutionError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_TASK_FAILED';
+}
+
+/**
+ * Teardown function failed
+ *
+ * Thrown when a benchmark teardown function fails.
+ */
+export class TeardownError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_TEARDOWN_FAILED';
+}
+
+/**
+ * Execution timeout exceeded
+ *
+ * Thrown when a benchmark operation exceeds the configured timeout.
+ */
+export class TimeoutError extends ModestBenchError {
+  readonly code = 'ERR_MB_EXECUTION_TIMEOUT';
+}

package/src/errors/file.ts

@@ -0,0 +1,53 @@
+/**
+ * File-related errors
+ *
+ * Errors that occur during file discovery, loading, and access.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * File discovery failed
+ *
+ * Thrown when file discovery/glob pattern matching fails.
+ */
+export class FileDiscoveryError extends ModestBenchError {
+  readonly code = 'ERR_MB_FILE_DISCOVERY_FAILED';
+}
+
+/**
+ * Failed to load benchmark file
+ *
+ * Thrown when loading a benchmark file fails (e.g., import errors, syntax
+ * errors).
+ */
+export class FileLoadError extends ModestBenchError {
+  readonly code = 'ERR_MB_FILE_LOAD_FAILED';
+}
+
+/**
+ * Benchmark file not found
+ *
+ * Thrown when a benchmark file cannot be found or does not exist.
+ */
+export class FileNotFoundError extends ModestBenchError {
+  readonly code = 'ERR_MB_FILE_NOT_FOUND';
+}
+
+/**
+ * File permission denied
+ *
+ * Thrown when file access is denied due to insufficient permissions.
+ */
+export class FilePermissionError extends ModestBenchError {
+  readonly code = 'ERR_MB_FILE_PERMISSION_DENIED';
+}
+
+/**
+ * Unsupported file extension
+ *
+ * Thrown when attempting to load a file with an unsupported extension.
+ */
+export class UnsupportedFileExtensionError extends ModestBenchError {
+  readonly code = 'ERR_MB_FILE_UNSUPPORTED_EXTENSION';
+}