modestbench 0.0.3 → 0.1.0

package/src/config/manager.ts

@@ -9,6 +9,7 @@
 import { cosmiconfig } from 'cosmiconfig';
 import { resolve } from 'node:path';
 
+import type { ModestBenchError } from '../errors/base.js';
 import type {
   ConfigurationManager,
   ModestBenchConfig,
@@ -17,6 +18,8 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import { ErrorCodes } from '../constants.js';
+import { ConfigLoadError, ConfigValidationError } from '../errors/index.js';
 import { safeParseConfig } from './schema.js';
 
 /**
@@ -175,15 +178,22 @@ export class ModestBenchConfigurationManager implements ConfigurationManager {
       // 3. Validate final configuration
       const validation = this.validate(finalConfig);
       if (!validation.valid) {
-        throw new Error(
+        throw new ConfigValidationError(
           `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
         );
       }
 
       return finalConfig;
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors
+      if (
+        (error as ModestBenchError).code === ErrorCodes.CONFIG_VALIDATION_FAILED
+      ) {
+        throw error;
+      }
+      throw new ConfigLoadError(
         `Failed to load configuration: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }

package/src/constants.ts

@@ -19,3 +19,63 @@ export const BENCHMARK_FILE_PATTERN = `.bench.{${Array.from(
 )
   .map((ext) => ext.slice(1))
   .join(',')}}`;
+
+/**
+ * Error codes for all ModestBench errors
+ *
+ * Use these constants to check error types instead of instanceof checks.
+ */
+export const ErrorCodes = {
+  //#region cli-errors
+  CLI_INVALID_ARGUMENT: 'ERR_MB_CLI_INVALID_ARGUMENT',
+  CLI_INVALID_DATE_FORMAT: 'ERR_MB_CLI_INVALID_DATE_FORMAT',
+  //#endregion
+
+  //#region config-errors
+  CONFIG_LOAD_FAILED: 'ERR_MB_CONFIG_LOAD_FAILED',
+  CONFIG_NOT_FOUND: 'ERR_MB_CONFIG_NOT_FOUND',
+  CONFIG_UNSUPPORTED_FORMAT: 'ERR_MB_CONFIG_UNSUPPORTED_FORMAT',
+  CONFIG_VALIDATION_FAILED: 'ERR_MB_CONFIG_VALIDATION_FAILED',
+  //#endregion
+
+  //#region execution-errors
+  EXECUTION_BENCHMARK_FAILED: 'ERR_MB_EXECUTION_BENCHMARK_FAILED',
+  EXECUTION_SETUP_FAILED: 'ERR_MB_EXECUTION_SETUP_FAILED',
+  EXECUTION_TASK_FAILED: 'ERR_MB_EXECUTION_TASK_FAILED',
+  EXECUTION_TEARDOWN_FAILED: 'ERR_MB_EXECUTION_TEARDOWN_FAILED',
+  EXECUTION_TIMEOUT: 'ERR_MB_EXECUTION_TIMEOUT',
+  EXECUTION_TOO_FAST: 'ERR_MB_EXECUTION_TOO_FAST',
+  //#endregion
+
+  //#region file-errors
+  FILE_DISCOVERY_FAILED: 'ERR_MB_FILE_DISCOVERY_FAILED',
+  FILE_LOAD_FAILED: 'ERR_MB_FILE_LOAD_FAILED',
+  FILE_NOT_FOUND: 'ERR_MB_FILE_NOT_FOUND',
+  FILE_PERMISSION_DENIED: 'ERR_MB_FILE_PERMISSION_DENIED',
+  FILE_UNSUPPORTED_EXTENSION: 'ERR_MB_FILE_UNSUPPORTED_EXTENSION',
+  //#endregion
+
+  //#region reporter-errors
+  REPORTER_ALREADY_REGISTERED: 'ERR_MB_REPORTER_ALREADY_REGISTERED',
+  REPORTER_OUTPUT_FAILED: 'ERR_MB_REPORTER_OUTPUT_FAILED',
+  REPORTER_UNKNOWN: 'ERR_MB_REPORTER_UNKNOWN',
+  //#endregion
+
+  //#region storage-errors
+  STORAGE_CORRUPTION: 'ERR_MB_STORAGE_CORRUPTION',
+  STORAGE_EXPORT_UNSUPPORTED: 'ERR_MB_STORAGE_EXPORT_UNSUPPORTED',
+  STORAGE_FAILED: 'ERR_MB_STORAGE_FAILED',
+  STORAGE_INDEX_CORRUPTION: 'ERR_MB_STORAGE_INDEX_CORRUPTION',
+  STORAGE_INSUFFICIENT_SPACE: 'ERR_MB_STORAGE_INSUFFICIENT_SPACE',
+  //#endregion
+
+  //#region misc-errors
+  UNKNOWN: 'ERR_MB_UNKNOWN',
+  //#endregion
+
+  //#region validation-errors
+  VALIDATION_SCHEMA_FAILED: 'ERR_MB_VALIDATION_SCHEMA_FAILED',
+  VALIDATION_STRUCTURE_INVALID: 'ERR_MB_VALIDATION_STRUCTURE_INVALID',
+  VALIDATION_TYPE_FAILED: 'ERR_MB_VALIDATION_TYPE_FAILED',
+  //#endregion
+} as const;

package/src/core/engine.ts

@@ -15,8 +15,6 @@ import type {
   CiInfo,
   ConfigurationManager,
   EnvironmentInfo,
-  ErrorManager,
-  ExecutionPhase,
   FileLoader,
   FileResult,
   GitInfo,
@@ -33,12 +31,19 @@ import type {
   ValidationWarning,
 } from '../types/index.js';
 
+import {
+  BenchmarkExecutionError,
+  FileDiscoveryError,
+  SchemaValidationError,
+  SetupError,
+  StructureValidationError,
+} from '../errors/index.js';
+
 /**
  * Dependencies required by the BenchmarkEngine
  */
 interface EngineDependencies {
   readonly configManager: ConfigurationManager;
-  readonly errorManager: ErrorManager;
   readonly fileLoader: FileLoader;
   readonly historyStorage: HistoryStorage;
   readonly progressManager: ProgressManager;
@@ -55,8 +60,6 @@ interface EngineDependencies {
 export abstract class ModestBenchEngine implements BenchmarkEngine {
   public readonly configManager: ConfigurationManager;
 
-  public readonly errorManager: ErrorManager;
-
   public readonly fileLoader: FileLoader;
 
   public readonly historyStorage: HistoryStorage;
@@ -71,7 +74,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     this.reporterRegistry = dependencies.reporterRegistry;
     this.historyStorage = dependencies.historyStorage;
     this.progressManager = dependencies.progressManager;
-    this.errorManager = dependencies.errorManager;
   }
 
   /**
@@ -86,13 +88,10 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     } catch (error) {
       const discoveryError =
         error instanceof Error ? error : new Error(String(error));
-      this.errorManager.handleError(discoveryError, {
-        metadata: { exclude, pattern },
-        phase: 'discovery',
-        timestamp: new Date(),
-      });
-
-      throw new Error(`File discovery failed: ${discoveryError.message}`);
+      throw new FileDiscoveryError(
+        `File discovery failed: ${discoveryError.message}`,
+        { cause: discoveryError },
+      );
     }
   }
 
@@ -105,11 +104,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
     signal?: AbortSignal,
   ): Promise<BenchmarkRun> {
     const startTime = new Date();
-    let currentPhase: ExecutionPhase = 'discovery';
 
     try {
       // 1. Merge configuration with defaults
-      currentPhase = 'discovery';
       const mergedConfig = await this.configManager.load(
         undefined, // No specific config path for now
         config as Record<string, unknown>,
@@ -125,30 +122,18 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         if (mergedConfig.exclude?.length) {
           msg += ` and excluding "${mergedConfig.exclude.join(', ')}"`;
         }
-        const error = new Error(msg);
-        this.errorManager.handleError(error, {
-          phase: currentPhase,
-          timestamp: new Date(),
-        });
-        throw error;
+        throw new FileDiscoveryError(msg);
       }
 
       // 3. Validate files
-      currentPhase = 'validation';
       const validationResult = await this.validate(files);
       if (!validationResult.valid) {
-        const error = new Error(
+        throw new SchemaValidationError(
           `Validation failed: ${validationResult.errors.map((e) => e.message).join(', ')}`,
         );
-        this.errorManager.handleError(error, {
-          phase: currentPhase,
-          timestamp: new Date(),
-        });
-        throw error;
       }
 
       // 4. Initialize progress tracking
-      currentPhase = 'setup';
       const runId = this.generateRunId();
 
       // Pre-calculate total tasks for progress tracking
@@ -236,7 +221,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       await this.callReporters(reporters, 'onStart', initialRun);
 
       // 6. Execute benchmark files
-      currentPhase = 'execution';
       const fileResults: FileResult[] = [];
 
       for (const filePath of files) {
@@ -263,11 +247,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         } catch (error) {
           const fileError =
             error instanceof Error ? error : new Error(String(error));
-          this.errorManager.handleError(fileError, {
-            file: filePath,
-            phase: currentPhase,
-            timestamp: new Date(),
-          });
 
           // Call reporter onError
           await this.callReporters(reporters, 'onError', fileError);
@@ -350,15 +329,23 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       // 9. Return completed run
       return finalRun;
     } catch (error) {
+      // Re-throw our custom errors
+      if (
+        error instanceof FileDiscoveryError ||
+        error instanceof SchemaValidationError ||
+        error instanceof BenchmarkExecutionError
+      ) {
+        throw error;
+      }
+
       const executionError =
         error instanceof Error ? error : new Error(String(error));
-      const handledError = this.errorManager.handleError(executionError, {
-        phase: currentPhase,
-        timestamp: new Date(),
-      });
 
       // Re-throw the original error with more context
-      throw new Error(`Benchmark execution failed: ${handledError.message}`);
+      throw new BenchmarkExecutionError(
+        `Benchmark execution failed: ${executionError.message}`,
+        { cause: executionError },
+      );
     }
   }
 
@@ -423,11 +410,6 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
         } catch (error) {
           const validationError =
             error instanceof Error ? error : new Error(String(error));
-          this.errorManager.handleError(validationError, {
-            file,
-            phase: 'validation',
-            timestamp: new Date(),
-          });
 
           errors.push({
             code: 'FILE_VALIDATION_ERROR',
@@ -516,7 +498,7 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
       const benchmarkDef = benchmarkFile.exports as BenchmarkDefinition;
 
       if (!benchmarkDef || typeof benchmarkDef !== 'object') {
-        throw new Error(
+        throw new StructureValidationError(
           'Benchmark file must export a default object with suites',
         );
       }
@@ -634,7 +616,9 @@ export abstract class ModestBenchEngine implements BenchmarkEngine {
             error instanceof Error
               ? error
               : new Error(`Setup failed: ${String(error)}`);
-          throw new Error(`Suite setup failed: ${setupError.message}`);
+          throw new SetupError(`Suite setup failed: ${setupError.message}`, {
+            cause: setupError,
+          });
         }
       }
 

package/src/core/engines/accurate-engine.ts

@@ -26,6 +26,7 @@ import type {
   TaskResult,
 } from '../../types/index.js';
 
+import { StructureValidationError } from '../../errors/index.js';
 import { ModestBenchEngine } from '../engine.js';
 import { calculateStatistics, removeOutliersIQR } from '../stats-utils.js';
 
@@ -59,7 +60,9 @@ export class AccurateEngine extends ModestBenchEngine {
   ): Promise<TaskResult> {
     try {
       if (!taskData.fn || typeof taskData.fn !== 'function') {
-        throw new Error('Benchmark task must have a "fn" function property');
+        throw new StructureValidationError(
+          'Benchmark task must have a "fn" function property',
+        );
       }
 
       // Check for V8 native syntax support

package/src/core/engines/tinybench-engine.ts

@@ -14,6 +14,11 @@ import type {
   TaskResult,
 } from '../../types/index.js';
 
+import {
+  BenchmarkExecutionError,
+  OperationTooFastError,
+  StructureValidationError,
+} from '../../errors/index.js';
 import { ModestBenchEngine } from '../engine.js';
 import { calculateStatistics, removeOutliersIQR } from '../stats-utils.js';
 
@@ -33,7 +38,9 @@ export class TinybenchEngine extends ModestBenchEngine {
   ): Promise<TaskResult> {
     try {
       if (!taskData.fn || typeof taskData.fn !== 'function') {
-        throw new Error('Benchmark task must have a "fn" function property');
+        throw new StructureValidationError(
+          'Benchmark task must have a "fn" function property',
+        );
       }
 
       // Determine effective time and iterations based on limitBy mode
@@ -137,13 +144,13 @@ export class TinybenchEngine extends ModestBenchEngine {
             await minimalBench.run();
           } catch {
             // If still failing, the operation is too fast even for tinybench
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark operation is too fast to measure reliably (execution time < 1ns)`,
             );
           }
           const minimalResults = minimalBench.results[0];
           if (!minimalResults || minimalResults.error) {
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark too fast to measure reliably: ${minimalResults?.error?.message || 'unknown error'}`,
             );
           }
@@ -180,7 +187,7 @@ export class TinybenchEngine extends ModestBenchEngine {
       // Get results
       const results = bench.results[0];
       if (!results) {
-        throw new Error('No benchmark results returned');
+        throw new BenchmarkExecutionError('No benchmark results returned');
       }
 
       // Check if the task was aborted
@@ -251,7 +258,7 @@ export class TinybenchEngine extends ModestBenchEngine {
 
           if (!minimalResults || minimalResults.error) {
             // If retry also fails, just accept it failed
-            throw new Error(
+            throw new OperationTooFastError(
               `Benchmark operation is too fast to measure reliably`,
             );
           }

package/src/core/loader.ts

@@ -22,6 +22,11 @@ import {
   BENCHMARK_FILE_EXTENSIONS,
   BENCHMARK_FILE_PATTERN,
 } from '../constants.js';
+import {
+  FileDiscoveryError,
+  FileLoadError,
+  StructureValidationError,
+} from '../errors/index.js';
 import { benchmarkFileSchema } from './benchmark-schema.js';
 
 /**
@@ -106,8 +111,9 @@ export class BenchmarkFileLoader implements FileLoader {
 
       return supportedFiles.sort();
     } catch (error) {
-      throw new Error(
+      throw new FileDiscoveryError(
         `File discovery failed: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }
@@ -120,7 +126,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Basic file checks (existence, extension)
       const basicValidation = await this.validate(filePath);
       if (!basicValidation.valid) {
-        throw new Error(
+        throw new FileLoadError(
           `Invalid benchmark file: ${basicValidation.errors.map((e) => e.message).join(', ')}`,
         );
       }
@@ -160,7 +166,7 @@ export class BenchmarkFileLoader implements FileLoader {
       // Validate the loaded exports structure with Zod
       const structureValidation = this.validateExports(filePath, exports);
       if (!structureValidation.valid || !structureValidation.data) {
-        throw new Error(
+        throw new StructureValidationError(
           `Invalid benchmark structure: ${structureValidation.errors.map((e) => e.message).join(', ')}`,
         );
       }
@@ -185,8 +191,16 @@ export class BenchmarkFileLoader implements FileLoader {
         },
       };
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors
+      if (
+        error instanceof FileLoadError ||
+        error instanceof StructureValidationError
+      ) {
+        throw error;
+      }
+      throw new FileLoadError(
         `Failed to load file ${filePath}: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }
@@ -199,8 +213,16 @@ export class BenchmarkFileLoader implements FileLoader {
       const loadPromises = filePaths.map((filePath) => this.load(filePath));
       return await Promise.all(loadPromises);
     } catch (error) {
-      throw new Error(
+      // Re-throw our custom errors (from individual load calls)
+      if (
+        error instanceof FileLoadError ||
+        error instanceof StructureValidationError
+      ) {
+        throw error;
+      }
+      throw new FileLoadError(
         `Failed to load files: ${error instanceof Error ? error.message : String(error)}`,
+        { cause: error },
       );
     }
   }

package/src/errors/base.ts

@@ -0,0 +1,152 @@
+/**
+ * ModestBench Custom Error System
+ *
+ * Base error classes providing structured error handling with error codes,
+ * documentation URLs, and consistent error display.
+ */
+
+/**
+ * Base URL for error documentation
+ */
+const ERROR_DOC_BASE_URL =
+  'https://boneskull.github.io/modestbench/reference/errors';
+
+/**
+ * Abstract base class for ModestBench aggregate errors
+ *
+ * Extends AggregateError to support multiple errors with ModestBench error
+ * system features.
+ */
+export abstract class ModestBenchAggregateError extends AggregateError {
+  /**
+   * Unique error code for this error type Must be in format:
+   * ERR_MB_CATEGORY_DESCRIPTION
+   */
+  abstract readonly code: string;
+
+  /**
+   * Error name (matches class name)
+   */
+  public override readonly name: string;
+
+  /**
+   * Create a new ModestBench aggregate error
+   *
+   * @param errors - Array of errors that occurred
+   * @param message - Human-readable error message
+   * @param options - Optional Error options (e.g., cause)
+   */
+  constructor(errors: unknown[], message: string, options?: ErrorOptions) {
+    super(errors, message, options);
+    this.name = this.constructor.name;
+  }
+
+  /**
+   * Get the documentation URL for this error
+   *
+   * Returns a URL to the error reference page with an anchor to the specific
+   * error.
+   *
+   * @returns Documentation URL
+   */
+  getDocUrl(): string {
+    // Use the error class name as the anchor (e.g., ConfigValidationError -> #configvalidationerror)
+    const anchor = this.name.toLowerCase();
+    return `${ERROR_DOC_BASE_URL}#${anchor}`;
+  }
+
+  /**
+   * Convert error to string with code, documentation URL, and nested errors
+   *
+   * @returns Formatted error string
+   */
+  override toString(): string {
+    let result = `${this.name} [${this.code}]: ${this.message}\n`;
+    result += `See: ${this.getDocUrl()}\n`;
+
+    if (this.errors.length > 0) {
+      result += `\nContains ${this.errors.length} error(s):\n`;
+      this.errors.forEach((err, index) => {
+        const errMsg = err instanceof Error ? err.message : String(err);
+        result += `  ${index + 1}. ${errMsg}\n`;
+      });
+    }
+
+    return result;
+  }
+}
+
+/**
+ * Abstract base class for all ModestBench errors
+ *
+ * Provides:
+ *
+ * - Unique error codes with ERR_MB_ prefix
+ * - Documentation URL generation
+ * - Consistent error display format
+ * - TypeScript type safety
+ */
+export abstract class ModestBenchError extends Error {
+  /**
+   * Unique error code for this error type Must be in format:
+   * ERR_MB_CATEGORY_DESCRIPTION
+   */
+  abstract readonly code: string;
+
+  /**
+   * Error name (matches class name)
+   */
+  public override readonly name: string;
+
+  /**
+   * Create a new ModestBench error
+   *
+   * @param message - Human-readable error message
+   * @param options - Optional Error options (e.g., cause)
+   */
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = this.constructor.name;
+  }
+
+  /**
+   * Get the documentation URL for this error
+   *
+   * Returns a URL to the error reference page with an anchor to the specific
+   * error.
+   *
+   * @returns Documentation URL
+   */
+  getDocUrl(): string {
+    // Use the error class name as the anchor (e.g., ConfigValidationError -> #configvalidationerror)
+    const anchor = this.name.toLowerCase();
+    return `${ERROR_DOC_BASE_URL}#${anchor}`;
+  }
+
+  /**
+   * Convert error to string with code and documentation URL
+   *
+   * @returns Formatted error string
+   */
+  override toString(): string {
+    return `${this.name} [${this.code}]: ${this.message}\nSee: ${this.getDocUrl()}`;
+  }
+}
+
+/**
+ * Type guard to check if an error is a ModestBench error
+ *
+ * @param error - The error to check
+ * @returns True if the error is a ModestBench error
+ */
+export const isModestBenchError = (
+  error: unknown,
+): error is ModestBenchError => {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    'code' in error &&
+    typeof (error as { code: unknown }).code === 'string' &&
+    (error as { code: string }).code.startsWith('ERR_MB_')
+  );
+};

package/src/errors/cli.ts

@@ -0,0 +1,59 @@
+/**
+ * CLI-related errors
+ *
+ * Errors that occur during command-line interface operations.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Invalid CLI argument
+ *
+ * Thrown when a CLI argument is invalid or cannot be parsed.
+ */
+export class InvalidArgumentError extends ModestBenchError {
+  readonly code = 'ERR_MB_CLI_INVALID_ARGUMENT';
+}
+
+/**
+ * Invalid date format
+ *
+ * Thrown when a date string cannot be parsed into a valid date.
+ */
+export class InvalidDateFormatError extends ModestBenchError {
+  readonly code = 'ERR_MB_CLI_INVALID_DATE_FORMAT';
+}
+
+/**
+ * Unknown error
+ *
+ * Thrown at the CLI boundary to wrap unexpected errors that are not ModestBench
+ * errors. This ensures all errors have proper structure and documentation
+ * links.
+ */
+export class UnknownError extends ModestBenchError {
+  readonly code = 'ERR_MB_UNKNOWN';
+
+  /**
+   * Create a new UnknownError wrapping an unexpected error
+   *
+   * @param message - The error message (typically from the original error)
+   * @param options - Error options with the original error as the cause
+   */
+  constructor(message: string, options: ErrorOptions) {
+    super(message, options);
+  }
+
+  /**
+   * Override toString to show full details of the wrapped error
+   */
+  override toString(): string {
+    let result = super.toString();
+
+    if (this.cause instanceof Error && this.cause.stack) {
+      result += '\n\nOriginal error:\n' + this.cause.stack;
+    }
+
+    return result;
+  }
+}