modestbench 0.0.1

package/src/core/loader.ts

@@ -0,0 +1,324 @@
+/**
+ * ModestBench File Loader
+ *
+ * Handles discovery, loading, and validation of benchmark files. Supports glob
+ * pattern matching and file structure validation.
+ */
+
+import { glob } from 'glob';
+import { access, readFile, stat } from 'node:fs/promises';
+import { extname } from 'node:path';
+
+import type {
+  BenchmarkDefinition,
+  BenchmarkFile,
+  FileLoader,
+  ValidationError,
+  ValidationResult,
+  ValidationWarning,
+} from '../types/index.js';
+
+import {
+  BENCHMARK_FILE_EXTENSIONS,
+  BENCHMARK_FILE_PATTERN,
+} from '../constants.js';
+import { benchmarkFileSchema } from './benchmark-schema.js';
+
+/**
+ * File change notification for watch functionality
+ */
+interface FileChange {
+  readonly filePath: string;
+  readonly timestamp: Date;
+  readonly type: 'added' | 'deleted' | 'modified';
+}
+
+/**
+ * File watcher interface
+ */
+interface FileWatcher {
+  close(): void;
+}
+
+/**
+ * Implementation of FileLoader for benchmark files
+ */
+export class BenchmarkFileLoader implements FileLoader {
+  private readonly supportedExtensions = BENCHMARK_FILE_EXTENSIONS;
+
+  /**
+   * Discover benchmark files using glob patterns or explicit file paths
+   */
+  async discover(
+    pattern: string | string[],
+    exclude: string[] = [],
+  ): Promise<string[]> {
+    try {
+      let patterns = Array.isArray(pattern) ? pattern : [pattern];
+
+      // Handle empty patterns - use sensible defaults
+      if (patterns.length === 0) {
+        patterns = [
+          `*${BENCHMARK_FILE_PATTERN}`, // top-level current directory
+          `bench/*${BENCHMARK_FILE_PATTERN}`, // top-level bench/ directory
+        ];
+      }
+
+      // Expand directory paths to recursive glob patterns
+      const expandedPatterns: string[] = [];
+      for (const p of patterns) {
+        try {
+          const stats = await stat(p);
+          if (stats.isDirectory()) {
+            // Directory: search recursively
+            expandedPatterns.push(`${p}/**/*${BENCHMARK_FILE_PATTERN}`);
+          } else {
+            // File or doesn't exist: use as-is (glob will handle it)
+            expandedPatterns.push(p);
+          }
+        } catch {
+          // Path doesn't exist, treat as glob pattern
+          expandedPatterns.push(p);
+        }
+      }
+
+      const allFiles = new Set<string>();
+
+      // Process each pattern
+      for (const p of expandedPatterns) {
+        const files = await glob(p, {
+          absolute: true,
+          ignore: exclude,
+          nodir: true,
+        });
+
+        // Add discovered files to the set (automatic deduplication)
+        for (const file of files) {
+          allFiles.add(file);
+        }
+      }
+
+      // Filter to supported file extensions
+      const supportedFiles = Array.from(allFiles).filter((file: string) => {
+        const ext = extname(file);
+        return this.supportedExtensions.has(ext);
+      });
+
+      return supportedFiles.sort();
+    } catch (error) {
+      throw new Error(
+        `File discovery failed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /**
+   * Load a single benchmark file
+   */
+  async load(filePath: string): Promise<BenchmarkFile> {
+    try {
+      // Basic file checks (existence, extension)
+      const basicValidation = await this.validate(filePath);
+      if (!basicValidation.valid) {
+        throw new Error(
+          `Invalid benchmark file: ${basicValidation.errors.map((e) => e.message).join(', ')}`,
+        );
+      }
+
+      // Read file content
+      const content = await readFile(filePath, 'utf-8');
+
+      // Get file stats for metadata
+      const stats = await stat(filePath);
+
+      // Load the module using dynamic import
+      const ext = extname(filePath);
+      let module: { [key: string]: unknown; default?: unknown };
+
+      if (ext === '.ts' || ext === '.mts' || ext === '.cts') {
+        // For TypeScript files, use cosmiconfig-typescript-loader
+        const { TypeScriptLoader: createTypeScriptLoader } = await import(
+          'cosmiconfig-typescript-loader'
+        );
+        const loader = createTypeScriptLoader();
+        module = (await loader(filePath, content)) as {
+          [key: string]: unknown;
+          default?: unknown;
+        };
+      } else {
+        // Use native dynamic import for JavaScript files with cache busting
+        // Add timestamp to prevent module caching issues across multiple loads
+        const timestamp = Date.now();
+        module = (await import(`${filePath}?t=${timestamp}`)) as {
+          [key: string]: unknown;
+          default?: unknown;
+        };
+      }
+
+      const exports = module.default || module;
+
+      // Validate the loaded exports structure with Zod
+      const structureValidation = this.validateExports(filePath, exports);
+      if (!structureValidation.valid || !structureValidation.data) {
+        throw new Error(
+          `Invalid benchmark structure: ${structureValidation.errors.map((e) => e.message).join(', ')}`,
+        );
+      }
+
+      // Use the transformed/normalized data from Zod
+      // (this ensures shorthand functions are properly wrapped)
+      const normalizedExports = structureValidation.data;
+
+      // Analyze exports for metadata (simplified - structure already validated)
+      const hasDefaultExport = module.default !== undefined;
+      const exportNames = Object.keys(module);
+
+      return {
+        content,
+        exports: normalizedExports,
+        filePath,
+        metadata: {
+          exportNames,
+          hasDefaultExport,
+          mtime: stats.mtime,
+          size: stats.size,
+        },
+      };
+    } catch (error) {
+      throw new Error(
+        `Failed to load file ${filePath}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /**
+   * Load multiple files in parallel
+   */
+  async loadAll(filePaths: string[]): Promise<BenchmarkFile[]> {
+    try {
+      const loadPromises = filePaths.map((filePath) => this.load(filePath));
+      return await Promise.all(loadPromises);
+    } catch (error) {
+      throw new Error(
+        `Failed to load files: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /**
+   * Validate benchmark file (basic checks only - file existence and extension)
+   * Structure validation happens after loading in the load() method
+   */
+  async validate(filePath: string): Promise<ValidationResult> {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationWarning[] = [];
+
+    try {
+      // Check file existence
+      try {
+        await access(filePath);
+      } catch {
+        errors.push({
+          code: 'FILE_NOT_FOUND',
+          file: filePath,
+          message: 'File does not exist',
+          severity: 'error',
+        });
+        return { errors, files: [], valid: false, warnings };
+      }
+
+      // Check file extension
+      const ext = extname(filePath);
+      if (!this.supportedExtensions.has(ext)) {
+        errors.push({
+          code: 'UNSUPPORTED_EXTENSION',
+          file: filePath,
+          message: `Unsupported file extension: ${ext}. Supported extensions: ${Array.from(this.supportedExtensions).join(', ')}`,
+          severity: 'error',
+        });
+      }
+
+      return {
+        errors,
+        files: [filePath],
+        valid: errors.length === 0,
+        warnings,
+      };
+    } catch (error) {
+      errors.push({
+        code: 'VALIDATION_ERROR',
+        file: filePath,
+        message: `Validation failed: ${error instanceof Error ? error.message : String(error)}`,
+        severity: 'error',
+      });
+
+      return {
+        errors,
+        files: [filePath],
+        valid: false,
+        warnings,
+      };
+    }
+  }
+
+  /**
+   * Watch for file changes (placeholder implementation)
+   */
+  watch(
+    _pattern: string,
+    _callback: (changes: FileChange[]) => void,
+  ): FileWatcher {
+    // TODO: Implement file watching with chokidar or similar
+    // For now, return a no-op watcher
+    return {
+      close() {
+        // No-op
+      },
+    };
+  }
+
+  /**
+   * Validate the structure of loaded exports using Zod schema Returns the
+   * transformed/normalized data if validation succeeds
+   */
+  private validateExports(
+    filePath: string,
+    exports: unknown,
+  ): {
+    data?: BenchmarkDefinition;
+    errors: ValidationError[];
+    valid: boolean;
+    warnings: ValidationWarning[];
+  } {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationWarning[] = [];
+
+    try {
+      const result = benchmarkFileSchema.safeParse(exports);
+
+      if (!result.success) {
+        for (const issue of result.error.issues) {
+          const path = issue.path.length > 0 ? `${issue.path.join('.')}: ` : '';
+          errors.push({
+            code: 'INVALID_STRUCTURE',
+            file: filePath,
+            message: `${path}${issue.message}`,
+            severity: 'error',
+          });
+        }
+        return { errors, valid: false, warnings };
+      }
+
+      // Return the transformed data (with shorthand functions normalized)
+      return { data: result.data, errors, valid: true, warnings };
+    } catch (error) {
+      errors.push({
+        code: 'VALIDATION_ERROR',
+        file: filePath,
+        message: `Structure validation failed: ${error instanceof Error ? error.message : String(error)}`,
+        severity: 'error',
+      });
+      return { errors, valid: false, warnings };
+    }
+  }
+}

package/src/core/stats-utils.ts

@@ -0,0 +1,135 @@
+/**
+ * Statistical utility functions for benchmark result analysis
+ *
+ * Provides IQR-based outlier removal and comprehensive statistics calculation
+ * adapted from bench-node's StatisticalHistogram.
+ */
+
+/**
+ * Statistics computed from sample data
+ */
+export interface SampleStatistics {
+  /** Coefficient of variation (stdDev/mean × 100) */
+  readonly cv: number;
+  /** Margin of error at 95% confidence */
+  readonly marginOfError: number;
+  /** Maximum value */
+  readonly max: number;
+  /** Mean (average) value */
+  readonly mean: number;
+  /** Minimum value */
+  readonly min: number;
+  /** 95th percentile */
+  readonly p95: number;
+  /** 99th percentile */
+  readonly p99: number;
+  /** Standard deviation */
+  readonly stdDev: number;
+  /** Variance */
+  readonly variance: number;
+}
+
+/**
+ * Calculate comprehensive statistics from samples
+ *
+ * Adapted from bench-node's StatisticalHistogram. Assumes samples are already
+ * sorted (e.g., from removeOutliersIQR output).
+ *
+ * @param samples - Sample values (should be sorted for accurate percentiles)
+ * @returns Computed statistics
+ */
+export const calculateStatistics = (samples: number[]): SampleStatistics => {
+  if (samples.length === 0) {
+    return {
+      cv: 0,
+      marginOfError: 0,
+      max: 0,
+      mean: 0,
+      min: 0,
+      p95: 0,
+      p99: 0,
+      stdDev: 0,
+      variance: 0,
+    };
+  }
+
+  // Min/Max (samples are already sorted from removeOutliersIQR)
+  const min = samples[0]!;
+  const max = samples[samples.length - 1]!;
+
+  // Mean
+  const mean =
+    samples.length === 1
+      ? samples[0]!
+      : samples.reduce(
+          (acc, v) => Math.min(Number.MAX_SAFE_INTEGER, acc + v),
+          0,
+        ) / samples.length;
+
+  // Standard deviation and variance
+  let stdDev = 0;
+  let variance = 0;
+  if (samples.length >= 2) {
+    variance =
+      samples.reduce((acc, v) => acc + Math.pow(v - mean, 2), 0) /
+      (samples.length - 1);
+    stdDev = Math.sqrt(variance);
+  }
+
+  // Coefficient of Variation
+  const cv = mean === 0 || samples.length < 2 ? 0 : (stdDev / mean) * 100;
+
+  // Margin of Error (95% confidence)
+  const Z = 1.96;
+  const marginOfError =
+    samples.length === 0 ? 0 : (Z * stdDev) / Math.sqrt(samples.length);
+
+  // Percentiles (using standard formula: floor((n-1) * p))
+  const p95 = samples[Math.floor((samples.length - 1) * 0.95)] ?? 0;
+  const p99 = samples[Math.floor((samples.length - 1) * 0.99)] ?? 0;
+
+  return { cv, marginOfError, max, mean, min, p95, p99, stdDev, variance };
+};
+
+/**
+ * Remove outliers using Interquartile Range (IQR) method
+ *
+ * Adapted from bench-node's StatisticalHistogram.removeOutliers. Filters values
+ * outside [Q1 - 1.5×IQR, Q3 + 1.5×IQR] range.
+ *
+ * @param samples - Raw sample values (will be sorted internally)
+ * @returns Filtered samples with outliers removed
+ */
+export const removeOutliersIQR = (samples: number[]): number[] => {
+  if (samples.length < 4) {
+    return samples;
+  }
+
+  const sorted = samples.slice().sort((a, b) => a - b);
+
+  // Calculate Q1 and Q3
+  let q1: number;
+  let q3: number;
+  const size = sorted.length;
+
+  if (((size - 1) / 4) % 1 === 0 || (size / 4) % 1 === 0) {
+    q1 =
+      (1 / 2) *
+      (sorted[Math.floor(size / 4) - 1]! + sorted[Math.floor(size / 4)]!);
+    q3 =
+      (1 / 2) *
+      (sorted[Math.ceil((size * 3) / 4) - 1]! +
+        sorted[Math.ceil((size * 3) / 4)]!);
+  } else {
+    q1 = sorted[Math.floor(size / 4)]!;
+    q3 = sorted[Math.floor((size * 3) / 4)]!;
+  }
+
+  // Calculate IQR and bounds
+  const iqr = q3 - q1;
+  const minValue = q1 - iqr * 1.5;
+  const maxValue = q3 + iqr * 1.5;
+
+  // Filter outliers
+  return sorted.filter((value) => value <= maxValue && value >= minValue);
+};

package/src/index.ts

@@ -0,0 +1,46 @@
+/**
+ * ModestBench Public API
+ *
+ * Main entry point for programmatic usage of ModestBench. This module exports
+ * all core classes, utilities, and types needed to use ModestBench as a
+ * library.
+ */
+
+export { bootstrap as modestbench } from './bootstrap.js';
+// Configuration management
+export { ModestBenchConfigurationManager } from './config/manager.js';
+
+// Core engine and loader
+export { ModestBenchEngine } from './core/engine.js';
+export { AccurateEngine, TinybenchEngine } from './core/engines/index.js';
+
+// Error handling
+export { ModestBenchErrorManager } from './core/error-manager.js';
+
+export { BenchmarkFileLoader } from './core/loader.js';
+
+// Statistical utilities
+export {
+  calculateStatistics,
+  removeOutliersIQR,
+  type SampleStatistics,
+} from './core/stats-utils.js';
+
+// Progress tracking
+export { ModestBenchProgressManager } from './progress/manager.js';
+// Reporters
+export { CsvReporter } from './reporters/csv.js';
+export { HumanReporter } from './reporters/human.js';
+export { JsonReporter } from './reporters/json.js';
+
+export {
+  BaseReporter,
+  CompositeReporter,
+  ModestBenchReporterRegistry,
+} from './reporters/registry.js';
+
+// Storage
+export { FileHistoryStorage } from './storage/history.js';
+
+// Export all types
+export * from './types/index.js';