modestbench 0.0.1

package/src/config/manager.ts

@@ -0,0 +1,387 @@
+/**
+ * ModestBench Configuration Manager
+ *
+ * Handles loading, merging, and validation of configuration from multiple
+ * sources. Supports CLI arguments, config files (JSON/YAML/JS/TS), and
+ * defaults.
+ */
+
+import { cosmiconfig } from 'cosmiconfig';
+import { resolve } from 'node:path';
+
+import type {
+  ConfigurationManager,
+  ModestBenchConfig,
+  ValidationError,
+  ValidationResult,
+  ValidationWarning,
+} from '../types/index.js';
+
+import { safeParseConfig } from './schema.js';
+
+/**
+ * Get the default reporter based on TTY status and environment
+ */
+const getDefaultReporter = (): string => {
+  // Use simple reporter when stdout is not a TTY and color is not forced
+  if (!process.stdout.isTTY && !isColorForced()) {
+    return 'simple';
+  }
+  return 'human';
+};
+
+/**
+ * Check if color output has been forced via environment variables
+ */
+const isColorForced = (): boolean => {
+  return (
+    process.env.FORCE_COLOR !== undefined &&
+    process.env.FORCE_COLOR !== '0' &&
+    process.env.NO_COLOR === undefined
+  );
+};
+
+/**
+ * Default configuration values Using minimal values to reduce test overhead
+ * while maintaining functionality
+ */
+const DEFAULT_CONFIG: ModestBenchConfig = {
+  bail: false,
+  exclude: ['node_modules/**', '.git/**'],
+  excludeTags: [],
+  iterations: 100, // Sufficient iterations for reliable statistics
+  limitBy: 'iterations', // Default to limiting by iteration count
+  metadata: {},
+  outputDir: './benchmark-results',
+  pattern: '**/*.bench.{js,ts,mjs,cjs,mts,cts}',
+  quiet: false,
+  reporterConfig: {},
+  reporters: [getDefaultReporter()],
+  tags: [],
+  thresholds: {},
+  time: 1000, // 1 second minimum for tinybench to gather samples
+  timeout: 30000, // 30 seconds
+  verbose: false, // No verbose output by default
+  warmup: 0, // No warmup by default for test speed
+};
+
+/**
+ * Configuration precedence order (highest to lowest):
+ *
+ * 1. CLI arguments
+ * 2. Config file
+ * 3. Default values
+ */
+export class ModestBenchConfigurationManager implements ConfigurationManager {
+  /**
+   * Apply smart defaults for limitBy based on which flags were provided
+   */
+  public static applySmartDefaults(
+    merged: ModestBenchConfig,
+    cliArgs: Record<string, unknown>,
+    fileConfig: Partial<ModestBenchConfig>,
+  ): ModestBenchConfig {
+    // If limitBy was explicitly provided in CLI or file, use it
+    if (cliArgs['limit-by'] || cliArgs.limitBy || fileConfig.limitBy) {
+      return merged;
+    }
+
+    // Determine if user explicitly provided time or iterations
+    const userProvidedTime = 'time' in cliArgs || 't' in cliArgs;
+    const userProvidedIterations = 'iterations' in cliArgs || 'i' in cliArgs;
+
+    let smartDefault: 'any' | 'iterations' | 'time';
+
+    if (userProvidedTime && userProvidedIterations) {
+      // Both provided → stop at whichever comes first
+      smartDefault = 'any';
+    } else if (userProvidedTime) {
+      // Only time → limit by time
+      smartDefault = 'time';
+    } else {
+      // Only iterations (or neither) → limit by iterations
+      smartDefault = 'iterations';
+    }
+
+    return {
+      ...merged,
+      limitBy: smartDefault,
+    };
+  }
+
+  /**
+   * Get default configuration values
+   */
+  getDefaults(): ModestBenchConfig {
+    return { ...DEFAULT_CONFIG };
+  }
+
+  /**
+   * Load configuration from various sources with precedence
+   */
+  async load(
+    configPath?: string,
+    cliArgs?: Record<string, unknown>,
+  ): Promise<ModestBenchConfig> {
+    try {
+      // Create a fresh explorer for each load to avoid module caching issues
+      const explorer = this.createExplorer();
+
+      // 1. Load config file using cosmiconfig
+      let result;
+      if (configPath) {
+        const resolvedPath = resolve(configPath);
+        // For .js/.mjs/.cjs files, add cache busting to the import to avoid Node's module cache
+        if (
+          resolvedPath.endsWith('.js') ||
+          resolvedPath.endsWith('.mjs') ||
+          resolvedPath.endsWith('.cjs')
+        ) {
+          // Clear Node's module cache for this file to ensure fresh load
+          const moduleUrl = `${resolvedPath}?t=${Date.now()}`;
+          try {
+            const module = (await import(moduleUrl)) as {
+              [key: string]: unknown;
+              default?: unknown;
+            };
+            result = {
+              config: module.default || module,
+              filepath: resolvedPath,
+            };
+          } catch {
+            // Fall back to explorer.load if cache busting fails
+            result = await explorer.load(resolvedPath);
+          }
+        } else {
+          result = await explorer.load(resolvedPath);
+        }
+      } else {
+        result = await explorer.search();
+      }
+
+      const fileConfig = (result?.config || {}) as Partial<ModestBenchConfig>;
+
+      // 2. Merge: defaults <- file <- CLI args
+      const normalizedCliArgs = cliArgs ? this.normalizeCliArgs(cliArgs) : {};
+      const merged = this.merge(DEFAULT_CONFIG, fileConfig, normalizedCliArgs);
+
+      // 2.5. Apply smart defaults for limitBy if not explicitly provided
+      const finalConfig = ModestBenchConfigurationManager.applySmartDefaults(
+        merged,
+        cliArgs || {},
+        fileConfig,
+      );
+
+      // 3. Validate final configuration
+      const validation = this.validate(finalConfig);
+      if (!validation.valid) {
+        throw new Error(
+          `Configuration validation failed: ${validation.errors.map((e) => e.message).join(', ')}`,
+        );
+      }
+
+      return finalConfig;
+    } catch (error) {
+      throw new Error(
+        `Failed to load configuration: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /**
+   * Merge multiple configuration objects with precedence
+   */
+  merge(...configs: Partial<ModestBenchConfig>[]): ModestBenchConfig {
+    let result: Partial<ModestBenchConfig> = {};
+
+    for (const config of configs) {
+      result = {
+        ...result,
+        ...config,
+        // Special handling for arrays - replace rather than merge
+        // Allow empty arrays to override defaults (for pattern defaulting in loader)
+        ...(config.pattern !== undefined && {
+          pattern: Array.isArray(config.pattern)
+            ? [...config.pattern]
+            : config.pattern,
+        }),
+        ...(config.exclude && { exclude: [...config.exclude] }),
+        ...(config.excludeTags && { excludeTags: [...config.excludeTags] }),
+        ...(config.reporters && { reporters: [...config.reporters] }),
+        ...(config.tags && { tags: [...config.tags] }),
+        // Deep merge for objects
+        ...(config.reporterConfig && {
+          reporterConfig: {
+            ...result.reporterConfig,
+            ...config.reporterConfig,
+          },
+        }),
+        ...(config.metadata && {
+          metadata: { ...result.metadata, ...config.metadata },
+        }),
+        ...(config.thresholds && {
+          thresholds: { ...result.thresholds, ...config.thresholds },
+        }),
+      };
+    }
+
+    return { ...DEFAULT_CONFIG, ...result };
+  }
+
+  /**
+   * Validate configuration object using Zod schema
+   */
+  validate(config: ModestBenchConfig): ValidationResult {
+    const errors: ValidationError[] = [];
+    const warnings: ValidationWarning[] = [];
+
+    // Use Zod schema validation
+    const result = safeParseConfig(config);
+
+    if (!result.success) {
+      // Convert Zod errors to ValidationError format
+      for (const issue of result.error.issues) {
+        const path = issue.path.join('.');
+        errors.push({
+          code: `INVALID_${path.toUpperCase().replace(/\./g, '_') || 'CONFIG'}`,
+          file: 'configuration',
+          message: `${path ? `${path}: ` : ''}${issue.message}`,
+          severity: 'error',
+        });
+      }
+    }
+
+    // Additional logical validations and warnings
+    if (result.success) {
+      const validConfig = result.data;
+
+      // Warn about empty reporters
+      if (validConfig.reporters.length === 0) {
+        warnings.push({
+          code: 'NO_REPORTERS',
+          file: 'configuration',
+          message: 'no reporters specified, using default human reporter',
+          severity: 'warning',
+        });
+      }
+
+      // Warn about potentially long runtime
+      if (validConfig.iterations > 1000 && validConfig.time > 60000) {
+        warnings.push({
+          code: 'LONG_RUNTIME_WARNING',
+          file: 'configuration',
+          message:
+            'high iterations and time values may result in very long benchmark runs',
+          severity: 'warning',
+        });
+      }
+    }
+
+    return {
+      errors,
+      files: ['configuration'],
+      valid: errors.length === 0,
+      warnings,
+    };
+  }
+
+  /**
+   * Create a cosmiconfig explorer for loading configuration files
+   */
+  private createExplorer() {
+    return cosmiconfig('modestbench', {
+      cache: false, // Disable caching to prevent cross-contamination between different config files
+      loaders: {
+        '.ts': async (filepath: string): Promise<unknown> => {
+          // Use cosmiconfig-typescript-loader to load TypeScript files
+          // This works without tsx in the import chain
+          const { TypeScriptLoader: createTypeScriptLoader } = await import(
+            'cosmiconfig-typescript-loader'
+          );
+          const loader = createTypeScriptLoader();
+          const { readFile } = await import('node:fs/promises');
+          const content = await readFile(filepath, 'utf-8');
+          return (await loader(filepath, content)) as unknown;
+        },
+      },
+      searchPlaces: [
+        'package.json',
+        '.modestbenchrc',
+        '.modestbenchrc.json',
+        '.modestbenchrc.yaml',
+        '.modestbenchrc.yml',
+        '.modestbenchrc.js',
+        '.modestbenchrc.mjs',
+        '.modestbenchrc.cjs',
+        'modestbench.config.json',
+        'modestbench.config.yaml',
+        'modestbench.config.yml',
+        'modestbench.config.js',
+        'modestbench.config.mjs',
+        'modestbench.config.cjs',
+        'modestbench.config.ts',
+      ],
+    });
+  }
+
+  /**
+   * Normalize CLI arguments to configuration format
+   */
+  private normalizeCliArgs(
+    cliArgs: Record<string, unknown>,
+  ): Partial<ModestBenchConfig> {
+    const normalized: Record<string, unknown> = {};
+
+    // Map CLI argument names to config property names
+    const argMap: Record<string, keyof ModestBenchConfig> = {
+      bail: 'bail',
+      exclude: 'exclude',
+      'exclude-tags': 'excludeTags',
+      excludeTags: 'excludeTags',
+      i: 'iterations',
+      iterations: 'iterations',
+      'limit-by': 'limitBy',
+      limitBy: 'limitBy',
+      o: 'outputDir',
+      output: 'outputDir',
+      'output-dir': 'outputDir',
+      pattern: 'pattern',
+      q: 'quiet',
+      quiet: 'quiet',
+      r: 'reporters',
+      reporters: 'reporters',
+      t: 'time',
+      tags: 'tags',
+      time: 'time',
+      timeout: 'timeout',
+      v: 'verbose',
+      verbose: 'verbose',
+      w: 'warmup',
+      warmup: 'warmup',
+    };
+
+    for (const [cliKey, configKey] of Object.entries(argMap)) {
+      if (cliKey in cliArgs && cliArgs[cliKey] !== undefined) {
+        const value = cliArgs[cliKey];
+
+        // Handle array arguments that might come as strings
+        if (
+          configKey === 'exclude' ||
+          configKey === 'excludeTags' ||
+          configKey === 'reporters' ||
+          configKey === 'tags'
+        ) {
+          if (typeof value === 'string') {
+            normalized[configKey] = value.split(',').map((s) => s.trim());
+          } else if (Array.isArray(value)) {
+            normalized[configKey] = value;
+          }
+        } else {
+          normalized[configKey] = value;
+        }
+      }
+    }
+
+    return normalized as Partial<ModestBenchConfig>;
+  }
+}

package/src/config/schema.ts

@@ -0,0 +1,188 @@
+/**
+ * ModestBench Configuration Schemas
+ *
+ * Zod schemas for validating configuration. These schemas are constrained to
+ * match the TypeScript types defined in types/core.ts, ensuring type safety and
+ * enabling JSON Schema generation.
+ */
+
+import * as z from 'zod';
+
+import type { ModestBenchConfig } from '../types/core.js';
+
+import { BENCHMARK_FILE_PATTERN } from '../constants.js';
+
+/**
+ * Schema for threshold configuration
+ *
+ * Defines performance assertion thresholds for benchmark validation.
+ */
+const thresholdConfigSchema = z
+  .object({
+    maxMarginOfError: z
+      .number()
+      .positive()
+      .describe('Maximum allowed margin of error as a percentage')
+      .optional(),
+    maxMean: z
+      .number()
+      .positive()
+      .describe('Maximum allowed mean execution time in nanoseconds')
+      .optional(),
+    maxP95: z
+      .number()
+      .positive()
+      .describe('Maximum allowed 95th percentile execution time in nanoseconds')
+      .optional(),
+    maxP99: z
+      .number()
+      .positive()
+      .describe('Maximum allowed 99th percentile execution time in nanoseconds')
+      .optional(),
+    maxStdDev: z
+      .number()
+      .positive()
+      .describe('Maximum allowed standard deviation in nanoseconds')
+      .optional(),
+    minOpsPerSecond: z
+      .number()
+      .positive()
+      .describe('Minimum required operations per second')
+      .optional(),
+  })
+  .strict()
+  .describe('Performance assertion thresholds for benchmark validation')
+  .meta({
+    title: 'Threshold Configuration',
+  });
+
+/**
+ * Schema for the main ModestBench configuration
+ *
+ * This is the complete configuration schema used for validating benchmark
+ * configuration from all sources (files, CLI args, defaults).
+ */
+const modestBenchConfigSchema = z
+  .object({
+    $schema: z
+      .string()
+      .optional()
+      .describe(
+        'JSON Schema reference for IDE support (not used by ModestBench)',
+      ),
+    bail: z.boolean().describe('Stop benchmark execution on first failure'),
+    exclude: z
+      .array(z.string())
+      .describe(
+        'Glob patterns to exclude from benchmark file discovery (e.g., "node_modules/**", ".git/**")',
+      ),
+    excludeTags: z
+      .array(z.string())
+      .describe(
+        'Tags to exclude from benchmark execution. Benchmarks matching any of these tags will be skipped.',
+      ),
+    iterations: z
+      .number()
+      .int()
+      .positive()
+      .describe(
+        'Default number of iterations to run for each benchmark task. Higher values provide more accurate statistics but take longer to execute.',
+      ),
+    limitBy: z
+      .enum(['time', 'iterations', 'any', 'all'])
+      .describe(
+        'How to limit benchmark execution: "time" stops after time limit, "iterations" stops after iteration count, "any" stops at whichever comes first, "all" runs until both limits are reached',
+      ),
+    metadata: z
+      .record(z.string(), z.unknown())
+      .describe(
+        'Custom metadata to attach to benchmark runs. Can include project name, version, environment details, etc.',
+      ),
+    outputDir: z
+      .string()
+      .min(1)
+      .describe(
+        'Directory path where benchmark results and reports will be written',
+      ),
+    pattern: z
+      .union([z.string().min(1), z.array(z.string().min(1))])
+      .describe(
+        `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`,
+      ),
+    quiet: z
+      .boolean()
+      .describe(
+        'Run in quiet mode with minimal console output (only errors and final results)',
+      ),
+    reporterConfig: z
+      .record(z.string(), z.unknown())
+      .describe(
+        'Configuration options specific to individual reporters, keyed by reporter name',
+      ),
+    reporters: z
+      .array(z.string())
+      .min(1)
+      .describe(
+        'List of reporter names to use for output. Available reporters: "human", "json", "csv"',
+      ),
+    tags: z
+      .array(z.string())
+      .describe(
+        'Tags to filter which benchmarks to run. If empty, all benchmarks are included. Only benchmarks with matching tags will execute.',
+      ),
+    thresholds: thresholdConfigSchema,
+    time: z
+      .number()
+      .int()
+      .positive()
+      .describe(
+        'Maximum time to spend on each benchmark task in milliseconds. Tasks will run at least until this duration or iteration count is reached, depending on limitBy setting.',
+      ),
+    timeout: z
+      .number()
+      .int()
+      .positive()
+      .describe(
+        'Timeout for individual benchmark tasks in milliseconds. Tasks exceeding this duration will be terminated and marked as failed.',
+      ),
+    verbose: z
+      .boolean()
+      .describe(
+        'Enable verbose output. Provides more detailed console output including progress, intermediate results, and diagnostic information',
+      ),
+    warmup: z
+      .number()
+      .int()
+      .nonnegative()
+      .describe(
+        'Number of warmup iterations to run before measurement begins. Warmup helps stabilize performance by allowing JIT compilation and caching to occur.',
+      ),
+  })
+  .strict()
+  .describe(
+    'ModestBench configuration for controlling benchmark discovery, execution, and reporting',
+  )
+  .meta({
+    title: 'ModestBench Configuration',
+  });
+
+/**
+ * Validate a partial configuration object
+ *
+ * This is used for validating configuration from files or CLI args before
+ * merging with defaults.
+ */
+export const partialModestBenchConfigSchema: z.ZodType<
+  Partial<ModestBenchConfig>
+> = modestBenchConfigSchema.partial();
+
+/**
+ * Safely parse and validate a configuration object
+ *
+ * @param config - The configuration object to validate
+ * @returns A result object with either success: true and data, or success:
+ *   false and error
+ */
+export const safeParseConfig = (config: unknown) => {
+  return modestBenchConfigSchema.safeParse(config);
+};

package/src/constants.ts

@@ -0,0 +1,21 @@
+/**
+ * Supported benchmark file extensions
+ */
+export const BENCHMARK_FILE_EXTENSIONS = new Set([
+  '.cjs',
+  '.cts',
+  '.js',
+  '.mjs',
+  '.mts',
+  '.ts',
+]);
+
+/**
+ * Glob pattern fragment for benchmark file extensions. Example:
+ * ".bench.{js,mjs,cjs,ts,mts,cts}"
+ */
+export const BENCHMARK_FILE_PATTERN = `.bench.{${Array.from(
+  BENCHMARK_FILE_EXTENSIONS,
+)
+  .map((ext) => ext.slice(1))
+  .join(',')}}`;