modestbench 0.2.0 → 0.3.1

package/src/types/budgets.ts

@@ -0,0 +1,180 @@
+/**
+ * Performance budget types for benchmark thresholds
+ *
+ * @module types/budgets
+ */
+
+/**
+ * Absolute performance budgets (time-based)
+ */
+export interface AbsoluteBudget {
+  /** Maximum 99th percentile in nanoseconds */
+  readonly maxP99?: number;
+
+  /** Maximum mean execution time in nanoseconds */
+  readonly maxTime?: number;
+
+  /** Minimum operations per second */
+  readonly minOpsPerSec?: number;
+}
+
+/**
+ * Named baseline reference
+ */
+export interface BaselineReference {
+  /** Git branch (if available) */
+  readonly branch?: string;
+
+  /** Git commit (if available) */
+  readonly commit?: string;
+
+  /** Date baseline was created */
+  readonly date: Date;
+
+  /** Baseline name */
+  readonly name: string;
+
+  /** Run ID this baseline points to */
+  readonly runId: RunId;
+
+  /** Summary of benchmark results for quick comparison */
+  readonly summary: Record<TaskId, BaselineSummaryData>;
+}
+
+/**
+ * Baseline storage file format
+ */
+export interface BaselineStorage {
+  /** Named baselines */
+  readonly baselines: Record<string, BaselineReference>;
+
+  /** Default baseline name (optional) */
+  readonly default?: string;
+
+  /** Schema version */
+  readonly version: string;
+}
+
+/**
+ * Baseline summary data for a single task
+ */
+export interface BaselineSummaryData {
+  /** Mean execution time in nanoseconds */
+  readonly mean: number;
+
+  /** Operations per second */
+  readonly opsPerSecond: number;
+
+  /** 99th percentile (if available) */
+  readonly p99?: number;
+}
+
+/**
+ * Complete budget definition
+ */
+export interface Budget {
+  /** Absolute thresholds */
+  readonly absolute?: AbsoluteBudget;
+
+  /** Relative thresholds */
+  readonly relative?: RelativeBudget;
+}
+
+/**
+ * Budget evaluation result for a single task
+ */
+export interface BudgetResult {
+  /** Actual measured values */
+  readonly actual: {
+    readonly mean: number;
+    readonly opsPerSecond: number;
+    readonly p99?: number;
+  };
+
+  /** Baseline values (if relative budget) */
+  readonly baseline?: {
+    readonly mean: number;
+    readonly opsPerSecond: number;
+    readonly p99?: number;
+  };
+
+  /** Budget that was checked */
+  readonly budget: Budget;
+
+  /** Whether budget passed */
+  readonly passed: boolean;
+
+  /** Task identifier (file/suite/task) */
+  readonly taskId: TaskId;
+
+  /** Violations (what thresholds were exceeded) */
+  readonly violations: BudgetViolation[];
+}
+
+/**
+ * Budget evaluation summary for entire run
+ */
+export interface BudgetSummary {
+  /** Number failed */
+  readonly failed: number;
+
+  /** Number passed */
+  readonly passed: number;
+
+  /** Individual results */
+  readonly results: BudgetResult[];
+
+  /** Total budgets checked */
+  readonly total: number;
+}
+
+/**
+ * Specific budget violation
+ */
+export interface BudgetViolation {
+  /** Actual measured value */
+  readonly actual: number;
+
+  /** How much over/under threshold (as decimal, e.g., 0.15 = 15% over) */
+  readonly delta: number;
+
+  /** Human-readable message */
+  readonly message: string;
+
+  /** Expected threshold */
+  readonly threshold: number;
+
+  /** Type of budget that was violated */
+  readonly type: 'maxP99' | 'maxRegression' | 'maxTime' | 'minOpsPerSec';
+}
+
+/**
+ * Relative performance budgets (comparison-based)
+ */
+export interface RelativeBudget {
+  /** Name of baseline to compare against */
+  readonly baseline?: string;
+
+  /** Maximum performance regression as decimal (0.10 = 10%) */
+  readonly maxRegression?: number;
+}
+
+/**
+ * Branded type for benchmark run identifiers
+ *
+ * RunId is a 7-character alphanumeric string that uniquely identifies a
+ * benchmark run. Using a branded type prevents accidental mixing with regular
+ * strings or TaskIds.
+ */
+export type RunId = string & { readonly __brand: 'RunId' };
+
+/**
+ * Branded type for task identifiers
+ *
+ * TaskId follows the format: `{filePath}/{suiteName}/{taskName}` Example:
+ * `benchmarks/array.bench.js/Array Operations/Array.push()`
+ *
+ * Using a branded type prevents accidental mixing with regular strings or
+ * RunIds.
+ */
+export type TaskId = string & { readonly __brand: 'TaskId' };

package/src/types/cli.ts

@@ -5,17 +5,9 @@
  * definitions, argument parsing, and CLI-specific configuration structures.
  */
 
-/**
- * Exit codes used by the CLI
- */
-export const ExitCodes = {
-  ConfigurationError: 2,
-  ExecutionError: 5,
-  FileDiscoveryError: 3,
-  GeneralError: 1,
-  Success: 0,
-  ValidationError: 4,
-} as const;
+import { ExitCodes } from '../constants.js';
+
+export { ExitCodes };
 
 /**
  * CLI argument specification for a command
@@ -39,56 +31,6 @@ export interface ArgumentSpec {
   readonly validate?: (value: unknown) => boolean | string;
 }
 
-/**
- * Base command interface
- */
-export interface CliCommand {
-  /** Command aliases */
-  readonly aliases?: string[];
-  /** Command description */
-  readonly description: string;
-  /** Execute the command */
-  execute(args: CommandArguments): Promise<ExitCode>;
-  /** Command name */
-  readonly name: string;
-}
-
-/**
- * CLI configuration
- */
-export interface CliConfig {
-  /** Available commands */
-  readonly commands: CommandSpec[];
-  /** Application description */
-  readonly description: string;
-  /** Global options */
-  readonly globalOptions: ArgumentSpec[];
-  /** Application name */
-  readonly name: string;
-  /** Application version */
-  readonly version: string;
-}
-
-/**
- * Color theme for CLI output
- */
-export interface ColorTheme {
-  /** Error color */
-  readonly error: string;
-  /** Highlight color */
-  readonly highlight: string;
-  /** Info color */
-  readonly info: string;
-  /** Muted/secondary text color */
-  readonly muted: string;
-  /** Primary color for branding */
-  readonly primary: string;
-  /** Success color */
-  readonly success: string;
-  /** Warning color */
-  readonly warning: string;
-}
-
 /**
  * Parsed command-line arguments
  */
@@ -119,118 +61,9 @@ export interface CommandSpec {
   readonly subcommands?: CommandSpec[];
 }
 
-export type ExitCode = (typeof ExitCodes)[keyof typeof ExitCodes];
-
-/**
- * Global CLI options available to all commands
- */
-export interface GlobalOptions {
-  readonly c?: string;
-  /** Configuration file */
-  readonly config?: string;
-  /** Working directory */
-  readonly cwd?: string;
-  readonly h?: boolean;
-  /** Help flag */
-  readonly help?: boolean;
-  /** Log level */
-  readonly logLevel?: 'debug' | 'error' | 'info' | 'silent' | 'warn';
-  /** No color output */
-  readonly noColor?: boolean;
-  /** Version flag */
-  readonly version?: boolean;
-}
-
-/**
- * CLI help information
- */
-export interface HelpInfo {
-  /** Command name */
-  readonly command: string;
-  /** Command description */
-  readonly description: string;
-  /** Examples */
-  readonly examples?: string[];
-  /** Available options */
-  readonly options: Array<{
-    readonly default?: string;
-    readonly description: string;
-    readonly flags: string;
-  }>;
-  /** Available subcommands */
-  readonly subcommands?: Array<{
-    readonly description: string;
-    readonly name: string;
-  }>;
-  /** Usage string */
-  readonly usage: string;
-}
-
-/**
- * Arguments for the history command
- */
-export interface HistoryCommandArgs extends CommandArguments {
-  readonly f?: 'csv' | 'json' | 'table';
-  /** Output format */
-  readonly format?: 'csv' | 'json' | 'table';
-  readonly l?: number;
-  /** Limit number of results */
-  readonly limit?: number;
-  /** Pattern to match */
-  readonly pattern?: string;
-  /** Run IDs for show/compare commands */
-  readonly runIds?: string[];
-  /** Filter by date */
-  readonly since?: string;
-  /** History subcommand */
-  readonly subcommand?: 'clean' | 'compare' | 'list' | 'show' | 'trends';
-  /** Tags to filter by */
-  readonly tags?: string | string[];
-}
-
-/**
- * Arguments for the init command
- */
-export interface InitCommandArgs extends CommandArguments {
-  /** Configuration file type */
-  readonly configType?: 'js' | 'json' | 'ts' | 'yaml';
-  /** Create example files */
-  readonly examples?: boolean;
-  /** Force overwrite existing files */
-  readonly force?: boolean;
-}
-
-/**
- * Output formatting options
- */
-export interface OutputFormat {
-  /** Compact output mode */
-  readonly compact: boolean;
-  /** Progress display options */
-  readonly progress: ProgressDisplayOptions;
-  /** Color theme */
-  readonly theme: ColorTheme;
-  /** Use colors in output */
-  readonly useColors: boolean;
-  /** Use Unicode symbols */
-  readonly useUnicode: boolean;
-}
+export type Engine = 'accurate' | 'tinybench';
 
-/**
- * CLI parser result
- */
-export interface ParseResult {
-  /** Parsed arguments */
-  readonly args: CommandArguments;
-  /** Parsed command name */
-  readonly command: string;
-  /** Parsing errors */
-  readonly errors: string[];
-  /** Whether help was requested */
-  readonly help: boolean;
-  /** Whether version was requested */
-  readonly version: boolean;
-}
+export type ExitCode = (typeof ExitCodes)[keyof typeof ExitCodes];
 
 /**
  * Progress display options
@@ -247,69 +80,3 @@ export interface ProgressDisplayOptions {
   /** Update interval in milliseconds */
   readonly updateInterval: number;
 }
-
-/**
- * Arguments for the run command
- */
-export interface RunCommandArgs extends CommandArguments {
-  /** Stop on first failure */
-  readonly bail?: boolean;
-  readonly c?: string;
-  /** Configuration file path */
-  readonly config?: string;
-  readonly e?: 'accurate' | 'tinybench';
-  /** Benchmark engine to use */
-  readonly engine?: 'accurate' | 'tinybench';
-  /** Files to exclude */
-  readonly exclude?: string | string[];
-  /** Tags to exclude */
-  readonly excludeTags?: string | string[];
-  readonly i?: number;
-  /** Number of iterations */
-  readonly iterations?: number;
-  /** How to limit benchmark execution */
-  readonly limitBy?: 'all' | 'any' | 'iterations' | 'time';
-  readonly o?: string;
-  readonly of?: string;
-  /** Output directory */
-  readonly output?: string;
-  /** Custom output filename (works with single reporter only) */
-  readonly outputFile?: string;
-  /** Pattern for discovering benchmark files */
-  readonly pattern?: string;
-  readonly q?: boolean;
-  /** Quiet output */
-  readonly quiet?: boolean;
-  readonly r?: string | string[];
-  /** Reporters to use */
-  readonly reporters?: string | string[];
-  readonly t?: number;
-  /** Tags to include */
-  readonly tags?: string | string[];
-  /** Time limit in milliseconds */
-  readonly time?: number;
-  /** Timeout per task */
-  readonly timeout?: number;
-  readonly v?: boolean;
-  /** Verbose output */
-  readonly verbose?: boolean;
-  readonly w?: number;
-  /** Warmup iterations */
-  readonly warmup?: number;
-}
-
-/**
- * Terminal capabilities
- */
-export interface TerminalCapabilities {
-  /** Terminal height in rows */
-  readonly height: number;
-  /** Terminal supports interactive features */
-  readonly interactive: boolean;
-  /** Terminal supports colors */
-  readonly supportsColor: boolean;
-  /** Terminal supports Unicode characters */
-  readonly supportsUnicode: boolean;
-  /** Terminal width in columns */
-  readonly width: number;
-}

package/src/types/core.ts

@@ -1,3 +1,42 @@
+// Budget-related types
+import type {
+  AbsoluteBudget,
+  BaselineReference,
+  BaselineStorage,
+  BaselineSummaryData,
+  Budget,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  RelativeBudget,
+  RunId,
+  TaskId,
+} from './budgets.js';
+
+export type {
+  AbsoluteBudget,
+  BaselineReference,
+  BaselineStorage,
+  BaselineSummaryData,
+  Budget,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  RelativeBudget,
+  RunId,
+  TaskId,
+};
+
+// Re-export schema-derived types
+export type {
+  BenchmarkDefinition,
+  BenchmarkDefinitionInput,
+  BenchmarkSuite,
+  BenchmarkSuiteInput,
+  BenchmarkTask,
+  BenchmarkTaskInput,
+} from '../core/benchmark-schema.js';
+
 /**
  * ModestBench Core Types
  *
@@ -10,15 +49,8 @@
  * safety and consistency.
  */
 
-// Re-export schema-derived types
-export type {
-  BenchmarkDefinition,
-  BenchmarkDefinitionInput,
-  BenchmarkSuite,
-  BenchmarkSuiteInput,
-  BenchmarkTask,
-  BenchmarkTaskInput,
-} from '../core/benchmark-schema.js';
+// Re-export identifier helper functions
+export { createRunId, createTaskId } from '../utils/identifiers.js';
 
 /**
  * Benchmark file structure after parsing
@@ -38,6 +70,8 @@ export interface BenchmarkFile {
  * Represents a complete benchmark run across multiple files
  */
 export interface BenchmarkRun {
+  /** Budget evaluation summary (if budgets configured) */
+  readonly budgetSummary?: BudgetSummary;
   /** CI/CD information if available */
   readonly ci?: CiInfo;
   /** Configuration used for this run */
@@ -53,7 +87,7 @@ export interface BenchmarkRun {
   /** Git information if available */
   readonly git?: GitInfo;
   /** Unique identifier for this run */
-  readonly id: string;
+  readonly id: RunId;
   /** Custom run-level metadata */
   readonly metadata?: Record<string, unknown>;
   /** Timestamp when run started */
@@ -277,6 +311,12 @@ export interface ModestBenchConfig {
   readonly $schema?: string | undefined;
   /** Whether to stop on first failure */
   readonly bail: boolean;
+  /** Name of baseline to use for relative budget comparisons */
+  readonly baseline?: string | undefined;
+  /** How to handle budget violations: 'fail', 'warn', or 'report' */
+  readonly budgetMode?: 'fail' | 'report' | 'warn' | undefined;
+  /** Performance budgets mapped by task identifier */
+  readonly budgets?: Record<string, unknown> | undefined;
   /** Patterns to exclude from discovery */
   readonly exclude: string[];
   /** Tags to exclude from execution */
@@ -363,6 +403,8 @@ export interface SuiteResult {
  * Represents a single benchmark task execution result
  */
 export interface TaskResult {
+  /** Whether the task was aborted (via Ctrl+C or signal) */
+  readonly aborted?: boolean;
   /** Coefficient of variation (stdDev/mean × 100) */
   readonly cv: number;
   /** Any error that occurred during execution */

package/src/types/index.ts

@@ -10,9 +10,14 @@ export * from './cli.js';
 
 // Core data types
 export type * from './core.js';
+// Helper functions from core (value exports)
+export { createRunId, createTaskId } from './core.js';
 
 // Interface contracts
 export type * from './interfaces.js';
 
+// Profiler types
+export type * from './profiler.js';
+
 // Utility types and helpers
 export * from './utility.js';

package/src/types/interfaces.ts

@@ -9,6 +9,7 @@
 import type {
   BenchmarkFile,
   BenchmarkRun,
+  BudgetSummary,
   FileResult,
   ModestBenchConfig,
   SuiteResult,
@@ -76,6 +77,7 @@ export interface ConfigurationManager {
   load(
     configPath?: string,
     cliArgs?: Record<string, unknown>,
+    commandDefaults?: Partial<ModestBenchConfig>,
   ): Promise<ModestBenchConfig>;
 
   /**
@@ -266,8 +268,16 @@ export interface ProgressState {
 
 /**
  * Base reporter interface for benchmark output
+ *
+ * Required methods: onStart, onEnd, onError, onTaskResult Optional methods: All
+ * others (implement only what you need)
  */
 export interface Reporter {
+  /**
+   * Called when budget evaluation completes
+   */
+  onBudgetResult?(summary: BudgetSummary): Promise<void> | void;
+
   /**
    * Called when benchmark run completes
    */
@@ -281,17 +291,17 @@ export interface Reporter {
   /**
    * Called when a file completes
    */
-  onFileEnd(result: FileResult): Promise<void> | void;
+  onFileEnd?(result: FileResult): Promise<void> | void;
 
   /**
    * Called when a file starts execution
    */
-  onFileStart(file: string): Promise<void> | void;
+  onFileStart?(file: string): Promise<void> | void;
 
   /**
    * Called for progress updates
    */
-  onProgress(state: ProgressState): Promise<void> | void;
+  onProgress?(state: ProgressState): Promise<void> | void;
 
   /**
    * Called when benchmark run starts
@@ -301,12 +311,20 @@ export interface Reporter {
   /**
    * Called when a suite completes
    */
-  onSuiteEnd(result: SuiteResult): Promise<void> | void;
+  onSuiteEnd?(result: SuiteResult): Promise<void> | void;
+
+  /**
+   * Called before a suite starts, providing task names for pre-calculation
+   */
+  onSuiteInit?(
+    suite: string,
+    taskNames: readonly string[],
+  ): Promise<void> | void;
 
   /**
    * Called when a suite starts execution
    */
-  onSuiteStart(suite: string): Promise<void> | void;
+  onSuiteStart?(suite: string): Promise<void> | void;
 
   /**
    * Called when a task completes
@@ -316,7 +334,7 @@ export interface Reporter {
   /**
    * Called when a task starts execution
    */
-  onTaskStart(task: string): Promise<void> | void;
+  onTaskStart?(task: string): Promise<void> | void;
 }
 
 /**