modestbench 0.2.0 → 0.3.1

package/src/core/engines/accurate-engine.ts

@@ -40,6 +40,12 @@ export class AccurateEngine extends ModestBenchEngine {
    */
   private static readonly MAX_ITERATIONS_PER_ROUND = 10000;
 
+  /**
+   * Maximum iterations per round for async functions (much lower due to
+   * sequential await overhead)
+   */
+  private static readonly MAX_ITERATIONS_PER_ROUND_ASYNC = 100;
+
   private hasCheckedNativeSyntax = false;
 
   private nativeSyntaxErrorShown = false;
@@ -65,34 +71,75 @@ export class AccurateEngine extends ModestBenchEngine {
         );
       }
 
+      // Detect if function is async by testing it
+      const testResult = taskData.fn();
+      const isAsync =
+        testResult != null &&
+        typeof testResult === 'object' &&
+        'then' in testResult &&
+        typeof testResult.then === 'function';
+      if (isAsync) {
+        // Clean up the test promise/thenable
+        const thenable = testResult as PromiseLike<unknown>;
+        thenable.then(
+          () => {
+            /* ignore */
+          },
+          () => {
+            /* ignore */
+          },
+        );
+      }
+
+      // For async functions, cap iterations to prevent excessive sequential rounds
+      // Even with limitBy: 'iterations', 5000 samples = 5000 rounds of async calls
+      // which is impractically slow. Cap at 1000 for reasonable balance.
+      const effectiveConfig =
+        isAsync && config.iterations > 1000
+          ? { ...config, iterations: 1000 }
+          : config;
+
       // Check for V8 native syntax support
       const useOptGuards = this.checkNativeSyntax();
 
       // Show helpful error if native syntax not available
       if (!useOptGuards && !this.nativeSyntaxErrorShown) {
         if (!config.quiet) {
+          // Get the path to the modestbench executable from process.argv
+          const modestbenchPath = process.argv[1] || 'modestbench';
+
           console.warn(
-            '\n⚠️  AccurateEngine requires --allow-natives-syntax flag for best accuracy.',
+            '\n⚠️  AccurateEngine recommends --allow-natives-syntax flag for best accuracy.',
             '\nRunning in fallback mode (reduced accuracy).',
             '\n\nTo enable V8 optimization guards:',
-            '\n  node --allow-natives-syntax --test',
-            '\n  or add to package.json: "test": "node --allow-natives-syntax ..."',
+            `\n  node --allow-natives-syntax ${modestbenchPath}`,
             '\n',
           );
         }
         this.nativeSyntaxErrorShown = true;
       }
 
-      // Execute benchmark with or without opt guards
+      // Execute benchmark with or without opt guards, and with async support
       const rawSamples = useOptGuards
-        ? await this.executeBenchmarkWithOptGuards(taskData.fn, config, signal)
-        : await this.executeBenchmarkBasic(taskData.fn, config, signal);
+        ? await this.executeBenchmarkWithOptGuards(
+            taskData.fn,
+            effectiveConfig,
+            signal,
+            isAsync,
+          )
+        : await this.executeBenchmarkBasic(
+            taskData.fn,
+            effectiveConfig,
+            signal,
+            isAsync,
+          );
 
-      // Check if aborted
+      // Check if aborted - return minimal valid result marked as aborted
+      // (abort message is shown at run level, not per-task)
       if (signal?.aborted) {
         return {
+          aborted: true,
           cv: 0,
-          error: new Error('Benchmark aborted by user signal'),
           iterations: rawSamples.length,
           marginOfError: 0,
           max: 0,
@@ -166,18 +213,25 @@ export class AccurateEngine extends ModestBenchEngine {
   private async calculateInitialIterations(
     fn: (...args: unknown[]) => unknown,
     targetTime: number, // in seconds
+    isAsync = false,
   ): Promise<number> {
     // eslint-disable-next-line @typescript-eslint/unbound-method
     const timer = process.hrtime.bigint;
     const MIN_RESOLUTION = 0.5; // nanoseconds
     const SCALE = 1e9; // ns to seconds
 
-    // Run a quick test with 30 iterations
-    const testIterations = 30;
+    // Run a quick test with 30 iterations (fewer for async to keep it fast)
+    const testIterations = isAsync ? 10 : 30;
     const start = timer();
 
-    for (let i = 0; i < testIterations; i++) {
-      fn();
+    if (isAsync) {
+      for (let i = 0; i < testIterations; i++) {
+        await fn();
+      }
+    } else {
+      for (let i = 0; i < testIterations; i++) {
+        fn();
+      }
     }
 
     const duration = Number(timer() - start);
@@ -186,8 +240,13 @@ export class AccurateEngine extends ModestBenchEngine {
     // Calculate how many iterations we need for targetTime
     const totalOpsForTargetTime = targetTime / (durationPerOp / SCALE);
 
+    // Use appropriate max based on sync/async
+    const maxIterations = isAsync
+      ? AccurateEngine.MAX_ITERATIONS_PER_ROUND_ASYNC
+      : AccurateEngine.MAX_ITERATIONS_PER_ROUND;
+
     return Math.min(
-      Number.MAX_SAFE_INTEGER,
+      maxIterations,
       Math.max(1, Math.round(totalOpsForTargetTime)),
     );
   }
@@ -224,6 +283,7 @@ export class AccurateEngine extends ModestBenchEngine {
     fn: (...args: unknown[]) => unknown,
     config: ModestBenchConfig,
     signal?: AbortSignal,
+    isAsync = false,
   ): Promise<number[]> {
     // eslint-disable-next-line @typescript-eslint/unbound-method
     const timer = process.hrtime.bigint;
@@ -234,26 +294,70 @@ export class AccurateEngine extends ModestBenchEngine {
     const initialIterations = await this.calculateInitialIterations(
       fn,
       targetTime,
+      isAsync,
     );
 
     if (config.warmup > 0) {
       const warmupTime = Math.min(config.warmup / 1000, 0.05);
-      await this.runWarmup(fn, initialIterations, warmupTime);
+      await this.runWarmup(fn, initialIterations, warmupTime, isAsync);
     }
 
     const maxDuration = (config.time / 1000) * SCALE;
     let timeSpent = 0;
     let iterations = initialIterations;
 
-    while (timeSpent < maxDuration || samples.length < config.iterations) {
+    // Use appropriate max based on sync/async
+    const maxIterations = isAsync
+      ? AccurateEngine.MAX_ITERATIONS_PER_ROUND_ASYNC
+      : AccurateEngine.MAX_ITERATIONS_PER_ROUND;
+
+    // Determine loop condition based on limitBy setting
+    const shouldContinue = (): boolean => {
       if (signal?.aborted) {
-        break;
+        return false;
+      }
+
+      const timeRemaining = timeSpent < maxDuration;
+      const samplesRemaining = samples.length < config.iterations;
+
+      switch (config.limitBy) {
+        case 'all':
+          return timeRemaining && samplesRemaining;
+        case 'any':
+          return timeRemaining || samplesRemaining;
+        case 'iterations':
+          return samplesRemaining;
+        case 'time':
+          return timeRemaining;
+        default:
+          return timeRemaining && samplesRemaining;
       }
+    };
 
+    while (shouldContinue()) {
       const start = timer();
 
-      for (let i = 0; i < iterations; i++) {
-        fn();
+      if (isAsync) {
+        // For async functions, await each call individually and check for abort
+        for (let i = 0; i < iterations; i++) {
+          if (signal?.aborted) {
+            break;
+          }
+          await fn();
+        }
+      } else {
+        // For sync functions, check for abort every 50 iterations
+        for (let i = 0; i < iterations; i++) {
+          if (i % 50 === 0 && signal?.aborted) {
+            break;
+          }
+          fn();
+        }
+      }
+
+      // Early exit if aborted during inner loop
+      if (signal?.aborted) {
+        break;
       }
 
       const duration = Number(timer() - start);
@@ -268,10 +372,7 @@ export class AccurateEngine extends ModestBenchEngine {
 
       const remainingTime = Math.max(0, (maxDuration - timeSpent) / SCALE);
       iterations = Math.round(remainingTime / (durationPerOp / SCALE));
-      iterations = Math.max(
-        1,
-        Math.min(AccurateEngine.MAX_ITERATIONS_PER_ROUND, iterations),
-      );
+      iterations = Math.max(1, Math.min(maxIterations, iterations));
     }
 
     return samples;
@@ -284,6 +385,7 @@ export class AccurateEngine extends ModestBenchEngine {
     fn: (...args: unknown[]) => unknown,
     config: ModestBenchConfig,
     signal?: AbortSignal,
+    isAsync = false,
   ): Promise<number[]> {
     // eslint-disable-next-line @typescript-eslint/unbound-method
     const timer = process.hrtime.bigint;
@@ -295,12 +397,13 @@ export class AccurateEngine extends ModestBenchEngine {
     const initialIterations = await this.calculateInitialIterations(
       fn,
       targetTime,
+      isAsync,
     );
 
     // Run warmup
     if (config.warmup > 0) {
       const warmupTime = Math.min(config.warmup / 1000, 0.05); // Max 50ms warmup
-      await this.runWarmup(fn, initialIterations, warmupTime);
+      await this.runWarmup(fn, initialIterations, warmupTime, isAsync);
     }
 
     // Create DoNotOptimize wrapper using V8 intrinsics
@@ -321,17 +424,61 @@ export class AccurateEngine extends ModestBenchEngine {
     let timeSpent = 0;
     let iterations = initialIterations;
 
-    // Main benchmark loop
-    while (timeSpent < maxDuration || samples.length < config.iterations) {
+    // Use appropriate max based on sync/async
+    const maxIterations = isAsync
+      ? AccurateEngine.MAX_ITERATIONS_PER_ROUND_ASYNC
+      : AccurateEngine.MAX_ITERATIONS_PER_ROUND;
+
+    // Determine loop condition based on limitBy setting
+    const shouldContinue = (): boolean => {
       if (signal?.aborted) {
-        break;
+        return false;
       }
 
+      const timeRemaining = timeSpent < maxDuration;
+      const samplesRemaining = samples.length < config.iterations;
+
+      switch (config.limitBy) {
+        case 'all':
+          return timeRemaining && samplesRemaining;
+        case 'any':
+          return timeRemaining || samplesRemaining;
+        case 'iterations':
+          return samplesRemaining;
+        case 'time':
+          return timeRemaining;
+        default:
+          return timeRemaining && samplesRemaining;
+      }
+    };
+
+    // Main benchmark loop
+    while (shouldContinue()) {
       const start = timer();
 
-      for (let i = 0; i < iterations; i++) {
-        const result = fn();
-        guardedDoNotOptimize(result); // Prevent optimization
+      if (isAsync) {
+        // For async functions, await each call individually and check for abort
+        for (let i = 0; i < iterations; i++) {
+          if (signal?.aborted) {
+            break;
+          }
+          const result = await fn();
+          guardedDoNotOptimize(result); // Prevent optimization
+        }
+      } else {
+        // For sync functions, check for abort every 50 iterations
+        for (let i = 0; i < iterations; i++) {
+          if (i % 50 === 0 && signal?.aborted) {
+            break;
+          }
+          const result = fn();
+          guardedDoNotOptimize(result); // Prevent optimization
+        }
+      }
+
+      // Early exit if aborted during inner loop
+      if (signal?.aborted) {
+        break;
       }
 
       const duration = Number(timer() - start);
@@ -348,10 +495,7 @@ export class AccurateEngine extends ModestBenchEngine {
       // Adjust iterations for next round
       const remainingTime = Math.max(0, (maxDuration - timeSpent) / SCALE);
       iterations = Math.round(remainingTime / (durationPerOp / SCALE));
-      iterations = Math.max(
-        1,
-        Math.min(AccurateEngine.MAX_ITERATIONS_PER_ROUND, iterations),
-      );
+      iterations = Math.max(1, Math.min(maxIterations, iterations));
     }
 
     return samples;
@@ -365,6 +509,7 @@ export class AccurateEngine extends ModestBenchEngine {
     fn: Function,
     initialIterations: number,
     warmupTime: number, // in seconds
+    isAsync = false,
   ): Promise<void> {
     // eslint-disable-next-line @typescript-eslint/unbound-method
     const timer = process.hrtime.bigint;
@@ -373,19 +518,28 @@ export class AccurateEngine extends ModestBenchEngine {
     const maxDuration = warmupTime * SCALE;
     const minSamples = 10;
 
+    // Use appropriate max based on sync/async
+    const maxIterations = isAsync
+      ? AccurateEngine.MAX_ITERATIONS_PER_ROUND_ASYNC
+      : AccurateEngine.MAX_ITERATIONS_PER_ROUND;
+
     let timeSpent = 0n;
     let samples = 0;
-    let iterations = Math.min(
-      initialIterations,
-      AccurateEngine.MAX_ITERATIONS_PER_ROUND,
-    );
+    let iterations = Math.min(initialIterations, maxIterations);
 
     while (Number(timeSpent) < maxDuration || samples <= minSamples) {
       const start = timer();
 
-      for (let i = 0; i < iterations; i++) {
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-call
-        fn();
+      if (isAsync) {
+        for (let i = 0; i < iterations; i++) {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+          await fn();
+        }
+      } else {
+        for (let i = 0; i < iterations; i++) {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+          fn();
+        }
       }
 
       const duration = timer() - start;
@@ -402,10 +556,7 @@ export class AccurateEngine extends ModestBenchEngine {
         (maxDuration - Number(timeSpent)) / SCALE,
       );
       iterations = Math.round(remainingTime / (durationPerOp / SCALE));
-      iterations = Math.max(
-        1,
-        Math.min(AccurateEngine.MAX_ITERATIONS_PER_ROUND, iterations),
-      );
+      iterations = Math.max(1, Math.min(maxIterations, iterations));
     }
   }
 }

package/src/core/engines/tinybench-engine.ts

@@ -192,10 +192,11 @@ export class TinybenchEngine extends ModestBenchEngine {
 
       // Check if the task was aborted
       if (results.aborted) {
-        // Task was aborted via signal - return minimal valid result with error
+        // Task was aborted via signal - return minimal valid result marked as aborted
+        // (abort message is shown at run level, not per-task)
         const taskResult: TaskResult = {
+          aborted: true,
           cv: 0,
-          error: new Error('Benchmark aborted by user signal'),
           iterations: results.latency?.samples?.length || 0,
           marginOfError: 0,
           max: 0,

package/src/core/output-path-resolver.ts

@@ -1,4 +1,4 @@
-import { isAbsolute, join, resolve } from 'node:path';
+import { extname, isAbsolute, join, resolve } from 'node:path';
 
 /**
  * Resolves the final output path for a reporter
@@ -29,7 +29,15 @@ export const resolveOutputPath = (
     return resolve(process.cwd(), outputFile);
   }
 
-  // Fall back to default behavior
+  // If outputDir looks like a file (has extension), treat it as a file path
+  // This handles cases like: --output results.csv
+  if (outputDir && extname(outputDir)) {
+    return isAbsolute(outputDir)
+      ? outputDir
+      : resolve(process.cwd(), outputDir);
+  }
+
+  // Fall back to default behavior (outputDir is a directory)
   if (outputDir && defaultFilename) {
     return join(outputDir, defaultFilename);
   }

package/src/errors/base.ts

@@ -143,10 +143,19 @@ export const isModestBenchError = (
   error: unknown,
 ): error is ModestBenchError => {
   return (
-    typeof error === 'object' &&
-    error !== null &&
+    isError(error) &&
     'code' in error &&
     typeof (error as { code: unknown }).code === 'string' &&
     (error as { code: string }).code.startsWith('ERR_MB_')
   );
 };
+
+/**
+ * Type guard to check if an error is a standard Error
+ *
+ * @param error - The error to check
+ * @returns `true` if the error is an `Error`
+ */
+export const isError = (error: unknown): error is Error => {
+  return error instanceof Error;
+};

package/src/errors/budget.ts

@@ -0,0 +1,38 @@
+/**
+ * Budget-related errors
+ *
+ * Errors that occur during budget evaluation and enforcement.
+ */
+
+import type { BudgetSummary } from '../types/core.js';
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Error thrown when performance budgets are exceeded
+ *
+ * Thrown when budget evaluation fails and budgetMode is set to 'fail'. Contains
+ * the full budget summary for detailed reporting.
+ */
+export class BudgetExceededError extends ModestBenchError {
+  /**
+   * Budget summary containing details of all violations
+   */
+  public readonly budgetSummary: BudgetSummary;
+
+  /**
+   * Error code for budget exceeded errors
+   */
+  readonly code = 'ERR_MB_BUDGET_EXCEEDED';
+
+  /**
+   * Create a new budget exceeded error
+   *
+   * @param message - Human-readable error message
+   * @param budgetSummary - Budget evaluation results
+   */
+  constructor(message: string, budgetSummary: BudgetSummary) {
+    super(message);
+    this.budgetSummary = budgetSummary;
+  }
+}

package/src/errors/index.ts

@@ -13,6 +13,9 @@ export {
   ModestBenchError,
 } from './base.js';
 
+// Budget errors
+export { BudgetExceededError } from './budget.js';
+
 // CLI errors
 export {
   InvalidArgumentError,

package/src/index.ts

@@ -26,11 +26,17 @@ export * from './errors/index.js';
 export { CsvReporter } from './reporters/csv.js';
 export { HumanReporter } from './reporters/human.js';
 export { JsonReporter } from './reporters/json.js';
+export { ProfileHumanReporter } from './reporters/profile-human.js';
 
 // Services
 export { ModestBenchConfigurationManager } from './services/config-manager.js';
 export { BenchmarkFileLoader } from './services/file-loader.js';
 export { FileHistoryStorage } from './services/history-storage.js';
+// Profiler services
+export { filterProfile } from './services/profiler/profile-filter.js';
+export { parseProfile } from './services/profiler/profile-parser.js';
+
+export { runWithProfiling } from './services/profiler/profile-runner.js';
 export { ModestBenchProgressManager } from './services/progress-manager.js';
 export {
   BaseReporter,
@@ -40,3 +46,6 @@ export {
 
 // Export all types
 export * from './types/index.js';
+
+// Utilities
+export { findPackageRoot } from './utils/package.js';