modestbench 0.3.1 → 0.5.0

package/src/adapters/node-test-hooks.ts

@@ -0,0 +1,71 @@
+/**
+ * ModestBench Node.js Test Runner Loader
+ *
+ * ES module loader hook that intercepts `node:test` imports and returns our
+ * capturing mock from globalThis.
+ *
+ * Usage: node --import modestbench/node-test your-test.js
+ *
+ * This loader exports async `resolve` and `load` hooks that get registered via
+ * module.register() when imported.
+ */
+
+import type { LoadHook, ResolveHook } from 'node:module';
+
+// Generate the mock module source code
+// The generated module source uses top-level await to conditionally get the mock or real module
+// Note: Uses 'node:test?passthrough' to bypass our hook when falling back
+// Security: The globalThis mock is only installed by our own adapter code, so the generated
+// source is safe. No user input is interpolated into this template.
+const generateMockSource = (): string => `
+const mock = globalThis.__MODESTBENCH_NODE_TEST_MOCK__;
+
+// If no mock installed, fall through to real node:test
+// The '?passthrough' query tells our hook to not intercept this import
+const source = mock ?? await import('node:test?passthrough');
+
+export const test = source.test;
+export const describe = source.describe;
+export const it = source.it;
+export const before = source.before;
+export const after = source.after;
+export const beforeEach = source.beforeEach;
+export const afterEach = source.afterEach;
+export const suite = source.suite;
+export default source.default ?? source.test;
+`;
+
+/**
+ * Resolve hook - intercepts node:test specifier
+ *
+ * Uses query param '?passthrough' to prevent infinite recursion when falling
+ * back to real node:test (when no mock is installed).
+ */
+export const resolve: ResolveHook = async (specifier, context, nextResolve) => {
+  // Only intercept bare 'node:test', not 'node:test?passthrough'
+  if (specifier === 'node:test') {
+    return {
+      shortCircuit: true,
+      url: 'modestbench://capture/node-test',
+    };
+  }
+  // Strip passthrough query to resolve real node:test
+  if (specifier === 'node:test?passthrough') {
+    return nextResolve('node:test', context);
+  }
+  return nextResolve(specifier, context);
+};
+
+/**
+ * Load hook - returns mock module for our custom URL
+ */
+export const load: LoadHook = async (url, context, nextLoad) => {
+  if (url === 'modestbench://capture/node-test') {
+    return {
+      format: 'module',
+      shortCircuit: true,
+      source: generateMockSource(),
+    };
+  }
+  return nextLoad(url, context);
+};

package/src/adapters/node-test-register.ts

@@ -0,0 +1,5 @@
+import { register } from 'node:module';
+
+register('./node-test-hooks.js', {
+  parentURL: import.meta.url,
+});

package/src/adapters/types.ts

@@ -0,0 +1,281 @@
+/**
+ * ModestBench Test Framework Adapter Types
+ *
+ * Shared types for capturing test definitions from various test frameworks
+ * (Mocha, node:test, AVA) and converting them to modestbench benchmark format.
+ */
+
+import type {
+  BenchmarkDefinition,
+  BenchmarkSuite,
+} from '../core/benchmark-schema.js';
+
+/**
+ * A captured test suite (describe block) from a test framework
+ */
+export interface CapturedSuite {
+  /** Nested child suites */
+  readonly children: CapturedSuite[];
+  /** Lifecycle hooks for the suite */
+  readonly hooks: SuiteHooks;
+  /** Suite name */
+  readonly name: string;
+  /** Tests in this suite */
+  readonly tests: CapturedTest[];
+}
+
+/**
+ * A captured test function from a test framework
+ */
+export interface CapturedTest {
+  /** Test function body */
+  readonly fn: () => Promise<void> | void;
+  /** Test name */
+  readonly name: string;
+  /** Whether this test is marked .only */
+  readonly only?: boolean;
+  /** Whether this test is marked .skip */
+  readonly skip?: boolean;
+}
+
+/**
+ * Result of capturing tests from a single file
+ */
+export interface CapturedTestFile {
+  /** File path */
+  readonly filePath: string;
+  /** Test framework detected/used */
+  readonly framework: TestFramework;
+  /** Root-level suites (describe blocks) */
+  readonly rootSuites: CapturedSuite[];
+  /** Root-level tests (not in any describe block - e.g., AVA) */
+  readonly rootTests: CapturedTest[];
+}
+
+/**
+ * Converted benchmark suite structure
+ *
+ * This is what capturedToBenchmark produces for each suite.
+ */
+export type ConvertedBenchmarkSuite = Pick<
+  BenchmarkSuite,
+  'benchmarks' | 'setup' | 'teardown'
+>;
+
+/**
+ * Lifecycle hooks captured from a test suite
+ */
+export interface SuiteHooks {
+  /** Hooks to run once after all tests in suite */
+  readonly after: Array<() => Promise<void> | void>;
+  /** Hooks to run after each test */
+  readonly afterEach: Array<() => Promise<void> | void>;
+  /** Hooks to run once before all tests in suite */
+  readonly before: Array<() => Promise<void> | void>;
+  /** Hooks to run before each test */
+  readonly beforeEach: Array<() => Promise<void> | void>;
+}
+
+/**
+ * Supported test frameworks
+ */
+export type TestFramework = 'ava' | 'mocha' | 'node-test';
+
+/**
+ * Interface for test framework adapters
+ *
+ * Each adapter is responsible for:
+ *
+ * 1. Capturing test definitions from a test file
+ * 2. Normalizing them to the CapturedTestFile format
+ */
+export interface TestFrameworkAdapter {
+  /**
+   * Capture test definitions from a file
+   *
+   * This method loads the test file with appropriate hooks/mocks in place to
+   * capture test function definitions without executing the tests.
+   *
+   * @param filePath - Absolute path to the test file
+   * @returns Captured test structure
+   */
+  capture(filePath: string): Promise<CapturedTestFile>;
+
+  /** Framework this adapter handles */
+  readonly framework: TestFramework;
+}
+
+/**
+ * Convert captured test file to modestbench benchmark definition
+ *
+ * Transforms the CapturedTestFile structure into the BenchmarkDefinition format
+ * that modestbench's engine can execute.
+ *
+ * @example
+ *
+ * ```typescript
+ * const captured = await adapter.capture('/path/to/test.js');
+ * const benchmark = capturedToBenchmark(captured);
+ * // benchmark.suites contains test suites converted to benchmark suites
+ * ```
+ *
+ * @param captured - The captured test file structure from a test framework
+ *   adapter
+ * @returns A BenchmarkDefinition with suites containing benchmarks derived from
+ *   tests
+ */
+export const capturedToBenchmark = (
+  captured: CapturedTestFile,
+): BenchmarkDefinition => {
+  const suites: Record<string, ReturnType<typeof suiteToRecord>> = {};
+
+  // Convert root-level tests to a default suite (AVA-style)
+  if (captured.rootTests.length > 0) {
+    suites['default'] = {
+      benchmarks: Object.fromEntries(
+        captured.rootTests
+          .filter((t) => !t.skip)
+          .map((test) => [test.name, { fn: test.fn }]),
+      ),
+    };
+  }
+
+  // Convert captured suites to benchmark suites
+  for (const suite of captured.rootSuites) {
+    const converted = convertSuite(suite);
+    if (converted) {
+      suites[suite.name] = converted;
+    }
+  }
+
+  return { suites };
+};
+
+/**
+ * Convert a captured suite to benchmark suite format
+ */
+const convertSuite = (
+  suite: CapturedSuite,
+): null | ReturnType<typeof suiteToRecord> => {
+  // Skip empty suites
+  const nonSkippedTests = suite.tests.filter((t) => !t.skip);
+  if (nonSkippedTests.length === 0 && suite.children.length === 0) {
+    return null;
+  }
+
+  return suiteToRecord(suite);
+};
+
+/**
+ * Create a wrapped test function that runs beforeEach/afterEach hooks
+ *
+ * The hooks run with each benchmark iteration, matching test framework
+ * semantics where beforeEach runs before each test execution.
+ *
+ * Hook ordering follows standard test framework conventions:
+ *
+ * - BeforeEach: runs in declaration order (FIFO)
+ * - AfterEach: runs in reverse declaration order (LIFO)
+ */
+const createWrappedTestFn = (
+  testFn: () => Promise<void> | void,
+  hooks: SuiteHooks,
+): (() => Promise<void> | void) => {
+  // If no per-test hooks, return original function
+  if (hooks.beforeEach.length === 0 && hooks.afterEach.length === 0) {
+    return testFn;
+  }
+
+  // Wrap with hooks
+  return async () => {
+    // Run beforeEach hooks in declaration order (FIFO)
+    for (const hook of hooks.beforeEach) {
+      await hook();
+    }
+
+    // Run test
+    await testFn();
+
+    // Run afterEach hooks in reverse order (LIFO), like most test frameworks
+    for (const hook of [...hooks.afterEach].reverse()) {
+      await hook();
+    }
+  };
+};
+
+/**
+ * Flatten nested suites into a list of [prefixed-name, wrapped-fn] pairs
+ */
+const flattenSuiteTests = (
+  suite: CapturedSuite,
+  prefix: string,
+): Array<[string, () => Promise<void> | void]> => {
+  const results: Array<[string, () => Promise<void> | void]> = [];
+
+  // Add direct tests with prefix
+  for (const test of suite.tests) {
+    if (!test.skip) {
+      const wrappedFn = createWrappedTestFn(test.fn, suite.hooks);
+      results.push([`${prefix} > ${test.name}`, wrappedFn]);
+    }
+  }
+
+  // Recursively add nested suite tests
+  for (const child of suite.children) {
+    const childResults = flattenSuiteTests(child, `${prefix} > ${child.name}`);
+    results.push(...childResults);
+  }
+
+  return results;
+};
+
+/**
+ * Helper to build the suite record structure
+ */
+const suiteToRecord = (suite: CapturedSuite) => {
+  const benchmarks: Record<string, { fn: () => Promise<void> | void }> = {};
+
+  // Add tests as benchmarks
+  for (const test of suite.tests) {
+    if (!test.skip) {
+      // Wrap test function with beforeEach/afterEach hooks
+      const wrappedFn = createWrappedTestFn(test.fn, suite.hooks);
+      benchmarks[test.name] = { fn: wrappedFn };
+    }
+  }
+
+  // Flatten nested suites into flat benchmark names
+  // e.g., "Parent > Child > test" becomes "Child > test"
+  for (const child of suite.children) {
+    const childBenchmarks = flattenSuiteTests(child, child.name);
+    for (const [name, fn] of childBenchmarks) {
+      benchmarks[name] = { fn };
+    }
+  }
+
+  // Combine before hooks into a single setup function
+  const setup =
+    suite.hooks.before.length > 0
+      ? async () => {
+          for (const hook of suite.hooks.before) {
+            await hook();
+          }
+        }
+      : undefined;
+
+  // Combine after hooks into a single teardown function
+  const teardown =
+    suite.hooks.after.length > 0
+      ? async () => {
+          for (const hook of suite.hooks.after) {
+            await hook();
+          }
+        }
+      : undefined;
+
+  return {
+    benchmarks,
+    ...(setup && { setup }),
+    ...(teardown && { teardown }),
+  };
+};

package/src/cli/commands/init.ts

@@ -17,6 +17,11 @@ import { createInterface } from 'node:readline';
 
 import type { CliContext } from '../index.js';
 
+import {
+  DEFAULT_BENCHMARK_DIR,
+  DEFAULT_OUTPUT_DIR,
+  SITE_URL,
+} from '../../constants.js';
 import {
   InvalidArgumentError,
   UnsupportedConfigFormatError,
@@ -43,39 +48,44 @@ const PROJECT_TEMPLATES = {
   advanced: {
     configOptions: {
       iterations: 1000,
-      outputDir: '.modestbench',
-      pattern: 'bench/**/*.bench.{js,ts}',
+      outputDir: DEFAULT_OUTPUT_DIR,
+      pattern: `${DEFAULT_BENCHMARK_DIR}/**/*.bench.{js,ts}`,
       reporters: ['human', 'json'],
       time: 10000,
       warmup: 50,
     },
     description: 'Feature-rich setup with multiple reporters and configuration',
-    directories: ['bench', '.modestbench'],
+    directories: [DEFAULT_BENCHMARK_DIR, DEFAULT_OUTPUT_DIR],
     name: 'Advanced Project',
   },
   basic: {
     configOptions: {
       iterations: 100,
-      pattern: 'bench/**/*.bench.{js,ts}',
+      outputDir: DEFAULT_OUTPUT_DIR,
+      pattern: `${DEFAULT_BENCHMARK_DIR}/**/*.bench.{js,ts}`,
       reporters: ['human'],
       time: 5000,
     },
     description: 'Simple benchmark setup for small projects',
-    directories: ['bench'],
+    directories: [DEFAULT_BENCHMARK_DIR, DEFAULT_OUTPUT_DIR],
     name: 'Basic Project',
   },
   library: {
     configOptions: {
       bail: false,
       iterations: 5000,
-      outputDir: '.modestbench',
-      pattern: 'bench/**/*.bench.{js,ts}',
+      outputDir: DEFAULT_OUTPUT_DIR,
+      pattern: `${DEFAULT_BENCHMARK_DIR}/**/*.bench.{js,ts}`,
       reporters: ['human', 'json'],
       time: 15000,
       warmup: 100,
     },
     description: 'Optimized for library performance testing',
-    directories: ['bench', 'bench/suites', '.modestbench'],
+    directories: [
+      DEFAULT_BENCHMARK_DIR,
+      `${DEFAULT_BENCHMARK_DIR}/suites`,
+      DEFAULT_OUTPUT_DIR,
+    ],
     name: 'Library Project',
   },
 } as const;
@@ -374,7 +384,7 @@ const createAdditionalFiles = async (
   // Create README.md
   const readmeContent = `# Benchmark Project
 
-This project uses [ModestBench](https://github.com/your-org/modestbench) for performance testing.
+This project uses [ModestBench](${SITE_URL}) for performance testing.
 
 ## Getting Started
 
@@ -385,7 +395,7 @@ modestbench run
 
 Run specific benchmarks:
 \`\`\`bash
-modestbench run "bench/array-*.bench.js"
+modestbench run "${DEFAULT_BENCHMARK_DIR}/array-*.bench.js"
 \`\`\`
 
 View benchmark history:
@@ -399,7 +409,7 @@ See \`modestbench.config.*\` for benchmark configuration options.
 
 ## Writing Benchmarks
 
-Create new benchmark files in the \`bench/\` directory. See the examples for the expected format.
+Create new benchmark files in the \`${DEFAULT_BENCHMARK_DIR}/\` directory. See the examples for the expected format.
 `;
 
   try {
@@ -422,7 +432,7 @@ const createOrUpdateGitignore = async (
   options?: { quiet?: boolean; yes?: boolean },
 ): Promise<void> => {
   const gitignorePath = resolve(cwd, '.gitignore');
-  const modestbenchEntry = '.modestbench/';
+  const modestbenchEntry = `${DEFAULT_OUTPUT_DIR}/`;
   const modestbenchSection = `\n# ModestBench history\n${modestbenchEntry}\n`;
 
   try {
@@ -435,13 +445,13 @@ const createOrUpdateGitignore = async (
     }
 
     if (existingContent) {
-      // File exists, check if .modestbench/ is already present
+      // File exists, check if ${DEFAULT_OUTPUT_DIR}/ is already present
       if (existingContent.includes(modestbenchEntry)) {
         // Already present, nothing to do
         return;
       }
 
-      // Append .modestbench/ entry (--yes or --quiet means auto-accept)
+      // Append default output dir entry (--yes or --quiet means auto-accept)
       if (options?.yes || options?.quiet) {
         // Ensure content ends with newline
         const contentToAppend = existingContent.endsWith('\n')
@@ -463,7 +473,6 @@ build/
 coverage/
 
 # Benchmark output
-benchmark-results/
 ${modestbenchSection}`;
       await writeFile(gitignorePath, newContent, 'utf8');
     }
@@ -549,7 +558,7 @@ const createDirectories = async (
  * Create example benchmark files
  */
 const createExampleBenchmarks = async (cwd: string): Promise<void> => {
-  const benchmarksDir = resolve(cwd, 'bench');
+  const benchmarksDir = resolve(cwd, DEFAULT_BENCHMARK_DIR);
 
   for (const [name, example] of Object.entries(EXAMPLE_BENCHMARKS)) {
     const filePath = join(benchmarksDir, example.filename);

package/src/cli/commands/run.ts

@@ -88,6 +88,9 @@ export const handleRunCommand = async (
     }
 
     // Step 1: Load and merge configuration
+    if (verbose && !options.quiet) {
+      console.error('Loading configuration...');
+    }
     const config = await loadConfiguration(context, options);
 
     // Check if JSON reporter is being used (need quiet output for clean JSON)
@@ -270,10 +273,17 @@ const handleResults = (
   // Check if any files failed to load/execute
   const hasFileErrors = executionResult.files.some((file) => file.error);
 
+  // Check if any suites failed (e.g., setup errors)
+  const hasSuiteErrors = executionResult.files.some((file) =>
+    file.suites.some((suite) => suite.error),
+  );
+
   // Determine exit code based on results
   if (executionResult && executionResult.summary) {
-    // Return error if there are failed tasks OR file-level errors
-    return executionResult.summary.failedTasks > 0 || hasFileErrors
+    // Return error if there are failed tasks, file errors, OR suite errors
+    return executionResult.summary.failedTasks > 0 ||
+      hasFileErrors ||
+      hasSuiteErrors
       ? ExitCodes.BENCHMARK_FAILURES
       : ExitCodes.SUCCESS;
   }