modestbench 0.7.0 → 0.9.0

package/src/services/reporter-loader.ts

@@ -0,0 +1,323 @@
+/**
+ * ModestBench Reporter Loader
+ *
+ * Service for loading third-party reporter plugins from file paths or npm
+ * packages. Supports multiple export patterns: plain objects, classes, and
+ * factory functions (sync or async).
+ */
+
+import { isAbsolute, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import type { Logger, Reporter, ReporterContext } from '../types/index.js';
+
+import { Reporters } from '../constants.js';
+import {
+  ReporterLoadError,
+  ReporterValidationError,
+} from '../errors/reporter.js';
+import { getPackageVersion } from '../utils/package.js';
+import { reporterUtils } from '../utils/reporter-utils.js';
+
+/**
+ * Current plugin API version
+ *
+ * Increment this when making breaking changes to the plugin API.
+ */
+export const PLUGIN_API_VERSION = 1;
+
+/**
+ * Set of built-in reporter names
+ */
+const BUILT_IN_REPORTERS = new Set(Object.values(Reporters));
+
+/**
+ * Required methods that all reporters must implement
+ */
+const REQUIRED_REPORTER_METHODS = [
+  'onStart',
+  'onEnd',
+  'onError',
+  'onTaskResult',
+] as const;
+
+/**
+ * Default logger implementation using console
+ *
+ * This provides a simple console-based logger for reporter plugins.
+ */
+const defaultLogger: Logger = {
+  debug: (message, ...args) => console.debug(message, ...args),
+  error: (message, ...args) => console.error(message, ...args),
+  info: (message, ...args) => console.info(message, ...args),
+  trace: (message, ...args) => console.trace(message, ...args),
+  warn: (message, ...args) => console.warn(message, ...args),
+};
+
+/**
+ * Create a ReporterContext for passing to plugins
+ *
+ * @param logger - Optional logger to use (defaults to console-based logger)
+ * @returns ReporterContext with version info and utilities
+ */
+export const createReporterContext = (logger?: Logger): ReporterContext => {
+  return {
+    logger: logger ?? defaultLogger,
+    pluginApiVersion: PLUGIN_API_VERSION,
+    utils: reporterUtils,
+    version: getPackageVersion(),
+  };
+};
+
+/**
+ * Get the list of missing required methods from a reporter object
+ *
+ * @param obj - Object to check
+ * @returns Array of missing method names
+ */
+const getMissingMethods = (obj: unknown): string[] => {
+  if (typeof obj !== 'object' || obj === null) {
+    return [...REQUIRED_REPORTER_METHODS];
+  }
+
+  return REQUIRED_REPORTER_METHODS.filter(
+    (method) => typeof (obj as Record<string, unknown>)[method] !== 'function',
+  );
+};
+
+/**
+ * Check if a specifier refers to a built-in reporter
+ *
+ * @param specifier - Reporter name or path
+ * @returns True if the specifier is a built-in reporter name
+ */
+export const isBuiltInReporter = (specifier: string): boolean => {
+  return BUILT_IN_REPORTERS.has(
+    specifier as (typeof Reporters)[keyof typeof Reporters],
+  );
+};
+
+/**
+ * Check if a function is a class constructor
+ *
+ * Uses heuristics to distinguish classes from regular functions:
+ *
+ * - Classes have a non-writable prototype property
+ * - Class syntax produces different toString() output
+ *
+ * @param func - Function to check
+ * @returns True if the function appears to be a class constructor
+ */
+const isClass = (
+  func: unknown,
+): func is new (...args: unknown[]) => unknown => {
+  if (typeof func !== 'function') {
+    return false;
+  }
+
+  // Classes have a non-writable prototype
+  const protoDescriptor = Object.getOwnPropertyDescriptor(func, 'prototype');
+  if (!protoDescriptor || protoDescriptor.writable) {
+    return false;
+  }
+
+  // Check if it uses class syntax (handles both 'class Foo' and 'class{')
+  const funcStr = func.toString();
+  return /^class\b/.test(funcStr);
+};
+
+/**
+ * Check if a specifier looks like a file path
+ *
+ * @param specifier - Reporter name or path
+ * @returns True if the specifier appears to be a file path
+ */
+export const isFilePath = (specifier: string): boolean => {
+  return (
+    specifier.startsWith('.') ||
+    specifier.startsWith('/') ||
+    // isAbsolute handles Windows paths like 'C:\path\to\file.js'
+    isAbsolute(specifier)
+  );
+};
+
+/**
+ * Check if an object implements the Reporter interface
+ *
+ * Validates that all required methods are present and are functions.
+ *
+ * @param obj - Object to validate
+ * @returns True if the object has all required reporter methods
+ */
+const isReporterObject = (obj: unknown): obj is Reporter => {
+  if (typeof obj !== 'object' || obj === null) {
+    return false;
+  }
+
+  return REQUIRED_REPORTER_METHODS.every(
+    (method) => typeof (obj as Record<string, unknown>)[method] === 'function',
+  );
+};
+
+/**
+ * Load a reporter from a file path or npm package name
+ *
+ * Supports multiple export patterns:
+ *
+ * 1. Plain Reporter object (simplest, no options support)
+ * 2. Class constructor (instantiated with options and context)
+ * 3. Factory function (called with options and context, can be async)
+ *
+ * @example
+ *
+ * ```typescript
+ * // Load from file path
+ * const reporter = await loadReporter('./my-reporter.js', {
+ *   verbose: true,
+ * });
+ *
+ * // Load from npm package
+ * const reporter = await loadReporter('@company/custom-reporter', {
+ *   apiKey: 'xxx',
+ * });
+ * ```
+ *
+ * @param specifier - File path (relative or absolute) or npm package name
+ * @param options - Options to pass to the reporter factory/constructor
+ * @param cwd - Current working directory for resolving relative paths
+ * @returns Loaded reporter instance
+ * @throws ReporterLoadError if the module cannot be loaded
+ * @throws ReporterValidationError if the module doesn't implement Reporter
+ */
+export const loadReporter = async (
+  specifier: string,
+  options: Record<string, unknown> = {},
+  cwd: string = process.cwd(),
+): Promise<Reporter> => {
+  const context = createReporterContext();
+  const resolvedSpecifier = resolveSpecifier(specifier, cwd);
+
+  let module: unknown;
+
+  try {
+    module = await import(resolvedSpecifier);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new ReporterLoadError(message, specifier, { cause: error });
+  }
+
+  // Handle ESM/CJS interop - get default export if present
+  const exported = (module as { default?: unknown }).default ?? module;
+
+  // Case 1: Already a Reporter object (plain object export)
+  if (isReporterObject(exported)) {
+    return exported;
+  }
+
+  // Case 2: Class constructor
+  if (isClass(exported)) {
+    let instance: unknown;
+
+    try {
+      instance = new (exported as new (
+        options: Record<string, unknown>,
+        context: ReporterContext,
+      ) => unknown)(options, context);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ReporterLoadError(
+        `Constructor threw error: ${message}`,
+        specifier,
+        { cause: error },
+      );
+    }
+
+    validateReporter(instance, specifier);
+    return instance;
+  }
+
+  // Case 3: Factory function (sync or async)
+  if (typeof exported === 'function') {
+    let result: unknown;
+
+    try {
+      result = await (
+        exported as (
+          options: Record<string, unknown>,
+          context: ReporterContext,
+        ) => Promise<unknown>
+      )(options, context);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new ReporterLoadError(
+        `Factory function threw error: ${message}`,
+        specifier,
+        { cause: error },
+      );
+    }
+
+    validateReporter(result, specifier);
+    return result;
+  }
+
+  // None of the above - could be an object with missing methods or invalid type
+  if (typeof exported === 'object' && exported !== null) {
+    // It's an object but missing required methods
+    const missing = getMissingMethods(exported);
+    throw new ReporterValidationError(
+      'Module does not implement Reporter interface.',
+      specifier,
+      missing,
+    );
+  }
+
+  // Completely invalid export type
+  throw new ReporterValidationError(
+    'Module must export a Reporter object, class, or factory function.',
+    specifier,
+  );
+};
+
+/**
+ * Resolve a specifier to an importable URL or module name
+ *
+ * @param specifier - File path or npm package name
+ * @param cwd - Current working directory for resolving relative paths
+ * @returns Resolved module specifier
+ */
+const resolveSpecifier = (specifier: string, cwd: string): string => {
+  if (isFilePath(specifier)) {
+    const absolutePath = resolve(cwd, specifier);
+    return pathToFileURL(absolutePath).href;
+  }
+
+  // npm package name - return as-is for dynamic import
+  return specifier;
+};
+
+/**
+ * Validate that an object implements the Reporter interface
+ *
+ * @param obj - Object to validate
+ * @param specifier - Original specifier for error messages
+ * @throws ReporterValidationError if validation fails
+ */
+/**
+ * Type signature for the validateReporter assertion function
+ */
+type ValidateReporterFn = (
+  obj: unknown,
+  specifier: string,
+) => asserts obj is Reporter;
+
+const validateReporter: ValidateReporterFn = (obj, specifier) => {
+  const missing = getMissingMethods(obj);
+
+  if (missing.length > 0) {
+    throw new ReporterValidationError(
+      'Module does not implement Reporter interface.',
+      specifier,
+      missing,
+    );
+  }
+};

package/src/types/budgets.ts

@@ -80,6 +80,30 @@ export interface Budget {
   readonly relative?: RelativeBudget;
 }
 
+/**
+ * A budget pattern with glob support for files and simple wildcards for
+ * suite/task
+ *
+ * File patterns use minimatch glob syntax (e.g., `**\/*.bench.js`). Suite/task
+ * patterns use simple `*` wildcard (matches any value).
+ */
+export interface BudgetPattern {
+  /** The budget to apply when this pattern matches */
+  readonly budget: Budget;
+
+  /** Glob pattern for file matching (minimatch syntax) */
+  readonly filePattern: string;
+
+  /** Computed specificity score (higher = more specific) */
+  readonly specificity: number;
+
+  /** Suite name or `*` for wildcard */
+  readonly suitePattern: string;
+
+  /** Task name or `*` for wildcard */
+  readonly taskPattern: string;
+}
+
 /**
  * Budget evaluation result for a single task
  */
@@ -159,6 +183,20 @@ export interface RelativeBudget {
   readonly maxRegression?: number;
 }
 
+/**
+ * Resolved budgets structure with exact matches and patterns separated
+ *
+ * During evaluation, exact matches are checked first, then patterns are matched
+ * in order of specificity (highest first).
+ */
+export interface ResolvedBudgets {
+  /** Exact TaskId matches (no wildcards or globs) */
+  readonly exact: Record<string, Budget>;
+
+  /** Patterns with wildcards/globs, sorted by specificity descending */
+  readonly patterns: readonly BudgetPattern[];
+}
+
 /**
  * Branded type for benchmark run identifiers
  *

package/src/types/core.ts

@@ -1,3 +1,11 @@
+import type { z } from 'zod';
+
+import type {
+  jsonReporterConfigSchema,
+  ModestBenchConfig,
+  ModestBenchConfigInput,
+  reporterConfigSchema,
+} from '../config/schema.js';
 // Budget-related types
 import type {
   AbsoluteBudget,
@@ -5,38 +13,16 @@ import type {
   BaselineStorage,
   BaselineSummaryData,
   Budget,
+  BudgetPattern,
   BudgetResult,
   BudgetSummary,
   BudgetViolation,
   RelativeBudget,
+  ResolvedBudgets,
   RunId,
   TaskId,
 } from './budgets.js';
 
-export type {
-  AbsoluteBudget,
-  BaselineReference,
-  BaselineStorage,
-  BaselineSummaryData,
-  Budget,
-  BudgetResult,
-  BudgetSummary,
-  BudgetViolation,
-  RelativeBudget,
-  RunId,
-  TaskId,
-};
-
-// Re-export schema-derived types
-export type {
-  BenchmarkDefinition,
-  BenchmarkDefinitionInput,
-  BenchmarkSuite,
-  BenchmarkSuiteInput,
-  BenchmarkTask,
-  BenchmarkTaskInput,
-} from '../core/benchmark-schema.js';
-
 /**
  * Benchmark file structure after parsing
  */
@@ -51,21 +37,6 @@ export interface BenchmarkFile {
   readonly metadata: FileMetadata;
 }
 
-/**
- * ModestBench Core Types
- *
- * Defines the fundamental data structures used throughout the ModestBench
- * system. These types represent benchmark results, metadata, configuration, and
- * system state.
- *
- * Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are
- * derived from Zod schemas and re-exported from benchmark-schema.ts for type
- * safety and consistency.
- */
-
-// Re-export identifier helper functions
-export { createRunId, createTaskId } from '../utils/identifiers.js';
-
 /**
  * Represents a complete benchmark run across multiple files
  */
@@ -98,6 +69,34 @@ export interface BenchmarkRun {
   readonly tags?: string[];
 }
 
+export type {
+  AbsoluteBudget,
+  BaselineReference,
+  BaselineStorage,
+  BaselineSummaryData,
+  Budget,
+  BudgetPattern,
+  BudgetResult,
+  BudgetSummary,
+  BudgetViolation,
+  RelativeBudget,
+  ResolvedBudgets,
+  RunId,
+  TaskId,
+};
+
+export type { ModestBenchConfig, ModestBenchConfigInput };
+
+// Re-export schema-derived types
+export type {
+  BenchmarkDefinition,
+  BenchmarkDefinitionInput,
+  BenchmarkSuite,
+  BenchmarkSuiteInput,
+  BenchmarkTask,
+  BenchmarkTaskInput,
+} from '../config/benchmark-schema.js';
+
 /**
  * CI/CD environment information
  */
@@ -116,6 +115,21 @@ export interface CiInfo {
   readonly pullRequest?: string;
 }
 
+/**
+ * ModestBench Core Types
+ *
+ * Defines the fundamental data structures used throughout the ModestBench
+ * system. These types represent benchmark results, metadata, configuration, and
+ * system state.
+ *
+ * Note: BenchmarkDefinition, BenchmarkSuite, and BenchmarkTask types are
+ * derived from Zod schemas and re-exported from benchmark-schema.ts for type
+ * safety and consistency.
+ */
+
+// Re-export identifier helper functions
+export { createRunId, createTaskId } from '../utils/identifiers.js';
+
 /**
  * CPU information
  */
@@ -277,6 +291,11 @@ export interface GitInfo {
   readonly timestamp: Date;
 }
 
+/**
+ * Configuration options for the JSON reporter
+ */
+export type JsonReporterConfig = z.infer<typeof jsonReporterConfigSchema>;
+
 /**
  * Memory information
  */
@@ -290,68 +309,12 @@ export interface MemoryInfo {
 }
 
 /**
- * Benchmark configuration
- *
- * The JSON Schema for this configuration is available at
- * `dist/schema/modestbench-config.schema.json` after building the project.
+ * Reporter-specific configuration
  *
- * Config files can optionally include a `$schema` property pointing to the
- * schema file for IDE autocomplete and validation support.
- *
- * @example
- *
- * ```json
- * {
- *   "$schema": "./node_modules/modestbench/dist/schema/modestbench-config.schema.json",
- *   "iterations": 1000,
- *   "reporters": ["human", "json"],
- *   "time": 5000
- * }
- * ```
+ * Provides typed configuration for known reporters. Unknown reporter configs
+ * are allowed via index signature.
  */
-export interface ModestBenchConfig {
-  readonly $schema?: string | undefined;
-  /** Whether to stop on first failure */
-  readonly bail: boolean;
-  /** Name of baseline to use for relative budget comparisons */
-  readonly baseline?: string | undefined;
-  /** How to handle budget violations: 'fail', 'warn', or 'report' */
-  readonly budgetMode?: 'fail' | 'report' | 'warn' | undefined;
-  /** Performance budgets mapped by task identifier */
-  readonly budgets?: Record<string, unknown> | undefined;
-  /** Patterns to exclude from discovery */
-  readonly exclude: string[];
-  /** Tags to exclude from execution */
-  readonly excludeTags: string[];
-  /** Default number of iterations per task */
-  readonly iterations: number;
-  /** How to limit benchmark execution: 'time', 'iterations', 'any', or 'all' */
-  readonly limitBy: 'all' | 'any' | 'iterations' | 'time';
-  /** Custom metadata to attach to runs */
-  readonly metadata: Record<string, unknown>;
-  /** Output directory for reports (undefined means stdout for data reporters) */
-  readonly outputDir?: string;
-  /** Pattern(s) for discovering benchmark files */
-  readonly pattern: string | string[];
-  /** Whether to run in quiet mode */
-  readonly quiet: boolean;
-  /** Configuration for specific reporters */
-  readonly reporterConfig: Record<string, unknown>;
-  /** Reporters to use for output */
-  readonly reporters: string[];
-  /** Tags to include (if empty, include all) */
-  readonly tags: string[];
-  /** Threshold configuration for performance assertions */
-  readonly thresholds: ThresholdConfig;
-  /** Maximum time to spend on each task in milliseconds */
-  readonly time: number;
-  /** Timeout for individual tasks in milliseconds */
-  readonly timeout: number;
-  /** Whether to run in verbose mode */
-  readonly verbose: boolean;
-  /** Number of warmup iterations before measurement */
-  readonly warmup: number;
-}
+export type ReporterConfig = z.infer<typeof reporterConfigSchema>;
 
 /**
  * Summary statistics for a benchmark run
@@ -441,6 +404,8 @@ export interface TaskResult {
 
 /**
  * Threshold configuration for performance assertions
+ *
+ * TODO: Derive from thresholdConfigSchema via z.infer (see #15)
  */
 export interface ThresholdConfig {
   /** Maximum allowed margin of error percentage */

package/src/types/index.ts

@@ -13,6 +13,9 @@ export { createRunId, createTaskId } from './core.js';
 // Interface contracts
 export type * from './interfaces.js';
 
+// Plugin types (for third-party reporter authors)
+export type * from './plugin.js';
+
 // Profiler types
 export type * from './profiler.js';