modestbench 0.7.0 → 0.9.0

package/src/config/schema.ts

@@ -8,11 +8,46 @@
 
 import * as z from 'zod';
 
-import type { ModestBenchConfig } from '../types/core.js';
+import type {
+  Budget,
+  BudgetPattern,
+  ResolvedBudgets,
+} from '../types/budgets.js';
 
 import { BENCHMARK_FILE_PATTERN } from '../constants.js';
+import {
+  createBudgetPattern,
+  isGlobPattern,
+} from '../services/budget-resolver.js';
 import { parsePercentageString, parseTimeString } from './budget-schema.js';
 
+/**
+ * Schema for JSON reporter configuration options
+ */
+export const jsonReporterConfigSchema = z.object({
+  prettyPrint: z
+    .boolean()
+    .optional()
+    .describe('Whether to pretty-print JSON output (default: false)'),
+});
+
+/**
+ * Schema for reporter-specific configuration
+ *
+ * Allows typed configuration for known reporters while permitting unknown
+ * reporter configs via catchall.
+ */
+export const reporterConfigSchema = z
+  .object({
+    json: jsonReporterConfigSchema
+      .optional()
+      .describe('Configuration options for the JSON reporter'),
+  })
+  .catchall(z.unknown())
+  .describe(
+    'Configuration options specific to individual reporters, keyed by reporter name',
+  );
+
 /**
  * Schema for threshold configuration
  *
@@ -58,11 +93,11 @@ const thresholdConfigSchema = z
   });
 
 /**
- * Inline budget schema for configuration (no transforms for JSON Schema
- * compatibility - transforms are applied manually in transformBudgets
- * function)
+ * Input schema for budget values (before transformation)
+ *
+ * Accepts string values like "10ms" or "10%" for human-readable configuration.
  */
-const budgetSchema = z
+const budgetInputSchema = z
   .object({
     absolute: z
       .object({
@@ -72,6 +107,7 @@ const budgetSchema = z
           .describe('Maximum 99th percentile in nanoseconds or time string'),
         maxTime: z
           .union([z.number().positive(), z.string()])
+          .optional()
           .describe(
             'Maximum mean time in nanoseconds or time string (e.g., "10ms")',
           ),
@@ -98,324 +134,367 @@ const budgetSchema = z
   .describe('Performance budget with absolute and/or relative thresholds');
 
 /**
- * Schema for the main ModestBench configuration
+ * Transform budget values (parse time/percentage strings to numbers)
  *
- * This is the complete configuration schema used for validating benchmark
- * configuration from all sources (files, CLI args, defaults).
+ * Returns a Budget object with all string values converted to numbers.
  */
-const modestBenchConfigSchema = z
-  .object({
-    $schema: z
-      .string()
-      .optional()
-      .describe(
-        'JSON Schema reference for IDE support (not used by ModestBench)',
-      ),
-    bail: z.boolean().describe('Stop benchmark execution on first failure'),
-    baseline: z
-      .string()
-      .optional()
-      .describe(
-        'Name of baseline to use for relative budget comparisons. Must match a saved baseline name.',
-      ),
-    budgetMode: z
-      .enum(['fail', 'warn', 'report'])
-      .optional()
-      .describe(
-        'How to handle budget violations: "fail" exits with error (default), "warn" shows warnings, "report" includes in output without failing',
-      ),
-    budgets: z
-      .record(
-        z.string(),
-        z.record(z.string(), z.record(z.string(), budgetSchema)),
-      )
-      .optional()
-      .describe(
-        'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds.',
-      ),
-    exclude: z
-      .array(z.string())
-      .describe(
-        'Glob patterns to exclude from benchmark file discovery (e.g., "node_modules/**", ".git/**")',
-      ),
-    excludeTags: z
-      .array(z.string())
-      .describe(
-        'Tags to exclude from benchmark execution. Benchmarks matching any of these tags will be skipped.',
-      ),
-    iterations: z
-      .number()
-      .int()
-      .positive()
-      .describe(
-        'Default number of iterations to run for each benchmark task. Higher values provide more accurate statistics but take longer to execute.',
-      ),
-    limitBy: z
-      .enum(['time', 'iterations', 'any', 'all'])
-      .describe(
-        'How to limit benchmark execution: "time" stops after time limit, "iterations" stops after iteration count, "any" stops at whichever comes first, "all" runs until both limits are reached',
-      ),
-    metadata: z
-      .record(z.string(), z.unknown())
-      .describe(
-        'Custom metadata to attach to benchmark runs. Can include project name, version, environment details, etc.',
-      ),
-    outputDir: z
-      .string()
-      .min(1)
-      .optional()
-      .describe(
-        'Directory path where benchmark results and reports will be written. If not specified, data reporters will write to stdout.',
-      ),
-    pattern: z
-      .union([z.string().min(1), z.array(z.string().min(1))])
-      .describe(
-        `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`,
-      ),
-    profile: z
-      .object({
-        exclude: z
-          .array(z.string())
-          .optional()
-          .describe('Glob patterns to exclude from profiling results'),
-        focus: z
-          .array(z.string())
-          .optional()
-          .describe(
-            'Glob patterns to focus on in profiling results. If specified, only matching files will be shown',
-          ),
-        minCallCount: z
-          .number()
-          .int()
-          .nonnegative()
-          .optional()
-          .describe(
-            'Minimum number of times a function must be called to be included in results',
-          ),
-        minExecutionPercent: z
-          .number()
-          .nonnegative()
-          .max(100)
-          .default(1.0)
-          .describe(
-            'Minimum execution percentage threshold for including functions in results',
-          ),
-        outputFile: z
-          .string()
-          .optional()
-          .describe('Path to write profile report to file'),
-        smartDetection: z
-          .boolean()
-          .default(true)
-          .describe(
-            'Automatically detect and focus on user code, excluding node_modules and Node.js internals',
-          ),
-        topN: z
-          .number()
-          .int()
-          .positive()
-          .default(25)
-          .describe('Maximum number of top functions to show in results'),
-      })
-      .optional()
-      .describe(
-        'Configuration for profile command to identify benchmark candidates',
-      ),
-    quiet: z
-      .boolean()
-      .describe(
-        'Run in quiet mode with minimal console output (only errors and final results)',
-      ),
-    reporterConfig: z
-      .record(z.string(), z.unknown())
-      .describe(
-        'Configuration options specific to individual reporters, keyed by reporter name',
-      ),
-    reporters: z
-      .array(z.string())
-      .min(1)
-      .describe(
-        'List of reporter names to use for output. Available reporters: "human", "json", "csv"',
-      ),
-    tags: z
-      .array(z.string())
-      .describe(
-        'Tags to filter which benchmarks to run. If empty, all benchmarks are included. Only benchmarks with matching tags will execute.',
-      ),
-    thresholds: thresholdConfigSchema,
-    time: z
-      .number()
-      .int()
-      .positive()
-      .describe(
-        'Maximum time to spend on each benchmark task in milliseconds. Tasks will run at least until this duration or iteration count is reached, depending on limitBy setting.',
-      ),
-    timeout: z
-      .number()
-      .int()
-      .positive()
-      .describe(
-        'Timeout for individual benchmark tasks in milliseconds. Tasks exceeding this duration will be terminated and marked as failed.',
-      ),
-    verbose: z
-      .boolean()
-      .describe(
-        'Enable verbose output. Provides more detailed console output including progress, intermediate results, and diagnostic information',
-      ),
-    warmup: z
-      .number()
-      .int()
-      .nonnegative()
-      .describe(
-        'Number of warmup iterations to run before measurement begins. Warmup helps stabilize performance by allowing JIT compilation and caching to occur.',
-      ),
-  })
-  .strict()
-  .describe(
-    'ModestBench configuration for controlling benchmark discovery, execution, and reporting',
-  )
-  .meta({
-    title: 'ModestBench Configuration',
-  });
+const transformBudgetValues = (
+  budget: z.infer<typeof budgetInputSchema>,
+): Budget => {
+  return {
+    // Build absolute budget object if present
+    absolute: budget.absolute
+      ? {
+          maxP99:
+            budget.absolute.maxP99 !== undefined
+              ? typeof budget.absolute.maxP99 === 'string'
+                ? parseTimeString(budget.absolute.maxP99)
+                : budget.absolute.maxP99
+              : undefined,
+          maxTime:
+            budget.absolute.maxTime !== undefined
+              ? typeof budget.absolute.maxTime === 'string'
+                ? parseTimeString(budget.absolute.maxTime)
+                : budget.absolute.maxTime
+              : undefined,
+          minOpsPerSec: budget.absolute.minOpsPerSec,
+        }
+      : undefined,
+    // Build relative budget object if present
+    relative: budget.relative
+      ? {
+          maxRegression:
+            budget.relative.maxRegression !== undefined
+              ? typeof budget.relative.maxRegression === 'string'
+                ? parsePercentageString(budget.relative.maxRegression)
+                : budget.relative.maxRegression
+              : undefined,
+        }
+      : undefined,
+  };
+};
 
 /**
- * Validate a partial configuration object
+ * Budget schema with transform for string-to-number conversion
  *
- * This is used for validating configuration from files or CLI args before
- * merging with defaults.
+ * Input: Budget with string values like "10ms" or "10%" Output: Budget with
+ * numeric values only
  */
-export const partialModestBenchConfigSchema: z.ZodType<
-  Partial<ModestBenchConfig>
-> = modestBenchConfigSchema.partial();
+const budgetSchema = budgetInputSchema.transform(transformBudgetValues);
 
 /**
- * Input budget type (before transformation)
+ * Input schema for budgets (nested file → suite → task → budget structure)
+ * without transforms - used for JSON Schema generation.
  */
-interface BudgetInput {
-  absolute?: {
-    maxP99?: number | string;
-    maxTime?: number | string;
-    minOpsPerSec?: number;
-  };
-  relative?: {
-    maxRegression?: number | string;
-  };
-}
+const budgetsRawInputSchema = z.record(
+  z.string(),
+  z.record(z.string(), z.record(z.string(), budgetInputSchema)),
+);
 
 /**
- * Output budget type (after transformation)
+ * Input schema for budgets with individual budget transforms applied.
+ *
+ * Used to validate the human-readable nested format from config files.
  */
-interface BudgetOutput {
-  absolute?: {
-    maxP99?: number;
-    maxTime?: number;
-    minOpsPerSec?: number;
-  };
-  relative?: {
-    maxRegression?: number;
-  };
-}
+const budgetsInputSchema = z.record(
+  z.string(),
+  z.record(z.string(), z.record(z.string(), budgetSchema)),
+);
 
 /**
- * Transform budget values (parse time/percentage strings)
+ * Check if a suite or task name is a wildcard
+ *
+ * @param name - The suite or task name
+ * @returns True if the name is a wildcard (`*`)
  */
-const transformBudgetValues = (budget: BudgetInput): BudgetOutput => {
-  const transformed: BudgetOutput = {};
+const isWildcard = (name: string): boolean => name === '*';
 
-  if (budget.absolute) {
-    transformed.absolute = {};
+/**
+ * Check if a budget entry contains any wildcards or glob patterns
+ *
+ * @param file - File pattern
+ * @param suite - Suite name or wildcard
+ * @param task - Task name or wildcard
+ * @returns True if any part contains wildcards
+ */
+const hasWildcards = (file: string, suite: string, task: string): boolean => {
+  return isGlobPattern(file) || isWildcard(suite) || isWildcard(task);
+};
 
-    // Copy minOpsPerSec as-is (already a number)
-    if (budget.absolute.minOpsPerSec !== undefined) {
-      transformed.absolute.minOpsPerSec = budget.absolute.minOpsPerSec;
-    }
+/**
+ * Transform nested budget structure to ResolvedBudgets with exact matches and
+ * patterns separated
+ *
+ * @param nested - Nested budgets structure (file → suite → task → budget)
+ * @returns ResolvedBudgets with exact matches and wildcard patterns
+ */
+const flattenBudgets = (
+  nested: z.infer<typeof budgetsInputSchema>,
+): ResolvedBudgets => {
+  const exact: Record<string, Budget> = {};
+  const patterns: BudgetPattern[] = [];
 
-    // Parse time strings
-    if (budget.absolute.maxTime !== undefined) {
-      transformed.absolute.maxTime =
-        typeof budget.absolute.maxTime === 'string'
-          ? parseTimeString(budget.absolute.maxTime)
-          : budget.absolute.maxTime;
-    }
-    if (budget.absolute.maxP99 !== undefined) {
-      transformed.absolute.maxP99 =
-        typeof budget.absolute.maxP99 === 'string'
-          ? parseTimeString(budget.absolute.maxP99)
-          : budget.absolute.maxP99;
+  for (const [file, suites] of Object.entries(nested)) {
+    for (const [suite, tasks] of Object.entries(suites)) {
+      for (const [task, budget] of Object.entries(tasks)) {
+        if (hasWildcards(file, suite, task)) {
+          // This is a pattern budget
+          patterns.push(createBudgetPattern(file, suite, task, budget));
+        } else {
+          // This is an exact match
+          const taskId = `${file}/${suite}/${task}`;
+          exact[taskId] = budget;
+        }
+      }
     }
   }
 
-  if (budget.relative) {
-    transformed.relative = {};
+  // Sort patterns by specificity descending for consistent iteration order
+  patterns.sort((a, b) => b.specificity - a.specificity);
 
-    // Parse percentage strings
-    if (budget.relative.maxRegression !== undefined) {
-      transformed.relative.maxRegression =
-        typeof budget.relative.maxRegression === 'string'
-          ? parsePercentageString(budget.relative.maxRegression)
-          : budget.relative.maxRegression;
-    }
-  }
+  return { exact, patterns };
+};
+
+/**
+ * Budgets schema with transform for nested-to-ResolvedBudgets conversion
+ *
+ * Input: { [file]: { [suite]: { [task]: Budget } } } Output: ResolvedBudgets {
+ * exact: { [taskId]: Budget }, patterns: BudgetPattern[] }
+ */
+const budgetsSchema = budgetsInputSchema.transform(flattenBudgets);
 
-  return transformed;
+/**
+ * Shared configuration properties (everything except budgets)
+ *
+ * These properties are identical between the runtime schema (with transforms)
+ * and the JSON Schema generation schema (without transforms).
+ */
+const baseConfigProperties = {
+  $schema: z
+    .string()
+    .optional()
+    .describe(
+      'JSON Schema reference for IDE support (not used by ModestBench)',
+    ),
+  bail: z.boolean().describe('Stop benchmark execution on first failure'),
+  baseline: z
+    .string()
+    .optional()
+    .describe(
+      'Name of baseline to use for relative budget comparisons. Must match a saved baseline name.',
+    ),
+  budgetMode: z
+    .enum(['fail', 'warn', 'report'])
+    .optional()
+    .describe(
+      'How to handle budget violations: "fail" exits with error (default), "warn" shows warnings, "report" includes in output without failing',
+    ),
+  exclude: z
+    .array(z.string())
+    .describe(
+      'Glob patterns to exclude from benchmark file discovery (e.g., "node_modules/**", ".git/**")',
+    ),
+  excludeTags: z
+    .array(z.string())
+    .describe(
+      'Tags to exclude from benchmark execution. Benchmarks matching any of these tags will be skipped.',
+    ),
+  iterations: z
+    .number()
+    .int()
+    .positive()
+    .describe(
+      'Default number of iterations to run for each benchmark task. Higher values provide more accurate statistics but take longer to execute.',
+    ),
+  limitBy: z
+    .enum(['time', 'iterations', 'any', 'all'])
+    .describe(
+      'How to limit benchmark execution: "time" stops after time limit, "iterations" stops after iteration count, "any" stops at whichever comes first, "all" runs until both limits are reached',
+    ),
+  metadata: z
+    .record(z.string(), z.unknown())
+    .describe(
+      'Custom metadata to attach to benchmark runs. Can include project name, version, environment details, etc.',
+    ),
+  outputDir: z
+    .string()
+    .min(1)
+    .optional()
+    .describe(
+      'Directory path where benchmark results and reports will be written. If not specified, data reporters will write to stdout.',
+    ),
+  pattern: z
+    .union([z.string().min(1), z.array(z.string().min(1))])
+    .describe(
+      `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`,
+    ),
+  profile: z
+    .object({
+      exclude: z
+        .array(z.string())
+        .optional()
+        .describe('Glob patterns to exclude from profiling results'),
+      focus: z
+        .array(z.string())
+        .optional()
+        .describe(
+          'Glob patterns to focus on in profiling results. If specified, only matching files will be shown',
+        ),
+      minCallCount: z
+        .number()
+        .int()
+        .nonnegative()
+        .optional()
+        .describe(
+          'Minimum number of times a function must be called to be included in results',
+        ),
+      minExecutionPercent: z
+        .number()
+        .nonnegative()
+        .max(100)
+        .default(1.0)
+        .describe(
+          'Minimum execution percentage threshold for including functions in results',
+        ),
+      outputFile: z
+        .string()
+        .optional()
+        .describe('Path to write profile report to file'),
+      smartDetection: z
+        .boolean()
+        .default(true)
+        .describe(
+          'Automatically detect and focus on user code, excluding node_modules and Node.js internals',
+        ),
+      topN: z
+        .number()
+        .int()
+        .positive()
+        .default(25)
+        .describe('Maximum number of top functions to show in results'),
+    })
+    .optional()
+    .describe(
+      'Configuration for profile command to identify benchmark candidates',
+    ),
+  quiet: z
+    .boolean()
+    .describe(
+      'Run in quiet mode with minimal console output (only errors and final results)',
+    ),
+  reporterConfig: reporterConfigSchema,
+  reporters: z
+    .array(z.string())
+    .min(1)
+    .describe(
+      'List of reporter names to use for output. Available reporters: "human", "json", "csv"',
+    ),
+  tags: z
+    .array(z.string())
+    .describe(
+      'Tags to filter which benchmarks to run. If empty, all benchmarks are included. Only benchmarks with matching tags will execute.',
+    ),
+  thresholds: thresholdConfigSchema,
+  time: z
+    .number()
+    .int()
+    .positive()
+    .describe(
+      'Maximum time to spend on each benchmark task in milliseconds. Tasks will run at least until this duration or iteration count is reached, depending on limitBy setting.',
+    ),
+  timeout: z
+    .number()
+    .int()
+    .positive()
+    .describe(
+      'Timeout for individual benchmark tasks in milliseconds. Tasks exceeding this duration will be terminated and marked as failed.',
+    ),
+  verbose: z
+    .boolean()
+    .describe(
+      'Enable verbose output. Provides more detailed console output including progress, intermediate results, and diagnostic information',
+    ),
+  warmup: z
+    .number()
+    .int()
+    .nonnegative()
+    .describe(
+      'Number of warmup iterations to run before measurement begins. Warmup helps stabilize performance by allowing JIT compilation and caching to occur.',
+    ),
 };
 
+/** Description for the budgets field */
+const budgetsDescription =
+  'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds. Supports wildcards (* for suite/task, glob patterns for files).';
+
+/** Description and metadata for the config schema */
+const configSchemaDescription =
+  'ModestBench configuration for controlling benchmark discovery, execution, and reporting';
+const configSchemaMeta = { title: 'ModestBench Configuration' };
+
 /**
- * Transform nested budget structure to flat TaskId → Budget mapping Also parses
- * time and percentage strings
+ * Schema for the main ModestBench configuration
+ *
+ * This is the complete configuration schema used for validating benchmark
+ * configuration from all sources (files, CLI args, defaults).
  *
- * @internal
+ * The budgets field uses transforms to:
+ *
+ * 1. Parse string values like "10ms" or "10%" to numbers
+ * 2. Separate exact matches from wildcard patterns into ResolvedBudgets
  */
-const transformBudgets = (
-  nested: Record<string, Record<string, Record<string, unknown>>> | undefined,
-): Record<string, unknown> | undefined => {
-  if (!nested) {
-    return undefined;
-  }
+const modestBenchConfigSchema = z
+  .object({
+    ...baseConfigProperties,
+    budgets: budgetsSchema.optional().describe(budgetsDescription),
+  })
+  .strict()
+  .describe(configSchemaDescription)
+  .meta(configSchemaMeta);
 
-  const flat: Record<string, unknown> = {};
+/**
+ * Input schema for configuration (without transforms)
+ *
+ * This schema is used for JSON Schema generation. It validates the same input
+ * structure but without transforms, which can't be represented in JSON Schema
+ * format.
+ */
+const modestBenchConfigInputSchema = z
+  .object({
+    ...baseConfigProperties,
+    budgets: budgetsRawInputSchema.optional().describe(budgetsDescription),
+  })
+  .strict()
+  .describe(configSchemaDescription)
+  .meta(configSchemaMeta);
 
-  for (const [file, suites] of Object.entries(nested)) {
-    for (const [suite, tasks] of Object.entries(suites)) {
-      for (const [task, budget] of Object.entries(tasks)) {
-        const taskId = `${file}/${suite}/${task}`;
-        // Transform budget values (parse strings)
-        flat[taskId] = transformBudgetValues(budget as BudgetInput);
-      }
-    }
-  }
+/**
+ * Partial input schema for JSON Schema generation
+ *
+ * This is used for generating JSON Schema for IDE autocomplete in config files.
+ */
+export const partialModestBenchConfigInputSchema =
+  modestBenchConfigInputSchema.partial();
 
-  return flat;
-};
+/**
+ * Validate a partial configuration object
+ *
+ * This is used for validating configuration from files or CLI args before
+ * merging with defaults.
+ */
+export const partialModestBenchConfigSchema: z.ZodType<
+  Partial<ModestBenchConfig>
+> = modestBenchConfigSchema.partial();
 
 /**
- * Safely parse and validate a partial configuration object with budget
- * transformation
+ * Safely parse and validate a partial configuration object
  *
  * @param config - The configuration object to validate
  * @returns A result object with either success: true and data, or success:
  *   false and error
  */
 export const safeParsePartialConfig = (config: unknown) => {
-  const result = partialModestBenchConfigSchema.safeParse(config);
-
-  // Transform nested budgets to flat structure after validation
-  if (result.success && result.data.budgets) {
-    return {
-      ...result,
-      data: {
-        ...result.data,
-        budgets: transformBudgets(
-          result.data.budgets as Record<
-            string,
-            Record<string, Record<string, unknown>>
-          >,
-        ),
-      },
-    };
-  }
-
-  return result;
+  return partialModestBenchConfigSchema.safeParse(config);
 };
 
 /**
@@ -426,23 +505,21 @@ export const safeParsePartialConfig = (config: unknown) => {
  *   false and error
  */
 export const safeParseConfig = (config: unknown) => {
-  const result = modestBenchConfigSchema.safeParse(config);
+  return modestBenchConfigSchema.safeParse(config);
+};
 
-  // Transform nested budgets to flat structure after validation
-  if (result.success && result.data.budgets) {
-    return {
-      ...result,
-      data: {
-        ...result.data,
-        budgets: transformBudgets(
-          result.data.budgets as Record<
-            string,
-            Record<string, Record<string, unknown>>
-          >,
-        ),
-      },
-    };
-  }
+/**
+ * Configuration type after parsing (output type)
+ *
+ * This is the type you get after parsing a config file - budgets are
+ * transformed to ResolvedBudgets and string values are converted to numbers.
+ */
+export type ModestBenchConfig = z.infer<typeof modestBenchConfigSchema>;
 
-  return result;
-};
+/**
+ * Configuration type before parsing (input type)
+ *
+ * This is the type of config files written by users - budgets are nested (file
+ * → suite → task) and values can be strings like "10ms" or "10%".
+ */
+export type ModestBenchConfigInput = z.input<typeof modestBenchConfigSchema>;