modestbench 0.2.0 → 0.3.1

package/src/config/budget-schema.ts

@@ -0,0 +1,189 @@
+import { z } from 'zod';
+
+import type { BaselineStorage } from '../types/budgets.js';
+
+/**
+ * Zod schema for budget configuration
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Parse time string to nanoseconds Supports: "10ms", "5s", "100us", "50ns"
+ */
+export const parseTimeString = (value: string): number => {
+  const match = value.match(/^(\d+(?:\.\d+)?)(ns|us|ms|s)$/i);
+  if (!match) {
+    throw new Error(
+      `Invalid time format: "${value}". Expected format like "10ms", "5s", "100us", "50ns"`,
+    );
+  }
+
+  const num = parseFloat(match[1]!);
+  const unit = match[2]!.toLowerCase();
+
+  const multipliers = {
+    ms: 1_000_000,
+    ns: 1,
+    s: 1_000_000_000,
+    us: 1_000,
+  };
+
+  return num * multipliers[unit as keyof typeof multipliers];
+};
+
+/**
+ * Parse percentage string to decimal Supports: "10%", "5.5%"
+ *
+ * Note: Does not round result - preserves full precision from input
+ */
+export const parsePercentageString = (value: string): number => {
+  const match = value.match(/^(\d+(?:\.\d+)?)%$/);
+  if (!match) {
+    throw new Error(
+      `Invalid percentage format: "${value}". Expected format like "10%", "5.5%"`,
+    );
+  }
+
+  return parseFloat(match[1]!) / 100;
+};
+
+/**
+ * Time or nanoseconds
+ */
+const timeSchema = z
+  .union([
+    z.number().int().nonnegative().describe('Time in nanoseconds'),
+    z
+      .string()
+      .transform(parseTimeString)
+      .describe('Time string (e.g., "10ms")'),
+  ])
+  .describe('Time value as nanoseconds or time string');
+
+/**
+ * Percentage or decimal
+ */
+const percentageSchema = z
+  .union([
+    z
+      .number()
+      .min(0)
+      .max(1)
+      .describe('Percentage as decimal (e.g., 0.1 for 10%)'),
+    z
+      .string()
+      .transform(parsePercentageString)
+      .describe('Percentage string (e.g., "10%")'),
+  ])
+  .describe('Percentage value as decimal or percentage string');
+
+/**
+ * Absolute budget schema
+ */
+const absoluteBudgetSchema = z
+  .object({
+    maxP99: timeSchema
+      .optional()
+      .describe('Maximum 99th percentile execution time'),
+    maxTime: timeSchema.optional().describe('Maximum mean execution time'),
+    minOpsPerSec: z
+      .number()
+      .positive()
+      .optional()
+      .describe('Minimum operations per second'),
+  })
+  .describe('Absolute performance budget thresholds');
+
+/**
+ * Relative budget schema
+ */
+const relativeBudgetSchema = z
+  .object({
+    baseline: z
+      .string()
+      .optional()
+      .describe('Name of baseline to compare against'),
+    maxRegression: percentageSchema
+      .optional()
+      .describe('Maximum allowed performance regression'),
+  })
+  .describe('Relative performance budget thresholds compared to baseline');
+
+/**
+ * Complete budget schema
+ */
+export const budgetSchema = z
+  .object({
+    absolute: absoluteBudgetSchema
+      .optional()
+      .describe('Absolute performance thresholds'),
+    relative: relativeBudgetSchema
+      .optional()
+      .describe('Relative performance thresholds'),
+  })
+  .describe('Performance budget configuration');
+
+/**
+ * Baseline reference schema
+ *
+ * Note: This validates and transforms to branded types (RunId, TaskId). The
+ * transforms are safe as they only add compile-time type information.
+ */
+const baselineReferenceSchema = z
+  .object({
+    branch: z.string().optional().describe('Git branch name'),
+    commit: z
+      .string()
+      .length(40)
+      .regex(/^[0-9a-f]{40}$/)
+      .optional()
+      .describe('Full Git commit hash (40 hex characters)'),
+    date: z.coerce.date().describe('Date baseline was created'),
+    name: z.string().describe('Baseline name identifier'),
+    runId: z
+      .string()
+      .length(7)
+      .regex(/^[0-9a-z]{7}$/)
+      .describe('Benchmark run ID (7 lowercase alphanumeric characters)'),
+    summary: z
+      .record(
+        z.string(),
+        z.object({
+          mean: z.number().describe('Mean execution time in nanoseconds'),
+          opsPerSecond: z.number().describe('Operations per second'),
+          p99: z
+            .number()
+            .optional()
+            .describe('99th percentile execution time in nanoseconds'),
+        }),
+      )
+      .describe('Summary of benchmark results for each task'),
+  })
+  .describe('Named baseline reference with benchmark results summary');
+
+/**
+ * Baseline storage schema
+ */
+export const baselineStorageSchema = z
+  .object({
+    baselines: z
+      .record(z.string(), baselineReferenceSchema)
+      .describe('Map of baseline names to baseline references'),
+    default: z.string().optional().describe('Default baseline name'),
+    version: z.string().describe('Schema version'),
+  })
+  .describe('Baseline storage file format');
+
+/**
+ * Validate baseline storage
+ *
+ * Note: The parsed data contains plain strings that are cast to branded types
+ * (RunId, TaskId). This is safe because branded types are compile-time only
+ * constructs.
+ */
+export const validateBaselineStorage = (storage: unknown): BaselineStorage => {
+  const parsed = baselineStorageSchema.parse(storage);
+  // Cast is safe: branded types are erased at runtime
+  return parsed as unknown as BaselineStorage;
+};

package/src/config/schema.ts

@@ -11,6 +11,7 @@ import * as z from 'zod';
 import type { ModestBenchConfig } from '../types/core.js';
 
 import { BENCHMARK_FILE_PATTERN } from '../constants.js';
+import { parsePercentageString, parseTimeString } from './budget-schema.js';
 
 /**
  * Schema for threshold configuration
@@ -56,6 +57,46 @@ const thresholdConfigSchema = z
     title: 'Threshold Configuration',
   });
 
+/**
+ * Inline budget schema for configuration (no transforms for JSON Schema
+ * compatibility - transforms are applied manually in transformBudgets
+ * function)
+ */
+const budgetSchema = z
+  .object({
+    absolute: z
+      .object({
+        maxP99: z
+          .union([z.number().positive(), z.string()])
+          .optional()
+          .describe('Maximum 99th percentile in nanoseconds or time string'),
+        maxTime: z
+          .union([z.number().positive(), z.string()])
+          .describe(
+            'Maximum mean time in nanoseconds or time string (e.g., "10ms")',
+          ),
+        minOpsPerSec: z
+          .number()
+          .positive()
+          .optional()
+          .describe('Minimum operations per second'),
+      })
+      .optional()
+      .describe('Absolute performance thresholds'),
+    relative: z
+      .object({
+        maxRegression: z
+          .union([z.number().min(0).max(1), z.string()])
+          .optional()
+          .describe(
+            'Maximum regression as decimal (0.1) or percentage string ("10%")',
+          ),
+      })
+      .optional()
+      .describe('Relative performance thresholds vs baseline'),
+  })
+  .describe('Performance budget with absolute and/or relative thresholds');
+
 /**
  * Schema for the main ModestBench configuration
  *
@@ -71,6 +112,27 @@ const modestBenchConfigSchema = z
         'JSON Schema reference for IDE support (not used by ModestBench)',
       ),
     bail: z.boolean().describe('Stop benchmark execution on first failure'),
+    baseline: z
+      .string()
+      .optional()
+      .describe(
+        'Name of baseline to use for relative budget comparisons. Must match a saved baseline name.',
+      ),
+    budgetMode: z
+      .enum(['fail', 'warn', 'report'])
+      .optional()
+      .describe(
+        'How to handle budget violations: "fail" exits with error (default), "warn" shows warnings, "report" includes in output without failing',
+      ),
+    budgets: z
+      .record(
+        z.string(),
+        z.record(z.string(), z.record(z.string(), budgetSchema)),
+      )
+      .optional()
+      .describe(
+        'Performance budgets organized by file → suite → task. Budgets define acceptable performance thresholds.',
+      ),
     exclude: z
       .array(z.string())
       .describe(
@@ -109,6 +171,55 @@ const modestBenchConfigSchema = z
       .describe(
         `Glob pattern(s) for discovering benchmark files. Can be a single pattern string or array of patterns (e.g., "**/*${BENCHMARK_FILE_PATTERN}")`,
       ),
+    profile: z
+      .object({
+        exclude: z
+          .array(z.string())
+          .optional()
+          .describe('Glob patterns to exclude from profiling results'),
+        focus: z
+          .array(z.string())
+          .optional()
+          .describe(
+            'Glob patterns to focus on in profiling results. If specified, only matching files will be shown',
+          ),
+        minCallCount: z
+          .number()
+          .int()
+          .nonnegative()
+          .optional()
+          .describe(
+            'Minimum number of times a function must be called to be included in results',
+          ),
+        minExecutionPercent: z
+          .number()
+          .nonnegative()
+          .max(100)
+          .default(1.0)
+          .describe(
+            'Minimum execution percentage threshold for including functions in results',
+          ),
+        outputFile: z
+          .string()
+          .optional()
+          .describe('Path to write profile report to file'),
+        smartDetection: z
+          .boolean()
+          .default(true)
+          .describe(
+            'Automatically detect and focus on user code, excluding node_modules and Node.js internals',
+          ),
+        topN: z
+          .number()
+          .int()
+          .positive()
+          .default(25)
+          .describe('Maximum number of top functions to show in results'),
+      })
+      .optional()
+      .describe(
+        'Configuration for profile command to identify benchmark candidates',
+      ),
     quiet: z
       .boolean()
       .describe(
@@ -176,6 +287,136 @@ export const partialModestBenchConfigSchema: z.ZodType<
   Partial<ModestBenchConfig>
 > = modestBenchConfigSchema.partial();
 
+/**
+ * Input budget type (before transformation)
+ */
+interface BudgetInput {
+  absolute?: {
+    maxP99?: number | string;
+    maxTime?: number | string;
+    minOpsPerSec?: number;
+  };
+  relative?: {
+    maxRegression?: number | string;
+  };
+}
+
+/**
+ * Output budget type (after transformation)
+ */
+interface BudgetOutput {
+  absolute?: {
+    maxP99?: number;
+    maxTime?: number;
+    minOpsPerSec?: number;
+  };
+  relative?: {
+    maxRegression?: number;
+  };
+}
+
+/**
+ * Transform budget values (parse time/percentage strings)
+ */
+const transformBudgetValues = (budget: BudgetInput): BudgetOutput => {
+  const transformed: BudgetOutput = {};
+
+  if (budget.absolute) {
+    transformed.absolute = {};
+
+    // Copy minOpsPerSec as-is (already a number)
+    if (budget.absolute.minOpsPerSec !== undefined) {
+      transformed.absolute.minOpsPerSec = budget.absolute.minOpsPerSec;
+    }
+
+    // Parse time strings
+    if (budget.absolute.maxTime !== undefined) {
+      transformed.absolute.maxTime =
+        typeof budget.absolute.maxTime === 'string'
+          ? parseTimeString(budget.absolute.maxTime)
+          : budget.absolute.maxTime;
+    }
+    if (budget.absolute.maxP99 !== undefined) {
+      transformed.absolute.maxP99 =
+        typeof budget.absolute.maxP99 === 'string'
+          ? parseTimeString(budget.absolute.maxP99)
+          : budget.absolute.maxP99;
+    }
+  }
+
+  if (budget.relative) {
+    transformed.relative = {};
+
+    // Parse percentage strings
+    if (budget.relative.maxRegression !== undefined) {
+      transformed.relative.maxRegression =
+        typeof budget.relative.maxRegression === 'string'
+          ? parsePercentageString(budget.relative.maxRegression)
+          : budget.relative.maxRegression;
+    }
+  }
+
+  return transformed;
+};
+
+/**
+ * Transform nested budget structure to flat TaskId → Budget mapping Also parses
+ * time and percentage strings
+ *
+ * @internal
+ */
+const transformBudgets = (
+  nested: Record<string, Record<string, Record<string, unknown>>> | undefined,
+): Record<string, unknown> | undefined => {
+  if (!nested) {
+    return undefined;
+  }
+
+  const flat: Record<string, unknown> = {};
+
+  for (const [file, suites] of Object.entries(nested)) {
+    for (const [suite, tasks] of Object.entries(suites)) {
+      for (const [task, budget] of Object.entries(tasks)) {
+        const taskId = `${file}/${suite}/${task}`;
+        // Transform budget values (parse strings)
+        flat[taskId] = transformBudgetValues(budget as BudgetInput);
+      }
+    }
+  }
+
+  return flat;
+};
+
+/**
+ * Safely parse and validate a partial configuration object with budget
+ * transformation
+ *
+ * @param config - The configuration object to validate
+ * @returns A result object with either success: true and data, or success:
+ *   false and error
+ */
+export const safeParsePartialConfig = (config: unknown) => {
+  const result = partialModestBenchConfigSchema.safeParse(config);
+
+  // Transform nested budgets to flat structure after validation
+  if (result.success && result.data.budgets) {
+    return {
+      ...result,
+      data: {
+        ...result.data,
+        budgets: transformBudgets(
+          result.data.budgets as Record<
+            string,
+            Record<string, Record<string, unknown>>
+          >,
+        ),
+      },
+    };
+  }
+
+  return result;
+};
+
 /**
  * Safely parse and validate a configuration object
  *
@@ -184,5 +425,23 @@ export const partialModestBenchConfigSchema: z.ZodType<
  *   false and error
  */
 export const safeParseConfig = (config: unknown) => {
-  return modestBenchConfigSchema.safeParse(config);
+  const result = modestBenchConfigSchema.safeParse(config);
+
+  // Transform nested budgets to flat structure after validation
+  if (result.success && result.data.budgets) {
+    return {
+      ...result,
+      data: {
+        ...result.data,
+        budgets: transformBudgets(
+          result.data.budgets as Record<
+            string,
+            Record<string, Record<string, unknown>>
+          >,
+        ),
+      },
+    };
+  }
+
+  return result;
 };

package/src/constants.ts

@@ -1,3 +1,5 @@
+import { type Engine } from './types/cli.js';
+
 /**
  * Supported benchmark file extensions
  */
@@ -20,21 +22,71 @@ export const BENCHMARK_FILE_PATTERN = `.bench.{${Array.from(
   .map((ext) => ext.slice(1))
   .join(',')}}`;
 
+/**
+ * Timeout before we force-quit when aborting benchmarks (ms)
+ */
+export const ABORT_TIMEOUT = 500;
+
+/**
+ * Exit codes for the CLI
+ */
+export const ExitCodes = {
+  BENCHMARK_FAILURES: 1,
+  CONFIG_ERROR: 2,
+  DISCOVERY_ERROR: 3,
+  RUNTIME_ERROR: 5,
+  SUCCESS: 0,
+  UNKNOWN_ERROR: 99,
+  VALIDATION_ERROR: 4,
+} as const;
+
+/**
+ * Supported benchmark engines
+ */
+export const Engines = {
+  ACCURATE: 'accurate',
+  TINYBENCH: 'tinybench',
+} as const satisfies Record<string, Engine>;
+
+/**
+ * Default benchmark engine
+ */
+export const DEFAULT_ENGINE = Engines.TINYBENCH;
+
+/**
+ * Supported reporters
+ */
+export const Reporters = {
+  CSV: 'csv',
+  HUMAN: 'human',
+  JSON: 'json',
+  SIMPLE: 'simple',
+} as const;
+
+/**
+ * Default reporter
+ */
+export const DEFAULT_REPORTER = Reporters.HUMAN;
+
 /**
  * Error codes for all ModestBench errors
  *
  * Use these constants to check error types instead of instanceof checks.
  */
 export const ErrorCodes = {
+  //#region budget-errors
+  BUDGET_EXCEEDED: 'ERR_MB_BUDGET_EXCEEDED',
   //#region cli-errors
   CLI_INVALID_ARGUMENT: 'ERR_MB_CLI_INVALID_ARGUMENT',
-  CLI_INVALID_DATE_FORMAT: 'ERR_MB_CLI_INVALID_DATE_FORMAT',
   //#endregion
 
+  CLI_INVALID_DATE_FORMAT: 'ERR_MB_CLI_INVALID_DATE_FORMAT',
   //#region config-errors
   CONFIG_LOAD_FAILED: 'ERR_MB_CONFIG_LOAD_FAILED',
   CONFIG_NOT_FOUND: 'ERR_MB_CONFIG_NOT_FOUND',
   CONFIG_UNSUPPORTED_FORMAT: 'ERR_MB_CONFIG_UNSUPPORTED_FORMAT',
+  //#endregion
+
   CONFIG_VALIDATION_FAILED: 'ERR_MB_CONFIG_VALIDATION_FAILED',
   //#endregion