modestbench 0.0.3 → 0.2.0

package/src/errors/index.ts

@@ -0,0 +1,71 @@
+/**
+ * ModestBench Error Classes
+ *
+ * Custom error classes for structured error handling throughout ModestBench.
+ * All errors extend ModestBenchError and include error codes and documentation
+ * URLs.
+ */
+
+// Base error classes and utilities
+export {
+  isModestBenchError,
+  ModestBenchAggregateError,
+  ModestBenchError,
+} from './base.js';
+
+// CLI errors
+export {
+  InvalidArgumentError,
+  InvalidDateFormatError,
+  UnknownError,
+} from './cli.js';
+
+// Configuration errors
+export {
+  ConfigLoadError,
+  ConfigNotFoundError,
+  ConfigValidationError,
+  UnsupportedConfigFormatError,
+} from './configuration.js';
+
+// Execution errors
+export {
+  BenchmarkExecutionError,
+  OperationTooFastError,
+  SetupError,
+  TaskExecutionError,
+  TeardownError,
+  TimeoutError,
+} from './execution.js';
+
+// File errors
+export {
+  FileDiscoveryError,
+  FileLoadError,
+  FileNotFoundError,
+  FilePermissionError,
+  UnsupportedFileExtensionError,
+} from './file.js';
+
+// Reporter errors
+export {
+  ReporterAlreadyRegisteredError,
+  ReporterOutputError,
+  UnknownReporterError,
+} from './reporter.js';
+
+// Storage errors
+export {
+  StorageCorruptionError,
+  StorageError,
+  StorageIndexError,
+  StorageSpaceError,
+  UnsupportedExportFormatError,
+} from './storage.js';
+
+// Validation errors
+export {
+  SchemaValidationError,
+  StructureValidationError,
+  TypeValidationError,
+} from './validation.js';

package/src/errors/reporter.ts

@@ -0,0 +1,35 @@
+/**
+ * Reporter-related errors
+ *
+ * Errors that occur during reporter operations.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Reporter already registered
+ *
+ * Thrown when attempting to register a reporter with a name that is already in
+ * use.
+ */
+export class ReporterAlreadyRegisteredError extends ModestBenchError {
+  readonly code = 'ERR_MB_REPORTER_ALREADY_REGISTERED';
+}
+
+/**
+ * Reporter output failed
+ *
+ * Thrown when a reporter fails to write output.
+ */
+export class ReporterOutputError extends ModestBenchError {
+  readonly code = 'ERR_MB_REPORTER_OUTPUT_FAILED';
+}
+
+/**
+ * Unknown reporter
+ *
+ * Thrown when attempting to use a reporter that is not registered.
+ */
+export class UnknownReporterError extends ModestBenchError {
+  readonly code = 'ERR_MB_REPORTER_UNKNOWN';
+}

package/src/errors/storage.ts

@@ -0,0 +1,52 @@
+/**
+ * Storage-related errors
+ *
+ * Errors that occur during history storage operations.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Storage data corruption
+ *
+ * Thrown when stored data is found to be corrupted or invalid.
+ */
+export class StorageCorruptionError extends ModestBenchError {
+  readonly code = 'ERR_MB_STORAGE_CORRUPTION';
+}
+
+/**
+ * Storage operation failed
+ *
+ * Thrown when a general storage operation fails.
+ */
+export class StorageError extends ModestBenchError {
+  readonly code = 'ERR_MB_STORAGE_FAILED';
+}
+
+/**
+ * Storage index corruption
+ *
+ * Thrown when the storage index is corrupted or cannot be read.
+ */
+export class StorageIndexError extends ModestBenchError {
+  readonly code = 'ERR_MB_STORAGE_INDEX_CORRUPTION';
+}
+
+/**
+ * Insufficient storage space
+ *
+ * Thrown when there is insufficient disk space for storage operations.
+ */
+export class StorageSpaceError extends ModestBenchError {
+  readonly code = 'ERR_MB_STORAGE_INSUFFICIENT_SPACE';
+}
+
+/**
+ * Unsupported export format
+ *
+ * Thrown when attempting to export data in an unsupported format.
+ */
+export class UnsupportedExportFormatError extends ModestBenchError {
+  readonly code = 'ERR_MB_STORAGE_EXPORT_UNSUPPORTED';
+}

package/src/errors/validation.ts

@@ -0,0 +1,35 @@
+/**
+ * Validation-related errors
+ *
+ * Errors that occur during benchmark file structure and data validation.
+ */
+
+import { ModestBenchError } from './base.js';
+
+/**
+ * Schema validation failed
+ *
+ * Thrown when data fails to validate against a Zod schema or other schema
+ * validation.
+ */
+export class SchemaValidationError extends ModestBenchError {
+  readonly code = 'ERR_MB_VALIDATION_SCHEMA_FAILED';
+}
+
+/**
+ * Invalid benchmark structure
+ *
+ * Thrown when a benchmark file has an invalid or unexpected structure.
+ */
+export class StructureValidationError extends ModestBenchError {
+  readonly code = 'ERR_MB_VALIDATION_STRUCTURE_INVALID';
+}
+
+/**
+ * Type validation failed
+ *
+ * Thrown when data fails type validation checks.
+ */
+export class TypeValidationError extends ModestBenchError {
+  readonly code = 'ERR_MB_VALIDATION_TYPE_FAILED';
+}

package/src/formatters/history/base.ts

@@ -0,0 +1,28 @@
+/**
+ * Base Formatter Interface
+ *
+ * Defines the contract for history command formatters. Each formatter
+ * transforms processed data into human-readable or machine-readable output.
+ */
+
+/**
+ * Formatter interface for history command output
+ *
+ * @template TData - The data type this formatter accepts
+ */
+export interface HistoryFormatter<TData> {
+  /**
+   * Format data as CSV (optional, not all commands support CSV)
+   */
+  formatCsv?(data: TData): string;
+
+  /**
+   * Format data for human-readable terminal output
+   */
+  formatHuman(data: TData): string;
+
+  /**
+   * Format data as JSON
+   */
+  formatJson(data: TData): string;
+}

package/src/formatters/history/compare.ts

@@ -0,0 +1,186 @@
+/**
+ * Compare Formatter
+ *
+ * Formats benchmark run comparison results in human and JSON formats.
+ */
+
+import type { CompareResult } from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { colorize } from '../../utils/ansi.js';
+import { ansiChars } from '../../utils/ansi.js';
+
+/**
+ * Formatter for history compare command
+ */
+export class HistoryCompareFormatter
+  implements HistoryFormatter<CompareResult>
+{
+  /**
+   * Format as human-readable comparison
+   */
+  formatHuman(data: CompareResult): string {
+    const lines: string[] = [];
+
+    lines.push(colorize('brightMagenta', colorize('bold', 'Comparing runs:')));
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightWhite', colorize('bold', 'Run 1'))} ${colorize('dim', data.run1.id)} (${colorize('white', data.run1.startTime.toLocaleString())})`,
+    );
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightWhite', colorize('bold', 'Run 2'))} ${colorize('dim', data.run2.id)} (${colorize('white', data.run2.startTime.toLocaleString())})`,
+    );
+    lines.push('');
+
+    lines.push(colorize('cyan', 'Summary comparison:'));
+    lines.push('');
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Files: ${colorize('brightWhite', String(data.run1.summary.totalFiles))} vs ${colorize('brightWhite', String(data.run2.summary.totalFiles))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Tasks: ${colorize('brightWhite', String(data.run1.summary.totalTasks))} vs ${colorize('brightWhite', String(data.run2.summary.totalTasks))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Passed: ${colorize('brightCyan', String(data.run1.summary.passedTasks))} vs ${colorize('brightCyan', String(data.run2.summary.passedTasks))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Failed: ${colorize('brightRed', colorize('bold', String(data.run1.summary.failedTasks)))} vs ${colorize('brightRed', colorize('bold', String(data.run2.summary.failedTasks)))}`,
+    );
+    lines.push('');
+
+    // Detailed task comparison
+    if (data.tasksInBoth.length > 0) {
+      lines.push(colorize('cyan', 'Task-by-task comparison:'));
+      lines.push('');
+
+      for (const comparison of data.tasksInBoth) {
+        const mean1 = comparison.run1!.mean / 1000000; // Convert to ms
+        const mean2 = comparison.run2!.mean / 1000000;
+        const changeSign = comparison.percentChange >= 0 ? '+' : '';
+        const changeStr = `${changeSign}${comparison.percentChange.toFixed(1)}%`;
+
+        lines.push(
+          `  ${colorize('brightWhite', `${comparison.suite} › ${comparison.task}`)}`,
+        );
+
+        // Mean - highlight higher (slower/worse) number
+        const meanHigher = mean2 > mean1;
+        const mean1Str = meanHigher
+          ? colorize('magenta', `${mean1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${mean1.toFixed(3)}ms`);
+        const mean2Str = meanHigher
+          ? colorize('brightMagenta', `${mean2.toFixed(3)}ms`)
+          : colorize('magenta', `${mean2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Mean:')} ${mean1Str} ${colorize('dim', '→')} ${mean2Str} ${colorize('dim', `(${colorize('white', changeStr)}`)}`,
+        );
+
+        // Min - highlight higher number
+        const min1 = comparison.run1!.min / 1000000;
+        const min2 = comparison.run2!.min / 1000000;
+        const minHigher = min2 > min1;
+        const min1Str = minHigher
+          ? colorize('magenta', `${min1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${min1.toFixed(3)}ms`);
+        const min2Str = minHigher
+          ? colorize('brightMagenta', `${min2.toFixed(3)}ms`)
+          : colorize('magenta', `${min2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Min:')}  ${min1Str} ${colorize('dim', '→')} ${min2Str}`,
+        );
+
+        // Max - highlight higher number
+        const max1 = comparison.run1!.max / 1000000;
+        const max2 = comparison.run2!.max / 1000000;
+        const maxHigher = max2 > max1;
+        const max1Str = maxHigher
+          ? colorize('magenta', `${max1.toFixed(3)}ms`)
+          : colorize('brightMagenta', `${max1.toFixed(3)}ms`);
+        const max2Str = maxHigher
+          ? colorize('brightMagenta', `${max2.toFixed(3)}ms`)
+          : colorize('magenta', `${max2.toFixed(3)}ms`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Max:')}  ${max1Str} ${colorize('dim', '→')} ${max2Str}`,
+        );
+
+        // Iterations - highlight higher number
+        const iter1 = comparison.run1!.iterations;
+        const iter2 = comparison.run2!.iterations;
+        const iterHigher = iter2 > iter1;
+        const iter1Str = iterHigher
+          ? colorize('brightWhite', String(iter1))
+          : colorize('bold', colorize('brightWhite', String(iter1)));
+        const iter2Str = iterHigher
+          ? colorize('bold', colorize('brightWhite', String(iter2)))
+          : colorize('brightWhite', String(iter2));
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'Iterations:')} ${iter1Str} ${colorize('dim', 'vs')} ${iter2Str}`,
+        );
+
+        // CV (Coefficient of Variation) - shows measurement consistency
+        const cv1 = comparison.run1!.cv * 100;
+        const cv2 = comparison.run2!.cv * 100;
+        const cvHigher = cv2 > cv1;
+        const cv1Str = cvHigher
+          ? colorize('magenta', `${cv1.toFixed(2)}%`)
+          : colorize('brightMagenta', `${cv1.toFixed(2)}%`);
+        const cv2Str = cvHigher
+          ? colorize('brightMagenta', `${cv2.toFixed(2)}%`)
+          : colorize('magenta', `${cv2.toFixed(2)}%`);
+        lines.push(
+          `    ${colorize('dim', ansiChars.bullet)} ${colorize('white', 'CV:')}  ${cv1Str} ${colorize('dim', '→')} ${cv2Str}`,
+        );
+
+        lines.push('');
+      }
+    }
+
+    if (data.tasksOnlyInRun1.length > 0) {
+      lines.push(
+        colorize(
+          'cyan',
+          `Tasks only in run 1 (${data.tasksOnlyInRun1.length}):`,
+        ),
+      );
+      for (const task of data.tasksOnlyInRun1) {
+        lines.push(
+          `  ${colorize('dim', ansiChars.bullet)} ${colorize('brightWhite', `${task.suite} › ${task.task}`)}`,
+        );
+      }
+      lines.push('');
+    }
+
+    if (data.tasksOnlyInRun2.length > 0) {
+      lines.push(
+        colorize(
+          'cyan',
+          `Tasks only in run 2 (${data.tasksOnlyInRun2.length}):`,
+        ),
+      );
+      for (const task of data.tasksOnlyInRun2) {
+        lines.push(
+          `  ${colorize('dim', ansiChars.bullet)} ${colorize('brightWhite', `${task.suite} › ${task.task}`)}`,
+        );
+      }
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as JSON
+   */
+  formatJson(data: CompareResult): string {
+    return JSON.stringify(
+      {
+        run1: data.run1,
+        run2: data.run2,
+        taskComparisons: data.tasksInBoth,
+        tasksOnlyInRun1: data.tasksOnlyInRun1,
+        tasksOnlyInRun2: data.tasksOnlyInRun2,
+      },
+      null,
+      2,
+    );
+  }
+}

package/src/formatters/history/list.ts

@@ -0,0 +1,101 @@
+/**
+ * List Formatter
+ *
+ * Formats historical run listings in human, JSON, and CSV formats.
+ */
+
+import type { HistoryListResult } from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { ansiChars, colorize } from '../../utils/ansi.js';
+
+/**
+ * Formatter for history list command
+ */
+export class HistoryListFormatter
+  implements HistoryFormatter<HistoryListResult>
+{
+  /**
+   * Format as CSV
+   */
+  formatCsv(data: HistoryListResult): string {
+    const lines: string[] = ['id,startTime,duration,files,tasks,passed,failed'];
+
+    for (const run of data.runs) {
+      lines.push(
+        `${run.id},${run.startTime.toISOString()},${run.duration},${run.summary.totalFiles},${run.summary.totalTasks},${run.summary.passedTasks},${run.summary.failedTasks}`,
+      );
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as human-readable list
+   */
+  formatHuman(data: HistoryListResult): string {
+    if (data.runs.length === 0) {
+      return colorize('dim', 'No historical data found matching criteria.');
+    }
+
+    const lines: string[] = [
+      colorize('brightMagenta', colorize('bold', '\nRecent Benchmark Runs')),
+      '',
+    ];
+
+    for (const run of data.runs) {
+      const dateStr = run.startTime.toLocaleString();
+      const durationStr = `${(run.duration / 1000).toFixed(1)}s`;
+
+      // Status with colors and symbols
+      const hasFailures = run.summary.failedTasks > 0;
+      const statusIcon = hasFailures
+        ? colorize('brightRed', ansiChars.cross)
+        : colorize('brightCyan', ansiChars.checkmark);
+
+      const passedStr = colorize(
+        'brightCyan',
+        `${run.summary.passedTasks} passed`,
+      );
+      const statusStr = hasFailures
+        ? `${passedStr}, ${colorize('brightRed', `${run.summary.failedTasks} failed`)}`
+        : passedStr;
+
+      // Run ID in bright white and bold, date dimmed, duration in magenta
+      lines.push(
+        `  ${statusIcon} ${colorize('brightWhite', colorize('bold', run.id))} ${colorize('dim', ansiChars.bullet)} ${colorize('gray', dateStr)} ${colorize('dim', ansiChars.bullet)} ${colorize('brightMagenta', durationStr)}`,
+      );
+
+      // Files and tasks info
+      lines.push(
+        `     ${colorize('dim', `${run.summary.totalFiles} files, ${run.summary.totalTasks} tasks:`)} ${statusStr}`,
+      );
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as JSON array
+   */
+  formatJson(data: HistoryListResult): string {
+    if (data.runs.length === 0) {
+      return '[]';
+    }
+
+    return JSON.stringify(
+      data.runs.map((run) => ({
+        duration: run.duration,
+        failed: run.summary.failedTasks,
+        files: run.summary.totalFiles,
+        id: run.id,
+        passed: run.summary.passedTasks,
+        startTime: run.startTime,
+        tasks: run.summary.totalTasks,
+      })),
+      null,
+      2,
+    );
+  }
+}

package/src/formatters/history/show.ts

@@ -0,0 +1,155 @@
+/**
+ * Show Formatter
+ *
+ * Formats detailed benchmark run display in human and JSON formats.
+ */
+
+import { relative } from 'node:path';
+
+import type { ShowResult } from '../../services/history/models.js';
+import type { HistoryFormatter } from './base.js';
+
+import { ansiChars, colorize } from '../../utils/ansi.js';
+
+/**
+ * Formatter for history show command
+ */
+export class HistoryShowFormatter implements HistoryFormatter<ShowResult> {
+  /**
+   * Format as human-readable detailed view
+   */
+  formatHuman(data: ShowResult): string {
+    const lines: string[] = [];
+
+    // Header with run ID
+    lines.push(
+      colorize(
+        'cyan',
+        colorize(
+          'bold',
+          `\nBenchmark Run: ${colorize('brightWhite', colorize('bold', data.id))}`,
+        ),
+      ),
+    );
+
+    // Run details (indented by 2 spaces)
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('white', data.startTime.toLocaleString())}`,
+    );
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('white', 'Duration:')} ${colorize('magenta', `${(data.duration / 1000).toFixed(1)}s`)}`,
+    );
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} Node.js ${colorize('brightWhite', data.environment.nodeVersion)} on ${colorize('brightWhite', data.environment.platform)} (${colorize('brightWhite', data.environment.arch)})`,
+    );
+
+    // CPU and system info
+    lines.push(
+      `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightWhite', String(data.environment.cpu.cores))} cores @ ${colorize('brightWhite', `${data.environment.cpu.speed}MHz`)} on ${colorize('brightWhite', data.environment.cpu.model)}`,
+    );
+
+    if (data.git) {
+      lines.push(
+        `  ${colorize('brightCyan', ansiChars.bullet)} ${colorize('brightBlue', data.git.branch)}@${colorize('dim', data.git.commit.substring(0, 8))}`,
+      );
+    }
+
+    // Summary section
+    lines.push('');
+    lines.push(colorize('cyan', 'Summary'));
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Files: ${colorize('brightWhite', String(data.summary.totalFiles))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Suites: ${colorize('brightWhite', String(data.summary.totalSuites))}`,
+    );
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Tasks: ${colorize('brightWhite', String(data.summary.totalTasks))}`,
+    );
+
+    lines.push(
+      `  ${colorize('dim', ansiChars.smallSquare)} Passed: ${colorize('brightCyan', String(data.summary.passedTasks))}`,
+    );
+
+    if (data.summary.failedTasks > 0) {
+      lines.push(
+        `  ${colorize('dim', ansiChars.smallSquare)} Failed: ${colorize('brightRed', colorize('bold', String(data.summary.failedTasks)))}`,
+      );
+    }
+
+    // Detailed results
+    lines.push('');
+    lines.push(colorize('cyan', 'Results'));
+    lines.push('');
+
+    for (const file of data.files) {
+      // Display filepath as relative if within cwd, otherwise absolute
+      const displayPath = relative(process.cwd(), file.filePath);
+      const finalPath = displayPath.startsWith('..')
+        ? file.filePath
+        : displayPath;
+
+      lines.push(
+        `${colorize('dim', ansiChars.bullet)} ${colorize('brightMagenta', colorize('bold', finalPath))}`,
+      );
+
+      for (const suite of file.suites) {
+        lines.push(
+          `  ${colorize('dim', ansiChars.bullet)} ${colorize('brightWhite', suite.name)}`,
+        );
+
+        for (const task of suite.tasks) {
+          const statusIcon = task.error
+            ? colorize('brightRed', ansiChars.cross)
+            : colorize('brightCyan', ansiChars.checkmark);
+
+          if (task.error) {
+            lines.push(
+              `    ${statusIcon} ${colorize('white', task.name)} ${colorize('dim', ansiChars.bullet)} ${colorize('brightRed', 'failed')}`,
+            );
+          } else {
+            const mean = formatTime(task.mean);
+            const opsStr = task.opsPerSecond.toLocaleString('en-US', {
+              maximumFractionDigits: 0,
+            });
+            const rmeStr = task.marginOfError.toFixed(2);
+
+            lines.push(
+              `    ${statusIcon} ${colorize('white', task.name)}: ${colorize('brightMagenta', mean)} ${colorize('dim', ansiChars.bullet)} ${colorize('brightBlue', `${ansiChars.plusMinus}${rmeStr}`)} ${colorize('dim', ansiChars.bullet)} ${colorize('magenta', opsStr)} ops/sec`,
+            );
+
+            if (task.iterations > 0) {
+              lines.push(
+                `      ${colorize('dim', `${task.iterations} iterations, cv: ${task.cv.toFixed(1)}%`)}`,
+              );
+            }
+          }
+        }
+      }
+
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format as complete JSON
+   */
+  formatJson(data: ShowResult): string {
+    return JSON.stringify(data, null, 2);
+  }
+}
+
+/**
+ * Format nanoseconds as a human-readable time string
+ */
+const formatTime = (ns: number): string => {
+  if (ns < 1000) {
+    return `${ns.toFixed(2)}ns`;
+  }
+  if (ns < 1000000) {
+    return `${(ns / 1000).toFixed(3)}µs`;
+  }
+  return `${(ns / 1000000).toFixed(3)}ms`;
+};