modestbench 0.0.1

package/dist/core/engine.cjs

@@ -0,0 +1,669 @@
+"use strict";
+/**
+ * ModestBench Core Engine
+ *
+ * Main orchestrator for benchmark discovery, validation, and execution.
+ * Implements the BenchmarkEngine interface with dependency injection
+ * architecture.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ModestBenchEngine = void 0;
+/**
+ * Abstract benchmark execution engine with dependency injection
+ *
+ * Provides generic orchestration logic for benchmark discovery, validation, and
+ * execution. Concrete implementations must provide the task execution logic via
+ * the executeBenchmarkTask method.
+ */
+class ModestBenchEngine {
+    configManager;
+    errorManager;
+    fileLoader;
+    historyStorage;
+    progressManager;
+    reporterRegistry;
+    constructor(dependencies) {
+        this.configManager = dependencies.configManager;
+        this.fileLoader = dependencies.fileLoader;
+        this.reporterRegistry = dependencies.reporterRegistry;
+        this.historyStorage = dependencies.historyStorage;
+        this.progressManager = dependencies.progressManager;
+        this.errorManager = dependencies.errorManager;
+    }
+    /**
+     * Discover benchmark files matching the pattern(s)
+     */
+    async discover(pattern, exclude) {
+        try {
+            return await this.fileLoader.discover(pattern, exclude);
+        }
+        catch (error) {
+            const discoveryError = error instanceof Error ? error : new Error(String(error));
+            this.errorManager.handleError(discoveryError, {
+                metadata: { exclude, pattern },
+                phase: 'discovery',
+                timestamp: new Date(),
+            });
+            throw new Error(`File discovery failed: ${discoveryError.message}`);
+        }
+    }
+    /**
+     * Execute benchmarks with the given configuration
+     */
+    async execute(config, reporters = [], signal) {
+        const startTime = new Date();
+        let currentPhase = 'discovery';
+        try {
+            // 1. Merge configuration with defaults
+            currentPhase = 'discovery';
+            const mergedConfig = await this.configManager.load(undefined, // No specific config path for now
+            config);
+            // 2. Discover files if not explicitly provided
+            const files = config.files ||
+                (await this.discover(mergedConfig.pattern, mergedConfig.exclude));
+            if (files.length === 0) {
+                const error = new Error('No benchmark files found matching the pattern');
+                this.errorManager.handleError(error, {
+                    phase: currentPhase,
+                    timestamp: new Date(),
+                });
+                throw error;
+            }
+            // 3. Validate files
+            currentPhase = 'validation';
+            const validationResult = await this.validate(files);
+            if (!validationResult.valid) {
+                const error = new Error(`Validation failed: ${validationResult.errors.map((e) => e.message).join(', ')}`);
+                this.errorManager.handleError(error, {
+                    phase: currentPhase,
+                    timestamp: new Date(),
+                });
+                throw error;
+            }
+            // 4. Initialize progress tracking
+            currentPhase = 'setup';
+            const runId = this.generateRunId();
+            // Pre-calculate total tasks for progress tracking
+            let totalTasks = 0;
+            let totalSuites = 0;
+            for (const filePath of files) {
+                try {
+                    const benchmarkFile = await this.fileLoader.load(filePath);
+                    const benchmarkDef = benchmarkFile.exports;
+                    if (benchmarkDef?.suites && typeof benchmarkDef.suites === 'object') {
+                        const fileTags = benchmarkDef.tags;
+                        for (const [_suiteName, suiteData] of Object.entries(benchmarkDef.suites)) {
+                            // Use shared filtering logic
+                            const { anyTaskMatches, suiteMatches, tasksToRun } = this.getFilteredTasksForSuite(suiteData, fileTags, mergedConfig.tags, mergedConfig.excludeTags);
+                            // Count suite only if it or any of its tasks match
+                            if (suiteMatches || anyTaskMatches) {
+                                totalSuites++;
+                                totalTasks += tasksToRun.length;
+                            }
+                        }
+                    }
+                }
+                catch (error) {
+                    // If we can't load a file for counting, we'll handle it during execution
+                    // Only show warning if not in quiet mode
+                    if (!mergedConfig.quiet) {
+                        console.warn(`Warning: Could not pre-load ${filePath} for task counting:`, error);
+                    }
+                }
+            }
+            // Create initial run structure for progress tracking
+            const gitInfo = await this.getGitInfo();
+            const ciInfo = await this.getCiInfo();
+            const initialRun = {
+                config: mergedConfig,
+                duration: 0,
+                endTime: startTime,
+                environment: await this.getEnvironmentInfo(),
+                files: [],
+                id: runId,
+                startTime,
+                ...(gitInfo && { git: gitInfo }),
+                ...(ciInfo && { ci: ciInfo }),
+                summary: {
+                    failedTasks: 0,
+                    fastest: null,
+                    overallMean: 0,
+                    passedTasks: 0,
+                    slowest: null,
+                    totalFiles: files.length,
+                    totalOperations: 0,
+                    totalSuites,
+                    totalTasks,
+                },
+            };
+            this.progressManager.initialize(initialRun);
+            // Register progress callbacks with reporters that support them
+            for (const reporter of reporters) {
+                if (typeof reporter.onProgress === 'function') {
+                    this.progressManager.onProgress((state) => {
+                        void reporter.onProgress(state);
+                    });
+                }
+            }
+            // 5. Call reporter onStart lifecycle method
+            await this.callReporters(reporters, 'onStart', initialRun);
+            // 6. Execute benchmark files
+            currentPhase = 'execution';
+            const fileResults = [];
+            for (const filePath of files) {
+                try {
+                    // Call reporter onFileStart
+                    await this.callReporters(reporters, 'onFileStart', filePath);
+                    const fileResult = await this.executeBenchmarkFile(filePath, mergedConfig, reporters, signal);
+                    fileResults.push(fileResult);
+                    // Call reporter onFileEnd
+                    await this.callReporters(reporters, 'onFileEnd', fileResult);
+                    // Update progress
+                    this.progressManager.update({
+                        currentFile: filePath,
+                        filesCompleted: fileResults.length,
+                    });
+                }
+                catch (error) {
+                    const fileError = error instanceof Error ? error : new Error(String(error));
+                    this.errorManager.handleError(fileError, {
+                        file: filePath,
+                        phase: currentPhase,
+                        timestamp: new Date(),
+                    });
+                    // Call reporter onError
+                    await this.callReporters(reporters, 'onError', fileError);
+                    // Create error result for this file
+                    const now = new Date();
+                    const errorResult = {
+                        duration: 0,
+                        endTime: now,
+                        error: fileError,
+                        filePath,
+                        startTime: now,
+                        suites: [],
+                    };
+                    fileResults.push(errorResult);
+                    // Call reporter onFileEnd for error case
+                    await this.callReporters(reporters, 'onFileEnd', errorResult);
+                }
+            }
+            // Calculate summary statistics
+            const finalTotalSuites = fileResults.reduce((sum, file) => sum + file.suites.length, 0);
+            const allTasks = fileResults.flatMap((file) => file.suites.flatMap((suite) => suite.tasks));
+            const finalTotalTasks = allTasks.length;
+            const failedTasks = allTasks.filter((task) => task.error).length;
+            const passedTasks = finalTotalTasks - failedTasks;
+            let fastest = null;
+            let slowest = null;
+            let totalOperations = 0;
+            let totalTime = 0;
+            for (const task of allTasks) {
+                if (!task.error) {
+                    totalOperations += task.iterations;
+                    totalTime += task.mean * task.iterations;
+                    if (!fastest || task.mean < fastest.mean) {
+                        fastest = task;
+                    }
+                    if (!slowest || task.mean > slowest.mean) {
+                        slowest = task;
+                    }
+                }
+            }
+            const overallMean = totalOperations > 0 ? totalTime / totalOperations : 0;
+            const endTime = new Date();
+            const finalRun = {
+                ...initialRun,
+                duration: endTime.getTime() - startTime.getTime(),
+                endTime,
+                files: fileResults,
+                summary: {
+                    failedTasks,
+                    fastest,
+                    overallMean,
+                    passedTasks,
+                    slowest,
+                    totalFiles: files.length,
+                    totalOperations,
+                    totalSuites: finalTotalSuites,
+                    totalTasks: finalTotalTasks,
+                },
+            };
+            // 7. Save to history
+            await this.historyStorage.saveRun(finalRun);
+            // 8. Call reporter onEnd lifecycle method
+            await this.callReporters(reporters, 'onEnd', finalRun);
+            // 9. Return completed run
+            return finalRun;
+        }
+        catch (error) {
+            const executionError = error instanceof Error ? error : new Error(String(error));
+            const handledError = this.errorManager.handleError(executionError, {
+                phase: currentPhase,
+                timestamp: new Date(),
+            });
+            // Re-throw the original error with more context
+            throw new Error(`Benchmark execution failed: ${handledError.message}`);
+        }
+    }
+    /**
+     * Get the configuration manager
+     */
+    getConfigManager() {
+        return this.configManager;
+    }
+    /**
+     * Get the file loader
+     */
+    getFileLoader() {
+        return this.fileLoader;
+    }
+    /**
+     * Get the history storage
+     */
+    getHistoryStorage() {
+        return this.historyStorage;
+    }
+    /**
+     * Get the progress manager
+     */
+    getProgressManager() {
+        return this.progressManager;
+    }
+    /**
+     * Get all available reporters
+     */
+    getReporters() {
+        return this.reporterRegistry.getAll();
+    }
+    /**
+     * Register a custom reporter
+     */
+    registerReporter(name, reporter) {
+        this.reporterRegistry.register(name, reporter);
+    }
+    /**
+     * Validate benchmark files without executing them
+     */
+    async validate(files) {
+        try {
+            const errors = [];
+            const warnings = [];
+            const validatedFiles = [];
+            // Validate each file
+            for (const file of files) {
+                try {
+                    const result = await this.fileLoader.validate(file);
+                    validatedFiles.push(file);
+                    errors.push(...result.errors);
+                    warnings.push(...result.warnings);
+                }
+                catch (error) {
+                    const validationError = error instanceof Error ? error : new Error(String(error));
+                    this.errorManager.handleError(validationError, {
+                        file,
+                        phase: 'validation',
+                        timestamp: new Date(),
+                    });
+                    errors.push({
+                        code: 'FILE_VALIDATION_ERROR',
+                        file,
+                        message: `Failed to validate file: ${validationError.message}`,
+                        severity: 'error',
+                    });
+                }
+            }
+            return {
+                errors,
+                files: validatedFiles,
+                valid: errors.length === 0,
+                warnings,
+            };
+        }
+        catch (error) {
+            throw new Error(`Validation failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * Helper method to call a lifecycle method on all reporters
+     */
+    async callReporters(reporters, method, ...args) {
+        for (const reporter of reporters) {
+            try {
+                const reporterMethod = reporter[method];
+                if (typeof reporterMethod === 'function') {
+                    const result = reporterMethod.call(reporter, ...args);
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+                    if (result && typeof result.then === 'function') {
+                        await result;
+                    }
+                }
+            }
+            catch (error) {
+                // Log reporter errors but don't fail the benchmark run
+                console.error(`Reporter error in ${method}:`, error);
+            }
+        }
+    }
+    /**
+     * Execute a single benchmark file and return its results
+     */
+    async executeBenchmarkFile(filePath, config, reporters = [], signal) {
+        const startTime = new Date();
+        try {
+            // Load the benchmark file using the file loader
+            const benchmarkFile = await this.fileLoader.load(filePath);
+            const benchmarkDef = benchmarkFile.exports;
+            if (!benchmarkDef || typeof benchmarkDef !== 'object') {
+                throw new Error('Benchmark file must export a default object with suites');
+            }
+            const suiteResults = [];
+            const fileTags = benchmarkDef.tags;
+            // Process each suite in the file
+            if (benchmarkDef.suites && typeof benchmarkDef.suites === 'object') {
+                for (const [suiteName, suiteData] of Object.entries(benchmarkDef.suites)) {
+                    // Use shared filtering logic
+                    const { anyTaskMatches, suiteMatches } = this.getFilteredTasksForSuite(suiteData, fileTags, config.tags, config.excludeTags);
+                    // Skip suite only if neither the suite nor any of its tasks match
+                    if (!suiteMatches && !anyTaskMatches) {
+                        continue;
+                    }
+                    await this.callReporters(reporters, 'onSuiteStart', suiteName);
+                    const suiteResult = await this.executeBenchmarkSuite(suiteName, suiteData, config, reporters, signal, fileTags);
+                    await this.callReporters(reporters, 'onSuiteEnd', suiteResult);
+                    suiteResults.push(suiteResult);
+                }
+            }
+            const endTime = new Date();
+            return {
+                config: benchmarkDef.config,
+                duration: endTime.getTime() - startTime.getTime(),
+                endTime,
+                filePath,
+                startTime,
+                suites: suiteResults,
+            };
+        }
+        catch (error) {
+            const endTime = new Date();
+            const executionError = error instanceof Error ? error : new Error(String(error));
+            return {
+                duration: endTime.getTime() - startTime.getTime(),
+                endTime,
+                error: executionError,
+                filePath,
+                startTime,
+                suites: [],
+            };
+        }
+    }
+    /**
+     * Execute a single benchmark suite and return its results
+     */
+    async executeBenchmarkSuite(suiteName, suiteData, config, reporters = [], signal, fileTags) {
+        const startTime = new Date();
+        try {
+            const taskResults = [];
+            // Use shared filtering logic to determine which tasks will run
+            const { tasksToRun } = this.getFilteredTasksForSuite(suiteData, fileTags, config.tags, config.excludeTags);
+            // Only run setup/teardown if there are tasks to execute
+            if (tasksToRun.length === 0) {
+                // No tasks match the filters, return empty suite result
+                const endTime = new Date();
+                return {
+                    duration: endTime.getTime() - startTime.getTime(),
+                    endTime,
+                    name: suiteName,
+                    startTime,
+                    tasks: [],
+                    ...(suiteData.config !== undefined && { config: suiteData.config }),
+                    ...(suiteData.metadata !== undefined && {
+                        metadata: suiteData.metadata,
+                    }),
+                    ...(suiteData.tags !== undefined && { tags: suiteData.tags }),
+                };
+            }
+            // Run suite setup if provided
+            if (suiteData.setup && typeof suiteData.setup === 'function') {
+                try {
+                    await suiteData.setup();
+                }
+                catch (error) {
+                    const setupError = error instanceof Error
+                        ? error
+                        : new Error(`Setup failed: ${String(error)}`);
+                    throw new Error(`Suite setup failed: ${setupError.message}`);
+                }
+            }
+            try {
+                // Process each task that passed filtering
+                for (const [taskName, taskData] of tasksToRun) {
+                    await this.callReporters(reporters, 'onTaskStart', taskName);
+                    // Mark task as in-progress (shows as 0.5 progress for current task)
+                    const currentState = this.progressManager.getState();
+                    this.progressManager.update({
+                        tasksCompleted: currentState.tasksCompleted + 0.5,
+                    });
+                    const taskResult = await this.executeBenchmarkTask(taskName, taskData, config, reporters, signal);
+                    await this.callReporters(reporters, 'onTaskResult', taskResult);
+                    taskResults.push(taskResult);
+                    // Update task-level progress - task is now complete (remove the 0.5 and add 1)
+                    this.progressManager.update({
+                        tasksCompleted: currentState.tasksCompleted + 1,
+                    });
+                }
+            }
+            finally {
+                // Run suite teardown if provided (always runs, even if benchmarks fail)
+                if (suiteData.teardown && typeof suiteData.teardown === 'function') {
+                    try {
+                        await suiteData.teardown();
+                    }
+                    catch (error) {
+                        // Log teardown errors but don't fail the suite
+                        const teardownError = error instanceof Error ? error : new Error(String(error));
+                        console.error(`Warning: Suite teardown failed for "${suiteName}":`, teardownError.message);
+                    }
+                }
+            }
+            const endTime = new Date();
+            return {
+                duration: endTime.getTime() - startTime.getTime(),
+                endTime,
+                name: suiteName,
+                startTime,
+                tasks: taskResults,
+                ...(suiteData.config !== undefined && { config: suiteData.config }),
+                ...(suiteData.metadata !== undefined && {
+                    metadata: suiteData.metadata,
+                }),
+                ...(suiteData.tags !== undefined && { tags: suiteData.tags }),
+            };
+        }
+        catch (error) {
+            const endTime = new Date();
+            const executionError = error instanceof Error ? error : new Error(String(error));
+            return {
+                duration: endTime.getTime() - startTime.getTime(),
+                endTime,
+                error: executionError,
+                name: suiteName,
+                startTime,
+                tasks: [],
+            };
+        }
+    }
+    /**
+     * Generate a unique run ID
+     */
+    generateRunId() {
+        return `run-${Date.now()}-${Math.random().toString(36).substring(2, 15)}`;
+    }
+    /**
+     * Get CI/CD information if available
+     */
+    async getCiInfo() {
+        const process = await Promise.resolve().then(() => __importStar(require('node:process')));
+        if (!process.env.CI) {
+            return undefined;
+        }
+        // Detect common CI providers
+        if (process.env.GITHUB_ACTIONS) {
+            return {
+                provider: 'GitHub Actions',
+                ...(process.env.GITHUB_RUN_NUMBER && {
+                    buildNumber: process.env.GITHUB_RUN_NUMBER,
+                }),
+                ...(process.env.GITHUB_REPOSITORY &&
+                    process.env.GITHUB_RUN_ID && {
+                    buildUrl: `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
+                }),
+                ...(process.env.GITHUB_EVENT_NAME === 'pull_request' &&
+                    process.env.GITHUB_REF_NAME && {
+                    pullRequest: process.env.GITHUB_REF_NAME,
+                }),
+                ...(process.env.GITHUB_REF_NAME && {
+                    branch: process.env.GITHUB_REF_NAME,
+                }),
+                ...(process.env.GITHUB_SHA && { commit: process.env.GITHUB_SHA }),
+            };
+        }
+        // Default CI info
+        return {
+            provider: 'Unknown CI',
+            ...(process.env.BRANCH && { branch: process.env.BRANCH }),
+            ...(process.env.COMMIT && { commit: process.env.COMMIT }),
+        };
+    }
+    /**
+     * Get environment information
+     */
+    async getEnvironmentInfo() {
+        const os = await Promise.resolve().then(() => __importStar(require('node:os')));
+        const process = await Promise.resolve().then(() => __importStar(require('node:process')));
+        return {
+            arch: process.arch,
+            availableMemory: os.freemem(),
+            cpu: {
+                cores: os.cpus().length,
+                model: os.cpus()[0]?.model || 'Unknown',
+                speed: os.cpus()[0]?.speed || 0,
+            },
+            env: {
+                CI: process.env.CI || 'false',
+                NODE_ENV: process.env.NODE_ENV || 'development',
+            },
+            hostname: os.hostname(),
+            memory: {
+                free: os.freemem(),
+                total: os.totalmem(),
+                used: os.totalmem() - os.freemem(),
+            },
+            nodeVersion: process.version,
+            platform: process.platform,
+        };
+    }
+    /**
+     * Get filtered tasks for a suite based on tag filtering Returns suite match
+     * status and list of tasks to run
+     */
+    getFilteredTasksForSuite(suiteData, fileTags, includeTags, excludeTags) {
+        // Check if suite itself matches filters
+        const mergedSuiteTags = this.mergeTags(fileTags, suiteData.tags);
+        const suiteMatches = this.matchesTags(mergedSuiteTags, includeTags, excludeTags);
+        // Check which tasks match filters
+        const tasksToRun = [];
+        if (suiteData.benchmarks && typeof suiteData.benchmarks === 'object') {
+            for (const [taskName, taskData] of Object.entries(suiteData.benchmarks)) {
+                // Merge task tags with suite and file tags (cascading)
+                const mergedTaskTags = this.mergeTags(mergedSuiteTags, taskData.tags);
+                // Check if task matches tag filters
+                if (this.matchesTags(mergedTaskTags, includeTags, excludeTags)) {
+                    tasksToRun.push([taskName, taskData]);
+                }
+            }
+        }
+        return {
+            anyTaskMatches: tasksToRun.length > 0,
+            suiteMatches,
+            tasksToRun,
+        };
+    }
+    /**
+     * Get Git information if available
+     */
+    async getGitInfo() {
+        // TODO: Implement Git information extraction
+        // This would use child_process to run git commands
+        return undefined;
+    }
+    /**
+     * Check if item tags match the filter criteria (OR logic)
+     */
+    matchesTags(itemTags, includeTags, excludeTags) {
+        const tags = itemTags || [];
+        // If exclude tags specified and any match, exclude this item
+        if (excludeTags.length > 0 &&
+            excludeTags.some((tag) => tags.includes(tag))) {
+            return false;
+        }
+        // If include tags specified, at least one must match
+        if (includeTags.length > 0) {
+            return includeTags.some((tag) => tags.includes(tag));
+        }
+        // No filters = include everything
+        return true;
+    }
+    /**
+     * Merge tags from parent to child (cascading)
+     */
+    mergeTags(parentTags, childTags) {
+        const merged = new Set();
+        if (parentTags) {
+            for (const tag of parentTags) {
+                merged.add(tag);
+            }
+        }
+        if (childTags) {
+            for (const tag of childTags) {
+                merged.add(tag);
+            }
+        }
+        return merged.size > 0 ? Array.from(merged) : undefined;
+    }
+}
+exports.ModestBenchEngine = ModestBenchEngine;
+//# sourceMappingURL=engine.js.map