@testsmith/perfornium 0.1.0

package/dist/core/test-runner.d.ts

@@ -0,0 +1,34 @@
+import { TestConfiguration } from '../config';
+import { MetricsCollector } from '../metrics/collector';
+export declare class TestRunner {
+    private config;
+    private metrics;
+    private handlers;
+    private outputs;
+    private activeVUs;
+    private isRunning;
+    private startTime;
+    constructor(config: TestConfiguration);
+    run(): Promise<void>;
+    private setupCSVBaseDirectory;
+    stop(): Promise<void>;
+    private initialize;
+    private initializeProtocolHandlers;
+    private getRequiredProtocols;
+    private findWSDLUrl;
+    private initializeOutputs;
+    /**
+     * Process template variables in file paths and automatically add timestamp if not present
+     */
+    private processTemplateFilePath;
+    private executeLoadPattern;
+    private getLoadPatternForPhase;
+    private createVUFactory;
+    private waitForVUsToComplete;
+    private cleanup;
+    private finalize;
+    private logSummary;
+    private logErrorDetails;
+    private generateReport;
+    getMetrics(): MetricsCollector;
+}

package/dist/core/test-runner.js

@@ -0,0 +1,465 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestRunner = void 0;
+const collector_1 = require("../metrics/collector");
+const virtual_user_1 = require("./virtual-user");
+const handler_1 = require("../protocols/rest/handler");
+const handler_2 = require("../protocols/soap/handler");
+const handler_3 = require("../protocols/web/handler");
+const basic_1 = require("../load-patterns/basic");
+const stepping_1 = require("../load-patterns/stepping");
+const arrivals_1 = require("../load-patterns/arrivals");
+const csv_1 = require("../outputs/csv");
+const json_1 = require("../outputs/json");
+const influxdb_1 = require("../outputs/influxdb");
+const graphite_1 = require("../outputs/graphite");
+const webhook_1 = require("../outputs/webhook");
+const logger_1 = require("../utils/logger");
+const time_1 = require("../utils/time");
+const csv_data_provider_1 = require("./csv-data-provider");
+const file_manager_1 = require("../utils/file-manager");
+class TestRunner {
+    constructor(config) {
+        this.handlers = new Map();
+        this.outputs = [];
+        this.activeVUs = [];
+        this.isRunning = false;
+        this.startTime = 0;
+        this.config = config;
+        this.metrics = new collector_1.MetricsCollector();
+        // Set log level based on debug config
+        if (config.debug?.log_level || config.global?.debug?.log_level) {
+            const logLevel = config.debug?.log_level || config.global?.debug?.log_level;
+            switch (logLevel) {
+                case 'debug':
+                    logger_1.logger.setLevel(logger_1.LogLevel.DEBUG);
+                    break;
+                case 'info':
+                    logger_1.logger.setLevel(logger_1.LogLevel.INFO);
+                    break;
+                case 'warn':
+                    logger_1.logger.setLevel(logger_1.LogLevel.WARN);
+                    break;
+                case 'error':
+                    logger_1.logger.setLevel(logger_1.LogLevel.ERROR);
+                    break;
+            }
+        }
+    }
+    async run() {
+        logger_1.logger.info(`🚀 Starting test: ${this.config.name}`);
+        this.isRunning = true;
+        this.startTime = Date.now();
+        try {
+            await this.initialize();
+            // NO CSV termination callback setup needed anymore
+            await this.executeLoadPattern();
+            await this.finalize();
+            const duration = Date.now() - this.startTime;
+            logger_1.logger.success(`✅ Test completed successfully in ${(duration / 1000).toFixed(1)}s`);
+        }
+        catch (error) {
+            logger_1.logger.error('❌ Test failed:', error);
+            throw error;
+        }
+        finally {
+            this.isRunning = false;
+        }
+    }
+    setupCSVBaseDirectory() {
+        const hasCSVScenarios = this.config.scenarios?.some((s) => s.csv_data);
+        if (hasCSVScenarios) {
+            const baseDir = process.cwd();
+            csv_data_provider_1.CSVDataProvider.setBaseDir(baseDir);
+            // Check if any scenario has cycleOnExhaustion: false
+            const hasStopOnExhaustion = this.config.scenarios?.some((s) => s.csv_data && s.csv_data.cycleOnExhaustion === false);
+            if (hasStopOnExhaustion) {
+                logger_1.logger.info('⏹️ CSV exhaustion handling enabled - individual VUs will stop when CSV data is exhausted');
+            }
+            logger_1.logger.debug(`Set CSV base directory: ${baseDir}`);
+        }
+    }
+    // FIXED: Added stop method that was missing
+    async stop() {
+        logger_1.logger.info('⏹️  Stopping test...');
+        this.isRunning = false;
+        // Stop all active VUs
+        this.activeVUs.forEach(vu => vu.stop());
+        // Wait for VUs to finish current operations
+        await this.waitForVUsToComplete(10000); // 10 second timeout
+        // Cleanup handlers
+        await this.cleanup();
+    }
+    async initialize() {
+        logger_1.logger.debug('🔧 Initializing test runner...');
+        // Initialize protocol handlers
+        await this.initializeProtocolHandlers();
+        // Initialize outputs
+        await this.initializeOutputs();
+        // Setup metrics collection
+        this.metrics.start();
+        this.metrics.on('result', (result) => {
+            this.outputs.forEach(output => {
+                if (output && typeof output.writeResult === 'function') {
+                    output.writeResult(result).catch(err => logger_1.logger.warn('⚠️  Output write failed:', err));
+                }
+            });
+        });
+        logger_1.logger.debug('✅ Test runner initialized');
+    }
+    async initializeProtocolHandlers() {
+        // Check which protocols are needed based on scenarios
+        const protocolsNeeded = this.getRequiredProtocols();
+        const debugConfig = this.config.debug || this.config.global?.debug;
+        // Initialize REST handler if needed
+        if (protocolsNeeded.has('rest')) {
+            const handler = new handler_1.RESTHandler(this.config.global?.base_url, this.config.global?.headers || {}, this.config.global?.timeout, debugConfig // Pass debug config to handler
+            );
+            this.handlers.set('rest', handler);
+            logger_1.logger.debug('🌐 REST handler initialized');
+        }
+        // Initialize SOAP handler if needed
+        if (protocolsNeeded.has('soap')) {
+            const wsdlUrl = this.findWSDLUrl();
+            if (wsdlUrl) {
+                const handler = new handler_2.SOAPHandler(wsdlUrl);
+                await handler.initialize();
+                this.handlers.set('soap', handler);
+                logger_1.logger.debug('🧼 SOAP handler initialized');
+            }
+        }
+        // Initialize Web handler if needed
+        if (protocolsNeeded.has('web')) {
+            const browserConfig = this.config.global?.web || this.config.global?.browser || {};
+            const webConfig = {
+                type: browserConfig.type || 'chromium',
+                headless: browserConfig.headless ?? true,
+                base_url: browserConfig.base_url || this.config.global?.base_url,
+                viewport: browserConfig.viewport,
+                slow_mo: browserConfig.slow_mo,
+                highlight: browserConfig.highlight,
+                clear_storage: browserConfig.clear_storage
+            };
+            const handler = new handler_3.WebHandler(webConfig);
+            await handler.initialize();
+            this.handlers.set('web', handler);
+            logger_1.logger.debug('🌐 Web handler initialized');
+        }
+    }
+    getRequiredProtocols() {
+        const protocols = new Set();
+        for (const scenario of this.config.scenarios) {
+            for (const step of scenario.steps) {
+                protocols.add(step.type || 'rest');
+            }
+        }
+        return protocols;
+    }
+    findWSDLUrl() {
+        // First check global config
+        if (this.config.global?.wsdl_url) {
+            return this.config.global.wsdl_url;
+        }
+        // Fallback: check individual steps (for backward compatibility)
+        for (const scenario of this.config.scenarios) {
+            for (const step of scenario.steps) {
+                if (step.type === 'soap' && 'wsdl' in step && step.wsdl) {
+                    return step.wsdl;
+                }
+            }
+        }
+        return null;
+    }
+    async initializeOutputs() {
+        if (!this.config.outputs)
+            return;
+        for (const outputConfig of this.config.outputs) {
+            // Skip disabled outputs
+            if (outputConfig.enabled === false) {
+                continue;
+            }
+            let output;
+            try {
+                // Process timestamp templates in file paths
+                const processedFilePath = this.processTemplateFilePath(outputConfig.file);
+                switch (outputConfig.type) {
+                    case 'csv':
+                        output = new csv_1.CSVOutput(processedFilePath);
+                        break;
+                    case 'json':
+                        output = new json_1.JSONOutput(processedFilePath);
+                        break;
+                    case 'influxdb':
+                        output = new influxdb_1.InfluxDBOutput(outputConfig.url, outputConfig.database, outputConfig.tags);
+                        break;
+                    case 'graphite':
+                        const [host, port] = (outputConfig.url || 'localhost:2003').split(':');
+                        output = new graphite_1.GraphiteOutput(host, parseInt(port || '2003'), 'perfornium');
+                        break;
+                    case 'webhook':
+                        output = new webhook_1.WebhookOutput(outputConfig.url, outputConfig.headers || {}, 'json', outputConfig.template);
+                        break;
+                    default:
+                        logger_1.logger.warn(`⚠️  Unsupported output type: ${outputConfig.type}`);
+                        continue;
+                }
+                await output.initialize();
+                this.outputs.push(output);
+                logger_1.logger.debug(`📊 ${outputConfig.type} output initialized`);
+            }
+            catch (error) {
+                logger_1.logger.warn(`⚠️  Failed to initialize ${outputConfig.type} output:`, error);
+            }
+        }
+    }
+    /**
+     * Process template variables in file paths and automatically add timestamp if not present
+     */
+    processTemplateFilePath(filePath) {
+        if (!filePath) {
+            return `results/${this.config.name}-{{timestamp}}.csv`;
+        }
+        // If no timestamp placeholder exists, automatically add one before the extension
+        let processedPath = filePath;
+        if (!filePath.includes('{{timestamp}}')) {
+            const lastDot = filePath.lastIndexOf('.');
+            if (lastDot > 0) {
+                const name = filePath.substring(0, lastDot);
+                const ext = filePath.substring(lastDot);
+                processedPath = `${name}-{{timestamp}}${ext}`;
+            }
+            else {
+                processedPath = `${filePath}-{{timestamp}}`;
+            }
+        }
+        // Use FileManager to process timestamp templates
+        return file_manager_1.FileManager.processFilePath(processedPath);
+    }
+    async executeLoadPattern() {
+        // Setup base directory for CSV files BEFORE creating VUs
+        this.setupCSVBaseDirectory();
+        const vuFactory = this.createVUFactory();
+        // Support both single load config and array of phases
+        const loadPhases = Array.isArray(this.config.load) ? this.config.load : [this.config.load];
+        logger_1.logger.info(`📈 Executing ${loadPhases.length} load phase(s)`);
+        // Execute each load phase sequentially
+        for (let i = 0; i < loadPhases.length; i++) {
+            const phase = loadPhases[i];
+            const phaseName = phase.name || `Phase ${i + 1}`;
+            logger_1.logger.info(`\n🔄 Starting ${phaseName}: ${phase.pattern} pattern`);
+            if (phase.virtual_users || phase.vus) {
+                logger_1.logger.info(`   Users: ${phase.virtual_users || phase.vus}, Duration: ${phase.duration}`);
+            }
+            const pattern = this.getLoadPatternForPhase(phase);
+            await pattern.execute(phase, vuFactory);
+            // Wait for phase to complete
+            if (phase.duration) {
+                await this.waitForVUsToComplete();
+            }
+            else {
+                // For "run once" mode, clear activeVUs
+                this.activeVUs.forEach(vu => vu.stop());
+                this.activeVUs.length = 0;
+            }
+            logger_1.logger.success(`✅ Completed ${phaseName}`);
+            // Small gap between phases
+            if (i < loadPhases.length - 1) {
+                logger_1.logger.info('⏸️  Pausing 2s between phases...');
+                await new Promise(resolve => setTimeout(resolve, 2000));
+            }
+        }
+        logger_1.logger.success(`\n🎉 All ${loadPhases.length} load phase(s) completed`);
+    }
+    getLoadPatternForPhase(phase) {
+        let pattern;
+        switch (phase.pattern) {
+            case 'basic':
+                pattern = new basic_1.BasicPattern();
+                if ('setTestRunningChecker' in pattern) {
+                    pattern.setTestRunningChecker(() => this.isRunning);
+                }
+                break;
+            case 'stepping':
+                pattern = new stepping_1.SteppingPattern();
+                break;
+            case 'arrivals':
+                pattern = new arrivals_1.ArrivalsPattern();
+                break;
+            default:
+                throw new Error(`Unsupported load pattern: ${phase.pattern}`);
+        }
+        return pattern;
+    }
+    createVUFactory() {
+        return {
+            create: async (id) => {
+                // Pass global think time to VU
+                const globalThinkTime = this.config.global?.think_time;
+                // Build global CSV options if configured
+                let globalCSV;
+                if (this.config.global?.csv_data) {
+                    globalCSV = {
+                        config: this.config.global.csv_data,
+                        mode: this.config.global.csv_mode || 'next'
+                    };
+                    logger_1.logger.debug(`VU ${id}: Global CSV configured from test config`);
+                }
+                const vu = new virtual_user_1.VirtualUser(id, this.metrics, this.handlers, this.config.name, undefined, // vuHooks
+                globalThinkTime, globalCSV);
+                // CRITICAL: Must await the CSV initialization
+                logger_1.logger.debug(`Initializing VU ${id} with scenarios (including CSV)...`);
+                await vu.setScenarios(this.config.scenarios);
+                logger_1.logger.debug(`VU ${id} fully initialized`);
+                this.activeVUs.push(vu);
+                return vu;
+            },
+            getMetrics: () => this.metrics
+        };
+    }
+    async waitForVUsToComplete(timeoutMs = 60000) {
+        const startTime = Date.now();
+        while (this.activeVUs.length > 0 && this.isRunning) {
+            const elapsed = Date.now() - startTime;
+            if (elapsed > timeoutMs) {
+                logger_1.logger.warn(`⚠️  Timeout waiting for ${this.activeVUs.length} VUs to complete`);
+                // Force stop remaining VUs
+                this.activeVUs.forEach(vu => vu.stop());
+                break;
+            }
+            await (0, time_1.sleep)(1000);
+            if (this.activeVUs.length > 0 && elapsed % 5000 === 0) { // Log every 5 seconds
+                logger_1.logger.debug(`⏳ Waiting for ${this.activeVUs.length} VUs to complete...`);
+            }
+        }
+        logger_1.logger.debug('✅ All VUs completed');
+    }
+    async cleanup() {
+        logger_1.logger.debug('🧹 Cleaning up handlers...');
+        // Cleanup protocol handlers
+        for (const [name, handler] of this.handlers) {
+            try {
+                if (handler.cleanup) {
+                    await handler.cleanup();
+                }
+            }
+            catch (error) {
+                logger_1.logger.warn(`⚠️  Error cleaning up ${name} handler:`, error);
+            }
+        }
+    }
+    async finalize() {
+        logger_1.logger.debug('🏁 Finalizing test...');
+        // Cleanup handlers
+        await this.cleanup();
+        // Generate summary and write to outputs
+        const summary = this.metrics.getSummary();
+        this.logSummary(summary);
+        this.logErrorDetails(summary);
+        // Write summary to outputs
+        for (const output of this.outputs) {
+            try {
+                await output.writeSummary(summary);
+                await output.finalize();
+            }
+            catch (error) {
+                logger_1.logger.warn('⚠️  Output finalization failed:', error);
+            }
+        }
+        // Generate HTML report if configured
+        if (this.config.report?.generate) {
+            await this.generateReport(summary);
+        }
+    }
+    logSummary(summary) {
+        logger_1.logger.info('📊 Test Summary:');
+        logger_1.logger.info(`   Total Requests: ${summary.total_requests}`);
+        logger_1.logger.info(`   Success Rate: ${summary.success_rate.toFixed(2)}%`);
+        logger_1.logger.info(`   Avg Response Time: ${summary.avg_response_time.toFixed(2)}ms`);
+        logger_1.logger.info(`   Requests/sec: ${summary.requests_per_second.toFixed(2)}`);
+        logger_1.logger.info(`   Duration: ${(summary.total_duration / 1000).toFixed(1)}s`);
+    }
+    logErrorDetails(summary) {
+        if (summary.failed_requests > 0) {
+            logger_1.logger.warn(`❌ ${summary.failed_requests} requests failed`);
+            // Log status code distribution
+            logger_1.logger.info('📊 Status Code Distribution:');
+            Object.entries(summary.status_distribution)
+                .sort(([a], [b]) => parseInt(a) - parseInt(b))
+                .forEach(([status, count]) => {
+                const statusNum = parseInt(status);
+                const isError = statusNum >= 400;
+                const emoji = isError ? '❌' : '✅';
+                logger_1.logger.info(`   ${emoji} ${status}: ${count} requests`);
+            });
+            // Log top errors
+            if (summary.error_details && summary.error_details.length > 0) {
+                logger_1.logger.warn('🔍 Top Error Details:');
+                summary.error_details.slice(0, 5).forEach((error, index) => {
+                    logger_1.logger.warn(`   ${index + 1}. ${error.error} (${error.count}x)`);
+                    if (error.status) {
+                        logger_1.logger.warn(`      Status: ${error.status}`);
+                    }
+                    if (error.request_url) {
+                        logger_1.logger.warn(`      URL: ${error.request_url}`);
+                    }
+                    if (error.response_body && error.response_body.length < 200) {
+                        logger_1.logger.warn(`      Response: ${error.response_body}`);
+                    }
+                });
+            }
+        }
+    }
+    async generateReport(summary) {
+        try {
+            const { EnhancedHTMLReportGenerator } = await Promise.resolve().then(() => __importStar(require('../reporting/enhanced-html-generator')));
+            const generator = new EnhancedHTMLReportGenerator();
+            // Process report path with automatic timestamp
+            const reportPath = this.processTemplateFilePath(this.config.report.output);
+            await generator.generate({
+                testName: this.config.name,
+                summary,
+                results: this.metrics.getResults()
+            }, reportPath);
+        }
+        catch (error) {
+            logger_1.logger.error('❌ Report generation failed:', error);
+        }
+    }
+    getMetrics() {
+        return this.metrics;
+    }
+}
+exports.TestRunner = TestRunner;

package/dist/core/threshold-evaluator.d.ts

@@ -0,0 +1,43 @@
+import { ThresholdConfig } from '../config';
+import { TestResult } from '../metrics/types';
+export interface ThresholdEvaluationResult {
+    passed: boolean;
+    failures: ThresholdFailure[];
+}
+export interface ThresholdFailure {
+    threshold: ThresholdConfig;
+    actualValue: number | string;
+    expectedValue: number | string;
+    message: string;
+    severity: 'warning' | 'error' | 'critical';
+}
+export declare class ThresholdEvaluator {
+    /**
+     * Evaluate all thresholds against a test result
+     */
+    static evaluate(thresholds: ThresholdConfig[], result: TestResult, stepName?: string): ThresholdEvaluationResult;
+    /**
+     * Evaluate a single threshold
+     */
+    private static evaluateThreshold;
+    /**
+     * Extract the actual metric value from test result
+     */
+    private static extractMetricValue;
+    /**
+     * Parse expected value, handling units like "1s", "500ms", etc.
+     */
+    private static parseExpectedValue;
+    /**
+     * Compare actual vs expected values using the specified operator
+     */
+    private static compareValues;
+    /**
+     * Get human-readable operator symbol
+     */
+    private static getOperatorSymbol;
+    /**
+     * Execute threshold action based on severity and configuration
+     */
+    static executeThresholdActions(evaluationResult: ThresholdEvaluationResult, stepName?: string): Promise<void>;
+}