@testsmith/perfornium 0.6.4 → 0.6.6

package/dist/dashboard/services/metrics-parser.js

@@ -0,0 +1,209 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsParser = void 0;
+class MetricsParser {
+    constructor(options) {
+        this.influxEnabled = options?.influxEnabled ?? false;
+    }
+    parseOutputForMetrics(line, test) {
+        // Parse [RT] JSON data for individual response times
+        const rtMatch = line.match(/\[RT\]\s*(.+)/);
+        if (rtMatch) {
+            try {
+                const rtData = JSON.parse(rtMatch[1]);
+                const newRTs = rtData.map((r) => ({
+                    timestamp: r.t,
+                    value: r.v,
+                    success: r.s === 1,
+                    stepName: r.n || 'unknown'
+                }));
+                // When InfluxDB is enabled, keep all response times (data is persisted)
+                // Otherwise limit to 500 to prevent memory issues
+                if (this.influxEnabled) {
+                    test.responseTimes = [...test.responseTimes, ...newRTs];
+                }
+                else {
+                    test.responseTimes = [...test.responseTimes, ...newRTs].slice(-500);
+                }
+                return true;
+            }
+            catch (e) {
+                // Ignore JSON parse errors
+            }
+            return false;
+        }
+        // Parse [STEPS] JSON data for step statistics
+        const stepsMatch = line.match(/\[STEPS\]\s*(.+)/);
+        if (stepsMatch) {
+            try {
+                const stepData = JSON.parse(stepsMatch[1]);
+                test.stepStats = stepData.map((s) => ({
+                    stepName: s.n,
+                    scenario: s.s,
+                    requests: s.r,
+                    errors: s.e,
+                    avgResponseTime: s.a,
+                    p50: s.p50,
+                    p95: s.p95,
+                    p99: s.p99,
+                    successRate: s.sr
+                }));
+                return true;
+            }
+            catch (e) {
+                // Ignore JSON parse errors
+            }
+            return false;
+        }
+        // Parse [ERRORS] JSON data for top errors
+        const topErrorsMatch = line.match(/\[ERRORS\]\s*(.+)/);
+        if (topErrorsMatch) {
+            try {
+                const errorData = JSON.parse(topErrorsMatch[1]);
+                test.topErrors = errorData.map((e) => ({
+                    scenario: e.scenario,
+                    action: e.action,
+                    status: e.status,
+                    error: e.error,
+                    url: e.url,
+                    count: e.count
+                }));
+                return true;
+            }
+            catch (e) {
+                // Ignore JSON parse errors
+            }
+            return false;
+        }
+        // Parse [NETWORK] JSON data for captured HTTP calls
+        const networkMatch = line.match(/\[NETWORK\]\s*(.+)/);
+        if (networkMatch) {
+            try {
+                const networkData = JSON.parse(networkMatch[1]);
+                // Initialize network calls array if not present
+                if (!test.networkCalls) {
+                    test.networkCalls = [];
+                }
+                test.networkCalls.push({
+                    id: networkData.id,
+                    vuId: networkData.vu,
+                    url: networkData.url,
+                    method: networkData.method,
+                    status: networkData.status,
+                    statusText: networkData.statusText,
+                    duration: networkData.duration,
+                    size: networkData.size,
+                    type: networkData.type,
+                    success: networkData.success,
+                    error: networkData.error,
+                    timestamp: Date.now(),
+                    requestHeaders: networkData.requestHeaders,
+                    requestBody: networkData.requestBody,
+                    responseHeaders: networkData.responseHeaders,
+                    responseBody: networkData.responseBody
+                });
+                // Keep last 100 network calls to prevent memory issues
+                if (test.networkCalls.length > 100) {
+                    test.networkCalls = test.networkCalls.slice(-100);
+                }
+                return true;
+            }
+            catch (e) {
+                // Ignore JSON parse errors
+            }
+            return false;
+        }
+        // Parse the extended [PROGRESS] format with percentiles
+        // Format: [PROGRESS] VUs: 5 | Requests: 100 | Errors: 2 | Avg RT: 150ms | RPS: 10.5 | P50: 100ms | P90: 200ms | P95: 300ms | P99: 500ms | Success: 98.5%
+        const progressLineMatch = line.match(/\[PROGRESS\]\s*VUs:\s*(\d+)\s*\|\s*Requests:\s*(\d+)\s*\|\s*Errors:\s*(\d+)\s*\|\s*Avg RT:\s*(\d+(?:\.\d+)?)\s*ms\s*\|\s*RPS:\s*(\d+(?:\.\d+)?)/i);
+        if (progressLineMatch) {
+            test.metrics.currentVUs = parseInt(progressLineMatch[1]);
+            test.metrics.requests = parseInt(progressLineMatch[2]);
+            test.metrics.errors = parseInt(progressLineMatch[3]);
+            test.metrics.avgResponseTime = parseFloat(progressLineMatch[4]);
+            test.metrics.requestsPerSecond = parseFloat(progressLineMatch[5]);
+            // Parse percentiles if present
+            const p50Match = line.match(/P50:\s*(\d+(?:\.\d+)?)\s*ms/i);
+            const p90Match = line.match(/P90:\s*(\d+(?:\.\d+)?)\s*ms/i);
+            const p95Match = line.match(/P95:\s*(\d+(?:\.\d+)?)\s*ms/i);
+            const p99Match = line.match(/P99:\s*(\d+(?:\.\d+)?)\s*ms/i);
+            const successMatch = line.match(/Success:\s*(\d+(?:\.\d+)?)\s*%/i);
+            if (p50Match)
+                test.metrics.p50ResponseTime = parseFloat(p50Match[1]);
+            if (p90Match)
+                test.metrics.p90ResponseTime = parseFloat(p90Match[1]);
+            if (p95Match)
+                test.metrics.p95ResponseTime = parseFloat(p95Match[1]);
+            if (p99Match)
+                test.metrics.p99ResponseTime = parseFloat(p99Match[1]);
+            if (successMatch)
+                test.metrics.successRate = parseFloat(successMatch[1]);
+            // Add to history
+            this.addHistoryEntry(test);
+            return true;
+        }
+        // Fallback: Parse various loose output formats for metrics
+        const vusMatch = line.match(/VUs?[:\s]+(\d+)/i);
+        const requestsMatch = line.match(/(?:total\s+)?requests?[:\s]+(\d+)/i);
+        const errorsMatch = line.match(/(?:failed|errors?)[:\s]+(\d+)/i);
+        const avgRtMatch = line.match(/(?:avg|average)\s*(?:rt|response\s*time)?[:\s]+(\d+(?:\.\d+)?)\s*ms/i);
+        const rpsMatch = line.match(/(?:rps|req\/s|requests\/s(?:ec)?)[:\s]+(\d+(?:\.\d+)?)/i);
+        let updated = false;
+        if (vusMatch) {
+            test.metrics.currentVUs = parseInt(vusMatch[1]);
+            updated = true;
+        }
+        if (requestsMatch) {
+            test.metrics.requests = parseInt(requestsMatch[1]);
+            updated = true;
+        }
+        if (errorsMatch) {
+            test.metrics.errors = parseInt(errorsMatch[1]);
+            updated = true;
+        }
+        if (avgRtMatch) {
+            test.metrics.avgResponseTime = parseFloat(avgRtMatch[1]);
+            updated = true;
+        }
+        if (rpsMatch) {
+            test.metrics.requestsPerSecond = parseFloat(rpsMatch[1]);
+            updated = true;
+        }
+        if (updated) {
+            this.addHistoryEntry(test);
+        }
+        return updated;
+    }
+    addHistoryEntry(test) {
+        const now = Date.now();
+        const lastHistory = test.history[test.history.length - 1];
+        const rps = lastHistory && (now - lastHistory.timestamp) > 0
+            ? (test.metrics.requests - lastHistory.requests) / ((now - lastHistory.timestamp) / 1000)
+            : (test.metrics.requestsPerSecond || 0);
+        test.history.push({
+            timestamp: now,
+            requests: test.metrics.requests,
+            errors: test.metrics.errors,
+            avgResponseTime: test.metrics.avgResponseTime,
+            p95ResponseTime: test.metrics.p95ResponseTime || 0,
+            p99ResponseTime: test.metrics.p99ResponseTime || 0,
+            vus: test.metrics.currentVUs,
+            rps: Math.max(0, rps)
+        });
+        if (test.history.length > 120)
+            test.history.shift();
+    }
+    parseNetworkData(line) {
+        const networkMatch = line.match(/\[NETWORK\]\s*(.+)/);
+        if (networkMatch) {
+            try {
+                return JSON.parse(networkMatch[1]);
+            }
+            catch (e) {
+                return null;
+            }
+        }
+        return null;
+    }
+}
+exports.MetricsParser = MetricsParser;

package/dist/dashboard/services/results-manager.d.ts

@@ -0,0 +1,17 @@
+import { TestResult } from '../types';
+export declare class ResultsManager {
+    private resultsDir;
+    constructor(resultsDir: string);
+    scanResults(): Promise<TestResult[]>;
+    loadFullResult(id: string): Promise<TestResult | null>;
+    deleteResult(id: string): Promise<void>;
+    saveResult(id: string, data: any): Promise<{
+        id: string;
+        name: string;
+    }>;
+    generateComparison(results: (TestResult | null)[]): any;
+    private extractSummary;
+    private extractNetworkCalls;
+    private generateStepComparisons;
+    private calcDiff;
+}

package/dist/dashboard/services/results-manager.js

@@ -0,0 +1,311 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResultsManager = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const logger_1 = require("../../utils/logger");
+class ResultsManager {
+    constructor(resultsDir) {
+        this.resultsDir = resultsDir;
+    }
+    async scanResults() {
+        const results = [];
+        try {
+            const files = await fs.readdir(this.resultsDir);
+            const excludePatterns = ['metrics', 'live-results', 'summary-incremental'];
+            const jsonFiles = files.filter(f => f.endsWith('.json') && !excludePatterns.some(p => f.includes(p)));
+            for (const file of jsonFiles) {
+                try {
+                    const filePath = path.join(this.resultsDir, file);
+                    const content = await fs.readFile(filePath, 'utf-8');
+                    const data = JSON.parse(content);
+                    const stat = await fs.stat(filePath);
+                    const fileId = file.replace('.json', '');
+                    // Extract test name from filename if not in data
+                    let testName = data.name || data.test_name;
+                    if (!testName) {
+                        const match = fileId.match(/^(.+)-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}/);
+                        testName = match ? match[1] : fileId;
+                    }
+                    // Extract timestamp from filename or metadata
+                    let timestamp = data.timestamp || data.metadata?.generated_at;
+                    if (!timestamp) {
+                        const tsMatch = fileId.match(/(\d{4}-\d{2}-\d{2})_(\d{2})-(\d{2})-(\d{2})/);
+                        if (tsMatch) {
+                            timestamp = `${tsMatch[1]}T${tsMatch[2]}:${tsMatch[3]}:${tsMatch[4]}Z`;
+                        }
+                        else {
+                            timestamp = stat.mtime.toISOString();
+                        }
+                    }
+                    results.push({
+                        id: fileId,
+                        name: testName,
+                        timestamp: timestamp,
+                        duration: data.duration || data.total_duration || data.summary?.total_duration || 0,
+                        summary: this.extractSummary(data),
+                        scenarios: data.scenarios || [],
+                        step_statistics: data.step_statistics || data.summary?.step_statistics || []
+                    });
+                }
+                catch (e) {
+                    // Log skipped files so users know why results don't appear
+                    logger_1.logger.warn(`Skipping invalid result file ${file}: ${e.message}`);
+                }
+            }
+        }
+        catch (e) {
+            // Results dir might not exist
+        }
+        results.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
+        return results;
+    }
+    async loadFullResult(id) {
+        try {
+            const decodedId = decodeURIComponent(id);
+            const filePath = path.join(this.resultsDir, `${decodedId}.json`);
+            logger_1.logger.debug(`Loading result from: ${filePath}`);
+            const content = await fs.readFile(filePath, 'utf-8');
+            const data = JSON.parse(content);
+            const stat = await fs.stat(filePath);
+            // Extract network calls from results
+            const networkCalls = this.extractNetworkCalls(data);
+            // Extract test name from filename if not in data
+            // Filename format: "Test Name-YYYY-MM-DD_HH-MM-SS-YYYYMMDD-HHMMSS-mmm.json"
+            let testName = data.name || data.test_name;
+            if (!testName) {
+                // Try to extract from filename by removing timestamp suffix
+                const match = decodedId.match(/^(.+)-\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}/);
+                testName = match ? match[1] : decodedId;
+            }
+            // Extract timestamp from filename or metadata
+            let timestamp = data.timestamp || data.metadata?.generated_at;
+            if (!timestamp) {
+                // Try to extract from filename
+                const tsMatch = decodedId.match(/(\d{4}-\d{2}-\d{2})_(\d{2})-(\d{2})-(\d{2})/);
+                if (tsMatch) {
+                    timestamp = `${tsMatch[1]}T${tsMatch[2]}:${tsMatch[3]}:${tsMatch[4]}Z`;
+                }
+                else {
+                    timestamp = stat.mtime.toISOString();
+                }
+            }
+            return {
+                id: decodedId,
+                name: testName,
+                timestamp: timestamp,
+                duration: data.duration || data.total_duration || data.summary?.total_duration || 0,
+                summary: this.extractSummary(data),
+                scenarios: data.scenarios || [],
+                step_statistics: data.step_statistics || data.summary?.step_statistics || [],
+                timeline_data: data.timeline_data || data.summary?.timeline_data || [],
+                vu_ramp_up: data.vu_ramp_up || data.summary?.vu_ramp_up || [],
+                response_time_distribution: data.response_time_distribution || [],
+                timeseries: data.timeseries || data.time_series || [],
+                error_details: data.error_details || data.summary?.error_details || [],
+                network_calls: networkCalls,
+                infrastructure_metrics: data.infrastructure_metrics || null,
+                raw: data
+            };
+        }
+        catch (e) {
+            logger_1.logger.error(`Failed to load result ${id}:`, e.message);
+            return null;
+        }
+    }
+    async deleteResult(id) {
+        const decodedId = decodeURIComponent(id);
+        const filePath = path.join(this.resultsDir, `${decodedId}.json`);
+        logger_1.logger.debug(`Deleting result file: ${filePath}`);
+        await fs.unlink(filePath);
+        logger_1.logger.info(`Deleted result: ${decodedId}`);
+    }
+    async saveResult(id, data) {
+        // Ensure results directory exists
+        try {
+            await fs.access(this.resultsDir);
+        }
+        catch {
+            await fs.mkdir(this.resultsDir, { recursive: true });
+        }
+        const sanitizedId = id.replace(/[<>:"/\\|?*]/g, '-');
+        const filePath = path.join(this.resultsDir, `${sanitizedId}.json`);
+        logger_1.logger.debug(`Saving result to: ${filePath}`);
+        await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8');
+        logger_1.logger.info(`Saved result: ${sanitizedId}`);
+        return { id: sanitizedId, name: data.name || sanitizedId };
+    }
+    generateComparison(results) {
+        const valid = results.filter(r => r !== null);
+        if (valid.length < 2)
+            return null;
+        const baseline = valid[0];
+        const comparisons = valid.slice(1).map(result => ({
+            id: result.id,
+            name: result.name,
+            timestamp: result.timestamp,
+            diff: {
+                avg_response_time: this.calcDiff(baseline.summary.avg_response_time, result.summary.avg_response_time),
+                p50_response_time: this.calcDiff(baseline.summary.p50_response_time, result.summary.p50_response_time),
+                p95_response_time: this.calcDiff(baseline.summary.p95_response_time, result.summary.p95_response_time),
+                p99_response_time: this.calcDiff(baseline.summary.p99_response_time, result.summary.p99_response_time),
+                requests_per_second: this.calcDiff(baseline.summary.requests_per_second, result.summary.requests_per_second, true),
+                error_rate: {
+                    value: result.summary.error_rate,
+                    baseline: baseline.summary.error_rate,
+                    change: (result.summary.error_rate - baseline.summary.error_rate).toFixed(2) + '%',
+                    improved: result.summary.error_rate < baseline.summary.error_rate
+                }
+            }
+        }));
+        // Generate step-level comparisons
+        const stepComparisons = this.generateStepComparisons(valid);
+        // Get timeline data for line graphs
+        const timelineComparisons = valid.map(result => ({
+            id: result.id,
+            name: result.name,
+            timeline: result.timeline_data || []
+        }));
+        return {
+            baseline: { id: baseline.id, name: baseline.name, timestamp: baseline.timestamp },
+            comparisons,
+            stepComparisons,
+            timelineComparisons
+        };
+    }
+    extractSummary(data) {
+        const s = data.summary || data;
+        const percentiles = s.percentiles || {};
+        const totalReq = s.total_requests || 0;
+        const failedReq = s.failed_requests || 0;
+        return {
+            total_requests: totalReq,
+            successful_requests: s.successful_requests || (totalReq - failedReq),
+            failed_requests: failedReq,
+            avg_response_time: s.avg_response_time || s.mean_response_time || 0,
+            min_response_time: s.min_response_time || 0,
+            max_response_time: s.max_response_time || 0,
+            p50_response_time: percentiles['50'] || s.p50_response_time || s.median_response_time || 0,
+            p75_response_time: percentiles['75'] || s.p75_response_time || 0,
+            p90_response_time: percentiles['90'] || s.p90_response_time || 0,
+            p95_response_time: percentiles['95'] || s.p95_response_time || 0,
+            p99_response_time: percentiles['99'] || s.p99_response_time || 0,
+            requests_per_second: s.requests_per_second || s.throughput || 0,
+            error_rate: s.error_rate ?? (failedReq / Math.max(1, totalReq) * 100),
+            success_rate: s.success_rate ?? ((totalReq - failedReq) / Math.max(1, totalReq) * 100)
+        };
+    }
+    extractNetworkCalls(data) {
+        const calls = [];
+        // Extract from results array (each result may have custom_metrics.network_calls)
+        const results = data.results || [];
+        for (const result of results) {
+            if (result.custom_metrics?.network_calls) {
+                calls.push(...result.custom_metrics.network_calls);
+            }
+        }
+        // Also check for direct network_calls array
+        if (data.network_calls) {
+            calls.push(...data.network_calls);
+        }
+        return calls;
+    }
+    generateStepComparisons(results) {
+        // Collect all unique step names across all results
+        const allSteps = new Set();
+        results.forEach(result => {
+            (result.step_statistics || []).forEach((step) => {
+                allSteps.add(step.step_name);
+            });
+        });
+        // For each step, gather metrics from all results
+        const stepComparisons = [];
+        allSteps.forEach(stepName => {
+            const stepData = {
+                step_name: stepName,
+                results: results.map(result => {
+                    const step = (result.step_statistics || []).find((s) => s.step_name === stepName);
+                    if (!step)
+                        return null;
+                    return {
+                        testId: result.id,
+                        testName: result.name,
+                        total_requests: step.total_requests,
+                        failed_requests: step.failed_requests,
+                        success_rate: step.success_rate,
+                        avg_response_time: step.avg_response_time,
+                        min_response_time: step.min_response_time,
+                        max_response_time: step.max_response_time,
+                        p50: step.percentiles?.[50] || step.p50 || 0,
+                        p95: step.percentiles?.[95] || step.p95 || 0,
+                        p99: step.percentiles?.[99] || step.p99 || 0
+                    };
+                })
+            };
+            // Calculate diffs from baseline (first result)
+            const baseline = stepData.results[0];
+            if (baseline) {
+                stepData.diffs = stepData.results.slice(1).map((current) => {
+                    if (!current)
+                        return null;
+                    return {
+                        avg_response_time: this.calcDiff(baseline.avg_response_time, current.avg_response_time),
+                        p95: this.calcDiff(baseline.p95, current.p95),
+                        p99: this.calcDiff(baseline.p99, current.p99),
+                        success_rate: {
+                            value: current.success_rate,
+                            baseline: baseline.success_rate,
+                            change: (current.success_rate - baseline.success_rate).toFixed(2) + '%',
+                            improved: current.success_rate > baseline.success_rate
+                        }
+                    };
+                });
+            }
+            stepComparisons.push(stepData);
+        });
+        return stepComparisons;
+    }
+    calcDiff(baseline, current, higherIsBetter = false) {
+        const change = baseline ? ((current - baseline) / baseline * 100) : 0;
+        return {
+            value: current,
+            baseline,
+            change: change.toFixed(2) + '%',
+            improved: higherIsBetter ? current > baseline : current < baseline
+        };
+    }
+}
+exports.ResultsManager = ResultsManager;

package/dist/dashboard/services/test-executor.d.ts

@@ -0,0 +1,41 @@
+import { LiveTest, InfrastructureMetrics } from '../types';
+export interface TestRunOptions {
+    verbose?: boolean;
+    report?: boolean;
+    output?: string;
+    vus?: number;
+    iterations?: number;
+    duration?: string;
+    rampUp?: string;
+    headless?: boolean;
+    workers?: string;
+}
+export interface TestExecutorCallbacks {
+    onOutput: (testId: string, data: string) => void;
+    onLiveUpdate: (test: LiveTest) => void;
+    onNetworkUpdate: (testId: string, data: any) => void;
+    onTestComplete: (test: LiveTest) => void;
+    onTestFinished: (testId: string, exitCode: number | null) => void;
+    getInfraSnapshot?: () => Record<string, InfrastructureMetrics[]>;
+}
+export interface TestExecutorOptions {
+    /** When InfluxDB is enabled, don't limit response times in memory */
+    influxEnabled?: boolean;
+}
+export declare class TestExecutor {
+    private testsDir;
+    private resultsDir;
+    private runningProcesses;
+    private liveTests;
+    private metricsParser;
+    private callbacks;
+    constructor(testsDir: string, resultsDir: string, liveTests: Map<string, LiveTest>, callbacks: TestExecutorCallbacks, options?: TestExecutorOptions);
+    runTest(testPath: string, options: TestRunOptions): {
+        testId: string;
+        status: string;
+    };
+    stopTest(testId: string): boolean;
+    killAllProcesses(): void;
+    private injectInfraMetrics;
+    private parseOutputLine;
+}