@testsmith/perfornium 0.1.0

package/dist/protocols/soap/handler.d.ts

@@ -0,0 +1,12 @@
+import { ProtocolHandler, ProtocolResult } from '../base';
+import { VUContext, SOAPStep } from '../../config/types';
+export declare class SOAPHandler implements ProtocolHandler {
+    private client;
+    private wsdlUrl;
+    constructor(wsdlUrl: string);
+    initialize(): Promise<void>;
+    execute(operation: SOAPStep, context: VUContext): Promise<ProtocolResult>;
+    private executeRawSOAP;
+    private extractSOAPActionFromXML;
+    private parseSOAPResponse;
+}

package/dist/protocols/soap/handler.js

@@ -0,0 +1,165 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SOAPHandler = void 0;
+const soap = __importStar(require("soap"));
+const logger_1 = require("../../utils/logger");
+class SOAPHandler {
+    constructor(wsdlUrl) {
+        this.wsdlUrl = wsdlUrl;
+    }
+    async initialize() {
+        try {
+            this.client = await soap.createClientAsync(this.wsdlUrl);
+            logger_1.logger.debug(`🧼 SOAP client initialized for ${this.wsdlUrl}`);
+        }
+        catch (error) {
+            logger_1.logger.error(`❌ Failed to initialize SOAP client:`, error);
+            throw error;
+        }
+    }
+    async execute(operation, context) {
+        const startTime = performance.now();
+        try {
+            if (!this.client) {
+                await this.initialize();
+            }
+            // If body is provided, use raw XML SOAP request
+            if (operation.body) {
+                const result = await this.executeRawSOAP(operation.body);
+                const duration = performance.now() - startTime;
+                return {
+                    success: true,
+                    data: result,
+                    response_size: JSON.stringify(result).length,
+                    duration,
+                    request_url: this.wsdlUrl,
+                    request_method: 'SOAP',
+                    response_body: JSON.stringify(result),
+                    custom_metrics: {
+                        operation: operation.operation,
+                        wsdl_url: this.wsdlUrl,
+                        used_raw_xml: true
+                    }
+                };
+            }
+            // Use traditional SOAP client with args
+            const result = await new Promise((resolve, reject) => {
+                this.client[operation.operation](operation.args, (err, result) => {
+                    if (err)
+                        reject(err);
+                    else
+                        resolve(result);
+                });
+            });
+            const duration = performance.now() - startTime;
+            const responseData = JSON.stringify(result);
+            return {
+                success: true,
+                data: result,
+                response_size: responseData.length,
+                duration,
+                request_url: this.wsdlUrl,
+                request_method: 'SOAP',
+                response_body: responseData,
+                custom_metrics: {
+                    operation: operation.operation,
+                    wsdl_url: this.wsdlUrl
+                }
+            };
+        }
+        catch (error) {
+            const duration = performance.now() - startTime;
+            return {
+                success: false,
+                error: error.message,
+                error_code: 'SOAP_ERROR',
+                response_size: 0,
+                duration,
+                request_url: this.wsdlUrl,
+                request_method: 'SOAP',
+                custom_metrics: {
+                    operation: operation.operation,
+                    wsdl_url: this.wsdlUrl
+                }
+            };
+        }
+    }
+    async executeRawSOAP(xmlBody) {
+        try {
+            // Extract endpoint URL from WSDL URL
+            const endpointUrl = this.wsdlUrl.replace('?wsdl', '');
+            // Make direct HTTP request with the XML body
+            const response = await fetch(endpointUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'text/xml; charset=utf-8',
+                    'SOAPAction': this.extractSOAPActionFromXML(xmlBody)
+                },
+                body: xmlBody
+            });
+            const responseText = await response.text();
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${responseText}`);
+            }
+            return this.parseSOAPResponse(responseText);
+        }
+        catch (error) {
+            logger_1.logger.error('Raw SOAP execution failed:', error);
+            throw error;
+        }
+    }
+    extractSOAPActionFromXML(xmlBody) {
+        const operationMatch = xmlBody.match(/<(\w+)\s+xmlns=/);
+        const operationName = operationMatch ? operationMatch[1] : 'UnknownOperation';
+        return `"http://tempuri.org/${operationName}"`;
+    }
+    parseSOAPResponse(responseXml) {
+        try {
+            const resultMatch = responseXml.match(/<(\w+Result)[^>]*>([^<]*)<\/\1>/);
+            if (resultMatch) {
+                return {
+                    [resultMatch[1]]: resultMatch[2]
+                };
+            }
+            return { rawResponse: responseXml };
+        }
+        catch (error) {
+            logger_1.logger.error('Failed to parse SOAP response:', error);
+            return { rawResponse: responseXml };
+        }
+    }
+}
+exports.SOAPHandler = SOAPHandler;

package/dist/protocols/web/core-web-vitals.d.ts

@@ -0,0 +1,121 @@
+import { Page } from 'playwright';
+export interface CoreWebVitals {
+    lcp?: number;
+    cls?: number;
+    inp?: number;
+    ttfb?: number;
+    fcp?: number;
+    fid?: number;
+    tti?: number;
+    tbt?: number;
+    speedIndex?: number;
+}
+export interface WebVitalsThresholds {
+    lcp: {
+        good: number;
+        poor: number;
+    };
+    cls: {
+        good: number;
+        poor: number;
+    };
+    inp: {
+        good: number;
+        poor: number;
+    };
+    ttfb: {
+        good: number;
+        poor: number;
+    };
+    fcp: {
+        good: number;
+        poor: number;
+    };
+    fid?: {
+        good: number;
+        poor: number;
+    };
+    tti?: {
+        good: number;
+        poor: number;
+    };
+    tbt?: {
+        good: number;
+        poor: number;
+    };
+    speedIndex?: {
+        good: number;
+        poor: number;
+    };
+}
+export declare const DEFAULT_WEB_VITALS_THRESHOLDS: WebVitalsThresholds;
+export declare class CoreWebVitalsCollector {
+    private static webVitalsScript;
+    static injectVitalsCollector(page: Page): Promise<void>;
+    static collectVitals(page: Page, waitTime?: number): Promise<CoreWebVitals>;
+    static evaluateVitals(vitals: CoreWebVitals, thresholds?: WebVitalsThresholds): {
+        score: 'good' | 'needs-improvement' | 'poor';
+        details: {
+            [metric: string]: {
+                value: number;
+                score: 'good' | 'needs-improvement' | 'poor';
+            };
+        };
+    };
+    static generateVitalsThresholds(vitals: CoreWebVitals): Array<{
+        metric: string;
+        value: number;
+        operator: string;
+        severity: string;
+        description: string;
+    }>;
+}
+export interface VerificationStepMetrics {
+    step_name: string;
+    step_type: string;
+    selector?: string;
+    duration: number;
+    success: boolean;
+    error_message?: string;
+    element_count?: number;
+    expected_text?: string;
+    actual_text?: string;
+    screenshot_size?: number;
+    dom_ready_time?: number;
+    network_idle_time?: number;
+}
+export declare class VerificationMetricsCollector {
+    static measureVerificationStep<T>(stepName: string, stepType: string, operation: () => Promise<T>, additionalMetrics?: Partial<VerificationStepMetrics>): Promise<{
+        result: T;
+        metrics: VerificationStepMetrics;
+    }>;
+    static generateVerificationThresholds(metrics: VerificationStepMetrics[]): Array<{
+        metric: string;
+        value: number;
+        operator: string;
+        severity: string;
+        description: string;
+    }>;
+}
+export interface WebPerformanceMetrics {
+    core_web_vitals: CoreWebVitals;
+    vitals_score: 'good' | 'needs-improvement' | 'poor';
+    vitals_details: {
+        [metric: string]: {
+            value: number;
+            score: 'good' | 'needs-improvement' | 'poor';
+        };
+    };
+    verification_metrics?: VerificationStepMetrics;
+    page_load_time: number;
+    dom_content_loaded: number;
+    network_requests: number;
+    failed_requests: number;
+    total_transfer_size: number;
+    page_size: number;
+    javascript_execution_time: number;
+    style_recalculation_time: number;
+}
+export declare class WebPerformanceCollector {
+    static collectAllMetrics(page: Page, verificationMetrics?: VerificationStepMetrics): Promise<WebPerformanceMetrics>;
+}

package/dist/protocols/web/core-web-vitals.js

@@ -0,0 +1,373 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebPerformanceCollector = exports.VerificationMetricsCollector = exports.CoreWebVitalsCollector = exports.DEFAULT_WEB_VITALS_THRESHOLDS = void 0;
+const logger_1 = require("../../utils/logger");
+exports.DEFAULT_WEB_VITALS_THRESHOLDS = {
+    lcp: { good: 2500, poor: 4000 },
+    cls: { good: 0.1, poor: 0.25 },
+    inp: { good: 200, poor: 500 },
+    ttfb: { good: 800, poor: 1800 },
+    fcp: { good: 1800, poor: 3000 },
+    fid: { good: 100, poor: 300 },
+    tti: { good: 3800, poor: 7300 },
+    tbt: { good: 200, poor: 600 },
+    speedIndex: { good: 3400, poor: 5800 }
+};
+class CoreWebVitalsCollector {
+    static async injectVitalsCollector(page) {
+        try {
+            await page.addInitScript(this.webVitalsScript);
+            logger_1.logger.debug('Core Web Vitals collector injected');
+        }
+        catch (error) {
+            logger_1.logger.warn('Failed to inject Web Vitals collector:', error);
+        }
+    }
+    static async collectVitals(page, waitTime = 3000) {
+        try {
+            // First ensure the script is injected on the current page
+            await page.evaluate(this.webVitalsScript);
+            // Wait for vitals to be collected
+            await page.waitForTimeout(waitTime);
+            // Get the collected vitals
+            const vitals = await page.evaluate(() => {
+                const collected = window.__webVitals || {};
+                const interactions = window.__interactions || [];
+                // Ensure INP is calculated if we have interactions
+                if (!collected.inp && interactions.length > 0) {
+                    if (interactions.length >= 10) {
+                        const sorted = [...interactions].sort((a, b) => b.latency - a.latency);
+                        const index = Math.min(Math.floor(interactions.length * 0.98), sorted.length - 1);
+                        collected.inp = sorted[index].latency;
+                    }
+                    else {
+                        collected.inp = Math.max(...interactions.map((i) => i.latency));
+                    }
+                }
+                // Calculate additional metrics
+                const navigation = performance.getEntriesByType('navigation')[0];
+                if (navigation) {
+                    // Time to Interactive approximation
+                    collected.tti = navigation.domInteractive - navigation.fetchStart;
+                    // Total Blocking Time approximation (simplified)
+                    const longTasks = performance.getEntriesByType('longtask');
+                    collected.tbt = longTasks.reduce((total, task) => {
+                        const blockingTime = Math.max(0, task.duration - 50);
+                        return total + blockingTime;
+                    }, 0);
+                    // Speed Index approximation (simplified)
+                    collected.speedIndex = navigation.domContentLoadedEventStart - navigation.fetchStart;
+                    // Ensure TTFB is captured
+                    if (!collected.ttfb) {
+                        collected.ttfb = navigation.responseStart - navigation.fetchStart;
+                    }
+                    // Ensure FCP is captured from paint timing if not already
+                    if (!collected.fcp) {
+                        const paintEntries = performance.getEntriesByType('paint');
+                        const fcpEntry = paintEntries.find((entry) => entry.name === 'first-contentful-paint');
+                        if (fcpEntry) {
+                            collected.fcp = fcpEntry.startTime;
+                        }
+                    }
+                }
+                // Get LCP from performance entries if not already captured
+                if (!collected.lcp) {
+                    const lcpEntries = performance.getEntriesByType('largest-contentful-paint');
+                    console.log('LCP fallback - checking entries. Found:', lcpEntries.length);
+                    if (lcpEntries.length > 0) {
+                        // Use the latest LCP entry (most recent candidate)
+                        const latestEntry = lcpEntries[lcpEntries.length - 1];
+                        collected.lcp = latestEntry.startTime;
+                        console.log('LCP fallback collected:', collected.lcp, 'from entry:', latestEntry);
+                    }
+                    else {
+                        // Try alternative approach - estimate LCP from paint timing
+                        const paintEntries = performance.getEntriesByType('paint');
+                        console.log('No LCP entries found. Paint entries:', paintEntries.length);
+                        if (paintEntries.length > 0) {
+                            const fcpEntry = paintEntries.find(entry => entry.name === 'first-contentful-paint');
+                            if (fcpEntry) {
+                                // Use FCP as a rough approximation for LCP if no LCP is available
+                                collected.lcp = fcpEntry.startTime;
+                                console.log('LCP approximated from FCP:', collected.lcp);
+                            }
+                        }
+                    }
+                }
+                // Get CLS from layout-shift entries if not captured
+                if (collected.cls === undefined) {
+                    const layoutShiftEntries = performance.getEntriesByType('layout-shift');
+                    collected.cls = layoutShiftEntries.reduce((total, entry) => {
+                        if (!entry.hadRecentInput) {
+                            return total + entry.value;
+                        }
+                        return total;
+                    }, 0);
+                    console.log('CLS fallback collected:', collected.cls, 'from', layoutShiftEntries.length, 'shifts');
+                }
+                console.log('Final collected vitals before return:', collected);
+                return collected;
+            });
+            logger_1.logger.debug('Core Web Vitals collection complete:', vitals);
+            return vitals;
+        }
+        catch (error) {
+            logger_1.logger.warn('Failed to collect Core Web Vitals:', error);
+            return {};
+        }
+    }
+    static evaluateVitals(vitals, thresholds = exports.DEFAULT_WEB_VITALS_THRESHOLDS) {
+        const details = {};
+        let goodCount = 0;
+        let poorCount = 0;
+        let totalCount = 0;
+        const metrics = ['lcp', 'cls', 'inp', 'ttfb', 'fcp', 'fid', 'tti', 'tbt', 'speedIndex'];
+        for (const metric of metrics) {
+            const value = vitals[metric];
+            if (value !== undefined && value !== null) {
+                totalCount++;
+                const threshold = thresholds[metric];
+                let score;
+                if (value <= threshold.good) {
+                    score = 'good';
+                    goodCount++;
+                }
+                else if (value <= threshold.poor) {
+                    score = 'needs-improvement';
+                }
+                else {
+                    score = 'poor';
+                    poorCount++;
+                }
+                details[metric] = { value, score };
+            }
+        }
+        // Overall score calculation
+        let overallScore;
+        if (totalCount === 0) {
+            overallScore = 'needs-improvement';
+        }
+        else if (goodCount >= totalCount * 0.75) {
+            overallScore = 'good';
+        }
+        else if (poorCount > totalCount * 0.25) {
+            overallScore = 'poor';
+        }
+        else {
+            overallScore = 'needs-improvement';
+        }
+        return { score: overallScore, details };
+    }
+    static generateVitalsThresholds(vitals) {
+        const thresholds = [];
+        const baseThresholds = exports.DEFAULT_WEB_VITALS_THRESHOLDS;
+        for (const [metric, value] of Object.entries(vitals)) {
+            if (value !== undefined && value !== null && metric in baseThresholds) {
+                const threshold = baseThresholds[metric];
+                thresholds.push({
+                    metric: `web_vitals_${metric}`,
+                    value: threshold.good,
+                    operator: 'lte',
+                    severity: 'warning',
+                    description: `${metric.toUpperCase()} should be good (≤${threshold.good}${metric === 'cls' ? '' : 'ms'})`
+                });
+                thresholds.push({
+                    metric: `web_vitals_${metric}`,
+                    value: threshold.poor,
+                    operator: 'lte',
+                    severity: 'error',
+                    description: `${metric.toUpperCase()} should not be poor (≤${threshold.poor}${metric === 'cls' ? '' : 'ms'})`
+                });
+            }
+        }
+        return thresholds;
+    }
+}
+exports.CoreWebVitalsCollector = CoreWebVitalsCollector;
+CoreWebVitalsCollector.webVitalsScript = `
+    // Improved Web Vitals measurement script based on best practices
+    (() => {
+      const vitals = {};
+      const interactions = [];
+      
+      // LCP Observer with buffered entries
+      const lcpObserver = new PerformanceObserver((list) => {
+        const entries = list.getEntries();
+        const lcp = entries.at(-1);
+        if (lcp) {
+          vitals.lcp = Number(lcp.startTime);
+          console.log('LCP collected:', vitals.lcp);
+        }
+      });
+      
+      // TTFB Observer
+      const ttfbObserver = new PerformanceObserver((list) => {
+        const entries = list.getEntries();
+        if (entries.length > 0) {
+          const navigationEntry = entries[0];
+          vitals.ttfb = navigationEntry.responseStart - navigationEntry.fetchStart;
+          console.log('TTFB collected:', vitals.ttfb);
+        }
+      });
+      
+      // CLS Observer
+      const clsObserver = new PerformanceObserver((list) => {
+        let total = vitals.cls || 0;
+        for (const entry of list.getEntries()) {
+          if (!entry.hadRecentInput) {
+            total += entry.value;
+          }
+        }
+        vitals.cls = Number(total.toPrecision(4));
+        console.log('CLS updated:', vitals.cls);
+      });
+      
+      // FID Observer
+      const fidObserver = new PerformanceObserver((list) => {
+        for (const entry of list.getEntries()) {
+          vitals.fid = entry.processingStart - entry.startTime;
+          console.log('FID collected:', vitals.fid);
+        }
+      });
+
+      // Start observing with error handling
+      try {
+        lcpObserver.observe({ type: 'largest-contentful-paint', buffered: true });
+        console.log('LCP observer started');
+      } catch (e) { console.warn('LCP observer failed:', e); }
+      
+      try {
+        ttfbObserver.observe({ type: 'navigation', buffered: true });
+        console.log('TTFB observer started');  
+      } catch (e) { console.warn('TTFB observer failed:', e); }
+      
+      try {
+        clsObserver.observe({ type: 'layout-shift', buffered: true });
+        console.log('CLS observer started');
+      } catch (e) { console.warn('CLS observer failed:', e); }
+      
+      try {
+        fidObserver.observe({ type: 'first-input', buffered: true });
+        console.log('FID observer started');
+      } catch (e) { console.warn('FID observer failed:', e); }
+
+      // Get FCP from paint timings
+      const paintTimings = performance.getEntriesByType('paint');
+      for (const entry of paintTimings) {
+        if (entry.name === 'first-contentful-paint') {
+          vitals.fcp = Number(entry.startTime);
+          console.log('FCP collected:', vitals.fcp);
+        }
+      }
+
+      // Initialize CLS to 0 if not set
+      if (vitals.cls === undefined) {
+        vitals.cls = 0;
+      }
+
+      // Store vitals on window for retrieval
+      window.__webVitals = vitals;
+      window.__interactions = interactions;
+      
+      console.log('Improved Web Vitals collector initialized');
+      
+      return vitals;
+    })();
+  `;
+class VerificationMetricsCollector {
+    static async measureVerificationStep(stepName, stepType, operation, additionalMetrics) {
+        const startTime = performance.now();
+        let success = false;
+        let error_message;
+        let result;
+        try {
+            result = await operation();
+            success = true;
+        }
+        catch (error) {
+            error_message = error.message;
+            throw error;
+        }
+        finally {
+            // Round to 1 decimal place for cleaner output
+            const duration = Math.round((performance.now() - startTime) * 10) / 10;
+            const metrics = {
+                step_name: stepName,
+                step_type: stepType,
+                duration,
+                success,
+                error_message,
+                ...additionalMetrics
+            };
+            return { result: result, metrics };
+        }
+    }
+    static generateVerificationThresholds(metrics) {
+        const thresholds = [];
+        // Analyze verification step performance
+        const durations = metrics.map(m => m.duration);
+        const avgDuration = durations.reduce((a, b) => a + b, 0) / durations.length;
+        const maxDuration = Math.max(...durations);
+        const p95Duration = durations.sort((a, b) => a - b)[Math.floor(durations.length * 0.95)];
+        // Generate thresholds based on observed performance
+        thresholds.push({
+            metric: 'verification_step_duration',
+            value: Math.max(avgDuration * 2, 1000), // 2x average or 1 second minimum
+            operator: 'lte',
+            severity: 'warning',
+            description: 'Verification steps should complete within reasonable time'
+        });
+        thresholds.push({
+            metric: 'verification_step_duration',
+            value: Math.max(p95Duration * 1.5, 3000), // 1.5x p95 or 3 seconds minimum
+            operator: 'lte',
+            severity: 'error',
+            description: 'Verification steps should not take too long'
+        });
+        // Success rate threshold
+        const successRate = metrics.filter(m => m.success).length / metrics.length;
+        thresholds.push({
+            metric: 'verification_success_rate',
+            value: Math.max(successRate * 0.9, 0.95), // 90% of observed or 95% minimum
+            operator: 'gte',
+            severity: 'critical',
+            description: 'Verification steps should have high success rate'
+        });
+        return thresholds;
+    }
+}
+exports.VerificationMetricsCollector = VerificationMetricsCollector;
+class WebPerformanceCollector {
+    static async collectAllMetrics(page, verificationMetrics) {
+        // Collect Core Web Vitals
+        const coreWebVitals = await CoreWebVitalsCollector.collectVitals(page);
+        const vitalsEvaluation = CoreWebVitalsCollector.evaluateVitals(coreWebVitals);
+        // Collect performance metrics
+        const performanceMetrics = await page.evaluate(() => {
+            const navigation = performance.getEntriesByType('navigation')[0];
+            const resources = performance.getEntriesByType('resource');
+            return {
+                page_load_time: navigation ? navigation.loadEventEnd - navigation.fetchStart : 0,
+                dom_content_loaded: navigation ? navigation.domContentLoadedEventEnd - navigation.fetchStart : 0,
+                network_requests: resources.length,
+                failed_requests: resources.filter(r => 'responseStatus' in r && r.responseStatus >= 400).length,
+                total_transfer_size: resources.reduce((total, r) => total + (r.transferSize || 0), 0),
+                page_size: resources.reduce((total, r) => total + (r.decodedBodySize || 0), 0),
+                javascript_execution_time: performance.getEntriesByType('measure')
+                    .filter(m => m.name.includes('js'))
+                    .reduce((total, m) => total + m.duration, 0),
+                style_recalculation_time: performance.getEntriesByType('measure')
+                    .filter(m => m.name.includes('style'))
+                    .reduce((total, m) => total + m.duration, 0)
+            };
+        });
+        return {
+            core_web_vitals: coreWebVitals,
+            vitals_score: vitalsEvaluation.score,
+            vitals_details: vitalsEvaluation.details,
+            verification_metrics: verificationMetrics,
+            ...performanceMetrics
+        };
+    }
+}
+exports.WebPerformanceCollector = WebPerformanceCollector;