@yasserkhanorg/e2e-agents 1.8.5 → 1.10.0

package/dist/esm/prompts/impact.js

@@ -1,5 +1,6 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
+import { extractJsonFromResponse } from './json_extract.js';
 import { formatSpecsForPrompt } from '../knowledge/spec_index.js';
 import { formatApiSurfaceForPrompt } from '../knowledge/api_surface.js';
 export function buildImpactPrompt(ctx) {
@@ -23,7 +24,7 @@ export function buildImpactPrompt(ctx) {
     })
         .join('\n\n');
     return [
-        'You are analyzing code changes in Mattermost to identify impacted user-facing flows.',
+        `You are analyzing code changes in ${ctx.projectName || 'Mattermost'} to identify impacted user-facing flows.`,
         '',
         `ROUTE FAMILY: ${ctx.family.id}`,
         `ROUTES: ${familyRoutes}`,
@@ -58,25 +59,5 @@ export function buildImpactPrompt(ctx) {
     ].filter(Boolean).join('\n');
 }
 export function parseImpactResponse(text) {
-    // Try to extract JSON from the response
-    const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
-    const candidates = fenced ? [fenced[1], text] : [text];
-    for (const candidate of candidates) {
-        const start = candidate.indexOf('{');
-        const end = candidate.lastIndexOf('}');
-        if (start < 0 || end <= start) {
-            continue;
-        }
-        const raw = candidate.slice(start, end + 1);
-        try {
-            const parsed = JSON.parse(raw);
-            if (parsed && Array.isArray(parsed.flows)) {
-                return parsed;
-            }
-        }
-        catch {
-            continue;
-        }
-    }
-    return null;
+    return extractJsonFromResponse(text, (obj) => obj != null && typeof obj === 'object' && Array.isArray(obj.flows));
 }

package/dist/esm/prompts/json_extract.js

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Shared JSON extraction from LLM text responses.
+ * Handles fenced code blocks, bare JSON, and partial text.
+ */
+/**
+ * Extract and parse JSON from LLM response text.
+ * Tries fenced code blocks first, then raw text.
+ * Returns null if no valid JSON found.
+ *
+ * @param text - Raw LLM response text
+ * @param validate - Predicate to check if parsed object has the expected shape
+ */
+export function extractJsonFromResponse(text, validate) {
+    const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
+    const candidates = fenced ? [fenced[1], text] : [text];
+    for (const candidate of candidates) {
+        const start = candidate.indexOf('{');
+        const end = candidate.lastIndexOf('}');
+        if (start < 0 || end <= start) {
+            continue;
+        }
+        const raw = candidate.slice(start, end + 1);
+        try {
+            const parsed = JSON.parse(raw);
+            if (validate(parsed)) {
+                return parsed;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return null;
+}

package/dist/esm/prompts/strategist.js

@@ -1,6 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
 import { sanitizeForPrompt } from '../crew/sanitize.js';
+import { extractJsonFromResponse } from './json_extract.js';
 export function buildStrategistPrompt(ctx) {
     const flowsBlock = ctx.impactedFlows
         .map((f) => {
@@ -56,24 +57,5 @@ export function buildStrategistPrompt(ctx) {
     ].join('\n');
 }
 export function parseStrategistResponse(text) {
-    const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
-    const candidates = fenced ? [fenced[1], text] : [text];
-    for (const candidate of candidates) {
-        const start = candidate.indexOf('{');
-        const end = candidate.lastIndexOf('}');
-        if (start < 0 || end <= start) {
-            continue;
-        }
-        const raw = candidate.slice(start, end + 1);
-        try {
-            const parsed = JSON.parse(raw);
-            if (parsed && Array.isArray(parsed.strategy)) {
-                return parsed;
-            }
-        }
-        catch {
-            continue;
-        }
-    }
-    return null;
+    return extractJsonFromResponse(text, (obj) => obj != null && typeof obj === 'object' && Array.isArray(obj.strategy));
 }

package/dist/esm/prompts/test-designer.js

@@ -1,5 +1,6 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
+import { extractJsonFromResponse } from './json_extract.js';
 import { formatApiSurfaceForPrompt } from '../knowledge/api_surface.js';
 import { sanitizeForPrompt } from '../crew/sanitize.js';
 export function buildTestDesignerPrompt(ctx) {
@@ -24,7 +25,7 @@ export function buildTestDesignerPrompt(ctx) {
         : 'None detected.';
     const categories = ctx.strategy.testCategories.join(', ');
     return [
-        'You are a senior QA engineer designing comprehensive test cases for a Mattermost user flow.',
+        `You are a senior QA engineer designing comprehensive test cases for a ${ctx.profile?.projectName || 'Mattermost'} user flow.`,
         '',
         `FLOW: ${ctx.flow.flowName}`,
         `Flow ID: ${ctx.flow.flowId}`,
@@ -84,24 +85,8 @@ export function buildTestDesignerPrompt(ctx) {
     ].join('\n');
 }
 export function parseTestDesignerResponse(text) {
-    const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
-    const candidates = fenced ? [fenced[1], text] : [text];
-    for (const candidate of candidates) {
-        const start = candidate.indexOf('{');
-        const end = candidate.lastIndexOf('}');
-        if (start < 0 || end <= start) {
-            continue;
-        }
-        const raw = candidate.slice(start, end + 1);
-        try {
-            const parsed = JSON.parse(raw);
-            if (parsed?.testDesign?.testCases && Array.isArray(parsed.testDesign.testCases)) {
-                return parsed;
-            }
-        }
-        catch {
-            continue;
-        }
-    }
-    return null;
+    return extractJsonFromResponse(text, (obj) => {
+        const r = obj;
+        return r?.testDesign?.testCases != null && Array.isArray(r.testDesign.testCases);
+    });
 }

package/dist/esm/provider_factory.js

@@ -263,15 +263,17 @@ class HybridProvider {
         const primaryStats = this.primary.getUsageStats();
         const fallbackStats = this.fallback.getUsageStats();
         // Combine stats
+        const totalRequests = primaryStats.requestCount + fallbackStats.requestCount;
         return {
-            requestCount: primaryStats.requestCount + fallbackStats.requestCount,
+            requestCount: totalRequests,
             totalInputTokens: primaryStats.totalInputTokens + fallbackStats.totalInputTokens,
             totalOutputTokens: primaryStats.totalOutputTokens + fallbackStats.totalOutputTokens,
             totalTokens: primaryStats.totalTokens + fallbackStats.totalTokens,
             totalCost: primaryStats.totalCost + fallbackStats.totalCost,
-            averageResponseTimeMs: (primaryStats.averageResponseTimeMs * primaryStats.requestCount +
-                fallbackStats.averageResponseTimeMs * fallbackStats.requestCount) /
-                (primaryStats.requestCount + fallbackStats.requestCount),
+            averageResponseTimeMs: totalRequests > 0
+                ? (primaryStats.averageResponseTimeMs * primaryStats.requestCount +
+                    fallbackStats.averageResponseTimeMs * fallbackStats.requestCount) / totalRequests
+                : 0,
             failedRequests: primaryStats.failedRequests + fallbackStats.failedRequests,
             startTime: new Date(Math.min(primaryStats.startTime.getTime(), fallbackStats.startTime.getTime())),
             lastUpdated: new Date(Math.max(primaryStats.lastUpdated.getTime(), fallbackStats.lastUpdated.getTime())),

package/dist/esm/reporters/junit.js

@@ -0,0 +1,86 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+function escapeXml(str) {
+    return str
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+function buildTestCase(tc, flowName) {
+    const className = escapeXml(flowName.replace(/\s+/g, '.'));
+    const testName = escapeXml(tc.name);
+    return `      <testcase classname="${className}" name="${testName}" status="${escapeXml(tc.priority)}">\n` +
+        `        <properties>\n` +
+        `          <property name="type" value="${escapeXml(tc.type)}" />\n` +
+        `          <property name="priority" value="${escapeXml(tc.priority)}" />\n` +
+        `        </properties>\n` +
+        `      </testcase>`;
+}
+function buildFailureCase(finding) {
+    const name = escapeXml(finding.title);
+    return `      <testcase classname="findings" name="${name}">\n` +
+        `        <failure message="${escapeXml(finding.title)}" type="${escapeXml(finding.severity)}">${escapeXml(finding.description)}</failure>\n` +
+        `      </testcase>`;
+}
+export const junitReporter = {
+    name: 'junit',
+    extension: '.xml',
+    format(results) {
+        const suites = [];
+        // Build a lookup from flowName -> test cases
+        const designsByFlow = new Map();
+        for (const design of results.testDesigns) {
+            designsByFlow.set(design.flowName, design.testCases);
+        }
+        // High-severity findings as failure cases
+        const highFindings = results.findings.filter((f) => f.severity === 'high');
+        // Each strategy entry becomes a test suite
+        for (const entry of results.strategyEntries) {
+            const testCases = designsByFlow.get(entry.flowName) ?? [];
+            const failures = highFindings.filter((f) => f.title.includes(entry.flowName));
+            const totalTests = testCases.length + failures.length;
+            const casesXml = testCases.map((tc) => buildTestCase(tc, entry.flowName)).join('\n');
+            const failuresXml = failures.map((f) => buildFailureCase(f)).join('\n');
+            const allCases = [casesXml, failuresXml].filter(Boolean).join('\n');
+            // Warnings as system-out
+            const warningsText = results.warnings.length > 0
+                ? `      <system-out>${escapeXml(results.warnings.join('\n'))}</system-out>`
+                : '';
+            suites.push(`    <testsuite name="${escapeXml(entry.flowName)}" tests="${totalTests}" failures="${failures.length}" ` +
+                `id="${escapeXml(entry.flowId)}">\n` +
+                `      <properties>\n` +
+                `        <property name="priority" value="${escapeXml(entry.priority)}" />\n` +
+                `        <property name="approach" value="${escapeXml(entry.approach)}" />\n` +
+                `        <property name="rationale" value="${escapeXml(entry.rationale)}" />\n` +
+                `      </properties>\n` +
+                (allCases ? allCases + '\n' : '') +
+                (warningsText ? warningsText + '\n' : '') +
+                `    </testsuite>`);
+        }
+        // Remaining high findings not tied to a strategy entry
+        const coveredFlowNames = new Set(results.strategyEntries.map((e) => e.flowName));
+        const uncoveredFindings = highFindings.filter((f) => !Array.from(coveredFlowNames).some((name) => f.title.includes(name)));
+        if (uncoveredFindings.length > 0) {
+            const failureCases = uncoveredFindings.map((f) => buildFailureCase(f)).join('\n');
+            suites.push(`    <testsuite name="findings" tests="${uncoveredFindings.length}" failures="${uncoveredFindings.length}">\n` +
+                failureCases + '\n' +
+                `    </testsuite>`);
+        }
+        const totalTests = results.testDesigns.reduce((sum, d) => sum + d.testCases.length, 0) + highFindings.length;
+        const totalFailures = highFindings.length;
+        return `<?xml version="1.0" encoding="UTF-8"?>\n` +
+            `<testsuites name="e2e-agents: ${escapeXml(results.workflow)}" ` +
+            `tests="${totalTests}" failures="${totalFailures}" ` +
+            `time="0">\n` +
+            `  <properties>\n` +
+            `    <property name="changedFiles" value="${results.changedFiles}" />\n` +
+            `    <property name="impactedFlows" value="${results.impactedFlows}" />\n` +
+            `    <property name="cost" value="${results.cost}" />\n` +
+            `    <property name="tokens" value="${results.tokens}" />\n` +
+            `  </properties>\n` +
+            suites.join('\n') + '\n' +
+            `</testsuites>\n`;
+    },
+};

package/dist/esm/reporters/reporter.js

@@ -0,0 +1,3 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+export {};

package/dist/esm/reporters/sarif.js

@@ -0,0 +1,131 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+function severityToLevel(severity) {
+    switch (severity.toLowerCase()) {
+        case 'high':
+        case 'critical':
+            return 'error';
+        case 'medium':
+            return 'warning';
+        case 'low':
+        case 'info':
+            return 'note';
+        default:
+            return 'note';
+    }
+}
+function riskToLevel(risk) {
+    switch (risk.toLowerCase()) {
+        case 'high':
+            return 'warning';
+        case 'medium':
+            return 'note';
+        default:
+            return 'none';
+    }
+}
+export const sarifReporter = {
+    name: 'sarif',
+    extension: '.sarif',
+    format(results) {
+        const rules = [];
+        const sarifResults = [];
+        const ruleIds = new Set();
+        function ensureRule(id, description, level) {
+            if (!ruleIds.has(id)) {
+                ruleIds.add(id);
+                rules.push({
+                    id,
+                    shortDescription: { text: description },
+                    defaultConfiguration: { level },
+                });
+            }
+        }
+        // Findings -> results
+        for (const finding of results.findings) {
+            const level = severityToLevel(finding.severity);
+            const ruleId = `finding/${finding.severity}`;
+            ensureRule(ruleId, `Finding (${finding.severity})`, level);
+            sarifResults.push({
+                ruleId,
+                level,
+                message: { text: `${finding.title}: ${finding.description}` },
+                properties: {
+                    severity: finding.severity,
+                },
+            });
+        }
+        // Strategy entries without matching test designs -> coverage gap results
+        const designedFlows = new Set(results.testDesigns.map((d) => d.flowName));
+        for (const entry of results.strategyEntries) {
+            if (!designedFlows.has(entry.flowName)) {
+                const ruleId = 'coverage/gap';
+                ensureRule(ruleId, 'Missing test coverage for impacted flow', 'warning');
+                sarifResults.push({
+                    ruleId,
+                    level: 'warning',
+                    message: {
+                        text: `Flow "${entry.flowName}" (${entry.flowId}) has strategy but no test design. ` +
+                            `Priority: ${entry.priority}, approach: ${entry.approach}.`,
+                    },
+                    properties: {
+                        flowId: entry.flowId,
+                        priority: entry.priority,
+                    },
+                });
+            }
+        }
+        // High-risk cross-impacts -> warning results
+        for (const impact of results.crossImpacts) {
+            const level = riskToLevel(impact.riskLevel);
+            if (level === 'none') {
+                continue;
+            }
+            const ruleId = `cross-impact/${impact.riskLevel}`;
+            ensureRule(ruleId, `Cross-impact (${impact.riskLevel} risk)`, level);
+            sarifResults.push({
+                ruleId,
+                level,
+                message: {
+                    text: `Cross-impact: "${impact.sourceFamily}" affects "${impact.affectedFamily}" ` +
+                        `with ${impact.riskLevel} risk.`,
+                },
+                properties: {
+                    sourceFamily: impact.sourceFamily,
+                    affectedFamily: impact.affectedFamily,
+                    riskLevel: impact.riskLevel,
+                },
+            });
+        }
+        const run = {
+            tool: {
+                driver: {
+                    name: 'e2e-agents',
+                    version: '1.8.5',
+                    informationUri: 'https://github.com/mattermost/e2e-agents',
+                    rules,
+                },
+            },
+            results: sarifResults,
+            invocations: [
+                {
+                    executionSuccessful: true,
+                    properties: {
+                        workflow: results.workflow,
+                        changedFiles: results.changedFiles,
+                        impactedFlows: results.impactedFlows,
+                        cost: results.cost,
+                        tokens: results.tokens,
+                        warnings: results.warnings,
+                    },
+                },
+            ],
+        };
+        const sarif = {
+            $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json',
+            version: '2.1.0',
+            runs: [run],
+        };
+        return JSON.stringify(sarif, null, 2) + '\n';
+    },
+};

package/dist/esm/resilience/circuit_breaker.js

@@ -0,0 +1,78 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+const DEFAULT_CONFIG = {
+    failureThreshold: 3,
+    cooldownMs: 60000,
+};
+export class CircuitBreaker {
+    constructor(config = {}) {
+        this.state = 'closed';
+        this.failures = 0;
+        this.lastFailureTime = 0;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /** Returns the derived state without mutating internal state. */
+    get currentState() {
+        if (this.state === 'open' && Date.now() - this.lastFailureTime >= this.config.cooldownMs) {
+            return 'half-open';
+        }
+        return this.state;
+    }
+    get isOpen() {
+        return this.currentState === 'open';
+    }
+    /**
+     * Execute a function with circuit breaker protection.
+     * If the circuit is open, the fallback is called instead.
+     */
+    async call(fn, fallback) {
+        // Transition from open to half-open if cooldown has elapsed
+        if (this.state === 'open') {
+            if (Date.now() - this.lastFailureTime >= this.config.cooldownMs) {
+                this.state = 'half-open';
+            }
+            else {
+                return fallback();
+            }
+        }
+        // At this point state is 'closed' or 'half-open'
+        const stateBeforeCall = this.state;
+        try {
+            const result = await fn();
+            this.onSuccess();
+            return result;
+        }
+        catch (error) {
+            const shouldCount = !this.config.shouldCount || this.config.shouldCount(error);
+            if (shouldCount) {
+                this.onFailure();
+            }
+            // In half-open state, a failure re-opens the circuit
+            if (stateBeforeCall === 'half-open') {
+                throw error;
+            }
+            // In closed state, if failures hit threshold the circuit opened
+            if (shouldCount && this.failures >= this.config.failureThreshold) {
+                return fallback();
+            }
+            throw error;
+        }
+    }
+    onSuccess() {
+        this.failures = 0;
+        this.state = 'closed';
+    }
+    onFailure() {
+        this.failures++;
+        this.lastFailureTime = Date.now();
+        if (this.failures >= this.config.failureThreshold) {
+            this.state = 'open';
+        }
+    }
+    /** Reset the circuit breaker to closed state */
+    reset() {
+        this.state = 'closed';
+        this.failures = 0;
+        this.lastFailureTime = 0;
+    }
+}

package/dist/esm/resilience/retry.js

@@ -0,0 +1,56 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+const DEFAULT_RETRY_CONFIG = {
+    maxRetries: 2,
+    baseDelayMs: 1000,
+    maxDelayMs: 10000,
+    jitter: true,
+};
+/** Errors that should be retried (transient failures) */
+function isRetryable(error) {
+    if (!(error instanceof Error))
+        return false;
+    const msg = error.message.toLowerCase();
+    // Rate limits
+    if (msg.includes('rate limit') || msg.includes('429') || msg.includes('too many requests'))
+        return true;
+    // Server errors
+    if (msg.includes('500') || msg.includes('502') || msg.includes('503') || msg.includes('504'))
+        return true;
+    if (msg.includes('internal server error') || msg.includes('bad gateway') || msg.includes('service unavailable'))
+        return true;
+    // Network errors
+    if (msg.includes('econnreset') || msg.includes('econnrefused') || msg.includes('etimedout'))
+        return true;
+    if (msg.includes('socket hang up') || msg.includes('network error'))
+        return true;
+    // Overloaded
+    if (msg.includes('overloaded') || msg.includes('capacity'))
+        return true;
+    return false;
+}
+function computeDelay(attempt, config) {
+    const exponential = Math.min(config.baseDelayMs * Math.pow(2, attempt), config.maxDelayMs);
+    if (!config.jitter)
+        return exponential;
+    // Full jitter: random between 0 and exponential
+    return Math.floor(Math.random() * exponential);
+}
+export async function withRetry(fn, config = {}) {
+    const cfg = { ...DEFAULT_RETRY_CONFIG, ...config };
+    let lastError;
+    for (let attempt = 0; attempt <= cfg.maxRetries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt >= cfg.maxRetries || !isRetryable(error)) {
+                throw error;
+            }
+            const delay = computeDelay(attempt, cfg);
+            await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+    }
+    throw lastError;
+}

package/dist/esm/sanitize.js

@@ -0,0 +1,66 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Secret scanning and sanitization utilities.
+ * Prevents API keys and credentials from leaking into artifacts, logs, and output.
+ *
+ * Patterns are stored WITHOUT the global flag to avoid shared mutable lastIndex state.
+ * New RegExp instances with /g are created per call for safe concurrent usage.
+ */
+const SECRET_PATTERNS = [
+    // Anthropic API keys (must be checked before generic sk- pattern)
+    /sk-ant-[a-zA-Z0-9_-]{20,}/,
+    // OpenAI API keys (negative lookahead to avoid matching Anthropic keys)
+    /sk-(?!ant-)[a-zA-Z0-9]{20,}/,
+    // Generic API key patterns
+    /(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token)['":\s=]+['"]?([a-zA-Z0-9_\-./]{20,})['"]?/i,
+    // Bearer tokens
+    /Bearer\s+[a-zA-Z0-9_\-./]{20,}/,
+    // AWS keys
+    /AKIA[0-9A-Z]{16}/,
+    // GitHub tokens
+    /gh[ps]_[a-zA-Z0-9]{36,}/,
+    /github_pat_[a-zA-Z0-9_]{22,}/,
+];
+/**
+ * Sanitize a string by replacing detected secrets with [REDACTED].
+ */
+export function sanitizeSecrets(text) {
+    let result = text;
+    for (const pattern of SECRET_PATTERNS) {
+        result = result.replace(new RegExp(pattern, 'gi'), '[REDACTED]');
+    }
+    return result;
+}
+/**
+ * Check if a string contains any detectable secrets.
+ */
+export function containsSecrets(text) {
+    for (const pattern of SECRET_PATTERNS) {
+        if (new RegExp(pattern, 'i').test(text))
+            return true;
+    }
+    return false;
+}
+/**
+ * Deep-sanitize a JSON-serializable object.
+ * Recursively walks all string values and sanitizes them.
+ * Tracks seen objects to prevent stack overflow on circular references.
+ */
+export function sanitizeObject(obj, _seen) {
+    if (typeof obj === 'string')
+        return sanitizeSecrets(obj);
+    if (obj === null || typeof obj !== 'object')
+        return obj;
+    const seen = _seen ?? new WeakSet();
+    if (seen.has(obj))
+        return '[Circular]';
+    seen.add(obj);
+    if (Array.isArray(obj))
+        return obj.map((item) => sanitizeObject(item, seen));
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+        result[key] = sanitizeObject(value, seen);
+    }
+    return result;
+}