@yasserkhanorg/e2e-agents 0.3.2

package/dist/esm/anthropic_provider.js

@@ -0,0 +1,324 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import Anthropic from '@anthropic-ai/sdk';
+import { LLMProviderError } from './provider_interface.js';
+import { API_KEY_PATTERNS, sanitizeErrorMessage, withTimeout, validateAndSanitizeUrl } from './provider_utils.js';
+import { BaseProvider } from './base_provider.js';
+import { logger } from './logger.js';
+/**
+ * Anthropic Provider - Claude AI models
+ *
+ * Features:
+ * - Highest quality AI (98% accuracy in testing)
+ * - Vision support (analyze screenshots, compare UI)
+ * - Fast response times (<1 second)
+ * - 200K token context window
+ * - Prompt caching (reduces costs by 90% on repeated prompts)
+ *
+ * Costs (Claude Sonnet 4.5):
+ * - Input: $3 per 1M tokens
+ * - Output: $15 per 1M tokens
+ * - Cached input: $0.30 per 1M tokens
+ * - Estimated: ~$30-80/month for autonomous testing
+ *
+ * Use cases:
+ * - Vision tasks (screenshot comparison)
+ * - Complex failure diagnosis
+ * - High-stakes production testing
+ * - When quality is paramount
+ *
+ * Models:
+ * - claude-sonnet-4-5-20250929 (recommended - best balance)
+ * - claude-opus-4-5-20251101 (highest quality, slower, more expensive)
+ * - claude-haiku-4-0-20250430 (fastest, cheapest, lower quality)
+ */
+export class AnthropicProvider extends BaseProvider {
+    constructor(config) {
+        super();
+        this.name = 'anthropic';
+        this.capabilities = {
+            vision: true, // Full vision support
+            streaming: true,
+            maxTokens: 200000, // 200K context window
+            costPer1MInputTokens: 3, // $3 per 1M input tokens
+            costPer1MOutputTokens: 15, // $15 per 1M output tokens
+            supportsTools: true, // Function calling support
+            supportsPromptCaching: true, // Reduces costs by 90%
+            typicalResponseTimeMs: 800, // ~0.8 seconds
+        };
+        // SECURITY: Validate API key format
+        if (!API_KEY_PATTERNS.anthropic.test(config.apiKey)) {
+            throw new Error('Invalid API key format. Expected sk-ant-* format.');
+        }
+        // SECURITY: Validate and enforce HTTPS for remote connections
+        if (config.baseUrl) {
+            const validation = validateAndSanitizeUrl(config.baseUrl);
+            if (!validation.valid) {
+                throw new Error(`Invalid base URL: ${validation.warning}`);
+            }
+            if (validation.warning) {
+                logger.warn(`HTTPS required for remote URLs: ${validation.warning}`);
+            }
+        }
+        this.client = new Anthropic({
+            apiKey: config.apiKey,
+            baseURL: config.baseUrl,
+        });
+        this.model = config.model || 'claude-sonnet-4-5-20250929';
+    }
+    async generateText(prompt, options) {
+        const startTime = Date.now();
+        try {
+            // SECURITY: Validate prompt length to prevent resource exhaustion
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const response = await withTimeout(this.client.messages.create({
+                model: this.model,
+                max_tokens: options?.maxTokens || 4000,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop_sequences: options?.stopSequences,
+                system: options?.systemPrompt,
+                messages: [
+                    {
+                        role: 'user',
+                        content: prompt,
+                    },
+                ],
+            }), options?.timeout, 'generateText');
+            const responseTime = Date.now() - startTime;
+            const text = this.extractTextFromResponse(response);
+            // SECURITY: Type-safe usage extraction
+            const usage = this.extractUsageFromResponse(response.usage);
+            const cost = this.calculateCost(usage, this.capabilities.costPer1MInputTokens, this.capabilities.costPer1MOutputTokens);
+            // Update stats
+            this.updateStats(usage, responseTime, cost);
+            return {
+                text,
+                usage,
+                cost,
+                metadata: {
+                    model: this.model,
+                    responseTimeMs: responseTime,
+                    stopReason: response.stop_reason,
+                    stopSequence: response.stop_sequence,
+                },
+            };
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'generateText'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    async analyzeImage(images, prompt, options) {
+        const startTime = Date.now();
+        try {
+            // SECURITY: Validate image array size
+            if (images.length === 0 || images.length > 20) {
+                throw new Error('Image count must be between 1 and 20');
+            }
+            // SECURITY: Validate prompt length
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            // Build content array with text and images
+            const content = [];
+            // Add prompt text first
+            content.push({
+                type: 'text',
+                text: prompt,
+            });
+            // Add each image
+            for (const image of images) {
+                // Validate media type
+                const mediaType = (image.mimeType || image.mediaType || 'image/png');
+                if (!['image/png', 'image/jpeg', 'image/webp', 'image/gif'].includes(mediaType)) {
+                    throw new Error(`Unsupported image type: ${mediaType}`);
+                }
+                const data = image.data || image.base64 || '';
+                // SECURITY: Validate base64 data size (limit to 20MB per image)
+                if (data.length > 20 * 1024 * 1024) {
+                    throw new Error('Image data exceeds maximum size (20MB)');
+                }
+                content.push({
+                    type: 'image',
+                    source: {
+                        type: 'base64',
+                        media_type: mediaType,
+                        data: data,
+                    },
+                });
+                // Add description if provided
+                if (image.description) {
+                    content.push({
+                        type: 'text',
+                        text: `[Image: ${image.description}]`,
+                    });
+                }
+            }
+            const response = await withTimeout(this.client.messages.create({
+                model: this.model,
+                max_tokens: options?.maxTokens || 4000,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop_sequences: options?.stopSequences,
+                system: options?.systemPrompt,
+                messages: [
+                    {
+                        role: 'user',
+                        content,
+                    },
+                ],
+            }), options?.timeout, 'analyzeImage');
+            const responseTime = Date.now() - startTime;
+            const text = this.extractTextFromResponse(response);
+            // SECURITY: Type-safe usage extraction
+            const usage = this.extractUsageFromResponse(response.usage);
+            const cost = this.calculateCost(usage, this.capabilities.costPer1MInputTokens, this.capabilities.costPer1MOutputTokens);
+            // Update stats
+            this.updateStats(usage, responseTime, cost);
+            return {
+                text,
+                usage,
+                cost,
+                metadata: {
+                    model: this.model,
+                    responseTimeMs: responseTime,
+                    stopReason: response.stop_reason,
+                    imageCount: images.length,
+                },
+            };
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'analyzeImage'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    async *streamText(prompt, options) {
+        try {
+            // SECURITY: Validate prompt length
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const stream = await withTimeout(this.client.messages.create({
+                model: this.model,
+                max_tokens: options?.maxTokens || 4000,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop_sequences: options?.stopSequences,
+                system: options?.systemPrompt,
+                messages: [
+                    {
+                        role: 'user',
+                        content: prompt,
+                    },
+                ],
+                stream: true,
+            }), options?.timeout, 'streamText');
+            for await (const event of stream) {
+                if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
+                    yield event.delta.text;
+                }
+            }
+            // Note: Streaming doesn't provide detailed usage stats
+            // We increment request count but can't track exact tokens/cost
+            this.stats.requestCount++;
+            this.stats.lastUpdated = new Date();
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'streamText'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    extractTextFromResponse(response) {
+        const textBlocks = response.content.filter((block) => block.type === 'text');
+        return textBlocks.map((block) => {
+            if (block.type === 'text') {
+                return block.text;
+            }
+            return '';
+        }).join('\n');
+    }
+    /**
+     * SECURITY: Type-safe usage extraction
+     * Avoids unsafe `as any` casts
+     */
+    extractUsageFromResponse(usage) {
+        return {
+            inputTokens: usage.input_tokens || 0,
+            outputTokens: usage.output_tokens || 0,
+            totalTokens: (usage.input_tokens || 0) + (usage.output_tokens || 0),
+            cachedTokens: usage.cache_read_input_tokens ?? undefined,
+        };
+    }
+    /**
+     * SECURITY: Extract status code safely
+     */
+    extractStatusCode(error) {
+        if (error && typeof error === 'object') {
+            const err = error;
+            const status = err.status;
+            if (typeof status === 'number') {
+                return status;
+            }
+        }
+        return undefined;
+    }
+    /**
+     * Check if API key is valid and service is accessible
+     */
+    async checkHealth() {
+        try {
+            // Try a minimal request to verify API key
+            await withTimeout(this.client.messages.create({
+                model: this.model,
+                max_tokens: 10,
+                messages: [
+                    {
+                        role: 'user',
+                        content: 'Hi',
+                    },
+                ],
+            }), 5000, 'health check');
+            return {
+                healthy: true,
+                message: `Anthropic API is accessible`,
+            };
+        }
+        catch (error) {
+            return {
+                healthy: false,
+                message: `Anthropic API error: ${sanitizeErrorMessage(error, 'health check')}`,
+            };
+        }
+    }
+}
+/**
+ * Helper to check Anthropic setup
+ */
+export async function checkAnthropicSetup(apiKey) {
+    if (!apiKey) {
+        return {
+            valid: false,
+            message: 'No API key provided',
+            estimatedMonthlyCost: 'N/A',
+        };
+    }
+    try {
+        const provider = new AnthropicProvider({ apiKey });
+        const health = await provider.checkHealth();
+        return {
+            valid: health.healthy,
+            message: health.message,
+            estimatedMonthlyCost: '$30-80 for autonomous testing (24 cycles/day)',
+        };
+    }
+    catch (error) {
+        return {
+            valid: false,
+            message: `Setup check failed: ${sanitizeErrorMessage(error, 'setup check')}`,
+            estimatedMonthlyCost: 'N/A',
+        };
+    }
+}

package/dist/esm/api.js

@@ -0,0 +1,105 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { resolveConfig } from './agent/config.js';
+import { runGap, runImpact } from './agent/runner.js';
+import { attachDeveloperActions, buildPlanFromImpactReport, renderCiSummaryMarkdown, writeCiSummary, writePlanReport, } from './agent/plan.js';
+import { applyOperationalInsights } from './agent/operational_insights.js';
+import { finalizeGeneratedTests } from './agent/handoff.js';
+import { ingestTraceabilityInput, } from './agent/traceability_ingest.js';
+import { captureTraceabilityInput, } from './agent/traceability_capture.js';
+function readReportJson(reportPath) {
+    if (!existsSync(reportPath)) {
+        throw new Error(`Expected report not found: ${reportPath}`);
+    }
+    const raw = readFileSync(reportPath, 'utf-8');
+    return JSON.parse(raw);
+}
+function resolveAgent(options, mode) {
+    const cwd = options.cwd || process.cwd();
+    const { config } = resolveConfig(cwd, options.configPath, {
+        ...options,
+        mode,
+    });
+    if (options.allowFallback) {
+        config.impact.allowFallback = true;
+    }
+    return config;
+}
+function reportPathFor(configPath, mode) {
+    return join(configPath, '.e2e-ai-agents', mode === 'impact' ? 'impact.json' : 'gap.json');
+}
+export async function analyzeImpact(options = {}) {
+    const config = resolveAgent(options, 'impact');
+    await runImpact(config, { apply: options.apply ?? false });
+    const reportRoot = config.testsRoot || config.path;
+    const reportPath = reportPathFor(reportRoot, 'impact');
+    const report = readReportJson(reportPath);
+    return { report, reportPath };
+}
+export async function findGaps(options = {}) {
+    const config = resolveAgent(options, 'gap');
+    await runGap(config, { apply: options.apply ?? false });
+    const reportRoot = config.testsRoot || config.path;
+    const reportPath = reportPathFor(reportRoot, 'gap');
+    const report = readReportJson(reportPath);
+    return { report, reportPath };
+}
+export async function recommendTests(options = {}) {
+    const config = resolveAgent(options, 'impact');
+    await runImpact(config, { apply: options.apply ?? false });
+    const reportRoot = config.testsRoot || config.path;
+    const impactPath = reportPathFor(reportRoot, 'impact');
+    const report = readReportJson(impactPath);
+    const basePlan = buildPlanFromImpactReport(report, config.policy);
+    const withActions = attachDeveloperActions(basePlan, {
+        appPath: config.path,
+        testsRoot: reportRoot,
+        sinceRef: config.git.since,
+    });
+    const plan = applyOperationalInsights(withActions, reportRoot);
+    const planPath = writePlanReport(reportRoot, plan);
+    const ciSummaryMarkdown = renderCiSummaryMarkdown(plan);
+    const ciSummaryPath = writeCiSummary(reportRoot, ciSummaryMarkdown);
+    return {
+        report,
+        reportPath: impactPath,
+        plan,
+        planPath,
+        ciSummaryMarkdown,
+        ciSummaryPath,
+    };
+}
+export function handoffGeneratedTests(options) {
+    return finalizeGeneratedTests(options);
+}
+export function ingestTraceability(options) {
+    const cwd = options.cwd || process.cwd();
+    const { config } = resolveConfig(cwd, options.configPath, {
+        path: options.path,
+        testsRoot: options.testsRoot,
+        mode: 'impact',
+    });
+    const reportRoot = config.testsRoot || config.path;
+    return ingestTraceabilityInput(reportRoot, config.impact.traceability, options.payload, options.options);
+}
+export function captureTraceability(options) {
+    const cwd = options.cwd || process.cwd();
+    const { config } = resolveConfig(cwd, options.configPath, {
+        path: options.path,
+        testsRoot: options.testsRoot,
+        mode: 'impact',
+    });
+    const reportRoot = config.testsRoot || config.path;
+    const captureOptions = {
+        appPath: config.path,
+        testsRoot: reportRoot,
+        reportPath: options.reportPath,
+        sinceRef: options.sinceRef || config.git.since,
+        outputPath: options.outputPath,
+        coverageMapPath: options.coverageMapPath,
+        changedFilesPath: options.changedFilesPath,
+    };
+    return captureTraceabilityInput(captureOptions);
+}

package/dist/esm/base_provider.js

@@ -0,0 +1,77 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Abstract base class for all LLM providers
+ * Eliminates 240+ lines of duplicate stats management code
+ * Provides common functionality for token tracking, cost calculation, and stats management
+ */
+export class BaseProvider {
+    constructor() {
+        this.initializeStats();
+    }
+    /**
+     * Initialize stats object with default values
+     */
+    initializeStats() {
+        this.stats = {
+            requestCount: 0,
+            totalInputTokens: 0,
+            totalOutputTokens: 0,
+            totalTokens: 0,
+            totalCost: 0,
+            averageResponseTimeMs: 0,
+            failedRequests: 0,
+            startTime: new Date(),
+            lastUpdated: new Date(),
+        };
+    }
+    /**
+     * Update stats with new usage data
+     * Maintains rolling average for response time
+     */
+    updateStats(usage, responseTime, cost) {
+        this.stats.requestCount++;
+        this.stats.totalInputTokens += usage.inputTokens;
+        this.stats.totalOutputTokens += usage.outputTokens;
+        this.stats.totalTokens += usage.totalTokens;
+        this.stats.totalCost += cost;
+        // Update rolling average response time
+        const totalRequests = this.stats.requestCount;
+        this.stats.averageResponseTimeMs =
+            (this.stats.averageResponseTimeMs * (totalRequests - 1) + responseTime) / totalRequests;
+        this.stats.lastUpdated = new Date();
+    }
+    /**
+     * Get a copy of current usage stats
+     */
+    getUsageStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Reset all usage stats to initial state
+     */
+    resetUsageStats() {
+        this.initializeStats();
+    }
+    /**
+     * Calculate cost for token usage, accounting for prompt caching discounts
+     * Cached tokens cost 90% less than regular tokens
+     */
+    calculateCost(usage, costPer1MInputTokens, costPer1MOutputTokens) {
+        // Calculate input token cost
+        let inputCost = 0;
+        // Cached tokens cost 90% less
+        if (usage.cachedTokens) {
+            const cachedCost = (usage.cachedTokens / 1000000) * (costPer1MInputTokens * 0.1);
+            const uncachedInputTokens = usage.inputTokens - usage.cachedTokens;
+            const uncachedCost = (uncachedInputTokens / 1000000) * costPer1MInputTokens;
+            inputCost = cachedCost + uncachedCost;
+        }
+        else {
+            inputCost = (usage.inputTokens / 1000000) * costPer1MInputTokens;
+        }
+        // Calculate output token cost
+        const outputCost = (usage.outputTokens / 1000000) * costPer1MOutputTokens;
+        return inputCost + outputCost;
+    }
+}