@yasserkhanorg/e2e-agents 0.3.2

package/dist/esm/ollama_provider.js

@@ -0,0 +1,300 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import OpenAI from 'openai';
+import { LLMProviderError, UnsupportedCapabilityError } from './provider_interface.js';
+import { sanitizeErrorMessage, withTimeout } from './provider_utils.js';
+import { BaseProvider } from './base_provider.js';
+import { logger } from './logger.js';
+/**
+ * SECURITY: Validate Ollama base URL and enforce HTTPS for remote connections
+ */
+function normalizeOllamaBaseUrl(baseUrl) {
+    const raw = baseUrl || 'http://localhost:11434';
+    try {
+        const parsed = new URL(raw);
+        if (!parsed.pathname || parsed.pathname === '/') {
+            parsed.pathname = '/v1';
+        }
+        return parsed.toString().replace(/\/$/, '');
+    }
+    catch {
+        return 'http://localhost:11434/v1';
+    }
+}
+function validateOllamaUrl(baseUrl) {
+    const url = normalizeOllamaBaseUrl(baseUrl);
+    try {
+        const parsed = new URL(url);
+        // For non-localhost URLs, warn about HTTP risks
+        const isLocalhost = parsed.hostname === 'localhost' || parsed.hostname === '127.0.0.1' || parsed.hostname === '::1';
+        if (!isLocalhost && parsed.protocol === 'http:') {
+            logger.warn('Ollama connection over plaintext HTTP to remote server. ' +
+                'Prompts and responses will be transmitted unencrypted. Consider using HTTPS proxy or local Ollama.');
+        }
+        return { valid: true, url };
+    }
+    catch {
+        return {
+            valid: false,
+            url: 'http://localhost:11434/v1',
+            warning: `Invalid Ollama URL: ${baseUrl}. Using default: http://localhost:11434/v1`,
+        };
+    }
+}
+/**
+ * SECURITY: Validate model name to prevent injection issues
+ */
+function validateModelName(model) {
+    // Allow alphanumeric, dash, colon, underscore
+    // Typical format: deepseek-r1:7b, llama4:13b, etc.
+    return /^[a-z0-9_:.\-]+$/i.test(model) && model.length < 256;
+}
+/**
+ * SECURITY: Validate timeout value
+ */
+function validateTimeout(timeout) {
+    if (!timeout)
+        return 60000;
+    if (timeout < 1000 || timeout > 600000) {
+        logger.warn('Timeout out of valid range (1s-10m). Using 60 second default.');
+        return 60000;
+    }
+    return timeout;
+}
+/**
+ * Ollama Provider - Free, local LLM execution
+ *
+ * Features:
+ * - Zero cost (runs locally)
+ * - Full privacy (no data leaves your machine)
+ * - OpenAI-compatible API
+ * - Supports DeepSeek-R1, Llama 4, and other open models
+ *
+ * Limitations:
+ * - No vision support (most models)
+ * - Slower inference than cloud APIs (~2-5 sec vs <1 sec)
+ * - Requires local installation and model downloads
+ *
+ * Recommended models:
+ * - deepseek-r1:7b - Fast, good quality, low memory (4GB)
+ * - deepseek-r1:14b - Better quality, medium memory (8GB)
+ * - llama4:13b - High quality, medium memory (8GB)
+ * - deepseek-r1:7b-q4 - Quantized for speed, lower quality
+ *
+ * Setup:
+ * 1. Install Ollama: curl -fsSL https://ollama.com/install.sh | sh
+ * 2. Pull model: ollama pull deepseek-r1:7b
+ * 3. Start: ollama serve (runs on localhost:11434)
+ */
+export class OllamaProvider extends BaseProvider {
+    constructor(config) {
+        super();
+        this.name = 'ollama';
+        this.capabilities = {
+            vision: false, // Most Ollama models don't support vision
+            streaming: true,
+            maxTokens: 8000, // Varies by model
+            costPer1MInputTokens: 0, // Free!
+            costPer1MOutputTokens: 0, // Free!
+            supportsTools: true, // DeepSeek, Llama 4 support function calling
+            supportsPromptCaching: false,
+            typicalResponseTimeMs: 3000, // ~2-5 seconds on decent hardware
+        };
+        // SECURITY: Validate and sanitize URL
+        const urlValidation = validateOllamaUrl(config.baseUrl);
+        if (!urlValidation.valid && urlValidation.warning) {
+            console.warn(urlValidation.warning);
+        }
+        // SECURITY: Validate timeout
+        const timeout = validateTimeout(config.timeout);
+        // Ollama uses OpenAI-compatible API
+        this.client = new OpenAI({
+            baseURL: urlValidation.url,
+            apiKey: 'ollama', // Ollama doesn't require real API key
+            timeout,
+            maxRetries: 0, // Don't retry to avoid hanging on connection issues
+        });
+        const model = config.model || 'deepseek-r1:7b';
+        // SECURITY: Validate model name format
+        if (!validateModelName(model)) {
+            throw new Error('Invalid model name format');
+        }
+        this.model = model;
+    }
+    async generateText(prompt, options) {
+        const startTime = Date.now();
+        try {
+            // SECURITY: Validate prompt length
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const messages = [];
+            // Add system message if provided
+            if (options?.systemPrompt) {
+                messages.push({
+                    role: 'system',
+                    content: options.systemPrompt,
+                });
+            }
+            // Add user prompt
+            messages.push({
+                role: 'user',
+                content: prompt,
+            });
+            const response = await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                messages,
+                max_tokens: options?.maxTokens,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop: options?.stopSequences,
+            }), options?.timeout, 'generateText');
+            const responseTime = Date.now() - startTime;
+            const text = response.choices[0]?.message?.content || '';
+            const usage = {
+                inputTokens: response.usage?.prompt_tokens || 0,
+                outputTokens: response.usage?.completion_tokens || 0,
+                totalTokens: response.usage?.total_tokens || 0,
+            };
+            // Update stats
+            this.updateStats(usage, responseTime, 0); // Cost is always 0 for Ollama
+            return {
+                text,
+                usage,
+                cost: 0, // Free!
+                metadata: {
+                    model: this.model,
+                    responseTimeMs: responseTime,
+                    finishReason: response.choices[0]?.finish_reason,
+                },
+            };
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'generateText'), this.name, undefined, error);
+        }
+    }
+    /**
+     * Ollama does not support vision by default
+     * This method throws an error to help users understand the limitation
+     */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    async analyzeImage(images, prompt, options) {
+        throw new UnsupportedCapabilityError(this.name, 'vision');
+    }
+    /**
+     * Stream text generation for real-time feedback
+     */
+    async *streamText(prompt, options) {
+        try {
+            // SECURITY: Validate prompt length
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const messages = [];
+            if (options?.systemPrompt) {
+                messages.push({
+                    role: 'system',
+                    content: options.systemPrompt,
+                });
+            }
+            messages.push({
+                role: 'user',
+                content: prompt,
+            });
+            const stream = await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                messages,
+                max_tokens: options?.maxTokens,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop: options?.stopSequences,
+                stream: true,
+            }), options?.timeout, 'streamText');
+            for await (const chunk of stream) {
+                const content = chunk.choices[0]?.delta?.content;
+                if (content) {
+                    yield content;
+                }
+            }
+            // Note: Streaming doesn't provide detailed usage stats
+            // We increment request count but can't track exact tokens
+            this.stats.requestCount++;
+            this.stats.lastUpdated = new Date();
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'streamText'), this.name, undefined, error);
+        }
+    }
+    /**
+     * Check if Ollama is running and accessible
+     */
+    async checkHealth() {
+        try {
+            // Try a simple request
+            await withTimeout(this.client.models.list(), 5000, 'health check');
+            return {
+                healthy: true,
+                message: `Ollama is running with model: ${this.model}`,
+            };
+        }
+        catch (error) {
+            return {
+                healthy: false,
+                message: `Ollama not accessible: ${sanitizeErrorMessage(error, 'health check')}`,
+            };
+        }
+    }
+    /**
+     * List available models in Ollama
+     */
+    async listModels() {
+        try {
+            const response = await withTimeout(this.client.models.list(), 5000, 'listModels');
+            return response.data.map((model) => model.id);
+        }
+        catch (error) {
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'listModels'), this.name, undefined, error);
+        }
+    }
+}
+/**
+ * Helper function to check if Ollama is installed and suggest setup
+ */
+export async function checkOllamaSetup() {
+    const provider = new OllamaProvider({});
+    try {
+        const health = await provider.checkHealth();
+        const models = await provider.listModels();
+        return {
+            installed: true,
+            running: health.healthy,
+            modelAvailable: models.length > 0,
+            setupInstructions: health.healthy ? 'Ollama is ready to use!' : 'Run: ollama serve',
+        };
+    }
+    catch {
+        return {
+            installed: false,
+            running: false,
+            modelAvailable: false,
+            setupInstructions: `
+Ollama is not installed. To set up:
+
+1. Install Ollama:
+   curl -fsSL https://ollama.com/install.sh | sh
+
+2. Pull a model (choose one):
+   ollama pull deepseek-r1:7b          # Recommended: Fast, 4GB RAM
+   ollama pull deepseek-r1:14b         # Better quality, 8GB RAM
+   ollama pull llama4:13b              # Alternative, 8GB RAM
+
+3. Start Ollama:
+   ollama serve
+
+For more info: https://ollama.com
+            `.trim(),
+        };
+    }
+}

package/dist/esm/openai_provider.js

@@ -0,0 +1,242 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+import OpenAI from 'openai';
+import { LLMProviderError, UnsupportedCapabilityError } from './provider_interface.js';
+import { API_KEY_PATTERNS, sanitizeErrorMessage, withTimeout, validateAndSanitizeUrl } from './provider_utils.js';
+import { BaseProvider } from './base_provider.js';
+import { logger } from './logger.js';
+function inferVisionSupport(model) {
+    const lower = model.toLowerCase();
+    return lower.includes('vision') || lower.includes('4o') || lower.includes('omni');
+}
+export class OpenAIProvider extends BaseProvider {
+    constructor(config) {
+        super();
+        this.name = 'openai';
+        if (!API_KEY_PATTERNS.openai.test(config.apiKey)) {
+            throw new Error('Invalid API key format. Expected sk-* format.');
+        }
+        if (config.baseUrl) {
+            const validation = validateAndSanitizeUrl(config.baseUrl);
+            if (!validation.valid) {
+                throw new Error(`Invalid base URL: ${validation.warning}`);
+            }
+            if (validation.warning) {
+                logger.warn(`HTTPS required for remote URLs: ${validation.warning}`);
+            }
+        }
+        this.client = new OpenAI({
+            apiKey: config.apiKey,
+            baseURL: config.baseUrl,
+            organization: config.organizationId,
+        });
+        this.model = config.model || 'gpt-4';
+        const maxTokens = config.maxTokens || 128000;
+        const costPer1MInputTokens = config.costPer1MInputTokens ?? 0;
+        const costPer1MOutputTokens = config.costPer1MOutputTokens ?? 0;
+        this.capabilities = {
+            vision: inferVisionSupport(this.model),
+            streaming: true,
+            maxTokens,
+            costPer1MInputTokens,
+            costPer1MOutputTokens,
+            supportsTools: true,
+            supportsPromptCaching: false,
+            typicalResponseTimeMs: 1200,
+        };
+    }
+    async generateText(prompt, options) {
+        const startTime = Date.now();
+        try {
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const messages = [];
+            if (options?.systemPrompt) {
+                messages.push({ role: 'system', content: options.systemPrompt });
+            }
+            messages.push({ role: 'user', content: prompt });
+            const response = await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                messages,
+                max_tokens: options?.maxTokens,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop: options?.stopSequences,
+            }), options?.timeout, 'generateText');
+            const responseTime = Date.now() - startTime;
+            const text = response.choices[0]?.message?.content || '';
+            const usage = this.extractUsage(response.usage);
+            const cost = this.calculateCost(usage, this.capabilities.costPer1MInputTokens, this.capabilities.costPer1MOutputTokens);
+            this.updateStats(usage, responseTime, cost);
+            return {
+                text,
+                usage,
+                cost,
+                metadata: {
+                    model: this.model,
+                    responseTimeMs: responseTime,
+                    finishReason: response.choices[0]?.finish_reason,
+                },
+            };
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'generateText'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    async analyzeImage(images, prompt, options) {
+        if (!this.capabilities.vision) {
+            throw new UnsupportedCapabilityError(this.name, 'vision');
+        }
+        const startTime = Date.now();
+        try {
+            if (images.length === 0 || images.length > 20) {
+                throw new Error('Image count must be between 1 and 20');
+            }
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const content = [{ type: 'text', text: prompt }];
+            for (const image of images) {
+                const mediaType = (image.mimeType || image.mediaType || 'image/png');
+                if (!['image/png', 'image/jpeg', 'image/webp'].includes(mediaType)) {
+                    throw new Error(`Unsupported image type: ${mediaType}`);
+                }
+                const data = image.data || image.base64 || '';
+                if (data.length > 20 * 1024 * 1024) {
+                    throw new Error('Image data exceeds maximum size (20MB)');
+                }
+                const url = `data:${mediaType};base64,${data}`;
+                content.push({ type: 'image_url', image_url: { url } });
+                if (image.description) {
+                    content.push({ type: 'text', text: `[Image: ${image.description}]` });
+                }
+            }
+            const messages = [];
+            if (options?.systemPrompt) {
+                messages.push({ role: 'system', content: options.systemPrompt });
+            }
+            messages.push({ role: 'user', content });
+            const response = await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                messages,
+                max_tokens: options?.maxTokens,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop: options?.stopSequences,
+            }), options?.timeout, 'analyzeImage');
+            const responseTime = Date.now() - startTime;
+            const text = response.choices[0]?.message?.content || '';
+            const usage = this.extractUsage(response.usage);
+            const cost = this.calculateCost(usage, this.capabilities.costPer1MInputTokens, this.capabilities.costPer1MOutputTokens);
+            this.updateStats(usage, responseTime, cost);
+            return {
+                text,
+                usage,
+                cost,
+                metadata: {
+                    model: this.model,
+                    responseTimeMs: responseTime,
+                    finishReason: response.choices[0]?.finish_reason,
+                    imageCount: images.length,
+                },
+            };
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'analyzeImage'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    async *streamText(prompt, options) {
+        try {
+            if (prompt.length > 10 * 1024 * 1024) {
+                throw new Error('Prompt exceeds maximum size (10MB)');
+            }
+            const messages = [];
+            if (options?.systemPrompt) {
+                messages.push({ role: 'system', content: options.systemPrompt });
+            }
+            messages.push({ role: 'user', content: prompt });
+            const stream = await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                messages,
+                max_tokens: options?.maxTokens,
+                temperature: options?.temperature,
+                top_p: options?.topP,
+                stop: options?.stopSequences,
+                stream: true,
+            }), options?.timeout, 'streamText');
+            for await (const chunk of stream) {
+                const content = chunk.choices[0]?.delta?.content;
+                if (content) {
+                    yield content;
+                }
+            }
+            this.stats.requestCount++;
+            this.stats.lastUpdated = new Date();
+        }
+        catch (error) {
+            this.stats.failedRequests++;
+            throw new LLMProviderError(sanitizeErrorMessage(error, 'streamText'), this.name, this.extractStatusCode(error), error);
+        }
+    }
+    extractUsage(usage) {
+        return {
+            inputTokens: usage?.prompt_tokens || 0,
+            outputTokens: usage?.completion_tokens || 0,
+            totalTokens: usage?.total_tokens || 0,
+        };
+    }
+    extractStatusCode(error) {
+        if (error && typeof error === 'object') {
+            const err = error;
+            const status = err.status;
+            if (typeof status === 'number') {
+                return status;
+            }
+        }
+        return undefined;
+    }
+    async checkHealth() {
+        try {
+            await withTimeout(this.client.chat.completions.create({
+                model: this.model,
+                max_tokens: 5,
+                messages: [{ role: 'user', content: 'Hi' }],
+            }), 5000, 'health check');
+            return {
+                healthy: true,
+                message: 'OpenAI API is accessible',
+            };
+        }
+        catch (error) {
+            return {
+                healthy: false,
+                message: `OpenAI API error: ${sanitizeErrorMessage(error, 'health check')}`,
+            };
+        }
+    }
+}
+export async function checkOpenAISetup(apiKey) {
+    if (!apiKey) {
+        return {
+            valid: false,
+            message: 'No API key provided',
+        };
+    }
+    try {
+        const provider = new OpenAIProvider({ apiKey });
+        const health = await provider.checkHealth();
+        return {
+            valid: health.healthy,
+            message: health.message,
+        };
+    }
+    catch (error) {
+        return {
+            valid: false,
+            message: `Setup check failed: ${sanitizeErrorMessage(error, 'setup check')}`,
+        };
+    }
+}

package/dist/esm/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "module"
+}

package/dist/esm/plan-and-test-constants.js

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+/**
+ * Centralized constants for plan-and-test command
+ * Eliminates magic strings and hardcoded values
+ * Makes configuration easy to modify and maintain
+ */
+export const PLANNING_CONFIG = {
+    // Number of test scenarios per priority level
+    SCENARIO_COUNTS: {
+        P0: 3,
+        P1: 2,
+        P2: 1,
+    },
+    // Default parameter values
+    DEFAULTS: {
+        MAX_TESTS: 10,
+        COVERAGE_THRESHOLD: 50,
+        PRIORITY_FILTER: ['P0', 'P1'],
+    },
+    // Limit constraints
+    LIMITS: {
+        MIN_MAX_TESTS: 1,
+        MAX_MAX_TESTS: 100,
+        MIN_COVERAGE_THRESHOLD: 0,
+        MAX_COVERAGE_THRESHOLD: 100,
+    },
+};
+/**
+ * Display messages for plan-and-test command
+ */
+export const PLAN_AND_TEST_MESSAGES = {
+    HEADER: {
+        MAIN: '🚀 Planning and Generating Tests',
+        STEP_1_ANALYSIS: '📊 Step 1: Analyzing Code Changes...',
+        STEP_2_PLANNING: '💡 Step 2: Planning Test Generation...',
+        STEP_3_GENERATION: '⚡ Step 3: Generating Tests...',
+        STEP_4_SUMMARY: '📈 Generation Summary',
+        FOUND_FLOW_GROUPS: (count) => `📦 Found ${count} flow groups (end-to-end journeys):`,
+    },
+    ANALYSIS: {
+        FOUND_FLOWS: (count) => `✓ Found ${count} affected flows`,
+        MORE_FLOWS: (count) => `  ... and ${count} more`,
+        FOUND_FLOW_GROUPS: (count) => `📦 Found ${count} flow groups (end-to-end journeys)`,
+    },
+    PLANNING: {
+        PLAN_CREATED: (count) => `✓ Plan created: ${count} flows to test`,
+        SKIPPING_COUNT: (count) => `⊘ Skipping ${count} flows:`,
+    },
+    SKIP_REASONS: {
+        MAX_LIMIT_REACHED: (limit) => `Max tests limit reached (${limit})`,
+        ALREADY_COVERED: (count) => `Already covered (${count} existing tests)`,
+    },
+    COVERAGE_REASONS: {
+        NO_COVERAGE: (priority) => `${priority} - no coverage`,
+        PARTIAL_COVERAGE: (priority, gaps) => `${priority} - partial coverage (${gaps} gaps)`,
+    },
+    EXECUTION: {
+        TEST_COUNT: (index, total) => `[${index}/${total}]`,
+        GENERATION_FAILED: (error) => `  ⚠️ Generation failed: ${error}`,
+        DRY_RUN_MODE: '📋 DRY RUN: Not executing. Run without --dry-run to generate tests.',
+    },
+    SUMMARY: {
+        SEPARATOR: '═'.repeat(50),
+        TOTAL_GENERATED: (count) => `Total Tests Generated: ${count}`,
+        SUCCESSFUL: (successful, total) => `Successful: ${successful}/${total}`,
+        TOTAL_SCENARIOS: (count) => `Total Scenarios: ${count}`,
+        COMPLETION: '✅ Execution complete!',
+    },
+    NEXT_STEPS: [
+        '  • Run tests: npx playwright test --grep @smoke',
+        '  • Re-run impact: npx e2e-ai-agents impact --path <app-root> --tests-root <tests-root>',
+        '  • Check coverage: npm run test:impact',
+    ],
+    ERRORS: {
+        INVALID_PRIORITY: (priorities) => `Invalid priority levels: ${priorities.join(', ')}`,
+        INVALID_MAX_TESTS: (min, max) => `maxTests must be between ${min} and ${max}`,
+        INVALID_COVERAGE_THRESHOLD: (min, max) => `coverageThreshold must be between ${min} and ${max}`,
+        NO_CHANGES_DETECTED: '✓ No significant changes detected',
+    },
+};
+/**
+ * Priority levels in the system
+ */
+export const PRIORITY_LEVELS = {
+    CRITICAL: 'P0',
+    HIGH: 'P1',
+    MEDIUM: 'P2',
+};
+/**
+ * Test strategy types for flow groups
+ */
+export const TEST_STRATEGIES = {
+    SEQUENTIAL: 'sequential',
+    PARALLEL: 'parallel',
+    MIXED: 'mixed',
+};
+/**
+ * Flow group types
+ */
+export const FLOW_GROUP_TYPES = {
+    MESSAGING_LIFECYCLE: 'messaging-lifecycle',
+    CHANNEL_MANAGEMENT: 'channel-management',
+    MESSAGING_INTERACTIONS: 'messaging-interactions',
+};
+/**
+ * Utility function to get scenario count for a priority
+ * Provides type-safe access to scenario counts
+ */
+export function getScenarioCount(priority) {
+    const normalizedPriority = priority.toUpperCase();
+    return (PLANNING_CONFIG.SCENARIO_COUNTS[normalizedPriority] || PLANNING_CONFIG.SCENARIO_COUNTS.P1);
+}
+/**
+ * Utility function to validate priority level
+ */
+export function isValidPriority(priority) {
+    const validPriorities = Object.values(PRIORITY_LEVELS);
+    return validPriorities.includes(priority.toUpperCase());
+}
+/**
+ * Utility function to get all valid priority levels
+ */
+export function getValidPriorities() {
+    return Object.values(PRIORITY_LEVELS);
+}