@fasttest-ai/qa-agent 0.4.3 → 1.0.0

package/dist/runner.js

@@ -1,700 +0,0 @@
-/**
- * Test execution runner — executes test cases locally via Playwright
- * and reports results back to the Cloud.
- *
- * Flow:
- *   1. Cloud returns test cases (steps + assertions) from POST /run
- *   2. Runner executes each test case sequentially in the browser
- *   3. On selector-based failures, attempts self-healing
- *   4. After each test case, POST results back to Cloud
- *   5. When done, POST /complete to finalize the execution
- */
-import * as actions from "./actions.js";
-import { healSelector } from "./healer.js";
-import { resolveString, resolveStepVariables, resolveAssertionVariables, collectVariableNames } from "./variables.js";
-/**
- * Execute a full test suite run.
- */
-export async function executeRun(browserMgr, cloud, options, consoleLogs) {
-    // Apply device emulation (or reset to desktop when omitted)
-    await browserMgr.setDevice(options.device);
-    // 1. Ask Cloud to create execution + return test cases
-    const run = await cloud.startRun({
-        suite_id: options.suiteId,
-        environment_id: options.environmentId,
-        browser: "chromium",
-        test_case_ids: options.testCaseIds,
-        device: options.device,
-    });
-    const executionId = run.execution_id;
-    const testCases = run.test_cases;
-    // Resolve {{VAR}} placeholders in baseUrl
-    let baseUrl = options.appUrlOverride ?? run.base_url ?? "";
-    if (baseUrl) {
-        try {
-            baseUrl = resolveString(baseUrl);
-        }
-        catch (err) {
-            try {
-                await cloud.completeExecution(executionId);
-            }
-            catch { /* non-fatal */ }
-            return {
-                execution_id: executionId,
-                status: "failed",
-                total: testCases.length,
-                passed: 0,
-                failed: testCases.length,
-                skipped: 0,
-                duration_ms: 0,
-                results: testCases.map((tc) => ({
-                    id: tc.id, name: tc.name, status: "failed",
-                    duration_ms: 0, error: String(err), step_results: [],
-                })),
-                healed: [],
-            };
-        }
-    }
-    // Pre-flight: check that all {{VAR}} placeholders can be resolved
-    const allVarNames = [];
-    for (const tc of testCases) {
-        for (const v of collectVariableNames(tc.steps, tc.assertions)) {
-            if (!allVarNames.includes(v))
-                allVarNames.push(v);
-        }
-    }
-    if (allVarNames.length > 0) {
-        const resolved = [];
-        const missing = [];
-        for (const v of allVarNames) {
-            if (process.env[v] !== undefined) {
-                resolved.push(v);
-            }
-            else {
-                missing.push(v);
-            }
-        }
-        if (resolved.length > 0) {
-            process.stderr.write(`Environment variables resolved: ${resolved.join(", ")}\n`);
-        }
-        if (missing.length > 0) {
-            const errMsg = `Missing environment variable(s): ${missing.join(", ")}. ` +
-                `Set these before running tests. In GitHub Actions, add them as repository secrets.`;
-            process.stderr.write(`ERROR: ${errMsg}\n`);
-            try {
-                await cloud.completeExecution(executionId);
-            }
-            catch { /* non-fatal */ }
-            return {
-                execution_id: executionId,
-                status: "failed",
-                total: testCases.length,
-                passed: 0,
-                failed: testCases.length,
-                skipped: 0,
-                duration_ms: 0,
-                results: testCases.map((tc) => ({
-                    id: tc.id, name: tc.name, status: "failed",
-                    duration_ms: 0, error: errMsg, step_results: [],
-                })),
-                healed: [],
-            };
-        }
-    }
-    // 2. Order test cases: topo sort for dependencies, then fail-first
-    let sortedTestCases = topoSort(testCases);
-    if (run.previous_statuses) {
-        sortedTestCases = failFirst(sortedTestCases, run.previous_statuses);
-    }
-    const results = [];
-    const allHealed = [];
-    const runStart = Date.now();
-    let cancelled = false;
-    let reportFailures = 0;
-    const passedIds = new Set();
-    const runIdSet = new Set(sortedTestCases.map((tc) => tc.id));
-    // 3. Execute each test case sequentially (with smart retry)
-    for (const tc of sortedTestCases) {
-        // Skip if an in-run dependency failed (ignore external deps from other suites)
-        if (tc.depends_on && tc.depends_on.length > 0) {
-            const unmet = tc.depends_on.filter((depId) => runIdSet.has(depId) && !passedIds.has(depId));
-            if (unmet.length > 0) {
-                results.push({
-                    id: tc.id,
-                    name: tc.name,
-                    status: "skipped",
-                    duration_ms: 0,
-                    error: `Skipped: dependency not met (${unmet.join(", ")})`,
-                    step_results: [],
-                });
-                continue;
-            }
-        }
-        // Check for pause/cancel from dashboard
-        try {
-            const controlStatus = await cloud.checkControlStatus(executionId);
-            if (controlStatus === "cancelled") {
-                cancelled = true;
-                break;
-            }
-            if (controlStatus === "paused") {
-                let resumed = false;
-                const pauseStart = Date.now();
-                const MAX_PAUSE_MS = 30 * 60 * 1000; // 30 minutes
-                while (!resumed) {
-                    if (Date.now() - pauseStart > MAX_PAUSE_MS) {
-                        process.stderr.write("Pause exceeded 30-minute limit, auto-cancelling.\n");
-                        cancelled = true;
-                        break;
-                    }
-                    await new Promise((resolve) => setTimeout(resolve, 2000));
-                    const s = await cloud.checkControlStatus(executionId);
-                    if (s === "running") {
-                        resumed = true;
-                    }
-                    if (s === "cancelled") {
-                        cancelled = true;
-                        break;
-                    }
-                }
-                if (cancelled)
-                    break;
-            }
-        }
-        catch {
-            // Non-fatal — if we can't check, continue executing
-        }
-        // Smart retry loop
-        const maxRetries = tc.retry_count ?? 0;
-        let tcResult;
-        let attempt = 0;
-        // Notify War Room that this test case is starting (once, before retries)
-        await cloud.notifyTestStarted(executionId, tc.id, tc.name);
-        while (true) {
-            const timeoutMs = (tc.timeout_seconds || 30) * 1000;
-            let timerId;
-            const timeoutPromise = new Promise((_, reject) => {
-                timerId = setTimeout(() => reject(new Error(`Test case "${tc.name}" timed out after ${tc.timeout_seconds || 30}s`)), timeoutMs);
-            });
-            tcResult = await Promise.race([
-                executeTestCase(browserMgr, cloud, executionId, tc, baseUrl, consoleLogs, allHealed, options.aiFallback),
-                timeoutPromise,
-            ]).finally(() => clearTimeout(timerId)).catch((err) => ({
-                id: tc.id,
-                name: tc.name,
-                status: "failed",
-                duration_ms: timeoutMs,
-                error: String(err),
-                step_results: [],
-            }));
-            if (tcResult.status === "passed" || attempt >= maxRetries) {
-                break;
-            }
-            // Failed but have retries left
-            attempt++;
-            process.stderr.write(`Retrying ${tc.name} (attempt ${attempt}/${maxRetries})...\n`);
-        }
-        tcResult.retry_attempts = attempt;
-        if (tcResult.status === "passed")
-            passedIds.add(tc.id);
-        results.push(tcResult);
-        // Capture network calls for this test case and reset for next one
-        const rawNetwork = browserMgr.getNetworkSummary();
-        browserMgr.clearNetworkEntries();
-        const networkSummary = filterNetworkEntries(rawNetwork);
-        // 4. Report result to Cloud
-        try {
-            await cloud.reportResult(executionId, {
-                test_case_id: tc.id,
-                status: tcResult.status,
-                duration_ms: tcResult.duration_ms,
-                error_message: tcResult.error,
-                console_logs: consoleLogs.slice(-50),
-                retry_attempt: attempt,
-                step_results: tcResult.step_results.map((sr) => ({
-                    step_index: sr.step_index,
-                    action: sr.action,
-                    success: sr.success,
-                    error: sr.error,
-                    duration_ms: sr.duration_ms,
-                    screenshot_url: sr.screenshot_url,
-                    healed: sr.healed,
-                    heal_details: sr.heal_details,
-                })),
-                network_summary: networkSummary.length > 0 ? networkSummary : undefined,
-            });
-        }
-        catch (err) {
-            // Non-fatal — log and continue
-            reportFailures++;
-            process.stderr.write(`Failed to report result for ${tc.name}: ${err}\n`);
-        }
-    }
-    // Mark any unexecuted test cases as skipped (e.g. due to cancellation)
-    const completedIds = new Set(results.map((r) => r.id));
-    for (const tc of testCases) {
-        if (!completedIds.has(tc.id)) {
-            results.push({
-                id: tc.id,
-                name: tc.name,
-                status: "skipped",
-                duration_ms: 0,
-                step_results: [],
-            });
-        }
-    }
-    // 4. Auto-update healed selectors in saved test cases (>= 90% confidence)
-    const AUTO_HEAL_THRESHOLD = 0.90;
-    if (allHealed.length > 0) {
-        // Deduplicate: only one update per (test_case_id, original_selector) pair
-        const seen = new Set();
-        for (const h of allHealed) {
-            if (h.confidence < AUTO_HEAL_THRESHOLD)
-                continue;
-            const key = `${h.test_case_id}:${h.original_selector}`;
-            if (seen.has(key))
-                continue;
-            seen.add(key);
-            try {
-                await cloud.applyHealing(h.test_case_id, h.original_selector, h.new_selector);
-                process.stderr.write(`Auto-updated selector in "${h.test_case}": ${h.original_selector} → ${h.new_selector}\n`);
-            }
-            catch {
-                // Non-fatal — selector update failure shouldn't block run completion
-            }
-        }
-    }
-    // 5. Complete execution
-    const passed = results.filter((r) => r.status === "passed").length;
-    const failed = results.filter((r) => r.status === "failed").length;
-    const skipped = results.filter((r) => r.status === "skipped").length;
-    const durationMs = Date.now() - runStart;
-    try {
-        await cloud.completeExecution(executionId, cancelled ? "cancelled" : undefined);
-    }
-    catch (err) {
-        process.stderr.write(`Failed to complete execution: ${err}\n`);
-    }
-    if (reportFailures > 0) {
-        process.stderr.write(`Warning: ${reportFailures} result report(s) failed to send to cloud.\n`);
-    }
-    // Build AI fallback context from the first failure with diagnostic info
-    let aiFallback;
-    if (options.aiFallback) {
-        for (const r of results) {
-            if (r.status !== "failed")
-                continue;
-            const failedStep = r.step_results.find((sr) => !sr.success && sr.ai_context);
-            if (failedStep?.ai_context) {
-                // Find the original step data from the test case
-                const tc = sortedTestCases.find((t) => t.id === r.id);
-                const stepData = tc?.steps[failedStep.step_index] ?? {};
-                aiFallback = {
-                    test_case_id: r.id,
-                    test_case_name: r.name,
-                    step_index: failedStep.step_index,
-                    step: stepData,
-                    intent: failedStep.ai_context.intent,
-                    error: failedStep.error ?? r.error ?? "Unknown error",
-                    page_url: failedStep.ai_context.page_url,
-                    snapshot: failedStep.ai_context.snapshot,
-                };
-                break;
-            }
-        }
-    }
-    return {
-        execution_id: executionId,
-        status: cancelled ? "cancelled" : (failed === 0 ? "passed" : "failed"),
-        total: testCases.length,
-        passed,
-        failed,
-        skipped,
-        duration_ms: durationMs,
-        results,
-        healed: allHealed,
-        ai_fallback: aiFallback,
-    };
-}
-/**
- * Execute a single test case.
- */
-async function executeTestCase(browserMgr, cloud, executionId, tc, baseUrl, consoleLogs, allHealed, captureAiContext) {
-    const stepResults = [];
-    const start = Date.now();
-    try {
-        // Fresh context per test case to prevent session/cookie leakage
-        const page = await browserMgr.newContext();
-        // Execute steps
-        for (let i = 0; i < tc.steps.length; i++) {
-            const step = tc.steps[i];
-            const stepStart = Date.now();
-            // Resolve {{VAR}} placeholders from environment
-            let resolvedStep;
-            try {
-                resolvedStep = resolveStepVariables(step);
-            }
-            catch (err) {
-                stepResults.push({
-                    step_index: i, action: step.action, success: false,
-                    error: String(err), duration_ms: Date.now() - stepStart,
-                });
-                return {
-                    id: tc.id, name: tc.name, status: "failed",
-                    duration_ms: Date.now() - start,
-                    error: `Step ${i + 1} (${step.action}) failed: ${String(err)}`,
-                    step_results: stepResults,
-                };
-            }
-            let result = await executeStep(page, resolvedStep, baseUrl);
-            // Self-healing: if step failed and has a selector, try healing
-            if (!result.success && resolvedStep.selector && isSelectorFailure(result.error)) {
-                // Notify War Room that healing is starting
-                await cloud.notifyHealingStarted(executionId, tc.id, resolvedStep.selector);
-                const healResult = await healSelector(page, cloud, resolvedStep.selector, classifyStepError(result.error), result.error ?? "unknown", page.url(), { action: resolvedStep.action, description: resolvedStep.description, intent: resolvedStep.intent });
-                if (healResult.healed && healResult.newSelector) {
-                    // Retry with the healed selector
-                    const healedStep = { ...resolvedStep, selector: healResult.newSelector };
-                    result = await executeStep(page, healedStep, baseUrl);
-                    if (result.success) {
-                        allHealed.push({
-                            test_case_id: tc.id,
-                            test_case: tc.name,
-                            step_index: i,
-                            original_selector: step.selector,
-                            new_selector: healResult.newSelector,
-                            strategy: healResult.strategy ?? "unknown",
-                            confidence: healResult.confidence ?? 0,
-                        });
-                        // Capture screenshot after healed step
-                        const capture = await captureStepScreenshot(page);
-                        stepResults.push({
-                            step_index: i,
-                            action: step.action,
-                            success: true,
-                            duration_ms: Date.now() - stepStart,
-                            screenshot_url: capture?.dataUrl,
-                            healed: true,
-                            heal_details: {
-                                original_selector: step.selector,
-                                new_selector: healResult.newSelector,
-                                strategy: healResult.strategy ?? "unknown",
-                                confidence: healResult.confidence ?? 0,
-                            },
-                        });
-                        continue;
-                    }
-                }
-            }
-            // Capture screenshot after each step (especially important on failure for debugging)
-            const capture = await captureStepScreenshot(page);
-            // On failure, capture diagnostic snapshot for AI fallback (only when enabled)
-            let aiContext;
-            if (!result.success && captureAiContext) {
-                try {
-                    const snapshot = await actions.getSnapshot(page);
-                    aiContext = {
-                        intent: resolvedStep.intent ?? resolvedStep.description,
-                        page_url: page.url(),
-                        snapshot,
-                    };
-                }
-                catch {
-                    // Non-fatal — snapshot may fail if page crashed
-                }
-            }
-            stepResults.push({
-                step_index: i,
-                action: step.action,
-                success: result.success,
-                error: result.error,
-                duration_ms: Date.now() - stepStart,
-                screenshot_url: capture?.dataUrl,
-                ai_context: aiContext,
-            });
-            if (!result.success) {
-                return {
-                    id: tc.id,
-                    name: tc.name,
-                    status: "failed",
-                    duration_ms: Date.now() - start,
-                    error: `Step ${i + 1} (${step.action}) failed: ${result.error}`,
-                    step_results: stepResults,
-                };
-            }
-        }
-        // Execute assertions
-        for (let i = 0; i < tc.assertions.length; i++) {
-            const assertion = tc.assertions[i];
-            const assertStart = Date.now();
-            // Resolve {{VAR}} placeholders in assertion values
-            let resolvedAssertion;
-            try {
-                resolvedAssertion = resolveAssertionVariables(assertion);
-            }
-            catch (err) {
-                stepResults.push({
-                    step_index: tc.steps.length + i, action: `assert:${assertion.type}`,
-                    success: false, error: String(err), duration_ms: Date.now() - assertStart,
-                });
-                return {
-                    id: tc.id, name: tc.name, status: "failed",
-                    duration_ms: Date.now() - start,
-                    error: `Assertion ${i + 1} (${assertion.type}) failed: ${String(err)}`,
-                    step_results: stepResults,
-                };
-            }
-            const result = await executeAssertion(page, resolvedAssertion);
-            stepResults.push({
-                step_index: tc.steps.length + i,
-                action: `assert:${assertion.type}`,
-                success: result.pass,
-                error: result.error,
-                duration_ms: Date.now() - assertStart,
-            });
-            if (!result.pass) {
-                return {
-                    id: tc.id,
-                    name: tc.name,
-                    status: "failed",
-                    duration_ms: Date.now() - start,
-                    error: `Assertion ${i + 1} (${assertion.type}) failed: ${result.error ?? "expected value mismatch"}`,
-                    step_results: stepResults,
-                };
-            }
-        }
-        return {
-            id: tc.id,
-            name: tc.name,
-            status: "passed",
-            duration_ms: Date.now() - start,
-            step_results: stepResults,
-        };
-    }
-    catch (err) {
-        return {
-            id: tc.id,
-            name: tc.name,
-            status: "failed",
-            duration_ms: Date.now() - start,
-            error: String(err),
-            step_results: stepResults,
-        };
-    }
-}
-/**
- * Capture a screenshot after a test step.
- * Returns { dataUrl } or undefined if capture fails.
- */
-async function captureStepScreenshot(page) {
-    try {
-        const b64 = await actions.screenshot(page, false);
-        return { dataUrl: `data:image/jpeg;base64,${b64}` };
-    }
-    catch {
-        return undefined;
-    }
-}
-/**
- * Execute a single test step via Playwright.
- */
-async function executeStep(page, step, baseUrl) {
-    const action = step.action;
-    try {
-        switch (action) {
-            case "navigate": {
-                let url = step.url ?? step.value ?? "";
-                if (url && !url.startsWith("http")) {
-                    url = baseUrl.replace(/\/$/, "") + url;
-                }
-                return await actions.navigate(page, url);
-            }
-            case "click":
-                return await actions.click(page, step.selector ?? "");
-            case "type":
-            case "fill":
-                return await actions.fill(page, step.selector ?? "", step.value ?? "");
-            case "fill_form": {
-                const fields = step.fields ?? {};
-                return await actions.fillForm(page, fields);
-            }
-            case "drag":
-                return await actions.drag(page, step.selector ?? "", step.target ?? "");
-            case "resize":
-                return await actions.resize(page, step.width ?? 1280, step.height ?? 720);
-            case "hover":
-                return await actions.hover(page, step.selector ?? "");
-            case "select":
-                return await actions.selectOption(page, step.selector ?? "", step.value ?? "");
-            case "wait_for": {
-                if (step.condition === "navigation") {
-                    await page.waitForLoadState("domcontentloaded", { timeout: (step.timeout ?? 10) * 1000 });
-                    await page.waitForLoadState("networkidle", { timeout: 5000 }).catch(() => { });
-                    return { success: true };
-                }
-                return await actions.waitFor(page, step.selector ?? "", (step.timeout ?? 10) * 1000);
-            }
-            case "scroll": {
-                if (step.selector) {
-                    await page.locator(step.selector).scrollIntoViewIfNeeded();
-                }
-                else {
-                    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
-                }
-                return { success: true };
-            }
-            case "press_key":
-                return await actions.pressKey(page, step.key ?? step.value ?? "Enter");
-            case "upload_file": {
-                const filePaths = step.file_paths ?? (step.value ? [step.value] : null);
-                if (!filePaths || filePaths.length === 0) {
-                    return { success: false, error: "upload_file step missing file_paths" };
-                }
-                return await actions.uploadFile(page, step.selector ?? "", filePaths);
-            }
-            case "evaluate":
-                return await actions.evaluate(page, step.expression ?? step.value ?? "");
-            case "go_back":
-                return await actions.goBack(page);
-            case "go_forward":
-                return await actions.goForward(page);
-            case "assert":
-                // Step-level assertion — delegate to assertPage
-                return executeAssertion(page, step).then((r) => ({
-                    success: r.pass,
-                    error: r.error,
-                }));
-            default:
-                return { success: false, error: `Unknown action: ${action}` };
-        }
-    }
-    catch (err) {
-        return { success: false, error: String(err) };
-    }
-}
-/**
- * Execute an assertion.
- */
-async function executeAssertion(page, assertion) {
-    return actions.assertPage(page, {
-        type: assertion.type,
-        selector: assertion.selector,
-        text: assertion.text ?? assertion.expected_value,
-        url: assertion.url,
-        count: assertion.count,
-        attribute: assertion.attribute,
-        value: assertion.value ?? assertion.expected_value,
-    });
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/** Detect if a step error is selector-related (element not found / timeout). */
-function isSelectorFailure(error) {
-    if (!error)
-        return false;
-    const lower = error.toLowerCase();
-    return (lower.includes("selector") ||
-        lower.includes("not found") ||
-        lower.includes("waiting for selector") ||
-        lower.includes("no element") ||
-        lower.includes("timeout") ||
-        lower.includes("locator"));
-}
-/** Map a step error message to a failure type for the healing service. */
-function classifyStepError(error) {
-    if (!error)
-        return "UNKNOWN";
-    const lower = error.toLowerCase();
-    if (lower.includes("timeout"))
-        return "TIMEOUT";
-    if (lower.includes("not found") || lower.includes("no element") || lower.includes("selector")) {
-        return "ELEMENT_NOT_FOUND";
-    }
-    if (lower.includes("navigation") || lower.includes("net::"))
-        return "NAVIGATION_FAILED";
-    return "UNKNOWN";
-}
-/** Keep only API/document calls — skip static assets like images, fonts, CSS, JS bundles. */
-function filterNetworkEntries(entries) {
-    return entries.filter((e) => {
-        const mime = e.mimeType.toLowerCase();
-        if (mime.includes("json") || mime.includes("text/html") || mime.includes("text/plain"))
-            return true;
-        if (e.status >= 400)
-            return true;
-        return false;
-    });
-}
-/**
- * Reorder test cases so previously-failed ones run first (faster feedback).
- * Preserves relative order within each group. Respects dependency constraints
- * by not moving a test before its dependencies.
- */
-function failFirst(testCases, previousStatuses) {
-    const idSet = new Set(testCases.map((tc) => tc.id));
-    const depSet = new Set();
-    for (const tc of testCases) {
-        if (tc.depends_on) {
-            for (const dep of tc.depends_on)
-                depSet.add(dep);
-        }
-    }
-    const failed = [];
-    const rest = [];
-    for (const tc of testCases) {
-        const prev = previousStatuses[tc.id];
-        // Move to front if previously failed, not a dependency of other tests,
-        // and doesn't itself have in-run dependencies (which would need to run first)
-        const hasInRunDeps = tc.depends_on?.some((d) => idSet.has(d)) ?? false;
-        if (prev === "failed" && !depSet.has(tc.id) && !hasInRunDeps) {
-            failed.push(tc);
-        }
-        else {
-            rest.push(tc);
-        }
-    }
-    return [...failed, ...rest];
-}
-/**
- * Topological sort of test cases based on depends_on.
- * Falls back to original order if no dependencies or on cycles.
- */
-function topoSort(testCases) {
-    const idSet = new Set(testCases.map((tc) => tc.id));
-    const hasDeps = testCases.some((tc) => tc.depends_on && tc.depends_on.some((d) => idSet.has(d)));
-    if (!hasDeps)
-        return testCases;
-    const map = new Map(testCases.map((tc) => [tc.id, tc]));
-    const visited = new Set();
-    const visiting = new Set();
-    const sorted = [];
-    function visit(id) {
-        if (visited.has(id))
-            return true;
-        if (visiting.has(id))
-            return false; // cycle
-        visiting.add(id);
-        const tc = map.get(id);
-        if (tc?.depends_on) {
-            for (const dep of tc.depends_on) {
-                if (idSet.has(dep) && !visit(dep))
-                    return false;
-            }
-        }
-        visiting.delete(id);
-        visited.add(id);
-        if (tc)
-            sorted.push(tc);
-        return true;
-    }
-    for (const tc of testCases) {
-        if (!visit(tc.id)) {
-            // Cycle detected — fall back to original order
-            process.stderr.write("Warning: dependency cycle detected, using original test order.\n");
-            return testCases;
-        }
-    }
-    return sorted;
-}
-//# sourceMappingURL=runner.js.map