donobu 5.4.0 → 5.6.0

package/dist/esm/lib/test/utils/triageTestFailure.js

@@ -36,10 +36,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TreatmentPlan = exports.RemediationStepSchema = exports.FailureReasonSchema = exports.AdditionalDataRequestSchema = void 0;
-exports.buildTreatmentPlanFromHeuristics = buildTreatmentPlanFromHeuristics;
+exports.TRIAGE_PERSISTENCE_FILE_IDS = exports.TreatmentPlan = exports.RemediationStepSchema = exports.FailureReasonSchema = exports.AdditionalDataRequestSchema = void 0;
+exports.deriveHeuristicAssessment = deriveHeuristicAssessment;
+exports.deriveHistoricalSignals = deriveHistoricalSignals;
 exports.gatherTestFailureEvidence = gatherTestFailureEvidence;
 exports.generateTreatmentPlanFromEvidence = generateTreatmentPlanFromEvidence;
+exports.reconcileTreatmentPlan = reconcileTreatmentPlan;
+exports.summarizeFlowHistory = summarizeFlowHistory;
 const crypto_1 = require("crypto");
 const fs = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
@@ -52,12 +55,155 @@ const SummarizeLearningsTool_1 = require("../../../tools/SummarizeLearningsTool"
 const Logger_1 = require("../../../utils/Logger");
 const cacheLocator_1 = require("../../ai/cache/cacheLocator");
 /**
- * Utilities for transforming a Playwright test failure into a structured treatment plan
- * that Donobu Studio can surface to engineers or automation. The flow pivots between:
- *  1. Gathering heuristics from the failing run (errors, attachments, Donobu AI history).
- *  2. Asking the GPT-based triage agent for a canonical plan compliant with the schema.
- *  3. Falling back to heuristics when the LLM cannot respond, so automation still receives
- *     next steps. The helper functions below exist to keep these stages composable.
+ * # Test Failure Triage System
+ *
+ * Transforms Playwright test failures in Donobu-powered test suites into structured,
+ * actionable **treatment plans**. A treatment plan tells both humans and automation *why*
+ * the test failed, *how confident* the system is in that diagnosis, and *what to do next*
+ * — whether that is retrying the automation, deleting a stale page.ai cache, filing a
+ * product bug, or updating selectors in the test code.
+ *
+ * ---
+ *
+ * ## Architecture Overview
+ *
+ * The system operates in two phases that run in sequence:
+ *
+ * ### Phase 1 — Evidence Collection (`gatherTestFailureEvidence`)
+ *
+ * Called automatically by the Donobu test extension (`testExtension.ts`) in the
+ * Playwright `afterEach` hook whenever a test fails. This phase:
+ *
+ *   1. Extracts error messages, stack traces, and assertion details from `TestInfo`.
+ *   2. Loads the Donobu flow metadata (objective, run mode, state) and recent tool call
+ *      history from the persistence layer.
+ *   3. Fetches **historical runs** of the same flow (by name) from the flows manager to
+ *      detect flakiness, regression patterns, and prior self-heal success.
+ *   4. Captures the **failure screenshot** (last tool call screenshot from the current
+ *      run) and the **baseline screenshot** (last tool call screenshot from the most
+ *      recent successful historical run) for visual comparison.
+ *   5. Reads the source of the failing test case for contextual grounding.
+ *   6. Runs the **heuristic classifier** (`deriveHeuristicAssessment`) which uses
+ *      rule-based pattern matching over errors, tool calls, stale-cache indicators,
+ *      and historical signals to produce a preliminary diagnosis — including a failure
+ *      reason, confidence score, and retry recommendation.
+ *   7. Persists the complete evidence bundle (JSON + screenshots) to disk as a
+ *      `FailureEvidenceRecord`.
+ *
+ * ### Phase 2 — Treatment Plan Generation (`generateTreatmentPlanFromEvidence`)
+ *
+ * Called by the Donobu CLI (`donobu-cli.ts`) after evidence files are collected. This
+ * phase reads the persisted evidence and **requires a GPT client** — there is no
+ * heuristic-only fallback path. It:
+ *
+ *   1. Sends the full evidence bundle — including screenshots as vision input — to a
+ *      GPT model with a detailed system prompt, requesting a structured `TreatmentPlan`
+ *      response.
+ *   2. **Reconciles** the GPT plan with heuristic signals (`reconcileTreatmentPlan`) to
+ *      enforce invariants the LLM might miss (e.g., forcing `shouldRetryAutomation` when
+ *      historical data shows prior self-heal success, or overriding retry-step priority
+ *      for stale-cache scenarios).
+ *   3. Returns the final `TreatmentPlan` for the CLI to act on — potentially triggering
+ *      automatic retries, cache deletion, or surfacing remediation steps to the engineer.
+ *
+ * ---
+ *
+ * ## Data Signals
+ *
+ * The triage system draws from several complementary data sources, each targeting
+ * different failure modes:
+ *
+ * | Signal                  | Source                        | What it reveals                                    |
+ * |-------------------------|-------------------------------|----------------------------------------------------|
+ * | Error messages & stacks | `TestInfo.errors`             | Direct cause (assertion, timeout, selector)        |
+ * | Tool call history       | `FlowsPersistence`            | What actions the AI took and their outcomes        |
+ * | Tool call parameters    | `ToolCall.parameters`         | Exact selectors, URLs, and inputs attempted        |
+ * | Flow metadata           | `DonobuExtendedPage._dnb`     | Run mode, objective, allowed tools, timing         |
+ * | Stale cache indicators  | Derived from above            | Whether page.ai cache staleness is the root cause  |
+ * | Historical flow runs    | `DonobuFlowsManager.getFlows` | Flakiness, regression patterns, prior self-heal    |
+ * | Failure screenshot      | Last tool call screenshot     | Visual state of the page when the failure occurred |
+ * | Baseline screenshot     | Last successful run's screenshot | Visual reference for what the page *should* look like |
+ * | Test source snippet     | TypeScript AST parsing        | The test's expectations and structure               |
+ *
+ * ---
+ *
+ * ## Failure Classification
+ *
+ * Every treatment plan assigns one of the following failure reasons:
+ *
+ * - `SELECTOR_REGRESSION` — UI locators have gone stale.
+ * - `STALE_CACHE_OR_INSTRUCTIONS` — The page.ai deterministic cache is outdated.
+ * - `TIMING_OR_SYNCHRONISATION` — Race conditions, slow loads, or flaky waits.
+ * - `NETWORK_OR_DEPENDENCY` — External service failures or connectivity issues.
+ * - `APPLICATION_DEFECT` — A real bug in the product under test.
+ * - `ASSERTION_DRIFT` — Test expectations no longer match valid application behavior.
+ * - `AUTOMATION_SCRIPT_ISSUE` — The test script itself is incorrect.
+ * - `AUTHENTICATION_FAILURE` — Session/auth problems prevented the test from running.
+ * - `ENVIRONMENT_CONFIGURATION` — Infrastructure or environment misconfiguration.
+ * - `TEST_DATA_UNAVAILABLE` — Required test data is missing or invalid.
+ * - `UNKNOWN` — Insufficient signal to determine the cause.
+ *
+ * ---
+ *
+ * ## Getting the Most Out of This System
+ *
+ * ### 1. Name your flows consistently
+ *
+ * Historical analysis works by matching flows by name. If every test uses a unique,
+ * stable flow name, the system can compare the current failure against all prior runs
+ * of the same flow and detect flakiness, regressions, and self-heal patterns:
+ *
+ * ```ts
+ * test('checkout flow adds item and completes purchase', async ({ page }) => {
+ *   const ai = await page.ai('Checkout — add item and purchase', { ... });
+ *   // ...
+ * });
+ * ```
+ *
+ * ### 2. Let evidence persist to disk
+ *
+ * The default behavior writes evidence JSON and screenshots to the run directory. This
+ * enables the CLI's Phase 2 to enrich the diagnosis with GPT and visual comparison.
+ * Ensure `DONOBU_TRIAGE_DISABLED` is not set, and that `runDirectory` is writable:
+ *
+ * ```ts
+ * // Evidence is gathered automatically on failure — no extra code needed.
+ * // To customize the output directory:
+ * await gatherTestFailureEvidence(testInfo, page, {
+ *   runDirectory: '/path/to/custom/output',
+ * });
+ * ```
+ *
+ * ### 3. Ensure a GPT client is available
+ *
+ * A GPT client is **required** for treatment plan generation. The LLM performs semantic
+ * reasoning: it reads the test source, interprets tool call parameters, compares
+ * screenshots visually, and produces human-readable remediation steps. The CLI
+ * instantiates a GPT client automatically from configured credentials.
+ *
+ * ### 4. Use deterministic (cached) mode for stable flows
+ *
+ * When flows run in `DETERMINISTIC` mode with a page.ai cache, the triage system
+ * activates its stale-cache detection pipeline — a composite scoring system that
+ * weighs whether the cached instructions have gone stale versus whether the failure
+ * is a legitimate test issue. This is the system's strongest diagnostic capability.
+ *
+ * ### 5. Inspect the evidence files
+ *
+ * Each failure produces a `failure-evidence-<id>.json` file (plus optional PNG
+ * screenshots) in the run directory. These files are self-contained and can be
+ * re-processed, shared for debugging, or fed back into `generateTreatmentPlanFromEvidence`
+ * independently.
+ *
+ * ---
+ *
+ * ## Key Exports
+ *
+ * - `gatherTestFailureEvidence` — Phase 1 entry point. Call from a Playwright afterEach hook.
+ * - `generateTreatmentPlanFromEvidence` — Phase 2 entry point. Requires a `GptClient` and a
+ *   `FailureEvidenceRecord`.
+ * - `TreatmentPlan` — The Zod schema defining the treatment plan structure.
+ * - `FailureReasonSchema` — The Zod enum of all possible failure classifications.
  */
 const FailureReasonSchema = v4_1.z
     .enum([
@@ -178,8 +324,15 @@ const TreatmentPlan = v4_1.z.object({
 });
 exports.TreatmentPlan = TreatmentPlan;
 const MAX_SERIALIZED_STRING_LENGTH = 10000;
+const MAX_TOOL_CALL_PARAMETERS_LENGTH = 500;
 const MAX_TOOL_CALLS_TO_INCLUDE = 12;
-const TRIAGE_EVIDENCE_SCHEMA_VERSION = 1;
+const TRIAGE_EVIDENCE_SCHEMA_VERSION = 2;
+const TRIAGE_PERSISTENCE_FILE_IDS = {
+    evidence: 'triage-evidence.json',
+    failureScreenshot: 'triage-failure-screenshot.png',
+    baselineScreenshot: 'triage-baseline-screenshot.png',
+};
+exports.TRIAGE_PERSISTENCE_FILE_IDS = TRIAGE_PERSISTENCE_FILE_IDS;
 /**
  * Ensures text blobs captured from tool calls or error messages fit within storage
  * limits without losing useful context by padding or truncation artifacts.
@@ -251,7 +404,211 @@ function summarizeToolCalls(toolCalls) {
         page: toolCall.page,
         startedAtIso: new Date(toolCall.startedAt).toISOString(),
         completedAtIso: new Date(toolCall.completedAt).toISOString(),
+        parameters: safeStringify(toolCall.parameters, MAX_TOOL_CALL_PARAMETERS_LENGTH, 200),
+    }));
+}
+const HISTORY_QUERY_WINDOW_DAYS = 14;
+const HISTORY_QUERY_LIMIT = 20;
+const HISTORY_RECENT_RUNS_CAP = 10;
+/**
+ * Compresses a set of historical flow runs into an aggregate summary compact
+ * enough for both heuristic reasoning and inclusion in GPT prompts.
+ */
+function summarizeFlowHistory(flowName, flows) {
+    const sorted = [...flows].sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));
+    let successCount = 0;
+    let failureCount = 0;
+    let otherCount = 0;
+    for (const flow of sorted) {
+        if (flow.state === 'SUCCESS') {
+            successCount++;
+        }
+        else if (flow.state === 'FAILED') {
+            failureCount++;
+        }
+        else {
+            otherCount++;
+        }
+    }
+    const totalRuns = sorted.length;
+    const passRate = totalRuns > 0 ? successCount / totalRuns : 0;
+    const recentRuns = sorted
+        .slice(0, HISTORY_RECENT_RUNS_CAP)
+        .map((f) => ({
+        id: f.id,
+        state: f.state,
+        runMode: f.runMode,
+        startedAt: f.startedAt,
+        completedAt: f.completedAt,
+        durationMs: f.startedAt !== null && f.completedAt !== null
+            ? f.completedAt - f.startedAt
+            : null,
     }));
+    let streakState = 'MIXED';
+    let streakLength = 0;
+    if (recentRuns.length > 0) {
+        const firstState = recentRuns[0].state;
+        if (firstState === 'SUCCESS' || firstState === 'FAILED') {
+            streakState = firstState;
+            streakLength = 1;
+            for (let i = 1; i < recentRuns.length; i++) {
+                if (recentRuns[i].state === firstState) {
+                    streakLength++;
+                }
+                else {
+                    break;
+                }
+            }
+        }
+    }
+    const lastSuccessfulRun = sorted.find((f) => f.state === 'SUCCESS');
+    return {
+        flowName,
+        totalRuns,
+        successCount,
+        failureCount,
+        otherCount,
+        passRate,
+        recentRuns,
+        currentStreak: { state: streakState, length: streakLength },
+        lastSuccessfulRunId: lastSuccessfulRun?.id ?? null,
+        queryWindowDays: HISTORY_QUERY_WINDOW_DAYS,
+        queriedAt: new Date().toISOString(),
+    };
+}
+/**
+ * Derives actionable signals from historical flow run data to feed into the
+ * heuristic classifier: flakiness, regression likelihood, prior self-heal
+ * success, and whether the page.ai cache was recently validated.
+ */
+function deriveHistoricalSignals(history) {
+    const { recentRuns, passRate } = history;
+    // Flakiness: proportion of state transitions in recent runs
+    let transitions = 0;
+    for (let i = 1; i < recentRuns.length; i++) {
+        if (recentRuns[i].state !== recentRuns[i - 1].state) {
+            transitions++;
+        }
+    }
+    const maxTransitions = Math.max(1, recentRuns.length - 1);
+    const flakinessScore = recentRuns.length > 1 ? transitions / maxTransitions : 0;
+    // Regression: consecutive failures at the head after passes existed
+    let regressionLikelihood = 0;
+    if (history.currentStreak.state === 'FAILED' && history.successCount > 0) {
+        const streakWeight = Math.min(history.currentStreak.length / 5, 1);
+        regressionLikelihood = Math.min(1, passRate * 0.6 + streakWeight * 0.4);
+    }
+    // Prior self-heal: a FAILED run immediately followed by SUCCESS (newest-first)
+    let priorSelfHealSuccess = false;
+    for (let i = 0; i < recentRuns.length - 1; i++) {
+        if (recentRuns[i].state === 'SUCCESS' &&
+            recentRuns[i + 1].state === 'FAILED') {
+            priorSelfHealSuccess = true;
+            break;
+        }
+    }
+    // Cache recently valid: any recent DETERMINISTIC SUCCESS
+    const cacheWasRecentlyValid = recentRuns.some((r) => r.runMode === 'DETERMINISTIC' && r.state === 'SUCCESS');
+    return {
+        flakinessScore,
+        regressionLikelihood,
+        recentPassRate: passRate,
+        priorSelfHealSuccess,
+        cacheWasRecentlyValid,
+    };
+}
+/**
+ * Fetches historical runs of the same flow by name and returns a compact
+ * summary. Runs concurrently with other evidence collection and fails open
+ * so triage proceeds even if the history lookup encounters errors.
+ */
+async function fetchFlowHistory(page) {
+    const flowName = page._dnb?.donobuFlowMetadata?.name;
+    if (!flowName) {
+        return null;
+    }
+    const flowsManager = page._dnb?.donobuStack?.flowsManager;
+    if (!flowsManager) {
+        return null;
+    }
+    const cutoffMs = Date.now() - HISTORY_QUERY_WINDOW_DAYS * 24 * 60 * 60 * 1000;
+    const currentFlowId = page._dnb.donobuFlowMetadata.id;
+    try {
+        const result = await flowsManager.getFlows({
+            name: flowName,
+            startedAfter: cutoffMs,
+            limit: HISTORY_QUERY_LIMIT,
+        });
+        const historicalFlows = result.items.filter((f) => f.id !== currentFlowId);
+        if (historicalFlows.length === 0) {
+            return null;
+        }
+        return summarizeFlowHistory(flowName, historicalFlows);
+    }
+    catch (error) {
+        Logger_1.appLogger.warn(`Failed to fetch historical flow data for "${flowName}", proceeding without history.`, error);
+        return null;
+    }
+}
+/**
+ * Retrieves the screenshot from the last completed tool call in the current flow.
+ * Returns the raw PNG/JPEG buffer if available, or null. Fails open so triage
+ * proceeds even if the screenshot cannot be loaded.
+ */
+async function fetchLastToolCallScreenshot(page) {
+    const flowId = page._dnb?.donobuFlowMetadata?.id;
+    const persistence = page._dnb?.persistence;
+    if (!flowId || !persistence) {
+        return null;
+    }
+    try {
+        const toolCalls = await persistence.getToolCalls(flowId);
+        if (toolCalls.length === 0) {
+            return null;
+        }
+        // Walk backwards to find the last tool call with a screenshot
+        for (let i = toolCalls.length - 1; i >= 0; i--) {
+            const screenshotId = toolCalls[i].postCallImageId;
+            if (screenshotId) {
+                return await persistence.getScreenShot(flowId, screenshotId);
+            }
+        }
+        return null;
+    }
+    catch (error) {
+        Logger_1.appLogger.debug(`Failed to fetch last tool call screenshot for flow ${flowId}.`, error);
+        return null;
+    }
+}
+/**
+ * Loads the final screenshot from a historical successful run to serve as a
+ * visual baseline for comparison with the current failure state. This enables
+ * the GPT triage agent to detect page redesigns and stale cache scenarios by
+ * comparing "what the page looked like when it last worked" vs "what it looks
+ * like now." Fails open — returns null if the screenshot cannot be retrieved.
+ */
+async function fetchBaselineScreenshot(page, historicalFlowId) {
+    const persistence = page._dnb?.persistence;
+    if (!persistence) {
+        return null;
+    }
+    try {
+        const toolCalls = await persistence.getToolCalls(historicalFlowId);
+        if (toolCalls.length === 0) {
+            return null;
+        }
+        for (let i = toolCalls.length - 1; i >= 0; i--) {
+            const screenshotId = toolCalls[i].postCallImageId;
+            if (screenshotId) {
+                return await persistence.getScreenShot(historicalFlowId, screenshotId);
+            }
+        }
+        return null;
+    }
+    catch (error) {
+        Logger_1.appLogger.debug(`Failed to fetch baseline screenshot from historical flow ${historicalFlowId}.`, error);
+        return null;
+    }
 }
 /**
  * Loads the failing Playwright test file and extracts the statement block that
@@ -832,7 +1189,7 @@ function reasonAttributesFor(reason, context) {
  * Builds the heuristic triage assessment by combining rule-based inference,
  * contextual flags, and derived remediation guidance ahead of GPT enrichment.
  */
-function deriveHeuristicAssessment(testInfo, errorSummaries, toolCalls, flowMetadata) {
+function deriveHeuristicAssessment(testInfo, errorSummaries, toolCalls, flowMetadata, flowHistory = null) {
     const inference = inferFailureReason(errorSummaries, toolCalls);
     const occurredDuringPageAi = didFailureOccurDuringPageAi(errorSummaries, toolCalls);
     const staleCacheIndicators = analyzeStaleCacheIndicators(testInfo, errorSummaries, toolCalls, flowMetadata);
@@ -872,6 +1229,50 @@ function deriveHeuristicAssessment(testInfo, errorSummaries, toolCalls, flowMeta
             notes.push('Selector regression detected, but stale cache is unlikely. This may be a legitimate test failure requiring manual review.');
         }
     }
+    // ---- Historical signal adjustments ----
+    let historicalSignals = null;
+    if (flowHistory) {
+        historicalSignals = deriveHistoricalSignals(flowHistory);
+        // Flakiness + TIMING/NETWORK => boost confidence
+        if (historicalSignals.flakinessScore > 0.5 &&
+            (finalReason === 'TIMING_OR_SYNCHRONISATION' ||
+                finalReason === 'NETWORK_OR_DEPENDENCY')) {
+            finalConfidence = Math.min(0.9, finalConfidence + 0.15);
+            evidence.push(`Historical flakiness detected (score: ${historicalSignals.flakinessScore.toFixed(2)}). ` +
+                `Pass rate: ${(historicalSignals.recentPassRate * 100).toFixed(0)}%.`);
+        }
+        // Flakiness + APPLICATION_DEFECT => reclassify to timing
+        if (historicalSignals.flakinessScore > 0.6 &&
+            finalReason === 'APPLICATION_DEFECT') {
+            finalReason = 'TIMING_OR_SYNCHRONISATION';
+            finalConfidence = 0.55;
+            evidence.push('Reclassified from APPLICATION_DEFECT to TIMING_OR_SYNCHRONISATION: ' +
+                'historical alternating pass/fail pattern indicates flakiness, not a persistent defect.');
+        }
+        // Regression: consecutive failures after a history of passes
+        if (historicalSignals.regressionLikelihood > 0.6) {
+            if (finalReason === 'TIMING_OR_SYNCHRONISATION' ||
+                finalReason === 'UNKNOWN') {
+                finalConfidence = Math.max(finalConfidence, 0.6);
+                evidence.push(`Regression pattern detected: ${flowHistory.currentStreak.length} consecutive failures ` +
+                    `after prior pass rate of ${(historicalSignals.recentPassRate * 100).toFixed(0)}%.`);
+            }
+            else if (finalReason === 'SELECTOR_REGRESSION') {
+                finalConfidence = Math.min(0.9, finalConfidence + 0.1);
+                evidence.push('Historical pass-then-fail pattern corroborates selector regression.');
+            }
+        }
+        // Stale cache calibration: cache was recently valid
+        if (historicalSignals.cacheWasRecentlyValid &&
+            staleCacheScore.isLikelyStaleCache) {
+            finalConfidence = Math.min(0.9, finalConfidence + 0.1);
+            evidence.push('Recent DETERMINISTIC SUCCESS confirms cache was valid recently, strengthening stale-cache hypothesis.');
+        }
+        // Prior self-heal success
+        if (historicalSignals.priorSelfHealSuccess) {
+            notes.push('Historical data shows a prior run recovered on retry. Automated retry has high likelihood of success.');
+        }
+    }
     if (testInfo.status === 'timedOut') {
         notes.push('Playwright marked the test as timed out.');
     }
@@ -885,6 +1286,14 @@ function deriveHeuristicAssessment(testInfo, errorSummaries, toolCalls, flowMeta
     const attributes = reasonAttributesFor(finalReason, {
         occurredDuringPageAi,
     });
+    // Override retry recommendation based on historical signals
+    if (historicalSignals?.priorSelfHealSuccess && !attributes.shouldRetry) {
+        attributes.shouldRetry = true;
+        evidence.push('Overriding shouldRetryAutomation to true based on prior self-heal success.');
+    }
+    if (historicalSignals && historicalSignals.flakinessScore > 0.5) {
+        attributes.shouldRetry = true;
+    }
     return {
         failureReason: finalReason,
         evidence,
@@ -904,6 +1313,7 @@ function deriveHeuristicAssessment(testInfo, errorSummaries, toolCalls, flowMeta
         notes: notes.length > 0 ? notes.join(' ') : undefined,
         occurredDuringPageAi,
         staleCacheIndicators,
+        historicalSignals,
     };
 }
 /**
@@ -1028,6 +1438,9 @@ function reconcileTreatmentPlan(plan, heuristics) {
                 : step);
         }
     }
+    if (heuristics.historicalSignals?.priorSelfHealSuccess) {
+        adjusted.shouldRetryAutomation = true;
+    }
     adjusted.automationDirectives = deriveAutomationDirectives(plan.automationDirectives, heuristics, adjusted.failureReason);
     return adjusted;
 }
@@ -1035,21 +1448,6 @@ function reconcileTreatmentPlan(plan, heuristics) {
  * Ensures callers always receive a complete treatment plan even when the GPT
  * orchestration fails, leaning entirely on the heuristic assessment data.
  */
-function buildFallbackTreatmentPlan(heuristics) {
-    return {
-        failureSummary: heuristics.failureSummary,
-        failureReason: heuristics.failureReason,
-        confidence: heuristics.confidence,
-        observedIndicators: heuristics.evidence,
-        remediationSteps: heuristics.remediationSteps,
-        additionalDataRequests: heuristics.additionalDataRequests,
-        shouldRetryAutomation: heuristics.shouldRetryAutomation,
-        requiresCodeChange: heuristics.requiresCodeChange,
-        requiresProductFix: heuristics.requiresProductFix,
-        notes: heuristics.notes,
-        automationDirectives: deriveAutomationDirectives(undefined, heuristics, heuristics.failureReason),
-    };
-}
 function sanitizeFilenameSegment(value, maxLength = 48) {
     const sanitized = value
         .toLowerCase()
@@ -1099,6 +1497,8 @@ async function collectFailureContext(testInfo, page) {
         path: 'path' in attachment ? (attachment.path ?? null) : null,
     })) ?? [];
     const errorSummaries = buildErrorSummaries(testInfo);
+    // Launch history fetch concurrently with metadata/tool-call loading
+    const flowHistoryPromise = fetchFlowHistory(page);
     let flowMetadata = null;
     let recentToolCalls = [];
     const flowId = page._dnb?.donobuFlowMetadata?.id;
@@ -1123,8 +1523,9 @@ async function collectFailureContext(testInfo, page) {
     else {
         flowMetadata = page._dnb?.donobuFlowMetadata ?? null;
     }
+    const flowHistory = await flowHistoryPromise;
     const sanitizedMetadata = sanitizeFlowMetadata(flowMetadata);
-    const heuristics = deriveHeuristicAssessment(testInfo, errorSummaries, recentToolCalls, sanitizedMetadata);
+    const heuristics = deriveHeuristicAssessment(testInfo, errorSummaries, recentToolCalls, sanitizedMetadata, flowHistory);
     const testSnippet = await extractTestCaseSnippet(testInfo.file, testInfo.title);
     return {
         testCase: {
@@ -1151,6 +1552,7 @@ async function collectFailureContext(testInfo, page) {
         },
         testSnippet,
         heuristics,
+        flowHistory,
     };
 }
 function createEvidenceFilePath(testInfo, runDirectory, evidenceId) {
@@ -1160,9 +1562,6 @@ function createEvidenceFilePath(testInfo, runDirectory, evidenceId) {
     const filename = `failure-evidence-${projectSegment}-${titleSegment}-${workerSegment}-${evidenceId}.json`;
     return path.join(runDirectory, filename);
 }
-function buildTreatmentPlanFromHeuristics(evidence) {
-    return buildFallbackTreatmentPlan(evidence.failureContext.heuristics);
-}
 async function gatherTestFailureEvidence(testInfo, page, options = {}) {
     if (!options.force && process.env.DONOBU_TRIAGE_DISABLED === '1') {
         Logger_1.appLogger.debug('Skipping Donobu triage evidence gathering because DONOBU_TRIAGE_DISABLED=1.');
@@ -1171,6 +1570,14 @@ async function gatherTestFailureEvidence(testInfo, page, options = {}) {
     const failureContext = await collectFailureContext(testInfo, page);
     const runDirectory = resolveRunDirectory(testInfo, options);
     const evidenceId = (0, crypto_1.randomUUID)();
+    // Capture screenshots for visual triage: current failure + baseline from last success
+    const lastSuccessfulRunId = failureContext.flowHistory?.lastSuccessfulRunId ?? null;
+    const [screenshotBuffer, baselineBuffer] = await Promise.all([
+        fetchLastToolCallScreenshot(page),
+        lastSuccessfulRunId
+            ? fetchBaselineScreenshot(page, lastSuccessfulRunId)
+            : Promise.resolve(null),
+    ]);
     const evidence = {
         schemaVersion: TRIAGE_EVIDENCE_SCHEMA_VERSION,
         evidenceId,
@@ -1178,12 +1585,38 @@ async function gatherTestFailureEvidence(testInfo, page, options = {}) {
         runDirectory,
         collectedAtIso: new Date().toISOString(),
         failureContext,
+        failureScreenshotPath: null,
+        baselineScreenshotPath: null,
     };
     const persistToDisk = options.persistToDisk ?? true;
     let filePath = null;
     if (persistToDisk) {
         try {
             await fs.mkdir(runDirectory, { recursive: true });
+            // Save failure screenshot alongside evidence if available
+            if (screenshotBuffer) {
+                try {
+                    const screenshotFilename = `failure-screenshot-${evidenceId}.png`;
+                    const screenshotPath = path.join(runDirectory, screenshotFilename);
+                    await fs.writeFile(screenshotPath, screenshotBuffer);
+                    evidence.failureScreenshotPath = screenshotPath;
+                }
+                catch (screenshotError) {
+                    Logger_1.appLogger.debug('Failed to persist failure screenshot for triage evidence.', screenshotError);
+                }
+            }
+            // Save baseline screenshot from last successful run if available
+            if (baselineBuffer) {
+                try {
+                    const baselineFilename = `baseline-screenshot-${evidenceId}.png`;
+                    const baselinePath = path.join(runDirectory, baselineFilename);
+                    await fs.writeFile(baselinePath, baselineBuffer);
+                    evidence.baselineScreenshotPath = baselinePath;
+                }
+                catch (baselineError) {
+                    Logger_1.appLogger.debug('Failed to persist baseline screenshot for triage evidence.', baselineError);
+                }
+            }
             filePath = createEvidenceFilePath(testInfo, runDirectory, evidenceId);
             await fs.writeFile(filePath, JSON.stringify(evidence, null, 2), 'utf8');
             try {
@@ -1202,6 +1635,26 @@ async function gatherTestFailureEvidence(testInfo, page, options = {}) {
             filePath = null;
         }
     }
+    // Persist triage artifacts through the Donobu persistence layer (in addition
+    // to disk) so they are accessible through whatever backend is configured.
+    const flowId = page._dnb.donobuFlowMetadata.id;
+    const persistence = page._dnb.persistence;
+    if (flowId && persistence) {
+        const writes = [];
+        if (screenshotBuffer) {
+            writes.push(persistence.setFlowFile(flowId, TRIAGE_PERSISTENCE_FILE_IDS.failureScreenshot, screenshotBuffer));
+        }
+        if (baselineBuffer) {
+            writes.push(persistence.setFlowFile(flowId, TRIAGE_PERSISTENCE_FILE_IDS.baselineScreenshot, baselineBuffer));
+        }
+        writes.push(persistence.setFlowFile(flowId, TRIAGE_PERSISTENCE_FILE_IDS.evidence, Buffer.from(JSON.stringify(evidence, null, 2), 'utf8')));
+        try {
+            await Promise.allSettled(writes);
+        }
+        catch (error) {
+            Logger_1.appLogger.debug('Failed to persist triage artifacts through Donobu persistence layer.', error);
+        }
+    }
     return {
         evidence,
         filePath,
@@ -1293,8 +1746,75 @@ When assertionsFailedAfterSuccessfulPageAi + usedDeterministicMode + !isRetryAtt
 
 When isRetryAttempt === true:
   → Do NOT recommend cache deletion (already done).
-  → This is NOT a stale cache issue - focus on code review and legitimate failure investigation.`;
+  → This is NOT a stale cache issue - focus on code review and legitimate failure investigation.
+
+HISTORICAL RUN DATA:
+When failureContext.flowHistory is present, it contains aggregated data about prior runs of the
+same flow within a recent time window. Use this to improve your diagnosis:
+
+- passRate: The ratio of successful runs in the query window. A high pass rate (>0.8) combined
+  with the current failure suggests either a regression or transient issue, NOT a chronic problem.
+- currentStreak: If state is 'FAILED' with length > 1, consecutive failures suggest a real
+  regression. If state is 'MIXED' or 'SUCCESS' with length 1, this may be a one-off flake.
+- recentRuns: The most recent runs in newest-first order. Look for patterns:
+  * Alternating SUCCESS/FAILED = flaky test (likely TIMING_OR_SYNCHRONISATION or NETWORK_OR_DEPENDENCY)
+  * Sudden switch from SUCCESS to FAILED = regression (SELECTOR_REGRESSION or APPLICATION_DEFECT)
+  * All FAILED = persistent issue (likely APPLICATION_DEFECT or ENVIRONMENT_CONFIGURATION)
+
+When using historical context:
+- If history shows flakiness, prefer TIMING_OR_SYNCHRONISATION over APPLICATION_DEFECT.
+- If history shows a regression pattern, increase confidence in the classification.
+- If prior DETERMINISTIC runs succeeded, the cache was recently valid — a change occurred.
+- Set shouldRetryAutomation=true when history shows the flow has recovered on retry before.
+- Note any historical patterns in the observedIndicators array.
+
+TOOL CALL PARAMETERS:
+The failureContext.donobuFlow.recentToolCalls may include a parameters field with the arguments
+passed to each tool invocation. Use these to improve diagnosis:
+- For selector failures: the actual selector or element description attempted reveals what changed.
+- For navigation: the target URL helps distinguish navigation errors from selector issues.
+- For input actions: the values entered help assess whether the automation intent was correct.
+
+SCREENSHOT EVIDENCE:
+You may receive one or two screenshots:
+1. "FAILURE SCREENSHOT" — the state of the page at or near the point of failure in the current run.
+2. "BASELINE SCREENSHOT" — the state of the page at the end of the most recent successful run of
+   this same flow. This serves as a visual reference for what the page *should* look like.
+
+When both screenshots are provided, compare them to:
+- Detect UI changes (redesigns, layout shifts, new modals) that would explain selector or cache failures.
+- Identify whether the failure screenshot shows a fundamentally different page state (error page, login wall)
+  versus the expected baseline state.
+- Assess whether the page content matches but timing/loading is off (partial render vs fully loaded baseline).
+- Corroborate regression signals: if the baseline looks normal but the failure screenshot shows a different
+  UI structure, this strongly supports SELECTOR_REGRESSION or STALE_CACHE_OR_INSTRUCTIONS.
+
+When only the failure screenshot is provided (no baseline available), use it to:
+- Visually verify whether the UI matches the expected state described in the test assertions.
+- Detect page redesigns that would explain stale cache scenarios (elements exist but are wrong).
+- Identify error pages, login walls, or unexpected modals that explain the failure.
+- Corroborate or override text-based heuristic signals with visual evidence.`;
     const instructions = JSON.stringify(failureContext);
+    // Load screenshots if persisted, for visual grounding
+    const userItems = [{ type: 'text', text: instructions }];
+    if (evidence.baselineScreenshotPath) {
+        try {
+            const baselineBytes = await fs.readFile(evidence.baselineScreenshotPath);
+            userItems.push({ type: 'text', text: 'BASELINE SCREENSHOT (last successful run):' }, { type: 'png', bytes: new Uint8Array(baselineBytes) });
+        }
+        catch (screenshotError) {
+            Logger_1.appLogger.debug('Failed to load baseline screenshot for GPT triage, proceeding without it.', screenshotError);
+        }
+    }
+    if (evidence.failureScreenshotPath) {
+        try {
+            const failureBytes = await fs.readFile(evidence.failureScreenshotPath);
+            userItems.push({ type: 'text', text: 'FAILURE SCREENSHOT (current run):' }, { type: 'png', bytes: new Uint8Array(failureBytes) });
+        }
+        catch (screenshotError) {
+            Logger_1.appLogger.debug('Failed to load failure screenshot for GPT triage, proceeding with text only.', screenshotError);
+        }
+    }
     try {
         const response = await gptClient.getStructuredOutput([
             {
@@ -1303,20 +1823,15 @@ When isRetryAttempt === true:
             },
             {
                 type: 'user',
-                items: [
-                    {
-                        type: 'text',
-                        text: instructions,
-                    },
-                ],
+                items: userItems,
             },
         ], TreatmentPlan);
         const plan = TreatmentPlan.parse(response.output);
         return reconcileTreatmentPlan(plan, heuristics);
     }
     catch (error) {
-        Logger_1.appLogger.warn(`GPT-driven triage failed for evidence ${evidence.evidenceId}, returning heuristic fallback treatment plan.`, error);
-        return buildFallbackTreatmentPlan(heuristics);
+        Logger_1.appLogger.warn(`GPT-driven triage failed for evidence ${evidence.evidenceId}.`, error);
+        throw error;
     }
 }
 //# sourceMappingURL=triageTestFailure.js.map