donobu 5.60.1 → 5.60.2

package/dist/reporter/renderMarkdown.js

@@ -11,6 +11,34 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.renderMarkdown = renderMarkdown;
 const ansi_1 = require("../utils/ansi");
 const reportWalk_1 = require("./reportWalk");
+const STATUS_LABELS = {
+    passed: '✅ Passed',
+    flaky: '🔁 Flaky',
+    expectedFailure: '☑️ Expected Failure',
+    healed: '❤️‍🩹 Healed',
+    failed: '❌ Failed',
+    timedOut: '⏰ Timed Out',
+    skipped: '⏭️ Skipped',
+    interrupted: '⚡ Interrupted',
+    unknown: '⚠️ Unknown',
+};
+/**
+ * The attempt whose details (duration, errors) the per-test section shows.
+ * For surviving failures this is the last failing attempt — the final entry
+ * in `results` may be a heal-rerun skip that carries no error context.
+ */
+function displayResultOf(test, status) {
+    const results = test.results ?? [];
+    if (status === 'failed' ||
+        status === 'timedOut' ||
+        status === 'interrupted') {
+        const failures = results.filter((r) => ['failed', 'timedOut', 'interrupted'].includes(r?.status));
+        if (failures.length > 0) {
+            return failures.at(-1);
+        }
+    }
+    return results.at(-1);
+}
 function formatDuration(ms) {
     if (ms < 1000) {
         return `${ms}ms`;
@@ -29,69 +57,111 @@ function renderMarkdown(report) {
     if (report.metadata?.donobuMergedReport) {
         markdown += `> ⚙️ Auto-heal summary generated by Donobu (merged initial and retry runs).\n\n`;
     }
-    // Per-file summary table
+    // Per-file summary table. Counts are computed for all suites first so the
+    // rarely-used Expected Failures column can be omitted when it would be
+    // entirely empty.
     markdown += `## Summary\n\n`;
-    markdown += `| File | Passed | Self-Healed | Failed | Timed Out | Skipped | Interrupted | Duration |\n`;
-    markdown += `| - | - | - | - | - | - | - | - |\n`;
-    let totalPassed = 0;
-    let totalFailed = 0;
-    let totalTimedOut = 0;
-    let totalSkipped = 0;
-    let totalInterrupted = 0;
-    let totalSelfHealed = 0;
-    let totalDuration = 0;
-    suites.forEach((suite) => {
-        let passed = 0;
-        let failed = 0;
-        let timedOut = 0;
-        let skipped = 0;
-        let interrupted = 0;
-        let selfHealed = 0;
-        const allSpecs = (0, reportWalk_1.collectSpecs)(suite);
-        const fileDuration = allSpecs.reduce((total, spec) => total +
-            (spec.tests ?? []).reduce((testTotal, test) => {
-                const result = test.results?.at(-1);
-                const healed = (0, reportWalk_1.isSelfHealed)(test);
-                if (test.status === 'skipped' ||
-                    (!result && test.status === undefined)) {
-                    skipped++;
-                }
-                else if (result) {
-                    if (healed) {
-                        selfHealed++;
-                    }
-                    else {
-                        switch (result.status) {
-                            case 'passed':
-                                passed++;
-                                break;
-                            case 'failed':
-                                failed++;
-                                break;
-                            case 'timedOut':
-                                timedOut++;
-                                break;
-                            case 'skipped':
-                                skipped++;
-                                break;
-                            case 'interrupted':
-                                interrupted++;
-                                break;
-                        }
-                    }
+    const suiteRows = suites.map((suite) => {
+        const row = {
+            file: suite.file,
+            passed: 0,
+            flaky: 0,
+            expectedFailures: 0,
+            selfHealed: 0,
+            failed: 0,
+            timedOut: 0,
+            skipped: 0,
+            interrupted: 0,
+            duration: 0,
+        };
+        for (const spec of (0, reportWalk_1.collectSpecs)(suite)) {
+            for (const test of spec.tests ?? []) {
+                row.duration += test.results?.at(-1)?.duration || 0;
+                switch ((0, reportWalk_1.statusOf)(test)) {
+                    case 'passed':
+                        row.passed++;
+                        break;
+                    case 'flaky':
+                        row.flaky++;
+                        break;
+                    case 'expectedFailure':
+                        row.expectedFailures++;
+                        break;
+                    case 'healed':
+                        row.selfHealed++;
+                        break;
+                    case 'timedOut':
+                        row.timedOut++;
+                        break;
+                    case 'skipped':
+                        row.skipped++;
+                        break;
+                    case 'interrupted':
+                        row.interrupted++;
+                        break;
+                    default:
+                        // 'failed' and 'unknown' both belong in the Failed column —
+                        // dropping unknowns from the table would hide real problems.
+                        row.failed++;
+                        break;
                 }
-                return testTotal + (result?.duration || 0);
-            }, 0), 0);
-        totalPassed += passed;
-        totalFailed += failed;
-        totalTimedOut += timedOut;
-        totalSkipped += skipped;
-        totalInterrupted += interrupted;
-        totalSelfHealed += selfHealed;
-        totalDuration += fileDuration;
-        markdown += `| ${suite.file} | ${passed ? passed + ' ✅' : ''} | ${selfHealed ? selfHealed + ' ❤️‍🩹' : ''} | ${failed ? failed + ' ❌' : ''} | ${timedOut ? timedOut + ' ⏰' : ''} | ${skipped ? skipped + ' ⏭️' : ''} | ${interrupted ? interrupted + ' ⚡' : ''} | ${formatDuration(fileDuration)} |\n`;
+            }
+        }
+        return row;
     });
-    markdown += `| **TOTAL** | **${totalPassed + ' ✅'}** | **${totalSelfHealed + ' ❤️‍🩹'}** | **${totalFailed + ' ❌'}** | **${totalTimedOut + ' ⏰'}** | **${totalSkipped + ' ⏭️'}** | **${totalInterrupted + ' ⚡'}** | **${formatDuration(totalDuration)}** |\n`;
+    const totals = suiteRows.reduce((acc, row) => ({
+        file: '**TOTAL**',
+        passed: acc.passed + row.passed,
+        flaky: acc.flaky + row.flaky,
+        expectedFailures: acc.expectedFailures + row.expectedFailures,
+        selfHealed: acc.selfHealed + row.selfHealed,
+        failed: acc.failed + row.failed,
+        timedOut: acc.timedOut + row.timedOut,
+        skipped: acc.skipped + row.skipped,
+        interrupted: acc.interrupted + row.interrupted,
+        duration: acc.duration + row.duration,
+    }), {
+        file: '**TOTAL**',
+        passed: 0,
+        flaky: 0,
+        expectedFailures: 0,
+        selfHealed: 0,
+        failed: 0,
+        timedOut: 0,
+        skipped: 0,
+        interrupted: 0,
+        duration: 0,
+    });
+    const includeExpectedFailures = totals.expectedFailures > 0;
+    const expectedFailuresHeader = includeExpectedFailures
+        ? ` Expected Failures |`
+        : '';
+    markdown += `| File | Passed | Flaky |${expectedFailuresHeader} Self-Healed | Failed | Timed Out | Skipped | Interrupted | Duration |\n`;
+    markdown += `| - | - | - |${includeExpectedFailures ? ' - |' : ''} - | - | - | - | - | - |\n`;
+    const cell = (count, emoji, bold) => {
+        const text = bold
+            ? `**${count} ${emoji}**`
+            : count
+                ? `${count} ${emoji}`
+                : '';
+        return ` ${text} |`;
+    };
+    for (const row of [...suiteRows, totals]) {
+        const bold = row === totals;
+        markdown += `| ${row.file} |`;
+        markdown += cell(row.passed, '✅', bold);
+        markdown += cell(row.flaky, '🔁', bold);
+        if (includeExpectedFailures) {
+            markdown += cell(row.expectedFailures, '☑️', bold);
+        }
+        markdown += cell(row.selfHealed, '❤️‍🩹', bold);
+        markdown += cell(row.failed, '❌', bold);
+        markdown += cell(row.timedOut, '⏰', bold);
+        markdown += cell(row.skipped, '⏭️', bold);
+        markdown += cell(row.interrupted, '⚡', bold);
+        const duration = formatDuration(row.duration);
+        markdown += ` ${bold ? `**${duration}**` : duration} |\n`;
+    }
     markdown += `\n`;
     // Per-test details
     suites.forEach((suite) => {
@@ -100,9 +170,9 @@ function renderMarkdown(report) {
         (0, reportWalk_1.collectSpecs)(suite).forEach((spec) => {
             markdown += `### ${spec.title}\n\n`;
             (spec.tests ?? []).forEach((test) => {
-                const result = test.results?.at(-1);
-                if (test.status === 'skipped' ||
-                    (!result && test.status === undefined)) {
+                const status = (0, reportWalk_1.statusOf)(test);
+                const result = displayResultOf(test, status);
+                if (status === 'skipped') {
                     markdown += `**Status**: ⏭️ Skipped  \n`;
                     markdown += `**Duration**: N/A  \n`;
                     if (Array.isArray(test.tags) && test.tags.length > 0) {
@@ -116,47 +186,32 @@ function renderMarkdown(report) {
                     markdown += `\n---\n\n`;
                     return;
                 }
-                const healed = (0, reportWalk_1.isSelfHealed)(test);
-                let status;
-                if (healed) {
-                    status = '❤️‍🩹 Healed';
-                }
-                else {
-                    switch (result.status) {
-                        case 'passed':
-                            status = '✅ Passed';
-                            break;
-                        case 'failed':
-                            status = '❌ Failed';
-                            break;
-                        case 'timedOut':
-                            status = '⏰ Timed Out';
-                            break;
-                        case 'skipped':
-                            status = '⏭️ Skipped';
-                            break;
-                        case 'interrupted':
-                            status = '⚡ Interrupted';
-                            break;
-                        default:
-                            status = `⚠️ ${result.status || 'Unknown'}`;
-                    }
-                }
-                const duration = formatDuration(result.duration || 0);
-                markdown += `**Status**: ${status}  \n`;
+                const duration = formatDuration(result?.duration || 0);
+                markdown += `**Status**: ${STATUS_LABELS[status]}  \n`;
                 markdown += `**Duration**: ${duration}  \n`;
                 if (Array.isArray(test.tags) && test.tags.length > 0) {
                     markdown += `**Tags**: ${test.tags.join(' ')}  \n`;
                 }
-                if (healed) {
+                if (status === 'healed') {
                     markdown += `> ❤️‍🩹 This test was automatically healed by re-running with Donobu treatment plan directives.\n\n`;
                 }
+                if (status === 'flaky') {
+                    markdown += `> 🔁 This test failed and then passed on a retry.\n\n`;
+                }
+                const notReattempted = test.annotations?.find((a) => a.type === 'auto-heal-not-reattempted');
+                if (notReattempted) {
+                    markdown += `> ⚠️ ${notReattempted.description}\n\n`;
+                }
+                const rerunFailed = test.annotations?.find((a) => a.type === 'auto-heal-rerun-failed');
+                if (rerunFailed) {
+                    markdown += `> ⚠️ ${rerunFailed.description}\n\n`;
+                }
                 const objectiveAnnotation = test.annotations?.find((a) => a.type === 'objective');
                 if (objectiveAnnotation) {
                     const objective = (objectiveAnnotation.description || 'No objective provided').replace(/```/g, '\\`\\`\\`');
                     markdown += `**Objective**:\n\`\`\`\n${objective}\n\`\`\`\n`;
                 }
-                if (result.status === 'failed' && result.error) {
+                if (result?.status === 'failed' && result.error) {
                     markdown += `\n<details>\n<summary>⚠️ Error Details</summary>\n\n`;
                     markdown += `\`\`\`\n${(0, ansi_1.stripAnsi)(result.error.message || '') || 'No error message available'}\n\`\`\`\n\n`;
                     if (result.error.snippet) {

package/dist/reporter/renderSlack.js

@@ -17,6 +17,8 @@ function renderSlack(report, options = {}) {
         text: { type: 'plain_text', text: '🐵 Donobu Test Summary' },
     });
     let totalPassed = 0;
+    let totalFlaky = 0;
+    let totalExpectedFailures = 0;
     let totalFailed = 0;
     let totalTimedOut = 0;
     let totalSkipped = 0;
@@ -25,41 +27,50 @@ function renderSlack(report, options = {}) {
     suites.forEach((suite) => {
         (0, reportWalk_1.collectSpecs)(suite).forEach((spec) => {
             (spec.tests ?? []).forEach((test) => {
-                const result = test.results?.at(-1);
-                const healed = (0, reportWalk_1.isSelfHealed)(test);
-                if (test.status === 'skipped' ||
-                    (!result && test.status === undefined)) {
-                    totalSkipped++;
-                }
-                else if (result) {
-                    if (healed) {
+                switch ((0, reportWalk_1.statusOf)(test)) {
+                    case 'passed':
+                        totalPassed++;
+                        break;
+                    case 'flaky':
+                        totalFlaky++;
+                        break;
+                    case 'expectedFailure':
+                        totalExpectedFailures++;
+                        break;
+                    case 'healed':
                         totalSelfHealed++;
-                    }
-                    else {
-                        switch (result.status) {
-                            case 'passed':
-                                totalPassed++;
-                                break;
-                            case 'failed':
-                                totalFailed++;
-                                break;
-                            case 'timedOut':
-                                totalTimedOut++;
-                                break;
-                            case 'skipped':
-                                totalSkipped++;
-                                break;
-                            case 'interrupted':
-                                totalInterrupted++;
-                                break;
-                        }
-                    }
+                        break;
+                    case 'timedOut':
+                        totalTimedOut++;
+                        break;
+                    case 'skipped':
+                        totalSkipped++;
+                        break;
+                    case 'interrupted':
+                        totalInterrupted++;
+                        break;
+                    default:
+                        // 'failed' and 'unknown' both count as failures.
+                        totalFailed++;
+                        break;
                 }
             });
         });
     });
     const statusRows = [
         { name: 'Passed', emoji: '✅', count: totalPassed },
+        { name: 'Flaky', emoji: '🔁', count: totalFlaky },
+        // Expected failures are rare enough that a permanent zero row would be
+        // clutter; anyone using test.fail() will see the row when it matters.
+        ...(totalExpectedFailures > 0
+            ? [
+                {
+                    name: 'Expected Failures',
+                    emoji: '☑️',
+                    count: totalExpectedFailures,
+                },
+            ]
+            : []),
         { name: 'Self-Healed', emoji: '❤️‍🩹', count: totalSelfHealed },
         { name: 'Failed', emoji: '❌', count: totalFailed },
         { name: 'Timed Out', emoji: '⏰', count: totalTimedOut },

package/dist/reporter/reportWalk.d.ts

@@ -1,7 +1,8 @@
 /**
  * @fileoverview Shared helpers for walking a `DonobuReport` and classifying
- * the tests inside it. Used by every renderer (HTML, Markdown, Slack) so they
- * all agree on "what counts as self-healed" and how suites nest.
+ * the tests inside it. Used by every renderer (HTML, Markdown, Slack) and the
+ * merge step's stats so they all agree on "what counts as self-healed",
+ * "what counts as flaky", and how suites nest.
  */
 /**
  * Recursively collect all specs from a suite and its nested sub-suites. The
@@ -18,11 +19,20 @@ export declare function collectSpecs(suite: any): any[];
  */
 export declare function isSelfHealed(test: any): boolean;
 /** Normalized test-status the renderers use for counts and labels. */
-export type NormalizedStatus = 'passed' | 'failed' | 'healed' | 'timedOut' | 'skipped' | 'interrupted' | 'unknown';
+export type NormalizedStatus = 'passed' | 'flaky' | 'expectedFailure' | 'failed' | 'healed' | 'timedOut' | 'skipped' | 'interrupted' | 'unknown';
 /**
- * Derive the display status for a test. Prefers the final attempt's result
- * status; bumps to `'healed'` when the self-heal signal is present; falls
- * back to `'skipped'` when Playwright didn't emit a result.
+ * Derive the display status for a test from its full attempt history, not
+ * just the final attempt. The distinctions that matter:
+ *
+ * - A trailing `skipped` attempt must NOT erase an earlier real failure. An
+ *   auto-heal rerun in which the test skipped itself (e.g. a precondition
+ *   guard fired because a prerequisite test wasn't part of the rerun) appends
+ *   a skipped result after genuine failures — the failure stands.
+ * - A failed-then-passed history is `'flaky'` (Playwright's own term), not a
+ *   plain pass — unless the self-heal signal is present, which wins.
+ * - A test whose final attempt matches its `expectedStatus` (e.g.
+ *   `test.fail()` specs) is an `'expectedFailure'`: expected for stats and
+ *   exit-code purposes, but never presented as a pass.
  */
 export declare function statusOf(test: any): NormalizedStatus;
 //# sourceMappingURL=reportWalk.d.ts.map

package/dist/reporter/reportWalk.js

@@ -1,8 +1,9 @@
 "use strict";
 /**
  * @fileoverview Shared helpers for walking a `DonobuReport` and classifying
- * the tests inside it. Used by every renderer (HTML, Markdown, Slack) so they
- * all agree on "what counts as self-healed" and how suites nest.
+ * the tests inside it. Used by every renderer (HTML, Markdown, Slack) and the
+ * merge step's stats so they all agree on "what counts as self-healed",
+ * "what counts as flaky", and how suites nest.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.collectSpecs = collectSpecs;
@@ -32,28 +33,77 @@ function isSelfHealed(test) {
     const hasAnnotation = annotations.some((a) => a?.type === 'self-healed');
     return hasAnnotation || test?.donobuStatus === 'healed';
 }
+const FAILURE_RESULT_STATUSES = new Set(['failed', 'timedOut', 'interrupted']);
 /**
- * Derive the display status for a test. Prefers the final attempt's result
- * status; bumps to `'healed'` when the self-heal signal is present; falls
- * back to `'skipped'` when Playwright didn't emit a result.
+ * Derive the display status for a test from its full attempt history, not
+ * just the final attempt. The distinctions that matter:
+ *
+ * - A trailing `skipped` attempt must NOT erase an earlier real failure. An
+ *   auto-heal rerun in which the test skipped itself (e.g. a precondition
+ *   guard fired because a prerequisite test wasn't part of the rerun) appends
+ *   a skipped result after genuine failures — the failure stands.
+ * - A failed-then-passed history is `'flaky'` (Playwright's own term), not a
+ *   plain pass — unless the self-heal signal is present, which wins.
+ * - A test whose final attempt matches its `expectedStatus` (e.g.
+ *   `test.fail()` specs) is an `'expectedFailure'`: expected for stats and
+ *   exit-code purposes, but never presented as a pass.
  */
 function statusOf(test) {
-    const lastResult = test?.results?.at?.(-1);
-    if (test?.status === 'skipped' ||
-        (!lastResult && test?.status === undefined)) {
+    const results = test?.results ?? [];
+    const lastResult = results.at?.(-1);
+    // No attempts recorded — Playwright reports statically-skipped tests this
+    // way (and `test.status === 'skipped'` for runtime skips with no results).
+    if (!lastResult) {
         return 'skipped';
     }
     if (isSelfHealed(test)) {
         return 'healed';
     }
-    const status = lastResult?.status ?? 'unknown';
-    switch (status) {
-        case 'passed':
+    const expectedStatus = test?.expectedStatus;
+    // An attempt "succeeded" when it passed outright, or when it landed on the
+    // declared expected outcome of a `test.fail()`-style spec. The two are kept
+    // distinct end-to-end: expected failures count as expected for stats and
+    // exit codes, but are never presented as passes.
+    const successKindOf = (result) => {
+        if (result?.status === 'passed') {
+            return 'passed';
+        }
+        if (expectedStatus !== undefined &&
+            expectedStatus !== 'passed' &&
+            expectedStatus !== 'skipped' &&
+            result?.status === expectedStatus) {
+            return 'expectedFailure';
+        }
+        return null;
+    };
+    const hadRealFailure = results.some((r) => FAILURE_RESULT_STATUSES.has(r?.status) && successKindOf(r) === null);
+    // A success anywhere in the history. Within a single run retries stop at
+    // the first success, so a success followed by a failure/skip only arises
+    // from a merged auto-heal rerun — and the rerun is advisory: it may flip a
+    // failure to healed but never an initial success to failed/skipped.
+    const priorSuccess = results.map(successKindOf).find((kind) => kind !== null) ?? null;
+    // Skips are handled first: a runtime-skipped test has
+    // `expectedStatus: 'skipped'` and must stay skipped.
+    if (lastResult.status === 'skipped') {
+        // A skip can't launder an earlier failure; surface the worst attempt.
+        if (hadRealFailure) {
+            const failure = results.find((r) => FAILURE_RESULT_STATUSES.has(r?.status) && successKindOf(r) === null);
+            return failure.status;
+        }
+        return priorSuccess ?? 'skipped';
+    }
+    const lastSuccess = successKindOf(lastResult);
+    if (lastSuccess === 'passed') {
+        return hadRealFailure ? 'flaky' : 'passed';
+    }
+    if (lastSuccess === 'expectedFailure') {
+        return 'expectedFailure';
+    }
+    switch (lastResult.status) {
         case 'failed':
         case 'timedOut':
-        case 'skipped':
         case 'interrupted':
-            return status;
+            return priorSuccess ?? lastResult.status;
         default:
             return 'unknown';
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "donobu",
-  "version": "5.60.1",
+  "version": "5.60.2",
   "description": "Create browser automations with an LLM agent and replay them as Playwright scripts.",
   "main": "dist/main.js",
   "module": "dist/esm/main.js",