@testingbot/cli 1.0.9 → 1.1.0

package/dist/providers/maestro.js

@@ -54,6 +54,9 @@ const terminal_title_1 = require("../ui/terminal-title");
 const constants_1 = require("../config/constants");
 const FLOW_SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
 const FLOW_ANIMATION_MS = 120;
+// Single-column glyph so it does not disturb the row-width math used by the
+// in-place table redraw; marks flow rows that are retry attempts.
+const RETRY_ICON = '↻';
 class Maestro extends base_provider_1.default {
     URL = 'https://api.testingbot.com/v1/app-automate/maestro';
     detectedPlatform = undefined;
@@ -61,10 +64,21 @@ class Maestro extends base_provider_1.default {
     updateServer = null;
     updateKey = null;
     socketFallbackWarned = false;
+    otherAppUrls = [];
     flowAnimationFrame = 0;
     flowAnimationTimer = null;
     latestFlows = [];
     latestDisplayedLineCount = 0;
+    flowsTableDisplayed = false;
+    // Tracks whether the currently-drawn table header includes the "Fail reason"
+    // column. When failures appear after the header was first drawn (e.g. a flow
+    // fails while others still run, or during a retry), the header is redrawn so
+    // the live in-place rows can show fail reasons instead of waiting for the
+    // final summary.
+    flowTableHasFailuresColumn = false;
+    // Maps a flow's MaestroRunTest id to its 1-based attempt number within its
+    // logical group. Attempt > 1 means the row is a retry. Recomputed on render.
+    flowAttempts = new Map();
     constructor(credentials, options) {
         super(credentials, options);
     }
@@ -91,12 +105,35 @@ class Maestro extends base_provider_1.default {
         if (this.options.report && !this.options.reportOutputDir) {
             throw new testingbot_error_1.default(`--report-output-dir is required when --report is specified`);
         }
+        // Validate other-apps count and extensions
+        const otherApps = this.options.otherApps;
+        if (otherApps.length > 4) {
+            throw new testingbot_error_1.default(`Too many other apps (${otherApps.length}). Maximum is 4.`);
+        }
+        for (const otherAppEntry of otherApps) {
+            if (Maestro.isOtherAppUrl(otherAppEntry)) {
+                continue;
+            }
+            const otherExt = node_path_1.default.extname(otherAppEntry).toLowerCase();
+            if (!Maestro.SUPPORTED_APP_EXTENSIONS.includes(otherExt)) {
+                throw new testingbot_error_1.default(`Unsupported other-app file format: ${otherExt || '(no extension)'} for ${otherAppEntry}. ` +
+                    `Supported formats: ${Maestro.SUPPORTED_APP_EXTENSIONS.join(', ')}, or a tb:// / http(s):// URL`);
+            }
+        }
         // Build list of all file checks to run in parallel
         const fileChecks = [
             node_fs_1.default.promises.access(this.options.app, node_fs_1.default.constants.R_OK).catch(() => {
                 throw new testingbot_error_1.default(`Provided app path does not exist ${this.options.app}`);
             }),
         ];
+        for (const otherAppEntry of otherApps) {
+            if (Maestro.isOtherAppUrl(otherAppEntry)) {
+                continue;
+            }
+            fileChecks.push(node_fs_1.default.promises.access(otherAppEntry, node_fs_1.default.constants.R_OK).catch(() => {
+                throw new testingbot_error_1.default(`Provided other-app path does not exist ${otherAppEntry}`);
+            }));
+        }
         if (this.options.configFile) {
             fileChecks.push(node_fs_1.default.promises
                 .access(this.options.configFile, node_fs_1.default.constants.R_OK)
@@ -147,6 +184,12 @@ class Maestro extends base_provider_1.default {
             const metadata = this.options.metadata;
             // Process flows to show actual zip structure
             const flowResult = await this.collectFlows();
+            const otherApps = this.options.otherApps;
+            const otherAppPaths = otherApps.filter((v) => !Maestro.isOtherAppUrl(v));
+            let uploadCounter = 0;
+            const previewOtherAppUrls = otherApps.map((entry) => Maestro.isOtherAppUrl(entry)
+                ? entry
+                : `<tb://appkey-other-app-${++uploadCounter}>`);
             this.printDryRunSummary({
                 provider: 'Maestro',
                 apiUrl: this.URL,
@@ -156,6 +199,11 @@ class Maestro extends base_provider_1.default {
                         filePath: this.options.app,
                         endpoint: `${this.URL}/app`,
                     },
+                    ...otherAppPaths.map((p, i) => ({
+                        label: `Other App ${i + 1}`,
+                        filePath: p,
+                        endpoint: `${this.URL}/other-apps`,
+                    })),
                     {
                         label: 'Flows',
                         filePath: this.options.flows.join(', '),
@@ -169,6 +217,9 @@ class Maestro extends base_provider_1.default {
                         shardSplit: this.options.shardSplit,
                     }),
                     ...(metadata && { metadata }),
+                    ...(otherApps.length > 0 && {
+                        otherApps: previewOtherAppUrls,
+                    }),
                 },
             });
             // Show zip structure details
@@ -197,6 +248,10 @@ class Maestro extends base_provider_1.default {
             }
             (0, terminal_title_1.setTitle)('maestro · uploading app');
             await this.uploadApp();
+            if (this.options.otherApps.length > 0) {
+                (0, terminal_title_1.setTitle)('maestro · uploading other apps');
+                await this.uploadOtherApps();
+            }
             if (!this.options.quiet) {
                 logger_1.default.info('Uploading Maestro Flows');
             }
@@ -205,6 +260,9 @@ class Maestro extends base_provider_1.default {
             if (this.options.tunnel && this.options.async) {
                 throw new testingbot_error_1.default('Cannot use --tunnel with --async mode. The tunnel would close when the CLI exits. Use a standalone tunnel instead.');
             }
+            if (this.options.retry > 0 && this.options.async) {
+                throw new testingbot_error_1.default('Cannot use --retry with --async mode. Retrying failed flows requires waiting for results.');
+            }
             await this.startTunnel();
             if (!this.options.quiet) {
                 logger_1.default.info('Running Maestro Tests');
@@ -316,6 +374,58 @@ class Maestro extends base_provider_1.default {
             }
         }
     }
+    static isOtherAppUrl(value) {
+        return (value.startsWith('tb://') ||
+            value.startsWith('http://') ||
+            value.startsWith('https://'));
+    }
+    async uploadOtherApps() {
+        const others = this.options.otherApps;
+        if (!others || others.length === 0)
+            return;
+        const uploadable = others.filter((v) => !Maestro.isOtherAppUrl(v));
+        if (!this.options.quiet) {
+            if (uploadable.length > 0) {
+                logger_1.default.info(`Uploading ${uploadable.length} other app(s)`);
+            }
+            else {
+                logger_1.default.info(`Using ${others.length} other app URL(s)`);
+            }
+        }
+        for (let i = 0; i < others.length; i++) {
+            const entry = others[i];
+            if (Maestro.isOtherAppUrl(entry)) {
+                if (!this.options.quiet) {
+                    logger_1.default.info(`  [${i + 1}/${others.length}] ${entry} (URL)`);
+                }
+                this.otherAppUrls.push(entry);
+                continue;
+            }
+            const ext = node_path_1.default.extname(entry).toLowerCase();
+            const contentType = ext === '.apk'
+                ? 'application/vnd.android.package-archive'
+                : ext === '.ipa'
+                    ? 'application/octet-stream'
+                    : ext === '.zip' || ext === '.app'
+                        ? 'application/zip'
+                        : 'application/octet-stream';
+            if (!this.options.quiet) {
+                logger_1.default.info(`  [${i + 1}/${others.length}] ${node_path_1.default.basename(entry)}`);
+            }
+            const result = await this.upload.upload({
+                filePath: entry,
+                url: `${this.URL}/other-apps`,
+                credentials: this.credentials,
+                contentType,
+                showProgress: !this.options.quiet,
+                validateZipFormat: ext === '.zip' || ext === '.app',
+            });
+            if (!result.app_url) {
+                throw new testingbot_error_1.default(`Other-app upload returned no app_url for ${entry}`);
+            }
+            this.otherAppUrls.push(result.app_url);
+        }
+    }
     /**
      * Zip a .app bundle directory into a temporary zip file
      */
@@ -1186,6 +1296,9 @@ class Maestro extends base_provider_1.default {
                     shardSplit: this.options.shardSplit,
                 }),
                 ...(metadata && { metadata }),
+                ...(this.otherAppUrls.length > 0 && {
+                    otherApps: this.otherAppUrls,
+                }),
             }, {
                 headers: {
                     'Content-Type': 'application/json',
@@ -1249,6 +1362,215 @@ class Maestro extends base_provider_1.default {
         }
     }
     async waitForCompletion() {
+        let status;
+        if (this.options.retry > 0) {
+            // Retry failed flows the moment they settle — while the rest of the run
+            // is still executing — instead of waiting for the whole run to finish
+            // first. Each retry creates a new attempt that surfaces in the live
+            // table (marked with the ↻ icon) on the following poll.
+            const retriedFrom = new Set();
+            const abandoned = new Set();
+            status = await this.pollOnce((s) => this.retriesComplete(s, retriedFrom, abandoned), (s) => this.issueEligibleRetries(s, retriedFrom, abandoned));
+        }
+        else {
+            status = await this.pollOnce((s) => s.completed);
+        }
+        return this.finalize(status);
+    }
+    /**
+     * Poll completion condition used when --retry > 0. True once no flow is
+     * still running and no failed flow is still eligible for — or awaiting the
+     * result of — a retry. Replaces the run-level `completed` flag, which can
+     * flip prematurely while a freshly-queued retry has not yet surfaced.
+     */
+    retriesComplete(status, retriedFrom, abandoned) {
+        for (const run of status.runs) {
+            const flows = run.flows ?? [];
+            if (flows.length === 0) {
+                // No flows yet: only settled once the run itself reaches a terminal
+                // state (e.g. it failed before producing any flow).
+                if (run.status !== 'DONE' && run.status !== 'FAILED')
+                    return false;
+                continue;
+            }
+            const attempts = this.computeFlowAttempts(flows);
+            for (const latest of this.groupLatest(flows)) {
+                if (latest.status === 'WAITING' || latest.status === 'READY') {
+                    return false; // still running
+                }
+                if (this.isFlowFailed(latest) && !abandoned.has(latest.id)) {
+                    // A retry was issued from this attempt but its replacement has not
+                    // surfaced yet — wait for the new attempt to appear and settle.
+                    if (retriedFrom.has(latest.id))
+                        return false;
+                    // Still within the retry budget: a retry is about to be issued.
+                    const attemptNumber = attempts.get(latest.id) ?? 1;
+                    if (attemptNumber <= this.options.retry)
+                        return false;
+                }
+            }
+        }
+        return true;
+    }
+    /**
+     * Issues a retry for every failed flow/shard whose latest attempt has just
+     * settled and still has retries left — immediately, without waiting for the
+     * rest of the run. Each attempt id is retried at most once; a retry whose
+     * request permanently fails is abandoned so the poll loop can still finish.
+     */
+    async issueEligibleRetries(status, retriedFrom, abandoned) {
+        if (this.isShuttingDown)
+            return;
+        for (const run of status.runs) {
+            const flows = run.flows ?? [];
+            if (flows.length === 0)
+                continue;
+            const attempts = this.computeFlowAttempts(flows);
+            for (const latest of this.groupLatest(flows)) {
+                if (!this.isFlowFailed(latest))
+                    continue;
+                if (retriedFrom.has(latest.id) || abandoned.has(latest.id))
+                    continue;
+                const attemptNumber = attempts.get(latest.id) ?? 1;
+                if (attemptNumber > this.options.retry)
+                    continue; // budget exhausted
+                try {
+                    await this.retryFlow(run.id, latest.id);
+                    retriedFrom.add(latest.id);
+                    (0, terminal_title_1.setTitle)(`maestro · retry ${attemptNumber}/${this.options.retry}`);
+                }
+                catch (err) {
+                    // Couldn't queue the retry — give up on this attempt so completion
+                    // isn't blocked forever waiting for a replacement that never comes.
+                    abandoned.add(latest.id);
+                    logger_1.default.warn(`Could not retry flow "${latest.name}" (run ${run.id}): ${err instanceof Error ? err.message : err}`);
+                }
+            }
+        }
+    }
+    /**
+     * Triggers a retry of a single flow/shard and returns the new attempt's
+     * MaestroRunTest id.
+     */
+    async retryFlow(runId, flowId) {
+        return await this.withRetry('Retrying Maestro flow', async () => {
+            const response = await axios_1.default.post(`${this.URL}/${this.appId}/${runId}/${flowId}/retry`, {}, {
+                headers: {
+                    'Content-Type': 'application/json',
+                    'User-Agent': utils_1.default.getUserAgent(),
+                },
+                auth: {
+                    username: this.credentials.userName,
+                    password: this.credentials.accessKey,
+                },
+                timeout: constants_1.HTTP.TIMEOUT_MS,
+            });
+            const data = response.data;
+            if (data?.success === false) {
+                const errorMessage = data.errors?.join('\n') || data.error || 'Unknown error';
+                throw new testingbot_error_1.default('Retrying Maestro flow failed', {
+                    cause: errorMessage,
+                });
+            }
+            return data?.flow?.id;
+        });
+    }
+    /** Stable key identifying the logical flow/shard a flow attempt belongs to. */
+    flowGroupKey(flow) {
+        return flow.shard_index != null
+            ? `shard:${flow.shard_index}`
+            : `name:${flow.name}`;
+    }
+    /** Latest attempt per logical flow/shard (highest id wins). */
+    groupLatest(flows) {
+        const latest = new Map();
+        for (const flow of flows) {
+            const key = this.flowGroupKey(flow);
+            const current = latest.get(key);
+            if (!current || flow.id > current.id)
+                latest.set(key, flow);
+        }
+        return [...latest.values()];
+    }
+    /**
+     * Maps each flow's id to its 1-based attempt number within its logical
+     * group (ordered by id). The first attempt is 1; anything higher is a retry.
+     */
+    computeFlowAttempts(flows) {
+        const groups = new Map();
+        for (const flow of flows) {
+            const key = this.flowGroupKey(flow);
+            const arr = groups.get(key);
+            if (arr)
+                arr.push(flow);
+            else
+                groups.set(key, [flow]);
+        }
+        const attempts = new Map();
+        for (const arr of groups.values()) {
+            arr
+                .slice()
+                .sort((a, b) => a.id - b.id)
+                .forEach((flow, index) => attempts.set(flow.id, index + 1));
+        }
+        return attempts;
+    }
+    /**
+     * The flow name as shown in the table, prefixed with a retry marker when the
+     * row is a retry attempt. Reads the attempt map populated at render time.
+     */
+    flowRowName(flow) {
+        const attempt = this.flowAttempts.get(flow.id) ?? 1;
+        if (attempt > 1) {
+            return `${RETRY_ICON} ${flow.name} (retry ${attempt - 1})`;
+        }
+        return flow.name;
+    }
+    /**
+     * Colorizes the retry marker without affecting column width: the padding is
+     * computed on the plain string first, then the glyph is wrapped in color.
+     */
+    colorizeRetryIcon(text) {
+        return text.includes(RETRY_ICON)
+            ? text.replace(RETRY_ICON, picocolors_1.default.cyan(RETRY_ICON))
+            : text;
+    }
+    isFlowFailed(flow) {
+        return ((flow.status === 'DONE' && flow.success !== 1) ||
+            flow.status === 'FAILED' ||
+            (flow.error_messages != null && flow.error_messages.length > 0));
+    }
+    /** Failed flows/shards, considering only the latest attempt per group. */
+    failedLatestAttempts(runs) {
+        const failed = [];
+        for (const run of runs) {
+            for (const flow of this.groupLatest(run.flows ?? [])) {
+                if (this.isFlowFailed(flow))
+                    failed.push({ runId: run.id, flow });
+            }
+        }
+        return failed;
+    }
+    /** A run passes if every logical flow's latest attempt passed. */
+    runPassed(run) {
+        const groups = this.groupLatest(run.flows ?? []);
+        if (groups.length === 0)
+            return run.success === 1;
+        return groups.every((flow) => !this.isFlowFailed(flow));
+    }
+    computeOverallSuccess(runs) {
+        return runs.every((run) => this.runPassed(run));
+    }
+    /**
+     * Polls the run status, rendering the live flow table, until `isComplete`
+     * returns true. Returns the raw status without printing the final summary or
+     * fetching reports/artifacts — that is finalize()'s job.
+     *
+     * `onPoll`, when provided, runs after each poll is rendered and before the
+     * completion check — used by --retry to queue retries for flows that have
+     * just failed without waiting for the rest of the run to finish.
+     */
+    async pollOnce(isComplete, onPoll) {
         const startTime = Date.now();
         const previousStatus = new Map();
         const previousFlowStatus = new Map();
@@ -1317,70 +1639,17 @@ class Maestro extends base_provider_1.default {
                     this.displayRunStatus(status.runs, startTime, previousStatus);
                 }
             }
-            if (status.completed) {
+            // Queue any eligible retries from this poll's results before deciding
+            // whether we're done, so a flow that just failed is retried immediately
+            // rather than after the whole run completes.
+            if (onPoll) {
+                await onPoll(status);
+            }
+            if (isComplete(status)) {
                 this.stopFlowAnimation();
-                // Display final flows table with error messages if there are failures
-                if (!this.options.quiet && flowsTableDisplayed) {
-                    const allFlows = [];
-                    for (const run of status.runs) {
-                        if (run.flows && run.flows.length > 0) {
-                            allFlows.push(...run.flows);
-                        }
-                    }
-                    const hasFailures = this.hasAnyFlowFailed(allFlows);
-                    if (hasFailures) {
-                        // Move cursor up to overwrite the existing table
-                        // +2 for header and separator lines
-                        const linesToMove = displayedLineCount + 2;
-                        process.stdout.write(`\x1b[${linesToMove}A`);
-                        // Clear header line, write new header, then clear separator line
-                        process.stdout.write('\x1b[2K');
-                        console.log(picocolors_1.default.dim(` ${'Duration'.padEnd(10)} ${'Status'.padEnd(10)} Flow                              Fail reason`));
-                        process.stdout.write('\x1b[2K');
-                        console.log(picocolors_1.default.dim(` ${'─'.repeat(10)} ${'─'.repeat(10)} ${'─'.repeat(30)} ${'─'.repeat(80)}`));
-                        // Redraw all flows with error messages
-                        for (const flow of allFlows) {
-                            // Clear the line before writing
-                            process.stdout.write('\x1b[2K');
-                            this.displayFlowRow(flow, false, true);
-                        }
-                    }
-                }
-                // Print final summary
-                if (!this.options.quiet) {
-                    this.spinner.stop();
-                    console.log(); // Empty line before summary
-                    for (const run of status.runs) {
-                        const passed = run.success === 1;
-                        const symbol = passed ? picocolors_1.default.green('✔') : picocolors_1.default.red('✘');
-                        const statusText = passed
-                            ? picocolors_1.default.green('Test completed successfully')
-                            : picocolors_1.default.red('Test failed');
-                        console.log(`  ${symbol} Run ${run.id} ${picocolors_1.default.dim(`(${this.getRunDisplayName(run)})`)}: ${statusText}`);
-                    }
-                }
-                const allSucceeded = status.runs.every((run) => run.success === 1);
-                if (allSucceeded) {
-                    (0, terminal_title_1.setTitle)('maestro · ✔ passed');
-                    if (!this.options.quiet) {
-                        logger_1.default.info('All tests completed successfully!');
-                    }
-                }
-                else {
-                    const failedRuns = status.runs.filter((run) => run.success !== 1);
-                    (0, terminal_title_1.setTitle)(`maestro · ✘ ${failedRuns.length} failed`);
-                    logger_1.default.error(`${failedRuns.length} test run(s) failed`);
-                }
-                if (this.options.report && this.options.reportOutputDir) {
-                    await this.fetchReports(status.runs);
-                }
-                if (this.options.downloadArtifacts) {
-                    await this.downloadArtifacts(status.runs);
-                }
-                return {
-                    success: status.success,
-                    runs: status.runs,
-                };
+                this.flowsTableDisplayed = flowsTableDisplayed;
+                this.latestDisplayedLineCount = displayedLineCount;
+                return status;
             }
             // Checked after getStatus() so a run that completes during the final
             // sleep is returned as success on the next iteration instead of being
@@ -1400,6 +1669,85 @@ class Maestro extends base_provider_1.default {
             await this.sleep(pollInterval);
         }
     }
+    /**
+     * Renders the final table/summary, reports the last-attempt-wins result, and
+     * fetches reports/artifacts. Called once after the (optional) retry loop.
+     */
+    async finalize(status) {
+        this.stopFlowAnimation();
+        // Display final flows table with error messages if there are failures
+        if (!this.options.quiet && this.flowsTableDisplayed) {
+            const allFlows = [];
+            for (const run of status.runs) {
+                if (run.flows && run.flows.length > 0) {
+                    allFlows.push(...run.flows);
+                }
+            }
+            this.flowAttempts = this.computeFlowAttempts(allFlows);
+            const hasFailures = this.hasAnyFlowFailed(allFlows);
+            if (hasFailures) {
+                // Move cursor up to overwrite the existing table
+                // +2 for header and separator lines
+                const linesToMove = this.latestDisplayedLineCount + 2;
+                process.stdout.write(`\x1b[${linesToMove}A`);
+                // Clear header line, write new header, then clear separator line
+                const { header, separator } = this.buildFlowsTableHeader(true);
+                process.stdout.write('\x1b[2K');
+                console.log(header);
+                process.stdout.write('\x1b[2K');
+                console.log(separator);
+                this.flowTableHasFailuresColumn = true;
+                // Redraw all flows with error messages
+                for (const flow of allFlows) {
+                    // Clear the line before writing
+                    process.stdout.write('\x1b[2K');
+                    this.displayFlowRow(flow, false, true);
+                }
+            }
+        }
+        // Print final summary (last-attempt-wins per run)
+        if (!this.options.quiet) {
+            this.spinner.stop();
+            console.log(); // Empty line before summary
+            for (const run of status.runs) {
+                const passed = this.runPassed(run);
+                const symbol = passed ? picocolors_1.default.green('✔') : picocolors_1.default.red('✘');
+                const statusText = passed
+                    ? picocolors_1.default.green('Test completed successfully')
+                    : picocolors_1.default.red('Test failed');
+                console.log(`  ${symbol} Run ${run.id} ${picocolors_1.default.dim(`(${this.getRunDisplayName(run)})`)}: ${statusText}`);
+            }
+        }
+        const allSucceeded = this.computeOverallSuccess(status.runs);
+        if (allSucceeded) {
+            (0, terminal_title_1.setTitle)('maestro · ✔ passed');
+            if (!this.options.quiet) {
+                logger_1.default.info('All tests completed successfully!');
+            }
+        }
+        else {
+            const failedRuns = status.runs.filter((run) => !this.runPassed(run));
+            const failedFlows = this.failedLatestAttempts(status.runs);
+            if (failedFlows.length > 0) {
+                (0, terminal_title_1.setTitle)(`maestro · ✘ ${failedFlows.length} failed`);
+                logger_1.default.error(`${failedFlows.length} flow(s) failed across ${failedRuns.length} run(s)`);
+            }
+            else {
+                (0, terminal_title_1.setTitle)(`maestro · ✘ ${failedRuns.length} failed`);
+                logger_1.default.error(`${failedRuns.length} test run(s) failed`);
+            }
+        }
+        if (this.options.report && this.options.reportOutputDir) {
+            await this.fetchReports(status.runs);
+        }
+        if (this.options.downloadArtifacts) {
+            await this.downloadArtifacts(status.runs);
+        }
+        return {
+            success: allSucceeded,
+            runs: status.runs,
+        };
+    }
     displayRunStatus(runs, startTime, previousStatus) {
         const elapsedSeconds = Math.floor((Date.now() - startTime) / 1000);
         const elapsedStr = this.formatElapsedTime(elapsedSeconds);
@@ -1572,6 +1920,7 @@ class Maestro extends base_provider_1.default {
         return ` ... and ${remaining.length} more: ${parts.join(', ')}`;
     }
     displayFlowsWithLimit(flows, previousFlowStatus, hasFailures = false) {
+        this.flowAttempts = this.computeFlowAttempts(flows);
         const maxFlows = this.getMaxDisplayableFlows();
         const displayFlows = flows.slice(0, maxFlows);
         let linesWritten = 0;
@@ -1587,23 +1936,38 @@ class Maestro extends base_provider_1.default {
         }
         return linesWritten;
     }
-    displayFlowsTableHeader(hasFailures = false) {
+    /**
+     * Builds the (dimmed) header and separator lines for the flows table. When
+     * `hasFailures` is set, the "Fail reason" column is appended.
+     */
+    buildFlowsTableHeader(hasFailures) {
         let header = ` ${'Duration'.padEnd(10)} ${'Status'.padEnd(10)} Flow`;
         let separator = ` ${'─'.repeat(10)} ${'─'.repeat(10)} ${'─'.repeat(30)}`;
         if (hasFailures) {
             header += '                              Fail reason';
             separator += ` ${'─'.repeat(80)}`;
         }
-        console.log(picocolors_1.default.dim(header));
-        console.log(picocolors_1.default.dim(separator));
+        return { header: picocolors_1.default.dim(header), separator: picocolors_1.default.dim(separator) };
     }
-    displayFlowRow(flow, isUpdate = false, hasFailures = false) {
+    displayFlowsTableHeader(hasFailures = false) {
+        const { header, separator } = this.buildFlowsTableHeader(hasFailures);
+        console.log(header);
+        console.log(separator);
+        this.flowTableHasFailuresColumn = hasFailures;
+    }
+    /**
+     * Builds the single-line representation of a flow row: duration, status, the
+     * (padded) flow name, and — when the flow has failed — its first error
+     * message inline. Shared by the static render (displayFlowRow) and the live
+     * in-place updates (updateFlowsInPlace) so the fail reason shows
+     * consistently, including while a retry is still running.
+     */
+    buildFlowRowLine(flow, hasFailures) {
         const duration = this.calculateFlowDuration(flow).padEnd(10);
         const statusDisplay = this.getFlowStatusDisplay(flow);
         // Pad based on display text length, add extra for color codes
         const statusPadded = statusDisplay.colored +
             ' '.repeat(Math.max(0, 10 - statusDisplay.text.length));
-        let linesWritten = 0;
         const isFailed = flow.status === 'DONE' && flow.success !== 1;
         const errorMessages = flow.error_messages || [];
         const firstError = hasFailures && isFailed && errorMessages.length > 0
@@ -1611,13 +1975,21 @@ class Maestro extends base_provider_1.default {
             : '';
         const errorReserve = firstError ? firstError.length + 1 : 0;
         const maxName = this.getMaxNameLength(errorReserve);
-        const name = this.truncateForRow(flow.name, maxName).padEnd(Math.min(30, maxName));
-        // Build the main row
-        let row = ` ${duration} ${statusPadded} ${name}`;
+        const name = this.truncateForRow(this.flowRowName(flow), maxName).padEnd(Math.min(30, maxName));
+        // Build the main row (colorize the retry marker after padding so the
+        // column-width math stays based on the plain glyph).
+        let row = ` ${duration} ${statusPadded} ${this.colorizeRetryIcon(name)}`;
         // Add first error message on the same line if failed and has errors
         if (firstError) {
             row += ` ${picocolors_1.default.red(firstError)}`;
         }
+        return row;
+    }
+    displayFlowRow(flow, isUpdate = false, hasFailures = false) {
+        const row = this.buildFlowRowLine(flow, hasFailures);
+        let linesWritten = 0;
+        const isFailed = flow.status === 'DONE' && flow.success !== 1;
+        const errorMessages = flow.error_messages || [];
         if (isUpdate) {
             process.stdout.write(`\r${row}`);
         }
@@ -1679,23 +2051,33 @@ class Maestro extends base_provider_1.default {
         this.stopFlowAnimation();
     }
     updateFlowsInPlace(flows, previousFlowStatus, displayedLineCount) {
+        this.flowAttempts = this.computeFlowAttempts(flows);
         const maxFlows = this.getMaxDisplayableFlows();
         const displayFlows = flows.slice(0, maxFlows);
         const hasRemaining = flows.length > maxFlows;
-        // Move cursor up by the number of lines we PREVIOUSLY displayed
-        if (displayedLineCount > 0) {
+        const hasFailures = this.hasAnyFlowFailed(flows);
+        // If failures appeared after the header was first drawn (e.g. mid-run, or
+        // when re-rendering during a retry), redraw the header to add the "Fail
+        // reason" column. This also moves the cursor back up to the first flow row.
+        // Otherwise just move up by the number of lines we PREVIOUSLY displayed.
+        if (hasFailures && !this.flowTableHasFailuresColumn) {
+            const { header, separator } = this.buildFlowsTableHeader(true);
+            process.stdout.write(`\x1b[${displayedLineCount + 2}A`);
+            process.stdout.write('\x1b[2K');
+            console.log(header);
+            process.stdout.write('\x1b[2K');
+            console.log(separator);
+            this.flowTableHasFailuresColumn = true;
+        }
+        else if (displayedLineCount > 0) {
             process.stdout.write(`\x1b[${displayedLineCount}A`);
         }
         let linesWritten = 0;
-        // Redraw displayed flows
-        const maxName = this.getMaxNameLength();
+        // Redraw displayed flows. buildFlowRowLine appends the first error message
+        // inline for failed flows so fail reasons stay visible during live updates
+        // (including while a retry re-runs the failed flows).
         for (const flow of displayFlows) {
-            const duration = this.calculateFlowDuration(flow).padEnd(10);
-            const statusDisplay = this.getFlowStatusDisplay(flow);
-            const statusPadded = statusDisplay.colored +
-                ' '.repeat(Math.max(0, 10 - statusDisplay.text.length));
-            const name = this.truncateForRow(flow.name, maxName);
-            const row = ` ${duration} ${statusPadded} ${name}`;
+            const row = this.buildFlowRowLine(flow, hasFailures);
             process.stdout.write(`\r\x1b[2K${row}\n`);
             previousFlowStatus.set(flow.id, flow.status);
             linesWritten++;