@tickernelz/paperclip-pro-server 2026.929.0 → 2026.929.2

package/dist/services/recovery/service.js

@@ -1087,8 +1087,20 @@ export function recoveryService(db, deps) {
                     });
                     return null;
                 }
-                if (deps.scheduleRecoveryRetry)
-                    return deps.scheduleRecoveryRetry(predecessor.id);
+                if (deps.scheduleRecoveryRetry) {
+                    const retry = await deps.scheduleRecoveryRetry(predecessor.id);
+                    if (retry)
+                        return retry;
+                    // A spent budget is the one "no retry" the sweeper must not wait
+                    // out: nothing else will ever queue a successor for this run, so
+                    // it reports the exhaustion instead of leaving the issue with no
+                    // live path (2026-09-25: three deploy restarts in a row spent the
+                    // budget and the issue sat in_progress with no run, unescalated).
+                    if (input.outcome && deps.transientRetryBudgetSpent?.(predecessor)) {
+                        input.outcome.retryExhausted = true;
+                    }
+                    return null;
+                }
                 return null;
             }
         }
@@ -2816,6 +2828,7 @@ export function recoveryService(db, deps) {
             successfulContinuationObserved: 0,
             orphanBlockersAssigned: 0,
             successfulRunHandoffEscalated: 0,
+            successfulRunHandoffRetried: 0,
             reviewParticipantRequeued: 0,
             escalated: 0,
             waitingOnReviewResolved: 0,
@@ -3358,6 +3371,7 @@ export function recoveryService(db, deps) {
                     result.skipped += 1;
                     continue;
                 }
+                const reviewOutcome = {};
                 const queued = await enqueueStrandedIssueRecovery({
                     issueId: issue.id,
                     agentId: participantAgentId,
@@ -3370,11 +3384,36 @@ export function recoveryService(db, deps) {
                         currentStageType: pendingExecutionState.currentStageType ?? null,
                         reviewRecoveryInstruction: "The previous reviewer run ended while this execution-review stage was still pending. Submit the review decision now, or mark the issue blocked with the exact unblock action.",
                     },
+                    outcome: reviewOutcome,
                 });
                 if (queued) {
                     result.reviewParticipantRequeued += 1;
                     result.issueIds.push(issue.id);
                 }
+                else if (reviewOutcome.retryExhausted &&
+                    !(await hasActiveExecutionPath(issue.companyId, issue.id, null))) {
+                    // Same exhaustion as the other lanes: the reviewer run's bounded
+                    // retries are spent, so escalate as the review-recovery failure it
+                    // is instead of skipping on every sweep with no live path. The
+                    // live-path re-read guards the write against a run or wake that
+                    // started after the loop's check — for ANY agent, not only the
+                    // participant: `blocked` is issue-wide, so another agent still
+                    // working the issue must never be blocked under.
+                    const updated = await escalateStrandedAssignedIssue({
+                        issue,
+                        previousStatus: "in_review",
+                        latestRun: participantLatestRun,
+                        notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
+                        recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_REASON,
+                    });
+                    if (updated) {
+                        result.escalated += 1;
+                        result.issueIds.push(issue.id);
+                    }
+                    else {
+                        result.skipped += 1;
+                    }
+                }
                 else {
                     result.skipped += 1;
                 }
@@ -3441,6 +3480,7 @@ export function recoveryService(db, deps) {
                     result.skipped += 1;
                     continue;
                 }
+                const dispatchOutcome = {};
                 const queued = await enqueueStrandedIssueRecovery({
                     issueId: issue.id,
                     agentId,
@@ -3448,11 +3488,39 @@ export function recoveryService(db, deps) {
                     retryReason: "assignment_recovery",
                     source: "issue.assignment_recovery",
                     retryOfRunId: latestRun.id,
+                    outcome: dispatchOutcome,
                 });
                 if (queued) {
                     result.dispatchRequeued += 1;
                     result.issueIds.push(issue.id);
                 }
+                else if (dispatchOutcome.retryExhausted &&
+                    !(await hasActiveExecutionPath(issue.companyId, issue.id, null))) {
+                    // Same exhaustion as the in_progress lane: the lost dispatch's
+                    // bounded retries are spent, so escalate instead of skipping on
+                    // every sweep with no live path. The live-path re-read guards the
+                    // `blocked` write against a run or wake that started after the
+                    // loop's check.
+                    const updated = await escalateStrandedAssignedIssue({
+                        issue,
+                        previousStatus: "todo",
+                        latestRun,
+                        notice: {
+                            body: "Paperclip automatically retried dispatch for this assigned `todo` issue after a lost wake/run, " +
+                                "but the bounded retry budget is spent and it still has no live execution path. " +
+                                "Moving it to `blocked` so it is visible for intervention.",
+                            title: "No live execution path",
+                            tone: "danger",
+                        },
+                    });
+                    if (updated) {
+                        result.escalated += 1;
+                        result.issueIds.push(issue.id);
+                    }
+                    else {
+                        result.skipped += 1;
+                    }
+                }
                 else {
                     result.skipped += 1;
                 }
@@ -3488,6 +3556,38 @@ export function recoveryService(db, deps) {
                     result.skipped += 1;
                     continue;
                 }
+                // An interrupted corrective run is not evidence that the agent could
+                // not choose a disposition: a graceful server shutdown (a deploy
+                // restart, a lost process) ended the attempt before the agent
+                // finished. The attempt cap counts attempts the agent got to finish,
+                // so give the interrupted run the same bounded transient retry any
+                // interrupted run gets — the retry keeps the handoff context, so it
+                // is still the corrective run — and escalate only once that retry
+                // budget is spent or a finished attempt still leaves no disposition.
+                // Native-runtime corrective runs have no process-loss retry lane
+                // (a graceful shutdown suspends their controller for reattach
+                // instead of interrupting them; other native interruptions are
+                // reconciled by their own finalizer), so the recovery enqueue
+                // returns null for them and they escalate exactly as before.
+                if (latestRun?.status === "interrupted") {
+                    if (await isInvocationBudgetBlocked(issue, agentId)) {
+                        result.skipped += 1;
+                        continue;
+                    }
+                    const retried = await enqueueStrandedIssueRecovery({
+                        issueId: issue.id,
+                        agentId,
+                        reason: "issue_continuation_needed",
+                        retryReason: "issue_continuation_needed",
+                        source: "issue.successful_run_handoff_interrupted_retry",
+                        retryOfRunId: latestRun.id,
+                    });
+                    if (retried) {
+                        result.successfulRunHandoffRetried += 1;
+                        result.issueIds.push(issue.id);
+                        continue;
+                    }
+                }
                 const updated = await escalateStrandedAssignedIssue({
                     issue,
                     previousStatus: "in_progress",
@@ -3653,6 +3753,7 @@ export function recoveryService(db, deps) {
                 result.skipped += 1;
                 continue;
             }
+            const recoveryOutcome = {};
             const queued = await enqueueStrandedIssueRecovery({
                 issueId: issue.id,
                 agentId,
@@ -3660,11 +3761,40 @@ export function recoveryService(db, deps) {
                 retryReason: "issue_continuation_needed",
                 source: "issue.continuation_recovery",
                 retryOfRunId: latestRun?.id ?? issue.checkoutRunId ?? null,
+                outcome: recoveryOutcome,
             });
             if (queued) {
                 result.continuationRequeued += 1;
                 result.issueIds.push(issue.id);
             }
+            else if (recoveryOutcome.retryExhausted &&
+                !(await hasActiveExecutionPath(issue.companyId, issue.id, null))) {
+                // The failed run's bounded transient retries are all spent, so no
+                // successor will ever be queued for it. Escalate rather than skip
+                // on every sweep forever: the issue would otherwise stay
+                // `in_progress` with no run and no path until a person noticed.
+                // The live-path re-read guards the `blocked` write against a run or
+                // wake that started after the loop's check.
+                const updated = await escalateStrandedAssignedIssue({
+                    issue,
+                    previousStatus: "in_progress",
+                    latestRun,
+                    notice: {
+                        body: "Paperclip retried this issue's run after it ended without finishing, but the bounded retry budget " +
+                            "is spent and it still has no live execution path. " +
+                            "Moving it to `blocked` so it is visible for intervention.",
+                        title: "No live execution path",
+                        tone: "danger",
+                    },
+                });
+                if (updated) {
+                    result.escalated += 1;
+                    result.issueIds.push(issue.id);
+                }
+                else {
+                    result.skipped += 1;
+                }
+            }
             else {
                 result.skipped += 1;
             }