@evermore.work/server 2026.511.0-canary.2 → 2026.513.0-canary.0

package/dist/services/recovery/service.js

@@ -1,6 +1,6 @@
 import { and, asc, desc, eq, gt, inArray, isNull, notInArray, sql } from "drizzle-orm";
 import { DEFAULT_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, MAX_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, MIN_ISSUE_GRAPH_LIVENESS_AUTO_RECOVERY_LOOKBACK_HOURS, } from "@evermore.work/shared";
-import { agents, agentWakeupRequests, approvals, companies, heartbeatRunEvents, heartbeatRunWatchdogDecisions, heartbeatRuns, issueApprovals, issueRelations, issueThreadInteractions, issues, } from "@evermore.work/db";
+import { agents, agentWakeupRequests, approvals, companies, issueComments, heartbeatRunEvents, heartbeatRunWatchdogDecisions, heartbeatRuns, issueApprovals, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, } from "@evermore.work/db";
 import { parseObject, asBoolean, asNumber } from "../../adapters/utils.js";
 import { runningProcesses } from "../../adapters/index.js";
 import { forbidden, notFound } from "../../errors.js";
@@ -10,10 +10,11 @@ import { redactSensitiveText } from "../../redaction.js";
 import { logActivity } from "../activity-log.js";
 import { budgetService } from "../budgets.js";
 import { instanceSettingsService } from "../instance-settings.js";
+import { issueRecoveryActionService } from "../issue-recovery-actions.js";
 import { issueTreeControlService } from "../issue-tree-control.js";
 import { issueService } from "../issues.js";
 import { getRunLogStore } from "../run-log-store.js";
-import { DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS, FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, SUCCESSFUL_RUN_MISSING_STATE_REASON, buildSuccessfulRunHandoffExhaustedNotice, } from "./successful-run-handoff.js";
+import { DEFAULT_MAX_SUCCESSFUL_RUN_HANDOFF_ATTEMPTS, FINISH_SUCCESSFUL_RUN_HANDOFF_REASON, SUCCESSFUL_RUN_MISSING_STATE_REASON, buildSuccessfulRunHandoffExhaustedNotice, noticeMetadataReferencesRecoveryAction, } from "./successful-run-handoff.js";
 import { RECOVERY_ORIGIN_KINDS, buildIssueGraphLivenessLeafKey, isStrandedIssueRecoveryOriginKind, parseIssueGraphLivenessIncidentKey, } from "./origins.js";
 import { classifyIssueGraphLiveness, } from "./issue-graph-liveness.js";
 import { recoveryAssigneeAdapterOverrides, withRecoveryModelProfileHint, } from "./model-profile-hint.js";
@@ -242,6 +243,7 @@ function buildLivenessOriginalIssueComment(finding, escalation) {
 }
 export function recoveryService(db, deps) {
     const issuesSvc = issueService(db);
+    const recoveryActionsSvc = issueRecoveryActionService(db);
     const treeControlSvc = issueTreeControlService(db);
     const budgets = budgetService(db);
     const instanceSettings = instanceSettingsService(db);
@@ -1228,6 +1230,111 @@ export function recoveryService(db, deps) {
         });
         return recovery;
     }
+    function strandedRecoveryActionKind(cause) {
+        return cause === SUCCESSFUL_RUN_MISSING_STATE_REASON
+            ? "missing_disposition"
+            : "stranded_assigned_issue";
+    }
+    function strandedRecoveryActionFingerprint(input) {
+        return [
+            "source_scoped_recovery",
+            input.issue.companyId,
+            input.issue.id,
+            input.recoveryCause,
+        ].join(":");
+    }
+    function buildStrandedRecoveryActionEvidence(input) {
+        const context = parseObject(input.latestRun?.contextSnapshot);
+        return {
+            sourceIssueId: input.issue.id,
+            sourceIdentifier: input.issue.identifier,
+            previousStatus: input.previousStatus,
+            latestIssueStatus: input.issue.status,
+            latestRunId: input.latestRun?.id ?? null,
+            latestRunStatus: input.latestRun?.status ?? null,
+            latestRunErrorCode: input.latestRun?.errorCode ?? null,
+            retryReason: readNonEmptyString(context.retryReason) ?? null,
+            recoveryCause: input.recoveryCause,
+            sourceRunId: input.successfulRunHandoffEvidence?.sourceRunId ?? null,
+            correctiveRunId: input.successfulRunHandoffEvidence?.correctiveRunId ?? null,
+            missingDisposition: input.successfulRunHandoffEvidence?.missingDisposition ?? null,
+            handoffAttempt: input.successfulRunHandoffEvidence?.handoffAttempt ?? null,
+            maxHandoffAttempts: input.successfulRunHandoffEvidence?.maxHandoffAttempts ?? null,
+        };
+    }
+    async function ensureSourceScopedStrandedRecoveryAction(input) {
+        const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
+        const ownerAgentId = await resolveStrandedIssueRecoveryOwnerAgentId(input.issue);
+        const now = new Date();
+        const action = await recoveryActionsSvc.upsertSourceScoped({
+            companyId: input.issue.companyId,
+            sourceIssueId: input.issue.id,
+            kind: strandedRecoveryActionKind(recoveryCause),
+            ownerType: ownerAgentId ? "agent" : "board",
+            ownerAgentId,
+            previousOwnerAgentId: input.issue.assigneeAgentId,
+            returnOwnerAgentId: input.issue.assigneeAgentId,
+            cause: recoveryCause,
+            fingerprint: strandedRecoveryActionFingerprint({
+                issue: input.issue,
+                recoveryCause,
+            }),
+            evidence: buildStrandedRecoveryActionEvidence({
+                issue: input.issue,
+                latestRun: input.latestRun,
+                previousStatus: input.previousStatus,
+                recoveryCause,
+                successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
+            }),
+            nextAction: recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON
+                ? "Choose and record a valid issue disposition without copying transcript content."
+                : "Restore a live execution path, fix the runtime/adapter failure, or record an intentional manual resolution.",
+            wakePolicy: ownerAgentId
+                ? {
+                    type: "wake_owner",
+                    reason: "source_scoped_recovery_action",
+                    ownerAgentId,
+                }
+                : {
+                    type: "board_escalation",
+                    reason: "no_invokable_recovery_owner",
+                },
+            monitorPolicy: null,
+            maxAttempts: null,
+            lastAttemptAt: now,
+        });
+        return action;
+    }
+    async function enqueueSourceScopedStrandedRecoveryWake(input) {
+        if (!input.action.ownerAgentId)
+            return;
+        await deps.enqueueWakeup(input.action.ownerAgentId, {
+            source: "assignment",
+            triggerDetail: "system",
+            reason: "source_scoped_recovery_action",
+            idempotencyKey: `source_scoped_recovery_action:${input.action.id}:${input.action.attemptCount}`,
+            payload: withRecoveryModelProfileHint({
+                issueId: input.issue.id,
+                sourceIssueId: input.issue.id,
+                recoveryActionId: input.action.id,
+                strandedRunId: input.latestRun?.id ?? null,
+                recoveryCause: input.recoveryCause,
+            }),
+            requestedByActorType: "system",
+            requestedByActorId: null,
+            contextSnapshot: withRecoveryModelProfileHint({
+                issueId: input.issue.id,
+                taskId: input.issue.id,
+                wakeReason: "source_scoped_recovery_action",
+                skipIssueComment: true,
+                source: "issue_recovery_action",
+                recoveryActionId: input.action.id,
+                sourceIssueId: input.issue.id,
+                strandedRunId: input.latestRun?.id ?? null,
+                recoveryCause: input.recoveryCause,
+            }),
+        });
+    }
     function buildRecoveryIssueInPlaceEscalationComment(input) {
         const runLink = input.latestRun
             ? runUiLink({ id: input.latestRun.id, agentId: input.latestRun.agentId }, input.prefix)
@@ -1298,29 +1405,31 @@ export function recoveryService(db, deps) {
             .then((rows) => rows.map((row) => row.blockerIssueId));
     }
     async function escalateStrandedAssignedIssue(input) {
-        const nestedRecoverySuppressed = isStrandedIssueRecoveryIssue(input.issue);
-        let recoveryIssue = null;
-        if (!nestedRecoverySuppressed) {
-            recoveryIssue = await ensureStrandedIssueRecoveryIssue({
+        if (isStrandedIssueRecoveryIssue(input.issue)) {
+            return escalateStrandedRecoveryIssueInPlace({
                 issue: input.issue,
                 previousStatus: input.previousStatus,
                 latestRun: input.latestRun,
-                recoveryCause: input.recoveryCause,
-                successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
             });
         }
+        const recoveryCause = input.recoveryCause ?? "stranded_assigned_issue";
+        const recoveryAction = await ensureSourceScopedStrandedRecoveryAction({
+            issue: input.issue,
+            previousStatus: input.previousStatus,
+            latestRun: input.latestRun,
+            recoveryCause,
+            successfulRunHandoffEvidence: input.successfulRunHandoffEvidence,
+        });
         const blockerIds = await existingUnresolvedBlockerIssueIds(input.issue.companyId, input.issue.id);
-        const nextBlockerIds = recoveryIssue
-            ? [...new Set([...blockerIds, recoveryIssue.id])]
-            : blockerIds;
         const updated = await issuesSvc.update(input.issue.id, {
             status: "blocked",
-            blockedByIssueIds: nextBlockerIds,
+            blockedByIssueIds: blockerIds,
+            assigneeAgentId: recoveryAction.ownerAgentId ?? input.issue.assigneeAgentId,
         });
         if (!updated)
             return null;
         const prefix = await getCompanyIssuePrefix(input.issue.companyId);
-        const recoveryOwner = recoveryIssue?.assigneeAgentId ? await getAgent(recoveryIssue.assigneeAgentId) : null;
+        const recoveryOwner = recoveryAction.ownerAgentId ? await getAgent(recoveryAction.ownerAgentId) : null;
         const sourceAssignee = input.issue.assigneeAgentId ? await getAgent(input.issue.assigneeAgentId) : null;
         let notice = null;
         if (input.recoveryCause === SUCCESSFUL_RUN_MISSING_STATE_REASON && input.successfulRunHandoffEvidence) {
@@ -1331,41 +1440,51 @@ export function recoveryService(db, deps) {
                     : null,
                 correctiveRun: input.latestRun ? { id: input.latestRun.id, status: input.latestRun.status } : null,
                 sourceAssignee,
-                recoveryIssue,
+                recoveryIssue: null,
+                recoveryActionId: recoveryAction.id,
                 recoveryOwner,
                 latestIssueStatus: input.issue.status,
                 latestHandoffRunStatus: input.latestRun?.status ?? "unknown",
                 missingDisposition: input.successfulRunHandoffEvidence.missingDisposition,
             });
         }
-        let recoveryLine;
-        if (nestedRecoverySuppressed) {
-            recoveryLine = await buildNestedStrandedRecoveryLine(input.issue, prefix);
-        }
-        else if (recoveryIssue) {
-            recoveryLine = [
+        const recoveryLine = recoveryAction.ownerAgentId
+            ? [
                 "",
-                `- Recovery issue: ${issueUiLink({ identifier: recoveryIssue.identifier, id: recoveryIssue.id }, prefix)}`,
+                `- Recovery action: \`${recoveryAction.id}\``,
                 `- Recovery owner: ${agentUiLink(recoveryOwner, prefix)}`,
-                "- Next action: the recovery owner should either restore a live execution path or record the manual resolution, then mark the recovery issue done.",
-            ].join("\n");
-        }
-        else {
-            recoveryLine = [
+                "- Next action: the recovery owner should either restore a live execution path or record the manual resolution on the source issue.",
+            ].join("\n")
+            : [
                 "",
-                "- Recovery issue: none created because Evermore could not find an invokable manager, creator, or executive owner with budget available.",
+                `- Recovery action: \`${recoveryAction.id}\``,
+                "- Recovery owner: board escalation, because Evermore could not find an invokable manager, creator, or executive owner with budget available.",
                 "- Next action: a board operator should assign an invokable recovery owner, fix the agent/runtime state, or record an intentional manual resolution.",
             ].join("\n");
-        }
-        if (notice) {
-            await issuesSvc.addComment(input.issue.id, notice.body, {}, {
-                authorType: "system",
-                presentation: notice.presentation,
-                metadata: notice.metadata,
-            });
-        }
-        else {
-            await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {});
+        if (recoveryAction.attemptCount === 1) {
+            const escalationCommentMarker = `Recovery action: \`${recoveryAction.id}\``;
+            const hasEscalationComment = await db
+                .select({ id: issueComments.id, body: issueComments.body, metadata: issueComments.metadata })
+                .from(issueComments)
+                .where(and(eq(issueComments.issueId, input.issue.id), eq(issueComments.authorType, "system")))
+                .orderBy(desc(issueComments.createdAt))
+                .limit(50)
+                .then((rows) => rows.some((row) => (row.body ?? "").includes(escalationCommentMarker) ||
+                noticeMetadataReferencesRecoveryAction(row.metadata, recoveryAction.id)));
+            if (!hasEscalationComment) {
+                if (notice) {
+                    await issuesSvc.addComment(input.issue.id, notice.body, {}, {
+                        authorType: "system",
+                        presentation: notice.presentation,
+                        metadata: notice.metadata,
+                    });
+                }
+                else {
+                    await issuesSvc.addComment(input.issue.id, `${input.comment ?? ""}${recoveryLine}`, {}, {
+                        authorType: "system",
+                    });
+                }
+            }
         }
         await logActivity(db, {
             companyId: input.issue.companyId,
@@ -1389,11 +1508,40 @@ export function recoveryService(db, deps) {
                 latestRunId: input.latestRun?.id ?? null,
                 latestRunStatus: input.latestRun?.status ?? null,
                 latestRunErrorCode: input.latestRun?.errorCode ?? null,
-                recoveryIssueId: recoveryIssue?.id ?? null,
-                nestedRecoverySuppressed,
-                blockerIssueIds: nextBlockerIds,
+                recoveryActionId: recoveryAction.id,
+                recoveryOwnerAgentId: recoveryAction.ownerAgentId,
+                previousOwnerAgentId: recoveryAction.previousOwnerAgentId,
+                returnOwnerAgentId: recoveryAction.returnOwnerAgentId,
+                blockerIssueIds: blockerIds,
             },
         });
+        await enqueueSourceScopedStrandedRecoveryWake({
+            action: recoveryAction,
+            issue: input.issue,
+            latestRun: input.latestRun,
+            recoveryCause,
+        });
+        if (recoveryAction.ownerAgentId && recoveryAction.ownerAgentId === input.issue.assigneeAgentId) {
+            const [currentIssue] = await db
+                .select({
+                status: issues.status,
+                assigneeAgentId: issues.assigneeAgentId,
+            })
+                .from(issues)
+                .where(eq(issues.id, input.issue.id))
+                .limit(1);
+            if (currentIssue &&
+                (currentIssue.status !== "blocked" ||
+                    currentIssue.assigneeAgentId !== recoveryAction.ownerAgentId)) {
+                const reblocked = await issuesSvc.update(input.issue.id, {
+                    status: "blocked",
+                    blockedByIssueIds: blockerIds,
+                    assigneeAgentId: recoveryAction.ownerAgentId,
+                });
+                if (reblocked)
+                    return reblocked;
+            }
+        }
         return updated;
     }
     async function reconcileStrandedAssignedIssues() {
@@ -1629,28 +1777,29 @@ export function recoveryService(db, deps) {
         return result;
     }
     async function collectIssueGraphLivenessFindings() {
-        const [issueRows, relationRows, agentRows, activeRunRows, activeIssueRunRows, wakeRows, interactionRows, approvalRows, recoveryIssueRows,] = await Promise.all([
-            db
-                .select({
-                id: issues.id,
-                companyId: issues.companyId,
-                identifier: issues.identifier,
-                title: issues.title,
-                status: issues.status,
-                projectId: issues.projectId,
-                goalId: issues.goalId,
-                parentId: issues.parentId,
-                assigneeAgentId: issues.assigneeAgentId,
-                assigneeUserId: issues.assigneeUserId,
-                createdByAgentId: issues.createdByAgentId,
-                createdByUserId: issues.createdByUserId,
-                executionPolicy: issues.executionPolicy,
-                executionState: issues.executionState,
-                monitorNextCheckAt: issues.monitorNextCheckAt,
-                monitorAttemptCount: issues.monitorAttemptCount,
-            })
-                .from(issues)
-                .where(and(isNull(issues.hiddenAt), notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]))),
+        const issueRowsPromise = Promise.resolve(db
+            .select({
+            id: issues.id,
+            companyId: issues.companyId,
+            identifier: issues.identifier,
+            title: issues.title,
+            status: issues.status,
+            projectId: issues.projectId,
+            goalId: issues.goalId,
+            parentId: issues.parentId,
+            assigneeAgentId: issues.assigneeAgentId,
+            assigneeUserId: issues.assigneeUserId,
+            createdByAgentId: issues.createdByAgentId,
+            createdByUserId: issues.createdByUserId,
+            executionPolicy: issues.executionPolicy,
+            executionState: issues.executionState,
+            monitorNextCheckAt: issues.monitorNextCheckAt,
+            monitorAttemptCount: issues.monitorAttemptCount,
+        })
+            .from(issues)
+            .where(and(isNull(issues.hiddenAt), notInArray(issues.originKind, [RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation]))));
+        const [issueRows, relationRows, agentRows, activeRunRows, activeIssueRunRows, wakeRows, interactionRows, approvalRows, recoveryIssueRows, recoveryActionRows,] = await Promise.all([
+            issueRowsPromise,
             db
                 .select({
                 companyId: issueRelations.companyId,
@@ -1728,6 +1877,19 @@ export function recoveryService(db, deps) {
                 STRANDED_ISSUE_RECOVERY_ORIGIN_KIND,
                 RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
             ]), notInArray(issues.status, ["done", "cancelled"]))),
+            issueRowsPromise.then((rows) => {
+                const issueIdsUnderAnalysis = rows.map((row) => row.id);
+                return issueIdsUnderAnalysis.length === 0
+                    ? []
+                    : db
+                        .select({
+                        companyId: issueRecoveryActions.companyId,
+                        issueId: issueRecoveryActions.sourceIssueId,
+                        status: issueRecoveryActions.status,
+                    })
+                        .from(issueRecoveryActions)
+                        .where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), inArray(issueRecoveryActions.sourceIssueId, issueIdsUnderAnalysis)));
+            }),
         ]);
         const openRecoveryIssues = recoveryIssueRows.flatMap((row) => {
             if (row.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation) {
@@ -1781,7 +1943,7 @@ export function recoveryService(db, deps) {
             })),
             pendingInteractions: interactionRows,
             pendingApprovals: approvalRows,
-            openRecoveryIssues,
+            openRecoveryIssues: openRecoveryIssues.concat(recoveryActionRows),
             now: new Date(),
         });
     }