agent-dealer 1.2.14 → 1.2.16

package/bundle/server/dist/coordinator/commands.js

@@ -7,7 +7,7 @@
 // duplicate delivery is a no-op. The effect *work* itself runs elsewhere, through a leased
 // worker whose structured result comes back into applyCompletion.
 import fs from "node:fs";
-import { canTransitionIssue, ExecutionContractError, tryCompileContract } from "@agent-dealer/shared";
+import { canTransitionIssue, ExecutionContractError, isCursorKeychainStuckOutput, runtimeAuthClassificationForLog, tryCompileContract, } from "@agent-dealer/shared";
 import { getDb } from "../db/index.js";
 import { getIssue, incrementIssueRound, incrementIssueInfraAttempts, incrementIssueCiAttempts, resetIssueInfraAttempts, resetIssueCiAttempts, grantReviewRetry, transitionIssue, updateIssue, } from "../repository/issues.js";
 import { appendWorkflowEvent, completeWorkflowInstance, getActiveWorkflowInstance, getWorkflowInstance, listWorkflowEventsForIssue, startWorkflowInstance, WorkflowAlreadyActiveError, } from "../repository/workflow-events.js";
@@ -17,7 +17,7 @@ import { ensureIssueRepoCheckout } from "../adapters/managed-repo.js";
 import { reconcileFinding, resolveFindingsAbsentFromRound } from "../repository/findings.js";
 import { normalizeReviewerResult } from "./reviewer-result.js";
 import { getAgent } from "../repository/agents.js";
-import { githubIssuesSync, invalidateMuseHealthCache } from "../adapters/agent-health.js";
+import { githubIssuesSync, invalidateMuseHealthCache, runtimeIssuesUncached } from "../adapters/agent-health.js";
 import { recordMuseCapabilityOverride } from "../adapters/muse-capability.js";
 import { createIssueArtifact, latestIssueArtifact } from "../repository/artifacts.js";
 import { listSourceAttachments } from "../repository/source-attachments.js";
@@ -26,11 +26,12 @@ import { completeSession, getActiveWorkerSessionForIssue, getWorkerSession, list
 import { killRunProcess } from "../runners/spawn-cli.js";
 import { buildProfileSnapshot, serializeProfileSnapshot } from "./profile-snapshot.js";
 import { workerSessionPayload } from "./session-progress.js";
-import { reasonForWorkerFailedEvent } from "./failure-reason.js";
-import { recordCausesForWorkerFailedEvent } from "./failure-cause.js";
+import { reasonForWorkerFailedEvent, readSpawnLogFailureText } from "./failure-reason.js";
+import { classifyAttemptFailure, recordCausesForWorkerFailedEvent } from "./failure-cause.js";
 import { routeDeveloperOutcome, routeReviewerOutcome, } from "./routing.js";
 import { projectDeveloperRoute, projectReviewerRoute } from "./projection.js";
 import { MERGE_CONFLICT_FILES_EVIDENCE_KEY, MERGE_FAILURE_EVIDENCE_KEY, MERGE_FAILURE_RESPONSE_OPTIONS, PUSH_DIVERGENCE_EVIDENCE_KEY, PUSH_DIVERGENCE_RESPONSE_OPTIONS, WORKTREE_BLOCKER_EVIDENCE_KEY, normalizeResolutionNote, parseHumanResolution, resolveHumanActionOutcome, } from "./human-resolution.js";
+import { AUTH_TRANSIENT_RETRY_PAYLOAD_KEY, RUNTIME_AUTH_PARK_EVIDENCE_KEY, authProbeConfirmsFailure, defaultRemediationForRuntime, gateRuntimeAuthParkResume, parseRuntimeAuthParkEvidence, remediationFromProbe, } from "./runtime-auth-park.js";
 import { AUTO_MERGE_INTENT, finalizeAutoMerge } from "./auto-merge.js";
 import { triggerLinearPostMerge } from "./linear-merge-verify.js";
 import { conflictRepairSpent, queueConflictRepairRound } from "./merge-conflict-sync.js";
@@ -482,6 +483,25 @@ export async function applyCompletion(workItemId, leaseToken, outcome) {
     if (outcome.kind === "base_conflict") {
         return applyBaseConflictCompletion(workItemId, leaseToken, outcome);
     }
+    // NOT-368: classify + live-probe outside the write txn (network/CLI must never hold
+    // the SQLite lock). Developer and reviewer session deaths both participate — medium /
+    // unattributed keep today's infra-retry path inside the role's route function.
+    let authFailure;
+    const authKind = getWorkItem(workItemId)?.kind;
+    if ((outcome.kind === "session_failed" || outcome.kind === "timed_out") &&
+        (authKind === "developer" || authKind === "reviewer")) {
+        try {
+            authFailure = await prepareAuthFailureRouting(workItemId, {
+                kind: outcome.kind,
+                reason: "reason" in outcome ? outcome.reason : undefined,
+                logPath: "logPath" in outcome ? outcome.logPath : undefined,
+            });
+        }
+        catch (err) {
+            console.error("[coordinator] prepareAuthFailureRouting", workItemId, err);
+            authFailure = undefined;
+        }
+    }
     const routed = getDb().transaction(() => {
         const before = getWorkItem(workItemId);
         if (!before)
@@ -499,7 +519,7 @@ export async function applyCompletion(workItemId, leaseToken, outcome) {
         const item = finishWorkItem(workItemId, leaseToken, { status: "done", result: outcome });
         if (!item)
             return { applied: false, reason: "lease_lost" };
-        return routeAppliedOutcome(issue, instance, item, outcome);
+        return routeAppliedOutcome(issue, instance, item, outcome, authFailure);
     })();
     if (routed.applied && routed.pendingAutoMerge) {
         // Await async `gh` merge outside the routing txn — never block the event loop with spawnSync.
@@ -507,6 +527,74 @@ export async function applyCompletion(workItemId, leaseToken, outcome) {
     }
     return routed;
 }
+/**
+ * NOT-368: when a developer or reviewer session_failed / timed_out with a high-confidence
+ * attributed auth cause, run the runtime's live auth probe and gather consecutive-park /
+ * transient-retry state for the role's route function.
+ */
+async function prepareAuthFailureRouting(workItemId, outcome) {
+    const item = getWorkItem(workItemId);
+    if (!item || (item.kind !== "developer" && item.kind !== "reviewer"))
+        return undefined;
+    const session = item.workerSessionId ? getWorkerSession(item.workerSessionId) : null;
+    const logPath = session?.logPath ?? outcome.logPath ?? null;
+    const runtimeHint = (session?.runtime ?? null);
+    const causes = classifyAttemptFailure({
+        outcomeKind: outcome.kind,
+        outcomeReason: outcome.reason ?? null,
+        logPath,
+        runtime: runtimeHint,
+    });
+    const primary = causes.find((c) => c.primary) ?? causes[0];
+    if (!primary || primary.code !== "authentication_configuration")
+        return undefined;
+    // Medium / unattributed ("Not logged in" with no vendor) keep the generic infra retry.
+    if (primary.confidence !== "high")
+        return undefined;
+    const haystack = `${readSpawnLogFailureText(logPath)}\n${outcome.reason ?? ""}`;
+    const classified = isCursorKeychainStuckOutput(haystack)
+        ? { runtime: "cursor_local", issue: { code: "cursor_keychain", message: defaultRemediationForRuntime("cursor_local", true) } }
+        : runtimeAuthClassificationForLog(haystack, runtimeHint);
+    const runtime = classified?.runtime ?? runtimeHint;
+    if (!runtime)
+        return undefined;
+    const remediation = classified?.issue.message?.trim() ||
+        defaultRemediationForRuntime(runtime, classified?.issue.code === "cursor_keychain");
+    const probeIssues = await runtimeIssuesUncached(runtime);
+    const probeStillFailing = authProbeConfirmsFailure(probeIssues);
+    const probeRemediation = remediationFromProbe(probeIssues, remediation);
+    return {
+        confidence: "high",
+        runtime,
+        remediation: probeStillFailing ? probeRemediation : remediation,
+        rawCause: primary.rawReason,
+        probeStillFailing,
+        consecutiveAuthParks: countConsecutiveAuthParks(item.issueId),
+        authTransientRetrySpent: issueSpentAuthTransientRetry(item.issueId),
+    };
+}
+/** Count trailing auth-park human actions on the issue (newest streak). */
+function countConsecutiveAuthParks(issueId) {
+    const actions = listHumanActionsForIssue(issueId);
+    let n = 0;
+    for (let i = actions.length - 1; i >= 0; i--) {
+        if (parseRuntimeAuthParkEvidence(actions[i].evidenceJson) == null)
+            break;
+        n++;
+    }
+    return n;
+}
+/** True when any prior developer/reviewer work item on this issue spent the transient auth retry. */
+function issueSpentAuthTransientRetry(issueId) {
+    for (const w of listWorkItemsForIssue(issueId)) {
+        if (w.kind !== "developer" && w.kind !== "reviewer")
+            continue;
+        const payload = parseWorkItemPayload(w.payloadJson);
+        if (payload[AUTH_TRANSIENT_RETRY_PAYLOAD_KEY] === true)
+            return true;
+    }
+    return false;
+}
 function parseWorkItemPayload(json) {
     if (!json)
         return {};
@@ -906,10 +994,10 @@ export function routeCapEscalation(issue, instance, item, cap) {
         instanceCompleted: false,
     };
 }
-export function routeAppliedOutcome(issue, instance, item, outcome) {
+export function routeAppliedOutcome(issue, instance, item, outcome, authFailure) {
     return item.kind === "developer"
-        ? applyDeveloper(issue, instance, item, outcome)
-        : applyReviewer(issue, instance, item, outcome);
+        ? applyDeveloper(issue, instance, item, outcome, authFailure)
+        : applyReviewer(issue, instance, item, outcome, authFailure);
 }
 function eventEmitter(issue, instance, workerSessionId, stage, round) {
     let causation = null;
@@ -955,7 +1043,7 @@ function applyProjectionTransition(issue, projection, patch) {
         ...patch,
     });
 }
-function applyDeveloper(issue, instance, item, outcome) {
+function applyDeveloper(issue, instance, item, outcome, authFailure) {
     const route = routeDeveloperOutcome(outcome, {
         currentRound: issue.currentRound,
         maxReviewRounds: issue.maxReviewRounds,
@@ -963,6 +1051,7 @@ function applyDeveloper(issue, instance, item, outcome) {
         maxInfraAttempts: issue.maxInfraAttempts,
         ciAttempts: issue.ciAttempts,
         maxCiAttempts: issue.maxCiAttempts,
+        ...(authFailure ? { authFailure } : {}),
     });
     const { projection, effect, advance } = projectDeveloperRoute(route, issue.status, issue.currentRound);
     const ev = eventEmitter(issue, instance, item.workerSessionId, projection.issueStatus, issue.currentRound);
@@ -1051,13 +1140,14 @@ function applyDeveloper(issue, instance, item, outcome) {
     const issueNow = getIssue(issue.id);
     return applyEffect(issue, instance, effect, route, issueNow, ev, item.id);
 }
-function applyReviewer(issue, instance, item, outcome) {
+function applyReviewer(issue, instance, item, outcome, authFailure) {
     let route = routeReviewerOutcome(outcome, {
         currentRound: issue.currentRound,
         maxReviewRounds: issue.maxReviewRounds,
         infraAttempts: issue.infraAttempts,
         maxInfraAttempts: issue.maxInfraAttempts,
         autoMerge: issue.autoMerge,
+        ...(authFailure ? { authFailure } : {}),
     }, issue.headSha);
     // NOT-184: a repair loop that keeps finding new blocking issues on the same file is not
     // converging — hand it to a human instead of queuing another developer round.
@@ -1240,6 +1330,8 @@ function applyEffect(issue, instance, effect, route, issueNow, ev, causativeItem
                 ...(effect.retryReason ? { retryReason: effect.retryReason } : {}),
                 ...(effect.publishOnly ? { publishOnly: true } : {}),
                 ...(effect.branch ? { branch: effect.branch } : {}),
+                // NOT-368: persist that this infra retry spent the one transient auth retry.
+                ...(effect.authTransientRetry ? { [AUTH_TRANSIENT_RETRY_PAYLOAD_KEY]: true } : {}),
                 profileSnapshot: queuedProfileSnapshot(issue, kind),
             },
             idempotencyKey,
@@ -1261,6 +1353,10 @@ function applyEffect(issue, instance, effect, route, issueNow, ev, causativeItem
         const pushDivergence = actionType === "policy_escalation" && !resumeAsReviewer
             ? (effect.pushDivergence ?? null)
             : null;
+        // NOT-368: confirmed runtime login park — resolve re-probes before re-queueing.
+        // Reviewer-origin parks keep resumeAsReviewer (re-queue reviewer) AND auth-park
+        // evidence (so resolve still re-probes); the two are not mutually exclusive.
+        const runtimeAuthPark = actionType === "policy_escalation" ? (effect.runtimeAuthPark ?? null) : null;
         // NOT-280: an unchanged worktree blocker (same fingerprint) lands on the action it
         // already raised — still open, or reopened when a Resume changed nothing — instead of
         // opening an identical one on every Resume.
@@ -1284,9 +1380,11 @@ function applyEffect(issue, instance, effect, route, issueNow, ev, causativeItem
                 ? { review: reviewerOutcome.result, ...(nonConvergence ? { nonConvergence } : {}) }
                 : pushDivergence
                     ? { [PUSH_DIVERGENCE_EVIDENCE_KEY]: pushDivergence }
-                    : blockerFingerprint
-                        ? { [WORKTREE_BLOCKER_EVIDENCE_KEY]: { fingerprint: blockerFingerprint } }
-                        : undefined,
+                    : runtimeAuthPark
+                        ? { [RUNTIME_AUTH_PARK_EVIDENCE_KEY]: runtimeAuthPark }
+                        : blockerFingerprint
+                            ? { [WORKTREE_BLOCKER_EVIDENCE_KEY]: { fingerprint: blockerFingerprint } }
+                            : undefined,
             // issueNow.headSha, not issue.headSha: a stale outcome that itself exhausted the
             // infra budget already patched the newly observed head onto the issue above — the
             // pre-transition issue param would still carry the stale SHA a "resume" must not reuse.
@@ -2051,6 +2149,48 @@ export async function resolveHumanActionAndAdvanceAsync(actionId, resolvedBy, ch
             resumeLiveHeadSha = null;
         }
     }
+    // NOT-368: re-probe before resolving a runtime-auth park. A still-failing probe keeps
+    // the action open (no spawn); only a passing probe proceeds to resume on the salvaged
+    // branch without charging a round or infra attempt.
+    if (choice === "resume") {
+        const authParkAction = getHumanAction(actionId);
+        const authParkEvidence = authParkAction
+            ? parseRuntimeAuthParkEvidence(authParkAction.evidenceJson)
+            : null;
+        if (authParkAction?.status === "open" && authParkEvidence) {
+            let probeIssues = [];
+            try {
+                probeIssues = await runtimeIssuesUncached(authParkEvidence.runtime);
+            }
+            catch (err) {
+                console.error("[coordinator] auth-park resolve probe", actionId, err);
+                probeIssues = [
+                    {
+                        code: "runtime_auth",
+                        message: authParkEvidence.remediation,
+                    },
+                ];
+            }
+            const gate = gateRuntimeAuthParkResume({
+                evidence: authParkEvidence,
+                probeStillFailing: authProbeConfirmsFailure(probeIssues),
+                probeRemediation: remediationFromProbe(probeIssues, authParkEvidence.remediation),
+            });
+            if (!gate.proceed) {
+                // Refresh the open action's reason so the operator sees the remediation again.
+                try {
+                    updateOpenHumanAction(actionId, {
+                        reason: gate.message,
+                        question: questionFor("policy_escalation", gate.message, false),
+                    });
+                }
+                catch {
+                    // Best-effort — the 409 below is what keeps the action open.
+                }
+                return { ok: false, code: 409, error: gate.message };
+            }
+        }
+    }
     // NOT-196: the `gh` PR-state read runs here, outside any DB transaction — the sync
     // core below only consumes the pre-read state. Only close choices on issues with a
     // PR number pay for the call; anything unreadable resolves to "unknown" (or