@paperclipai/server 2026.824.0 → 2026.825.0-beta.1

package/dist/services/heartbeat.js

@@ -4,10 +4,12 @@ import { execFile as execFileCallback } from "node:child_process";
 import { promisify } from "node:util";
 import { createHash, randomUUID } from "node:crypto";
 import { and, asc, desc, eq, getTableColumns, gt, gte, inArray, isNull, lt, lte, ne, notInArray, or, sql } from "drizzle-orm";
-import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, MODEL_PROFILE_KEYS, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, envBindingSchema, } from "@paperclipai/shared";
+import { AGENT_DEFAULT_MAX_CONCURRENT_RUNS, ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY, ISSUE_DISPOSITION_REPAIR_RETRY_REASON, MODEL_PROFILE_KEYS, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, envBindingSchema, } from "@paperclipai/shared";
 import { agents, agentConfigRevisions, agentRuntimeState, agentTaskSessions, agentWakeupRequests, activityLog, approvals, companyMemberships, companySkillTestRuns, companySkillVersions, companySkills as companySkillsTable, companies, costEvents, documentAnnotationComments, documentAnnotationThreads, documentRevisions, environmentLeases, issueDocuments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueComments, issuePlanDecompositions, issueRecoveryActions, issueRelations, issueThreadInteractions, issues, issueWorkProducts, projects, projectWorkspaces, routineRevisions, routineRuns, routines, toolMcpGateways, toolMcpGatewayTokens, toolConnections, toolProfiles, workspaceOperations, } from "@paperclipai/db";
 import { conflict, HttpError, notFound } from "../errors.js";
-import { getStartupTraceContext } from "../instrumentation.js";
+import { getStartupTraceContext, getStartupTracer } from "../instrumentation.js";
+import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js";
+import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js";
 import { logger } from "../middleware/logger.js";
 import { createGitRemoteAuthProvider, describeGitAuthFailure, scrubGitCredentialText, } from "./git-credentials.js";
 // Re-exported because heartbeat's workspace surface exposed the scrubber before the
@@ -31,10 +33,11 @@ import { buildHeartbeatRunStopMetadata, mergeHeartbeatRunStopMetadata, normalize
 import { classifyRunLiveness, } from "./run-liveness.js";
 import { ISSUE_NEW_INPUT_ACTIVITY_ACTIONS, ISSUE_PROGRESS_ACTIVITY_ACTIONS, ISSUE_REWAKE_LOOKBACK_MS, ISSUE_REWAKE_RUN_SAMPLE_LIMIT, evaluateIssueRewakeThrottle, isThrottleCandidateIssueRewake, } from "./issue-rewake-throttle.js";
 import { logActivity, publishPluginDomainEvent } from "./activity-log.js";
-import { buildWorkspaceReadyComment, buildWorkspaceReadyMetadata, buildWorkspaceReadyPresentation, cleanupExecutionWorkspaceArtifacts, ensureGitWorktreeBranchCoherent, ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, formatManagedGitWorktreeBranchInspection, inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js";
+import { buildWorkspaceReadyComment, buildWorkspaceReadyMetadata, buildWorkspaceReadyPresentation, cleanupExecutionWorkspaceArtifacts, ensureGitWorktreeBranchCoherent, ensurePersistedExecutionWorkspaceAvailable, ensureRuntimeServicesForRun, formatManagedGitWorktreeBranchInspection, inspectManagedGitWorktreeBranch, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, isUnresolvedWorkspaceBaseRefError, sanitizeRuntimeServiceBaseEnv, } from "./workspace-runtime.js";
 import { readManagedWorktreeInstanceOwnership, WORKTREE_INSTANCE_ROOT_METADATA_KEY, } from "./workspace-instance-cleanup.js";
 import { issueService } from "./issues.js";
 import { projectService } from "./projects.js";
+import { getEnvironmentDriverTraits } from "./environment-driver-traits.js";
 import { authorizationService } from "./authorization.js";
 import { createToolGatewayService } from "./tool-gateway.js";
 import { toolAccessService } from "./tool-access.js";
@@ -45,6 +48,7 @@ import { ISSUE_TREE_CONTROL_INTERACTION_WAKE_REASONS, isVerifiedIssueTreeControl
 import { continuationSummaryParksExecutor, getIssueContinuationSummaryDocument, refreshIssueContinuationSummary, } from "./issue-continuation-summary.js";
 import { buildDocumentReviewContext, buildPlanReviewContext } from "./plan-review-context.js";
 import { executionWorkspaceService, mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
+import { GIT_BRANCH_OWNERSHIP_METADATA_KEY, GIT_BRANCH_OWNERSHIP_METADATA_VERSION, isRuntimeOwnedGitBranch, } from "./execution-workspace-branch-ownership.js";
 import { workspaceOperationService } from "./workspace-operations.js";
 import { isProcessGroupAlive, terminateLocalService } from "./local-service-supervisor.js";
 import { HEARTBEAT_RUN_SCRATCH_MARKER, buildHeartbeatRunScratchEnv, cleanupHeartbeatRunScratch, prepareHeartbeatRunScratch, } from "./run-scratch.js";
@@ -56,12 +60,14 @@ import { isAutomaticRecoverySuppressedByPauseHold } from "./recovery/pause-hold-
 import { buildConfigurationIncompleteRecoveryNoticeSeed, buildExecutionReviewParticipantRecoveryNoticeSeed, buildImmediateExecutionPathRecoveryNoticeSeed, buildWorkspaceValidationRecoveryNoticeSeed, } from "./recovery/stranded-notice.js";
 import { recoveryAssigneeAdapterOverrides, withRecoveryModelProfileHint, } from "./recovery/model-profile-hint.js";
 import { ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS as RECOVERY_ACTIVE_RUN_OUTPUT_SUSPICION_THRESHOLD_MS, recoveryService } from "./recovery/service.js";
+import { collectDispositionRepairSourceState } from "./recovery/disposition-repair.js";
 import { buildIssueReviewPathLostIdempotencyKey, decideIssueReviewPathRecovery, ISSUE_REVIEW_PATH_LOST_WAKE_REASON, isReviewPathRecoveryIdempotencyConflict, REVIEW_PATH_RECOVERY_INSTRUCTION, reviewPathConsumedRefFromRun, } from "./recovery/review-path-recovery.js";
 import { productivityReviewService } from "./productivity-review.js";
 import { resolveRequiredSuccessfulRunHandoffOnValidPath } from "./successful-run-handoff-state.js";
 import { taskWatchdogService } from "./task-watchdogs.js";
 import { withAgentStartLock } from "./agent-start-lock.js";
 import { evaluateAgentInvokability, evaluateAgentInvokabilityFromDb, shouldCancelRunsForNonInvokableAgent, DIRECT_NON_INVOKABLE_STATUSES, } from "./agent-invokability.js";
+import { isHeartbeatWakeOnDemandEnabled } from "./heartbeat-policy.js";
 import { redactQuarantinedBodyForHigherTrust, sanitizeQuarantinedCommentForHigherTrust, } from "./source-trust.js";
 import { redactCurrentUserText, redactCurrentUserValue, } from "../log-redaction.js";
 import { redactEventPayload, redactSensitiveText } from "../redaction.js";
@@ -202,6 +208,14 @@ const WORKSPACE_VALIDATION_FAILURE_CODE = "workspace_validation_failed";
 const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
 const CONFIGURATION_INCOMPLETE_FAILURE_CODE = "configuration_incomplete";
 const CONFIGURATION_INCOMPLETE_RECOVERY_CAUSE = "configuration_incomplete";
+// Error codes that mark a pre-dispatch setup failure. The adapter process never
+// started, so no agent could post an issue comment. The setup catch writes one
+// of these codes when a failure happens before `adapter.execute`.
+const PRE_ADAPTER_SETUP_FAILURE_CODES = new Set([
+    "setup_failed",
+    CONFIGURATION_INCOMPLETE_FAILURE_CODE,
+    WORKSPACE_VALIDATION_FAILURE_CODE,
+]);
 const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_RETRY_REASON = "execution_review_participant_recovery";
 const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_WAKE_REASON = "execution_review_participant_recovery";
 const EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE = "execution_review_participant_recovery";
@@ -216,6 +230,7 @@ const GIT_SENSITIVE_LOCAL_ADAPTER_TYPES = new Set([
     "gemini_local",
     "grok_local",
     "hermes_local",
+    "kimi_local",
     "opencode_local",
     "pi_local",
 ]);
@@ -268,6 +283,31 @@ export class ConfigurationIncompleteFailure extends Error {
         this.resultJson = resultJson;
     }
 }
+// Build the configuration-incomplete result payload for a workspace base ref
+// that never resolved to a commit. The setup catch maps this to errorCode
+// `configuration_incomplete`, so the recovery path routes it to a human owner
+// instead of a dispatched-then-failed run. The `fingerprint` uses the canonical
+// remote ref, not the operator spelling. Two equivalent spellings of one remote
+// branch (`fix/foo` and `origin/fix/foo`) share one fingerprint, so a repeated
+// failure reuses one active recovery action and does not reset the attempt
+// count or post a duplicate notice. A different branch makes a new action.
+function buildUnresolvedWorkspaceBaseRefResultJson(run, error) {
+    const context = parseObject(run.contextSnapshot);
+    return {
+        configurationIncomplete: {
+            reason: "workspace_base_ref_unresolved",
+            companyId: run.companyId,
+            agentId: run.agentId,
+            issueId: readNonEmptyString(context.issueId) ?? null,
+            projectId: readNonEmptyString(context.projectId) ?? null,
+            requestedRef: error.requestedRef,
+            attemptedRefs: error.attemptedRefs,
+            fetchError: error.fetchError,
+            fingerprint: `workspace_base_ref:${error.recoveryIdentityRef}`,
+            missingBindings: [],
+        },
+    };
+}
 // Pre-dispatch gate outcome: another running run currently holds the issue's
 // shared project workspace. Not a failure — the run is parked as a bounded
 // scheduled retry and re-attempted once the holder finishes, so two agents
@@ -450,6 +490,7 @@ const SESSIONED_LOCAL_ADAPTERS = new Set([
     "cursor",
     "gemini_local",
     "hermes_local",
+    "kimi_local",
     "opencode_local",
     "pi_local",
 ]);
@@ -931,6 +972,12 @@ export function mergeExecutionWorkspaceMetadataForPersistence(input) {
         source: input.source,
         createdByRuntime: input.createdByRuntime,
     };
+    if (input.strategyType === "git_worktree") {
+        base[GIT_BRANCH_OWNERSHIP_METADATA_KEY] = GIT_BRANCH_OWNERSHIP_METADATA_VERSION;
+    }
+    else {
+        delete base[GIT_BRANCH_OWNERSHIP_METADATA_KEY];
+    }
     const existingSnapshot = parseObject(base.baseRefSnapshot);
     if (typeof existingSnapshot.resolvedSha !== "string"
         && input.baseRefSha) {
@@ -953,6 +1000,9 @@ export function mergeExecutionWorkspaceMetadataForPersistence(input) {
     }
     return mergeExecutionWorkspaceConfig(base, input.configSnapshot);
 }
+export function resolveExecutionWorkspaceBranchOwnership(executionWorkspace) {
+    return executionWorkspace.branchCreatedByRuntime;
+}
 export function stripWorkspaceRuntimeFromExecutionRunConfig(config) {
     const nextConfig = { ...config };
     delete nextConfig.workspaceRuntime;
@@ -1902,7 +1952,7 @@ export function isMultiProjectWorkspaceSyncEnabled(env = process.env) {
  * treated as local.
  */
 export function isRemoteExecutionEnvironmentDriver(driver) {
-    return driver === "ssh" || driver === "sandbox" || driver === "plugin";
+    return getEnvironmentDriverTraits(driver)?.runsWorkspaceOffHost ?? false;
 }
 /**
  * Environment flag (kill-switch, default ON) that gates whether a *remote* run stages the
@@ -1931,7 +1981,7 @@ export function isMultiProjectWorkspaceSyncRemoteEnabled(env = process.env) {
  * confines each staged referenced tree.
  */
 export function isConfinedRemoteStagingDriver(driver) {
-    return driver === "sandbox";
+    return getEnvironmentDriverTraits(driver)?.confinesStagedProjects ?? false;
 }
 /**
  * Upper bound on how many additional (mentioned) projects a single run may materialize
@@ -2195,6 +2245,9 @@ export function buildReferencedProjectRunObservability(input) {
         referenced_project_failures: input.failures.map((failure) => ({
             project_id: failure.projectId,
             reason: failure.reason,
+            // Carry the error only when the layer produced one, so an authorization or resolution drop
+            // stays a two-field entry and a staging drop names its reason.
+            ...(failure.error !== undefined ? { error: failure.error } : {}),
         })),
     };
 }
@@ -3040,7 +3093,16 @@ const EFFECTIVE_RUN_WORKSPACE_CONFIG_CATEGORIES = [
 ];
 export function resolveExecutionWorkspaceReuseRequestForIssue(input) {
     const requestedExecutionWorkspaceId = readNonEmptyString(input.issueExecutionWorkspaceId);
-    const requestedShouldReuseExisting = input.issueExecutionWorkspacePreference === "reuse_existing" && requestedExecutionWorkspaceId !== null;
+    // An explicitly pinned existing branch outranks an inherited reuse_existing
+    // binding: a persisted workspace on any other branch (or with no recorded
+    // branch) is stale for this issue, so dispatch realizes the pinned branch
+    // instead of restoring the mismatched workspace.
+    const requestedExistingBranch = readNonEmptyString(input.requestedExistingBranch);
+    const existingWorkspaceMatchesRequestedBranch = requestedExistingBranch === null ||
+        readNonEmptyString(input.existingExecutionWorkspaceBranchName) === requestedExistingBranch;
+    const requestedShouldReuseExisting = input.issueExecutionWorkspacePreference === "reuse_existing" &&
+        requestedExecutionWorkspaceId !== null &&
+        existingWorkspaceMatchesRequestedBranch;
     return {
         requestedExecutionWorkspaceId,
         requestedShouldReuseExisting,
@@ -4799,6 +4861,7 @@ export function heartbeatService(db, options = {}) {
     const envOrchestrator = environmentRunOrchestrator(db, {
         pluginWorkerManager: options.pluginWorkerManager,
         environmentRuntime,
+        duplexAggregateByteLedger: options.duplexAggregateByteLedger,
     });
     const workspaceOperationsSvc = workspaceOperationService(db);
     const liveRunExecutions = {
@@ -7424,6 +7487,21 @@ export function heartbeatService(db, options = {}) {
             }
             return { outcome: "not_applicable", queuedRun: null };
         }
+        // A pre-dispatch setup failure means the adapter process never started (for
+        // example an unresolved workspace base ref). No agent could run, so no agent
+        // could post an issue comment. A missing-comment retry cannot help and would
+        // loop the identical pre-adapter failure, so mark the policy not_applicable
+        // and queue nothing.
+        if (run.errorCode != null && PRE_ADAPTER_SETUP_FAILURE_CODES.has(run.errorCode)) {
+            if (run.issueCommentStatus !== "not_applicable") {
+                await patchRunIssueCommentStatus(run.id, {
+                    issueCommentStatus: "not_applicable",
+                    issueCommentSatisfiedByCommentId: null,
+                    issueCommentRetryQueuedAt: null,
+                });
+            }
+            return { outcome: "not_applicable", queuedRun: null };
+        }
         const postedComment = await findRunIssueComment(run.id, run.companyId, issueId);
         if (postedComment) {
             await patchRunIssueCommentStatus(run.id, {
@@ -8077,15 +8155,28 @@ export function heartbeatService(db, options = {}) {
                 },
             };
         }
+        if (!isHeartbeatWakeOnDemandEnabled(agent)) {
+            return {
+                allowed: false,
+                reason: "Scheduled retry suppressed because on-demand agent wakes are disabled",
+                errorCode: "heartbeat_wake_on_demand_disabled",
+                issueId,
+                details: { agentId: agent.id },
+            };
+        }
         if (!issueId)
             return { allowed: true };
         const issue = await db
             .select({
             id: issues.id,
+            companyId: issues.companyId,
             status: issues.status,
             assigneeAgentId: issues.assigneeAgentId,
+            assigneeUserId: issues.assigneeUserId,
             executionRunId: issues.executionRunId,
+            executionPolicy: issues.executionPolicy,
             executionState: issues.executionState,
+            monitorNextCheckAt: issues.monitorNextCheckAt,
         })
             .from(issues)
             .where(and(eq(issues.id, issueId), eq(issues.companyId, run.companyId)))
@@ -8099,6 +8190,32 @@ export function heartbeatService(db, options = {}) {
                 details: { issueId },
             };
         }
+        if (retryReason === ISSUE_DISPOSITION_REPAIR_RETRY_REASON) {
+            const expectedFingerprint = readNonEmptyString(contextSnapshot.dispositionRepairFingerprint);
+            const sourceState = await collectDispositionRepairSourceState(db, {
+                issue,
+                excludeRunId: run.id,
+                excludeWakeupRequestId: run.wakeupRequestId,
+            });
+            if (!expectedFingerprint ||
+                sourceState.fingerprint !== expectedFingerprint ||
+                sourceState.hasActiveExecutionPath ||
+                sourceState.hasDurableWaitingPath) {
+                return {
+                    allowed: false,
+                    reason: "Scheduled disposition repair suppressed because the source state changed or gained a durable path",
+                    errorCode: "issue_disposition_repair_superseded",
+                    issueId,
+                    details: {
+                        issueId,
+                        expectedFingerprint,
+                        currentFingerprint: sourceState.fingerprint,
+                        hasActiveExecutionPath: sourceState.hasActiveExecutionPath,
+                        durablePathReason: sourceState.durablePathReason,
+                    },
+                };
+            }
+        }
         if (issue.assigneeAgentId !== run.agentId) {
             if (!isNonAssigneeWorkspaceBusyRetry(retryReason, contextSnapshot)) {
                 return {
@@ -9196,7 +9313,7 @@ export function heartbeatService(db, options = {}) {
         return {
             enabled: asBoolean(heartbeat.enabled, false),
             intervalSec: Math.max(0, asNumber(heartbeat.intervalSec, 0)),
-            wakeOnDemand: asBoolean(heartbeat.wakeOnDemand ?? heartbeat.wakeOnAssignment ?? heartbeat.wakeOnOnDemand ?? heartbeat.wakeOnAutomation, true),
+            wakeOnDemand: isHeartbeatWakeOnDemandEnabled(agent),
             maxConcurrentRuns: normalizeMaxConcurrentRuns(heartbeat.maxConcurrentRuns),
             skipTimerWhenNoActionableWork: asBoolean(heartbeat.skipTimerWhenNoActionableWork ??
                 heartbeat.requireActionableTimerWork ??
@@ -9624,9 +9741,19 @@ export function heartbeatService(db, options = {}) {
         const reviewParticipant = reviewExecutionState?.currentParticipant ?? null;
         const isCurrentReviewParticipant = reviewParticipant?.type === "agent" &&
             reviewParticipant.agentId === run.agentId;
+        const recoveryActionId = readNonEmptyString(context.recoveryActionId);
+        const authorizedSourceScopedRecovery = wakeReason === "source_scoped_recovery_action" && recoveryActionId
+            ? await db
+                .select({ id: issueRecoveryActions.id })
+                .from(issueRecoveryActions)
+                .where(and(eq(issueRecoveryActions.id, recoveryActionId), eq(issueRecoveryActions.companyId, run.companyId), eq(issueRecoveryActions.sourceIssueId, issue.id), eq(issueRecoveryActions.ownerAgentId, run.agentId), inArray(issueRecoveryActions.status, ["active", "escalated"])))
+                .limit(1)
+                .then((rows) => Boolean(rows[0]))
+            : false;
         if (issue.assigneeAgentId !== run.agentId &&
             !isInteractionWake &&
             !isCurrentReviewParticipant &&
+            !authorizedSourceScopedRecovery &&
             !isNonAssigneeWorkspaceBusyRetry(retryReason, context)) {
             return {
                 stale: true,
@@ -10939,6 +11066,8 @@ export function heartbeatService(db, options = {}) {
                 issueExecutionWorkspaceId: requestedExecutionWorkspaceId,
                 issueExecutionWorkspacePreference: issueRef?.executionWorkspacePreference ?? null,
                 existingExecutionWorkspaceStatus: existingExecutionWorkspace?.status ?? null,
+                requestedExistingBranch: issueExecutionWorkspaceSettings?.workspaceStrategy?.existingBranch ?? null,
+                existingExecutionWorkspaceBranchName: existingExecutionWorkspace?.branchName ?? null,
             });
             const requestedShouldReuseExisting = workspaceReuseRequest.requestedShouldReuseExisting;
             const reusableExistingExecutionWorkspace = workspaceReuseRequest.existingExecutionWorkspaceAvailable
@@ -11462,6 +11591,13 @@ export function heartbeatService(db, options = {}) {
                         name: agent.name,
                         companyId: agent.companyId,
                     },
+                    recordedBranchOwnership: existingExecutionWorkspace?.status !== "archived"
+                        && existingExecutionWorkspace?.branchName
+                        ? {
+                            branchName: existingExecutionWorkspace.branchName,
+                            createdByRuntime: isRuntimeOwnedGitBranch(existingExecutionWorkspace.metadata),
+                        }
+                        : null,
                     heartbeatRunId: run.id,
                     enableWorkspaceBranchReconcileForward: resolvedInstanceSettings.experimental.enableWorkspaceBranchReconcileForward,
                     enableWorkspaceDirtyQuarantineRepair: resolvedInstanceSettings.experimental.enableWorkspaceDirtyQuarantineRepair,
@@ -11477,7 +11613,11 @@ export function heartbeatService(db, options = {}) {
                     ? reusableExistingExecutionWorkspace?.metadata ?? null
                     : null,
                 source: executionWorkspace.source,
-                createdByRuntime: executionWorkspace.created,
+                // Branch ownership, not worktree freshness: attaching a worktree to a
+                // pre-existing branch reports created=true but must never make terminal
+                // cleanup delete that operator-owned branch.
+                createdByRuntime: resolveExecutionWorkspaceBranchOwnership(executionWorkspace),
+                strategyType: executionWorkspace.strategy,
                 configSnapshot,
                 shouldReuseExisting: resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,
                 shouldRefreshConfigSnapshot: resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
@@ -11686,6 +11826,33 @@ export function heartbeatService(db, options = {}) {
                 lease: acquiredEnvironment.lease,
                 leaseContext: acquiredEnvironment.leaseContext,
             };
+            // The host duplex observability recorder for this run. It binds the fixed duplex
+            // observability surface to real sinks: the spans to the OTel tracer, the
+            // guarded counters to the tool-runtime metric store, and the transport event
+            // to the run-event path. Each sink runs guarded and fire-and-forget, so a
+            // telemetry failure never breaks the run. The orchestrator stamps it on the
+            // sandbox target; a non-duplex run keeps the safe no-op default in the bridge.
+            const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder({
+                tracer: getStartupTracer(),
+                incrementCounter: (metric) => {
+                    void incrementToolRuntimeMetricCounter(db, {
+                        companyId: run.companyId,
+                        metric,
+                    }).catch(() => { });
+                },
+                emitTransportEvent: (event) => {
+                    void (async () => {
+                        const eventRun = run;
+                        const seq = await nextRunEventSeq(eventRun.id);
+                        await appendRunEvent(eventRun, seq, {
+                            eventType: event.name,
+                            stream: "system",
+                            level: event.dimensions.outcome === "error" ? "warn" : "info",
+                            payload: { ...event.dimensions },
+                        });
+                    })().catch(() => { });
+                },
+            });
             const realizationResult = await envOrchestrator.realizeForRun({
                 environment: selectedEnvironment,
                 lease: activeEnvironmentLease.lease,
@@ -11696,6 +11863,7 @@ export function heartbeatService(db, options = {}) {
                 executionWorkspace,
                 effectiveExecutionWorkspaceMode,
                 persistedExecutionWorkspace,
+                duplexObservabilityRecorder,
             });
             activeEnvironmentLease = {
                 ...activeEnvironmentLease,
@@ -12508,6 +12676,7 @@ export function heartbeatService(db, options = {}) {
                         failures: referencedProjectStagingFailures.map((failure) => ({
                             projectId: failure.projectId,
                             reason: "staging",
+                            error: failure.error,
                         })),
                     });
                     logger.info({
@@ -12967,9 +13136,15 @@ export function heartbeatService(db, options = {}) {
                 // recovery path routes it to a human owner instead of looping retries.
                 const workspaceValidationSetupFailure = isWorkspaceValidationFailure(outerErr) ? outerErr : null;
                 const configurationIncompleteSetupFailure = isConfigurationIncompleteFailure(outerErr) ? outerErr : null;
+                // A remote-only base ref that never resolved is a known pre-dispatch
+                // configuration gap, not an opaque setup crash. Map it to the same
+                // configuration-incomplete code so the recovery path routes it to a
+                // human owner and bounds the repeat by its per-ref fingerprint.
+                const unresolvedBaseRefSetupFailure = isUnresolvedWorkspaceBaseRefError(outerErr) ? outerErr : null;
                 const recordedResponsibleUserDenialCode = normalizeResponsibleUserDenialCode((await getRun(runId).catch(() => null))?.errorCode);
                 const setupFailureErrorCode = workspaceValidationSetupFailure?.code ??
                     configurationIncompleteSetupFailure?.code ??
+                    (unresolvedBaseRefSetupFailure ? CONFIGURATION_INCOMPLETE_FAILURE_CODE : null) ??
                     recordedResponsibleUserDenialCode ??
                     "setup_failed";
                 logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
@@ -12982,7 +13157,11 @@ export function heartbeatService(db, options = {}) {
                         resultJson: mergeRunStopMetadataForAgent(setupFailureAgent, "failed", {
                             errorCode: setupFailureErrorCode,
                             errorMessage: message,
-                            resultJson: workspaceValidationSetupFailure?.resultJson ?? configurationIncompleteSetupFailure?.resultJson ?? null,
+                            resultJson: workspaceValidationSetupFailure?.resultJson ??
+                                configurationIncompleteSetupFailure?.resultJson ??
+                                (unresolvedBaseRefSetupFailure
+                                    ? buildUnresolvedWorkspaceBaseRefResultJson(run, unresolvedBaseRefSetupFailure)
+                                    : null),
                         }),
                     } : {}),
                 }).catch(() => ({ run: null, updated: false }));
@@ -13502,7 +13681,6 @@ export function heartbeatService(db, options = {}) {
                         previousStatus: issue.status,
                         notice: buildExecutionReviewParticipantRecoveryNoticeSeed(),
                         recoveryCause: EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE,
-                        recoveryOwnerAgentId: currentParticipant.agentId,
                     };
                 }
                 const now = new Date();
@@ -13579,6 +13757,10 @@ export function heartbeatService(db, options = {}) {
                 !issue.assigneeUserId &&
                 issue.assigneeAgentId === run.agentId &&
                 (run.status === "failed" || run.status === "timed_out" || run.status === "cancelled");
+            if (readNonEmptyString(parseObject(run.contextSnapshot).retryReason) ===
+                ISSUE_DISPOSITION_REPAIR_RETRY_REASON) {
+                return { kind: "released" };
+            }
             if (!issueNeedsImmediateRecovery) {
                 return { kind: "released" };
             }
@@ -13729,7 +13911,6 @@ export function heartbeatService(db, options = {}) {
                         : promotionResult.recoveryCause === EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE
                             ? EXECUTION_REVIEW_PARTICIPANT_RECOVERY_CAUSE
                             : undefined,
-                recoveryOwnerAgentId: promotionResult.recoveryOwnerAgentId,
             });
             return;
         }