@rudderhq/server 0.7.11 → 0.7.13

package/dist/services/runtime-kernel/heartbeat.d.ts

@@ -7,6 +7,14 @@ import type { WakeupOptions } from "./heartbeat.core.js";
 export declare function heartbeatService(db: Db, testHooks?: {
     afterIssuePromotionCommitted?: (outcome: unknown) => Promise<void> | void;
     beforeRunClaim?: (run: typeof heartbeatRuns.$inferSelect) => Promise<void> | void;
+    /** Server-owned runners (currently Chat) may reattach a claimed wait run. */
+    onNetworkWaitingRun?: (run: typeof heartbeatRuns.$inferSelect & {
+        networkRecoveryExhausted?: boolean;
+        networkRecoveryFailure?: {
+            errorCode: "network_resume_unsafe" | "network_retry_exhausted";
+            error: string;
+        };
+    }) => Promise<boolean> | boolean;
     beforeRunExecutionLeaseRenewal?: (input: {
         runId: string;
         ownerToken: string;
@@ -19,6 +27,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
     list: (orgId: string, agentId?: string, limit?: number, filters?: {
         startDate?: Date;
         endDate?: Date;
+        goalId?: string;
     }) => Promise<HeartbeatRun[]>;
     getAgentSkillAnalytics: (agentId: string, opts?: {
         windowDays?: number;
@@ -42,6 +51,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
         invocationSource: string;
         triggerDetail: string | null;
         status: string;
+        runningSubstate: "executing" | "waiting_for_network" | null;
         startedAt: Date | null;
         finishedAt: Date | null;
         error: string | null;
@@ -66,9 +76,15 @@ export declare function heartbeatService(db: Db, testHooks?: {
         errorCode: string | null;
         externalRunId: string | null;
         chatConversationId: string | null;
+        goalId: string | null;
         processPid: number | null;
         processStartedAt: Date | null;
         processExitedAt: Date | null;
+        networkWaitStartedAt: Date | null;
+        networkWaitNextRetryAt: Date | null;
+        networkWaitAttemptCount: number;
+        networkWaitDurationMs: number;
+        recoveryCheckpoint: Record<string, unknown> | null;
         executionOwnerToken: string | null;
         executionLeaseExpiresAt: Date | null;
         terminalEffectsPending: boolean;
@@ -636,6 +652,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
         invocationSource: string;
         triggerDetail: string | null;
         status: string;
+        runningSubstate: "executing" | "waiting_for_network" | null;
         startedAt: Date | null;
         finishedAt: Date | null;
         error: string | null;
@@ -660,9 +677,15 @@ export declare function heartbeatService(db: Db, testHooks?: {
         errorCode: string | null;
         externalRunId: string | null;
         chatConversationId: string | null;
+        goalId: string | null;
         processPid: number | null;
         processStartedAt: Date | null;
         processExitedAt: Date | null;
+        networkWaitStartedAt: Date | null;
+        networkWaitNextRetryAt: Date | null;
+        networkWaitAttemptCount: number;
+        networkWaitDurationMs: number;
+        recoveryCheckpoint: Record<string, unknown> | null;
         executionOwnerToken: string | null;
         executionLeaseExpiresAt: Date | null;
         terminalEffectsPending: boolean;
@@ -706,6 +729,12 @@ export declare function heartbeatService(db: Db, testHooks?: {
         timedOut: number;
         runIds: string[];
     }>;
+    recoverNetworkWaitingRuns: (opts?: {
+        now?: Date;
+    }) => Promise<{
+        resumed: number;
+        runIds: string[];
+    }>;
     startNextQueuedRunForAgent: (agentId: string) => Promise<{
         status: string;
         agentId: string;
@@ -714,6 +743,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
         startedAt: Date | null;
         updatedAt: Date;
         wakeupRequestId: string | null;
+        goalId: string | null;
         contextSnapshot: Record<string, unknown> | null;
         orgId: string;
         createdAt: Date;
@@ -721,6 +751,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
         chatConversationId: string | null;
         finishedAt: Date | null;
         invocationSource: string;
+        runningSubstate: "executing" | "waiting_for_network" | null;
         exitCode: number | null;
         signal: string | null;
         usageJson: Record<string, unknown> | null;
@@ -743,6 +774,11 @@ export declare function heartbeatService(db: Db, testHooks?: {
         processPid: number | null;
         processStartedAt: Date | null;
         processExitedAt: Date | null;
+        networkWaitStartedAt: Date | null;
+        networkWaitNextRetryAt: Date | null;
+        networkWaitAttemptCount: number;
+        networkWaitDurationMs: number;
+        recoveryCheckpoint: Record<string, unknown> | null;
         executionOwnerToken: string | null;
         executionLeaseExpiresAt: Date | null;
         terminalEffectsPending: boolean;
@@ -784,6 +820,7 @@ export declare function heartbeatService(db: Db, testHooks?: {
         invocationSource: string;
         triggerDetail: string | null;
         status: string;
+        runningSubstate: "executing" | "waiting_for_network" | null;
         startedAt: Date | null;
         finishedAt: Date | null;
         error: string | null;
@@ -808,9 +845,15 @@ export declare function heartbeatService(db: Db, testHooks?: {
         errorCode: string | null;
         externalRunId: string | null;
         chatConversationId: string | null;
+        goalId: string | null;
         processPid: number | null;
         processStartedAt: Date | null;
         processExitedAt: Date | null;
+        networkWaitStartedAt: Date | null;
+        networkWaitNextRetryAt: Date | null;
+        networkWaitAttemptCount: number;
+        networkWaitDurationMs: number;
+        recoveryCheckpoint: Record<string, unknown> | null;
         executionOwnerToken: string | null;
         executionLeaseExpiresAt: Date | null;
         terminalEffectsPending: boolean;

package/dist/services/runtime-kernel/heartbeat.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"heartbeat.d.ts","sourceRoot":"","sources":["../../../src/services/runtime-kernel/heartbeat.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAQL,aAAa,EAGd,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAwB1E,OAAO,EAAiB,KAAK,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAa3E,OAAO,EAAE,0CAA0C,EAAE,KAAK,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAEnH,OAAO,KAAK,EAA0C,aAAa,EAAE,MAAM,qBAAqB,CAAC;AA2GjG,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,EAAE,EACN,SAAS,CAAC,EAAE;IACV,4BAA4B,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1E,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,aAAa,CAAC,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAClF,8BAA8B,CAAC,EAAE,CAAC,KAAK,EAAE;QACvC,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;KACpB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5B;sBAwiEyB,MAAM;;;;kBAiCrB,MAAM,YACH,MAAM,UACR,MAAM,YACL;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,OAAO,CAAC,EAAE,IAAI,CAAA;KAAE;sCA4EpC,MAAM,SACR;QACL,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,GAAG,CAAC,EAAE,IAAI,CAAC;QACX,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAChC,KACA,OAAO,CAAC,mBAAmB,CAAC;2CAOtB,MAAM,SACN;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,KAC/E,OAAO,CAAC,mBAAmB,CAAC;oBAhpEJ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+BA6pEA,MAAM;;;;;;;;;;;;;;;;;;gCAmBL,MAAM;;;;;;;;;;;;;mCAWH,MAAM,SAAS;QAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;;;;;;;;;;;;;;;;;;;wBAmC3D,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAOH,MAAM,SAAS;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;;;;;;;;;sBA+BnE,MAAM,WACP,OAAO,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,oBACrD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,kBACzB,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,UAChD;QAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;;;;;;;4BAznEvE,CAAC;0BAEO,CAAC;WAA0B,CAAC;;+BAyjBC,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BAw1Bb;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;8BAIhE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;8BAI/D;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;0CAuThD,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yCAwGR;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,OAAO,CAAA;KAAE;;;;;;;;;;;;uBA8X1E,MAAM;oCAEO,MAAM;;oCAIA,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkB/C"}
+{"version":3,"file":"heartbeat.d.ts","sourceRoot":"","sources":["../../../src/services/runtime-kernel/heartbeat.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAQL,aAAa,EAGd,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAwB1E,OAAO,EAAiB,KAAK,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAc3E,OAAO,EAAE,0CAA0C,EAAE,KAAK,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAEnH,OAAO,KAAK,EAA0C,aAAa,EAAE,MAAM,qBAAqB,CAAC;AA2GjG,wBAAgB,gBAAgB,CAC9B,EAAE,EAAE,EAAE,EACN,SAAS,CAAC,EAAE;IACV,4BAA4B,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1E,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,aAAa,CAAC,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAClF,6EAA6E;IAC7E,mBAAmB,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,aAAa,CAAC,YAAY,GAAG;QAC9D,wBAAwB,CAAC,EAAE,OAAO,CAAC;QACnC,sBAAsB,CAAC,EAAE;YACvB,SAAS,EAAE,uBAAuB,GAAG,yBAAyB,CAAC;YAC/D,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;KACH,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACjC,8BAA8B,CAAC,EAAE,CAAC,KAAK,EAAE;QACvC,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;KACpB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5B;sBAouEyB,MAAM;;;;kBAiCrB,MAAM,YACH,MAAM,UACR,MAAM,YACL;QAAE,SAAS,CAAC,EAAE,IAAI,CAAC;QAAC,OAAO,CAAC,EAAE,IAAI,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;sCA6ErD,MAAM,SACR;QACL,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,GAAG,CAAC,EAAE,IAAI,CAAC;QACX,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAChC,KACA,OAAO,CAAC,mBAAmB,CAAC;2CAOtB,MAAM,SACN;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,KAC/E,OAAO,CAAC,mBAAmB,CAAC;oBA70EJ,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+BA01EA,MAAM;;;;;;;;;;;;;;;;;;gCAmBL,MAAM;;;;;;;;;;;;;mCAWH,MAAM,SAAS;QAAE,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;;;;;;;;;;;;;;;;;;;wBAmC3D,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAOH,MAAM,SAAS;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE;;;;;;;;;sBA+BnE,MAAM,WACP,OAAO,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,oBACrD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,kBACzB,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,QAAQ,UAChD;QAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;;;;;;;4BAz0E1E,CAAP;0BACqB,CAAC;WACf,CAAJ;;+BAylB4C,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BAq2Bb;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;8BAIhE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;8BAI/D;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,IAAI,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE;;;;uCA6jBnD;QAAE,GAAG,CAAC,EAAE,IAAI,CAAA;KAAE;;;;0CAtQX,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yCA0QR;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,OAAO,CAAA;KAAE;;;;;;;;;;;;uBAiY1E,MAAM;oCAEO,MAAM;;oCAIA,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkB/C"}

package/dist/services/runtime-kernel/heartbeat.execute.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"heartbeat.execute.d.ts","sourceRoot":"","sources":["../../../src/services/runtime-kernel/heartbeat.execute.ts"],"names":[],"mappings":"AAmEA,OAAO,EAAE,0CAA0C,EAAE,KAAK,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAsCnH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,GAAG;wBA2CxB,MAAM,SAAS;QAAE,iBAAiB,CAAC,EAAE,OAAO,CAAA;KAAE;EAi8ChF"}
+{"version":3,"file":"heartbeat.execute.d.ts","sourceRoot":"","sources":["../../../src/services/runtime-kernel/heartbeat.execute.ts"],"names":[],"mappings":"AA6EA,OAAO,EAAE,0CAA0C,EAAE,KAAK,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAsCnH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,GAAG;wBA2CxB,MAAM,SAAS;QAAE,iBAAiB,CAAC,EAAE,OAAO,CAAA;KAAE;EAypDhF"}

package/dist/services/runtime-kernel/heartbeat.execute.js

@@ -1,3 +1,13 @@
+// @ts-nocheck
+/**
+ * @fileoverview Executes claimed heartbeat runs through runtime adapters,
+ * workspace realization, transcript persistence, and close-out release.
+ *
+ * @see doc/product/domains/execution/agent-runs.md - run execution state and evidence
+ * @see doc/product/domains/execution/run-admission-and-recovery.md - retry and recovery behavior
+ * @see doc/product/domains/agents/instruction-loading.md - AGENT.INSTRUCTIONS.001 runtime instruction frame
+ */
+import { buildModelAttemptSpecs, isAgentRuntimeNetworkSuspension, } from "@rudderhq/agent-runtime-utils";
 import { goals, heartbeatRuns, issues, projects } from "@rudderhq/db";
 import { resolveAgentRunScene, } from "@rudderhq/shared";
 import { and, eq, sql } from "drizzle-orm";
@@ -14,11 +24,12 @@ import { isManagedWorkspaceConfigurationError, isWorkspacePermissionPreflightErr
 import { listProjectResourceAttachments } from "../resource-catalog.js";
 import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun } from "../workspace-runtime.js";
 import { AssignmentDependencyPreflightError, createAssignmentRunFailureBudget, inspectAssignmentRunWorkspace, repairAssignmentRunWorkspace, resolveProjectWorkingSetCwd, } from "./assignment-run-guardrail.js";
+import { beginHeartbeatRunAttempt, finishHeartbeatRunAttempt, markHeartbeatRunAttemptWaiting, } from "./heartbeat-attempt-ledger.js";
 import { executeAdapterWithModelFallbacks } from "./model-fallback.js";
 export { prioritizeProjectWorkspaceCandidatesForRun } from "../agent-run-context.js";
 import * as heartbeatCore from "./heartbeat.core.js";
 import * as heartbeatSessions from "./heartbeat.sessions.js";
-const { MAX_LIVE_LOG_CHUNK_BYTES, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, DEFERRED_WAKE_CONTEXT_KEY, DETACHED_PROCESS_ERROR_CODE, ORPHANED_PROCESS_TERMINATION_GRACE_MS, ORPHANED_PROCESS_KILL_WAIT_MS, ORPHANED_PROCESS_POLL_INTERVAL_MS, startLocksByAgent, MAX_RECOVERY_CHAIN_DEPTH, ISSUE_PASSIVE_FOLLOWUP_REASON, ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE, ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS, ISSUE_REVIEW_CLOSEOUT_REASON, ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS, ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS, SESSIONED_LOCAL_ADAPTERS, heartbeatRunListColumns, appendExcerpt, appendTranscriptEntriesFromChunk, normalizeMaxConcurrentRuns, withAgentStartLock, readNonEmptyString, buildHeartbeatAdapterInvokePayload, sanitizeStartupContextContextForPersistence, sanitizeStartupContextPromptForPersistence, buildRecentDateKeys, buildDateKeysBetween, fallbackSkillLabel, normalizeLoadedSkill, normalizeLoadedSkillForPayload, emptySkillEvidenceCounts, incrementSkillEvidenceCount, strongestSkillEvidence, resolveSkillEvidence, readSkillEvidenceFromPayload, extractSkillSlugFromPath, collectSkillPathsFromText, collectStringValues, normalizeSkillUseFromPath, dedupeSkillUses, collectSkillUsesFromText, readToolCommandInput, isCommandTranscriptTool, isReadTranscriptTool, inferUsedSkillsFromTranscript, normalizeSkillCandidate, addSkillCandidate, readSkillReferenceSlug, collectSkillReferences, inferUsedSkillsFromPrompt, resolveForbiddenRuntimeSkillMarkers, detectForbiddenRuntimeSkillMarker, normalizeLedgerBillingType, resolveLedgerBiller, normalizeBilledCostCents, resolveLedgerScopeForRun } = heartbeatCore;
+const { MAX_LIVE_LOG_CHUNK_BYTES, HEARTBEAT_MAX_CONCURRENT_RUNS_DEFAULT, HEARTBEAT_MAX_CONCURRENT_RUNS_MIN, HEARTBEAT_MAX_CONCURRENT_RUNS_MAX, DEFERRED_WAKE_CONTEXT_KEY, DETACHED_PROCESS_ERROR_CODE, ORPHANED_PROCESS_TERMINATION_GRACE_MS, ORPHANED_PROCESS_KILL_WAIT_MS, ORPHANED_PROCESS_POLL_INTERVAL_MS, startLocksByAgent, MAX_RECOVERY_CHAIN_DEPTH, ISSUE_PASSIVE_FOLLOWUP_REASON, ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE, ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON, ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS, ISSUE_REVIEW_CLOSEOUT_REASON, ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS, ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT, ISSUE_PASSIVE_FOLLOWUP_TIMER_CONTINUITY_MAX_WINDOW_MS, networkWaitBackoffMs, SESSIONED_LOCAL_ADAPTERS, heartbeatRunListColumns, appendExcerpt, appendTranscriptEntriesFromChunk, normalizeMaxConcurrentRuns, withAgentStartLock, readNonEmptyString, buildHeartbeatAdapterInvokePayload, sanitizeStartupContextContextForPersistence, sanitizeStartupContextPromptForPersistence, buildRecentDateKeys, buildDateKeysBetween, fallbackSkillLabel, normalizeLoadedSkill, normalizeLoadedSkillForPayload, emptySkillEvidenceCounts, incrementSkillEvidenceCount, strongestSkillEvidence, resolveSkillEvidence, readSkillEvidenceFromPayload, extractSkillSlugFromPath, collectSkillPathsFromText, collectStringValues, normalizeSkillUseFromPath, dedupeSkillUses, collectSkillUsesFromText, readToolCommandInput, isCommandTranscriptTool, isReadTranscriptTool, inferUsedSkillsFromTranscript, normalizeSkillCandidate, addSkillCandidate, readSkillReferenceSlug, collectSkillReferences, inferUsedSkillsFromPrompt, resolveForbiddenRuntimeSkillMarkers, detectForbiddenRuntimeSkillMarker, normalizeLedgerBillingType, resolveLedgerBiller, normalizeBilledCostCents, resolveLedgerScopeForRun } = heartbeatCore;
 const { buildExplicitResumeSessionOverride, selectRunSessionLineage, normalizeUsageTotals, readRawUsageTotals, deriveNormalizedUsageDelta, formatCount, parseSessionCompactionPolicy, resolveRuntimeSessionParamsForWorkspace, parseIssueAssigneeAgentRuntimeOverrides, deriveTaskKey, shouldResetTaskSessionForWake, formatRuntimeWorkspaceWarningLog, describeSessionResetReason, deriveCommentId, enrichWakeContextSnapshot, mergeCoalescedContextSnapshot, issueCommentAuthorKind, issueCommentAuthorLabel, buildDeferredWakePayload, readDeferredWakeContext, readDeferredWakePayload, deriveDeferredWakeTaskKey, hydrateWakeContextSnapshot, firstNonEmptyLine, deriveRecoveryFailureKind, deriveRecoveryFailureSummary, mergeMissingRecoveryContextFields, hydrateRecoveryBaseContextSnapshot, buildRecoveryContextSnapshot, normalizePassiveFollowupContext, normalizeReviewCloseoutContext, passiveFollowupCooldownMs, issueHasReviewer, isAgentEligibleForTimerContinuation, hasCredibleTimerContinuation, buildPassiveFollowupContextSnapshot, runTaskKey, isSameTaskScope, isTrackedLocalChildProcessAdapter, isProcessAlive, waitForProcessExit, terminateOrphanedProcess, truncateDisplayId, normalizeAgentNameKey, defaultSessionCodec, getAgentRuntimeSessionCodec, normalizeSessionParams, resolveNextSessionState } = heartbeatSessions;
 function buildPersistableHeartbeatContext(context) {
     return sanitizeStartupContextContextForPersistence(context) ?? {};
@@ -123,6 +134,20 @@ export function createHeartbeatExecuteHandlers(context) {
         }
         const executionAbortController = runAbortControllers.get(run.id) ?? new AbortController();
         runAbortControllers.set(run.id, executionAbortController);
+        let activeAttemptRef = null;
+        let networkSuspended = false;
+        const finishActiveAttempt = async (input) => {
+            const ref = activeAttemptRef;
+            activeAttemptRef = null;
+            if (!ref)
+                return;
+            try {
+                await finishHeartbeatRunAttempt(db, ref, input);
+            }
+            catch (error) {
+                logger.warn({ err: error, runId: runId, attemptIndex: ref.attemptIndex }, "failed to persist heartbeat attempt terminal state");
+            }
+        };
         try {
             if (run.status === "queued") {
                 const claimed = await claimQueuedRun(run);
@@ -178,6 +203,7 @@ export function createHeartbeatExecuteHandlers(context) {
             let finalRunOutput = null;
             let ownsTerminalState = false;
             let shouldCompleteTerminalEffects = false;
+            let activeAttemptSpec = null;
             const finalizeExecutionTranscript = () => {
                 stdoutTranscriptBuffer = appendTranscriptEntriesFromChunk({
                     buffer: stdoutTranscriptBuffer,
@@ -592,6 +618,27 @@ export function createHeartbeatExecuteHandlers(context) {
                 sessionDisplayId: previousSessionDisplayId,
                 taskKey,
             };
+            const attemptStride = Math.max(1, buildModelAttemptSpecs(runtimeConfig, agent.agentRuntimeType).length);
+            const recoveryAttemptOrdinal = Math.max(0, Math.floor(Number(run.networkWaitAttemptCount) || 0));
+            const attemptResumeSource = recoveryAttemptOrdinal === 0
+                ? "fresh"
+                : run.recoveryCheckpoint?.continuation === "resume_same_session"
+                    ? "same_session"
+                    : "pristine_replay";
+            const resolveLedgerAttemptIndex = (attempt) => recoveryAttemptOrdinal * attemptStride + attempt.index;
+            const recoveryStartAttemptIndex = recoveryAttemptOrdinal > 0
+                && typeof run.recoveryCheckpoint?.fallbackIndex === "number"
+                ? Math.max(0, Math.floor(run.recoveryCheckpoint.fallbackIndex))
+                : 0;
+            const persistAttempt = async (label, operation) => {
+                try {
+                    return await operation();
+                }
+                catch (error) {
+                    logger.warn({ err: error, runId: run.id, label }, "failed to persist heartbeat attempt ledger state");
+                    return null;
+                }
+            };
             let handle = null;
             let stdoutExcerpt = "";
             let stderrExcerpt = "";
@@ -631,6 +678,7 @@ export function createHeartbeatExecuteHandlers(context) {
                 const startedAt = run.startedAt ?? new Date();
                 const runningWithSession = await persistRunningExecutionContext(run.id, context, {
                     startedAt,
+                    runningSubstate: "executing",
                     sessionIdBefore: runtimeForAdapter.sessionDisplayId ?? runtimeForAdapter.sessionId,
                     sessionParamsBeforeJson: runtimeForAdapter.sessionParams,
                     sessionReuseScope,
@@ -730,6 +778,7 @@ export function createHeartbeatExecuteHandlers(context) {
                     orgId: run.orgId,
                     agentId: run.agentId,
                     runId,
+                    append: Boolean(run.logRef),
                 });
                 await db
                     .update(heartbeatRuns)
@@ -946,11 +995,43 @@ export function createHeartbeatExecuteHandlers(context) {
                     requestApproval,
                     waitForApproval,
                 }, {
+                    startAttemptIndex: recoveryStartAttemptIndex,
                     resolveAdapter: findServerAdapter,
                     createAuthToken: (agentRuntimeType) => createLocalAgentJwt(agent.id, agent.orgId, agentRuntimeType, run.id) ?? undefined,
-                    onAttemptStart: (_attempt, attemptAdapter) => {
+                    onAttemptStart: async (attempt, attemptAdapter) => {
+                        activeAttemptSpec = {
+                            index: attempt.index,
+                            fallbackIndex: attempt.fallbackIndex,
+                        };
+                        activeAttemptRef = await persistAttempt("started", () => beginHeartbeatRunAttempt(db, {
+                            orgId: run.orgId,
+                            runId: run.id,
+                            agentId: run.agentId,
+                            attemptIndex: resolveLedgerAttemptIndex(attempt),
+                            fallbackIndex: attempt.fallbackIndex,
+                            runtimeType: attempt.agentRuntimeType ?? agent.agentRuntimeType,
+                            model: attempt.model,
+                            isFallback: attempt.isFallback,
+                            resumeSource: attemptResumeSource,
+                        }));
                         stdoutTranscriptParser = attemptAdapter.parseStdoutLine ?? null;
                     },
+                    onAttemptFailure: async (_attempt, failure) => {
+                        const failureRecord = failure && typeof failure === "object" ? failure : null;
+                        const failureMessage = failure instanceof Error
+                            ? failure.message
+                            : readNonEmptyString(failureRecord?.errorMessage) ?? "Adapter fallback attempt failed";
+                        await finishActiveAttempt({
+                            status: "failed",
+                            errorCode: readNonEmptyString(failureRecord?.errorCode) ?? "adapter_failed",
+                            error: failureMessage,
+                            usageDeltaJson: failureRecord?.usage,
+                            costUsd: failureRecord?.costUsd,
+                            sessionDisplayId: readNonEmptyString(failureRecord?.sessionDisplayId)
+                                ?? readNonEmptyString(failureRecord?.sessionId),
+                            sessionParamsJson: failureRecord?.sessionParams,
+                        });
+                    },
                 });
                 if (assignmentGuardrailCheckpoint) {
                     adapterResult.errorMessage = "Assignment run stopped after reaching its tool failure budget";
@@ -997,6 +1078,123 @@ export function createHeartbeatExecuteHandlers(context) {
                         }
                     }
                 }
+                const networkSuspension = isAgentRuntimeNetworkSuspension(adapterResult.networkSuspension)
+                    ? adapterResult.networkSuspension
+                    : isAgentRuntimeNetworkSuspension(adapterResult.suspension)
+                        ? adapterResult.suspension
+                        : null;
+                if (networkSuspension) {
+                    // A transport suspension is a non-terminal transition. Persist the
+                    // provider checkpoint before releasing the local execution lease so a
+                    // restart or another scheduler tick can safely claim this same Run.
+                    const now = new Date();
+                    const currentRunForWait = await getRun(run.id);
+                    const waitAttempt = (currentRunForWait?.networkWaitAttemptCount ?? 0) + 1;
+                    const backoff = networkWaitBackoffMs(waitAttempt);
+                    const jitteredBackoff = Math.max(1_000, Math.round(backoff * (0.9 + Math.random() * 0.2)));
+                    const nextRetryAt = new Date(now.getTime() + jitteredBackoff);
+                    const checkpoint = {
+                        kind: networkSuspension.kind,
+                        code: networkSuspension.code ?? "provider_transport_unavailable",
+                        transport: networkSuspension.transport,
+                        provider: networkSuspension.provider ?? adapterResult.provider ?? null,
+                        model: networkSuspension.model ?? adapterResult.model ?? null,
+                        submissionPhase: networkSuspension.submissionPhase,
+                        continuation: networkSuspension.continuation,
+                        sessionId: networkSuspension.sessionId ?? adapterResult.sessionDisplayId ?? adapterResult.sessionId ?? null,
+                        sessionParams: networkSuspension.sessionParams ?? adapterResult.sessionParams ?? null,
+                        progress: networkSuspension.progress ?? {
+                            modelOutputObserved: networkSuspension.modelOutputObserved,
+                            toolActivityObserved: networkSuspension.toolActivityObserved,
+                        },
+                        modelOutputObserved: networkSuspension.modelOutputObserved
+                            ?? networkSuspension.progress?.modelOutputObserved
+                            ?? false,
+                        toolActivityObserved: networkSuspension.toolActivityObserved
+                            ?? networkSuspension.progress?.toolActivityObserved
+                            ?? false,
+                        sideEffectRisk: networkSuspension.sideEffectRisk ?? null,
+                        attemptIndex: activeAttemptSpec?.index ?? 0,
+                        fallbackIndex: activeAttemptSpec?.fallbackIndex ?? null,
+                        message: networkSuspension.message,
+                        observedAt: now.toISOString(),
+                    };
+                    const resumedSessionId = readNonEmptyString(networkSuspension.sessionId)
+                        ?? readNonEmptyString(adapterResult.sessionDisplayId)
+                        ?? readNonEmptyString(adapterResult.sessionId)
+                        ?? previousSessionDisplayId
+                        ?? null;
+                    const resumedSessionParams = networkSuspension.sessionParams
+                        ?? adapterResult.sessionParams
+                        ?? runtimeForAdapter.sessionParams
+                        ?? null;
+                    const waitingRun = await db
+                        .update(heartbeatRuns)
+                        .set({
+                        runningSubstate: "waiting_for_network",
+                        networkWaitStartedAt: currentRunForWait?.networkWaitStartedAt ?? now,
+                        networkWaitNextRetryAt: nextRetryAt,
+                        networkWaitAttemptCount: waitAttempt,
+                        recoveryCheckpoint: checkpoint,
+                        sessionIdBefore: resumedSessionId,
+                        sessionParamsBeforeJson: resumedSessionParams,
+                        sessionReuseScope: resumedSessionParams || resumedSessionId ? "explicit" : "none",
+                        processExitedAt: now,
+                        processPid: null,
+                        processStartedAt: null,
+                        executionOwnerToken: null,
+                        executionLeaseExpiresAt: null,
+                        updatedAt: now,
+                    })
+                        .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ...(executionOwnerToken ? [eq(heartbeatRuns.executionOwnerToken, executionOwnerToken)] : [])))
+                        .returning()
+                        .then((rows) => rows[0] ?? null);
+                    if (!waitingRun)
+                        return;
+                    networkSuspended = true;
+                    const waitingAttemptRef = activeAttemptRef;
+                    activeAttemptRef = null;
+                    await persistAttempt("waiting_for_network", () => markHeartbeatRunAttemptWaiting(db, waitingAttemptRef, {
+                        submissionPhase: networkSuspension.submissionPhase,
+                        providerThreadId: networkSuspension.providerThreadId ?? adapterResult.providerThreadId,
+                        providerTurnId: networkSuspension.providerTurnId ?? adapterResult.providerTurnId,
+                        sessionDisplayId: resumedSessionId,
+                        sessionParamsJson: resumedSessionParams,
+                        checkpointJson: checkpoint,
+                        errorCode: networkSuspension.code ?? "provider_transport_unavailable",
+                        error: networkSuspension.message ?? null,
+                        suspendedAt: now,
+                    }));
+                    await appendRunEvent(waitingRun, {
+                        eventType: "network.waiting",
+                        stream: "system",
+                        level: "info",
+                        message: "run waiting for network",
+                        payload: {
+                            attempt: waitAttempt,
+                            nextRetryAt: nextRetryAt.toISOString(),
+                            backoffMs: jitteredBackoff,
+                            suspension: checkpoint,
+                        },
+                    });
+                    publishLiveEvent({
+                        orgId: waitingRun.orgId,
+                        type: "heartbeat.run.status",
+                        payload: {
+                            runId: waitingRun.id,
+                            agentId: waitingRun.agentId,
+                            status: waitingRun.status,
+                            executionPhase: "waiting_for_network",
+                            networkWaitAttemptCount: waitAttempt,
+                            networkWaitNextRetryAt: nextRetryAt.toISOString(),
+                        },
+                    });
+                    if (handle) {
+                        await runLogStore.finalize(handle).catch(() => undefined);
+                        handle = null;
+                    }
+                    return;
+                }
                 const nextSessionState = resolveNextSessionState({
                     codec: sessionCodec,
                     adapterResult,
@@ -1163,6 +1361,19 @@ export function createHeartbeatExecuteHandlers(context) {
                     expectedExecutionOwnerToken: executionOwnerToken,
                 });
                 ownsTerminalState = Boolean(claimedTerminalRun);
+                await finishActiveAttempt({
+                    status,
+                    submissionPhase: adapterResult.submissionPhase,
+                    providerThreadId: adapterResult.providerThreadId,
+                    providerTurnId: adapterResult.providerTurnId,
+                    sessionDisplayId: nextSessionState.displayId ?? nextSessionState.legacySessionId,
+                    sessionParamsJson: adapterResult.clearSession ? {} : nextSessionState.params,
+                    usageDeltaJson: usageJson ?? adapterResult.usage,
+                    costUsd: adapterResult.costUsd,
+                    errorCode: terminalEvidence.errorCode,
+                    error: terminalEvidence.error,
+                    finishedAt: terminalEvidence.finishedAt,
+                });
                 if (!claimedTerminalRun) {
                     await reconcileRunEvidence(run.id, terminalEvidence);
                     await reconcileTerminalEffectsIntent(run.id, {
@@ -1361,6 +1572,14 @@ export function createHeartbeatExecuteHandlers(context) {
                     expectedExecutionOwnerToken: executionOwnerToken,
                 });
                 ownsTerminalState = Boolean(claimedFailedRun);
+                await finishActiveAttempt({
+                    status: "failed",
+                    errorCode: failureEvidence.errorCode,
+                    error: failureEvidence.error,
+                    sessionDisplayId: previousSessionDisplayId,
+                    sessionParamsJson: previousSessionParams,
+                    finishedAt: failureEvidence.finishedAt,
+                });
                 if (!claimedFailedRun)
                     await reconcileRunEvidence(run.id, failureEvidence);
                 const failedRun = claimedFailedRun ?? await getRun(run.id);
@@ -1421,6 +1640,12 @@ export function createHeartbeatExecuteHandlers(context) {
             // The inner catch did not fire, so we must record the failure here.
             const message = outerErr instanceof Error ? outerErr.message : "Unknown setup failure";
             logger.error({ err: outerErr, runId }, "heartbeat execution setup failed");
+            await finishActiveAttempt({
+                status: "failed",
+                errorCode: "adapter_failed",
+                error: message,
+                finishedAt: new Date(),
+            });
             const latestRun = await getRun(runId).catch(() => null);
             const claimedFailedRun = latestRun?.status === "running"
                 ? await transitionRunToTerminal(runId, "failed", {
@@ -1458,7 +1683,8 @@ export function createHeartbeatExecuteHandlers(context) {
             await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
             runAbortControllers.delete(run.id);
             activeRunExecutions.delete(run.id);
-            await startNextQueuedRunForAgent(run.agentId);
+            if (!networkSuspended)
+                await startNextQueuedRunForAgent(run.agentId);
         }
     }
     return { executeRun };