agent-relay-server 0.121.5 → 0.122.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/openapi.json +1 -1
- package/package.json +6 -6
- package/public/assets/{MessageBubble-CLqxCG9b.js → MessageBubble-DRdUD-kW.js} +2 -2
- package/public/assets/{MessageBubble-CLqxCG9b.js.map → MessageBubble-DRdUD-kW.js.map} +1 -1
- package/public/assets/{PlanGraphPanel-BkkToFEb.js → PlanGraphPanel-siKMPCln.js} +2 -2
- package/public/assets/{PlanGraphPanel-BkkToFEb.js.map → PlanGraphPanel-siKMPCln.js.map} +1 -1
- package/public/assets/{activity-D-ocGQGs.js → activity-D3MNcSaV.js} +2 -2
- package/public/assets/{activity-D-ocGQGs.js.map → activity-D3MNcSaV.js.map} +1 -1
- package/public/assets/{agent-profiles-CRPgYTyb.js → agent-profiles-CKzttq6G.js} +2 -2
- package/public/assets/{agent-profiles-CRPgYTyb.js.map → agent-profiles-CKzttq6G.js.map} +1 -1
- package/public/assets/{agent-type-icon-CeEoMouY.js → agent-type-icon-0ZJRP1Vy.js} +2 -2
- package/public/assets/{agent-type-icon-CeEoMouY.js.map → agent-type-icon-0ZJRP1Vy.js.map} +1 -1
- package/public/assets/{agents-Chw5E_2q.js → agents-CuVj_rfG.js} +2 -2
- package/public/assets/{agents-Chw5E_2q.js.map → agents-CuVj_rfG.js.map} +1 -1
- package/public/assets/{analytics-BmNDG0hR.js → analytics-iob6C27d.js} +2 -2
- package/public/assets/{analytics-BmNDG0hR.js.map → analytics-iob6C27d.js.map} +1 -1
- package/public/assets/{automation-D7g90kTv.js → automation-B2ixxs4q.js} +2 -2
- package/public/assets/{automation-D7g90kTv.js.map → automation-B2ixxs4q.js.map} +1 -1
- package/public/assets/{branch-state-badge-Bv7DhLBZ.js → branch-state-badge-CafBJWKo.js} +2 -2
- package/public/assets/{branch-state-badge-Bv7DhLBZ.js.map → branch-state-badge-CafBJWKo.js.map} +1 -1
- package/public/assets/{channels-CLwPeEPi.js → channels-C-WsbL2H.js} +2 -2
- package/public/assets/{channels-CLwPeEPi.js.map → channels-C-WsbL2H.js.map} +1 -1
- package/public/assets/{chat-Cgj7VKzc.js → chat-DeoVTVob.js} +2 -2
- package/public/assets/{chat-Cgj7VKzc.js.map → chat-DeoVTVob.js.map} +1 -1
- package/public/assets/{connectors-Br6fiTny.js → connectors-C2Ac03LJ.js} +2 -2
- package/public/assets/{connectors-Br6fiTny.js.map → connectors-C2Ac03LJ.js.map} +1 -1
- package/public/assets/display-CJzSoTMq.js +3 -0
- package/public/assets/display-CJzSoTMq.js.map +1 -0
- package/public/assets/{formatted-body-CmkuD23M.js → formatted-body-35XBOY5L.js} +3 -3
- package/public/assets/{formatted-body-CmkuD23M.js.map → formatted-body-35XBOY5L.js.map} +1 -1
- package/public/assets/{formatted-body-impl-DbdFmbwW.js → formatted-body-impl-RmDxI5Ux.js} +2 -2
- package/public/assets/{formatted-body-impl-DbdFmbwW.js.map → formatted-body-impl-RmDxI5Ux.js.map} +1 -1
- package/public/assets/index-D0xMQQb_.js +25 -0
- package/public/assets/{index-D9OogaB5.js.map → index-D0xMQQb_.js.map} +1 -1
- package/public/assets/{insights-qvaPqWCV.js → insights-B643SFvm.js} +2 -2
- package/public/assets/{insights-qvaPqWCV.js.map → insights-B643SFvm.js.map} +1 -1
- package/public/assets/{integrations-DYEGSqq_.js → integrations-hFIHeCF4.js} +2 -2
- package/public/assets/{integrations-DYEGSqq_.js.map → integrations-hFIHeCF4.js.map} +1 -1
- package/public/assets/{maintenance-C0HIEB-J.js → maintenance-CURvdp_Z.js} +2 -2
- package/public/assets/{maintenance-C0HIEB-J.js.map → maintenance-CURvdp_Z.js.map} +1 -1
- package/public/assets/{managed-agents-DdS7aI38.js → managed-agents-DrZtdlSn.js} +2 -2
- package/public/assets/{managed-agents-DdS7aI38.js.map → managed-agents-DrZtdlSn.js.map} +1 -1
- package/public/assets/{markdown-preview-impl-BnXMH1z1.js → markdown-preview-impl-CVDO--ek.js} +2 -2
- package/public/assets/{markdown-preview-impl-BnXMH1z1.js.map → markdown-preview-impl-CVDO--ek.js.map} +1 -1
- package/public/assets/{memory-CSwx7bz8.js → memory-Bm7dt_Qf.js} +2 -2
- package/public/assets/{memory-CSwx7bz8.js.map → memory-Bm7dt_Qf.js.map} +1 -1
- package/public/assets/{messages-Be9CmvDv.js → messages-DTq4i2KZ.js} +2 -2
- package/public/assets/{messages-Be9CmvDv.js.map → messages-DTq4i2KZ.js.map} +1 -1
- package/public/assets/{orchestrators-BgtvvHvo.js → orchestrators-Dra318Ap.js} +2 -2
- package/public/assets/{orchestrators-BgtvvHvo.js.map → orchestrators-Dra318Ap.js.map} +1 -1
- package/public/assets/{overview-BvhbPWtF.js → overview-yr1Vmk64.js} +2 -2
- package/public/assets/{overview-BvhbPWtF.js.map → overview-yr1Vmk64.js.map} +1 -1
- package/public/assets/{pairs-CpXSOMx0.js → pairs-mnlKDBGp.js} +2 -2
- package/public/assets/{pairs-CpXSOMx0.js.map → pairs-mnlKDBGp.js.map} +1 -1
- package/public/assets/projects-Dzdf-gkK.js +2 -0
- package/public/assets/{projects-DdPNPFJI.js.map → projects-Dzdf-gkK.js.map} +1 -1
- package/public/assets/prompt-settings-ByL8SNCi.js +2 -0
- package/public/assets/{prompt-settings-BpZse7-X.js.map → prompt-settings-ByL8SNCi.js.map} +1 -1
- package/public/assets/{registry-pb1gTZEg.js → registry-CXWyOtpr.js} +2 -2
- package/public/assets/{registry-pb1gTZEg.js.map → registry-CXWyOtpr.js.map} +1 -1
- package/public/assets/{security-BgcX1n9D.js → security-0SBsc_4W.js} +2 -2
- package/public/assets/{security-BgcX1n9D.js.map → security-0SBsc_4W.js.map} +1 -1
- package/public/assets/{select-DZPVpKPv.js → select-Bw5z2KJ_.js} +2 -2
- package/public/assets/{select-DZPVpKPv.js.map → select-Bw5z2KJ_.js.map} +1 -1
- package/public/assets/{settings-hd5kzdRB.js → settings-78Cyq_tA.js} +4 -4
- package/public/assets/{settings-hd5kzdRB.js.map → settings-78Cyq_tA.js.map} +1 -1
- package/public/assets/{store-DKTLubBT.js → store-BPrVuKiv.js} +4 -4
- package/public/assets/store-BPrVuKiv.js.map +1 -0
- package/public/assets/{tasks-C5d2LU5E.js → tasks-Cg2MMeri.js} +2 -2
- package/public/assets/{tasks-C5d2LU5E.js.map → tasks-Cg2MMeri.js.map} +1 -1
- package/public/assets/teams-DWv6UwFV.js +2 -0
- package/public/assets/{teams-B8f2pGJe.js.map → teams-DWv6UwFV.js.map} +1 -1
- package/public/assets/{terminal-viewer-impl-CxAscH0U.js → terminal-viewer-impl-DBLDjbIg.js} +2 -2
- package/public/assets/{terminal-viewer-impl-CxAscH0U.js.map → terminal-viewer-impl-DBLDjbIg.js.map} +1 -1
- package/public/assets/{types-uVOXgvMT.js → types-BeJUSfDj.js} +2 -2
- package/public/assets/types-BeJUSfDj.js.map +1 -0
- package/public/assets/{user-avatar-Dw6mzWhv.js → user-avatar-BsOLyVJj.js} +2 -2
- package/public/assets/{user-avatar-Dw6mzWhv.js.map → user-avatar-BsOLyVJj.js.map} +1 -1
- package/public/assets/{work-queue-9imc7aNK.js → work-queue-DmwvnF6F.js} +2 -2
- package/public/assets/{work-queue-9imc7aNK.js.map → work-queue-DmwvnF6F.js.map} +1 -1
- package/public/assets/{workspaces-CLYc3yF3.js → workspaces-dIBivJN8.js} +2 -2
- package/public/assets/{workspaces-CLYc3yF3.js.map → workspaces-dIBivJN8.js.map} +1 -1
- package/public/index.html +6 -6
- package/runner/plugins/claude/.claude-plugin/plugin.json +1 -1
- package/runner/plugins/claude/monitors/relay-monitor.provisioned.mjs +44 -938
- package/runner/src/adapter.ts +53 -18
- package/runner/src/adapters/claude-delivery.ts +127 -0
- package/runner/src/adapters/claude-quota-harvest.ts +92 -0
- package/runner/src/adapters/claude-session-probe.ts +225 -0
- package/runner/src/adapters/claude-status-detectors.ts +147 -0
- package/runner/src/adapters/claude-transcript-tail.ts +128 -0
- package/runner/src/adapters/claude-transcript.ts +414 -0
- package/runner/src/adapters/claude.ts +672 -0
- package/runner/src/adapters/codex-agent-message-capture.ts +139 -0
- package/runner/src/adapters/codex-client.ts +279 -0
- package/runner/src/adapters/codex.ts +1345 -0
- package/runner/src/attachment-cache.ts +189 -0
- package/runner/src/busy-reconciler.ts +102 -0
- package/runner/src/claim-tracker.ts +140 -0
- package/runner/src/claude-prompt-gates.ts +268 -0
- package/runner/src/config.ts +1 -4
- package/runner/src/context-probe-state.ts +6 -0
- package/runner/src/continuation-archive.ts +179 -0
- package/runner/src/control-server.ts +639 -0
- package/runner/src/index.ts +396 -0
- package/runner/src/launch-assembly.ts +508 -0
- package/runner/src/liveness.ts +50 -0
- package/runner/src/logger.ts +99 -0
- package/runner/src/mcp-outbox.ts +120 -0
- package/runner/src/message-body-config/config.ts +3 -0
- package/runner/src/message-body-config.ts +1 -0
- package/runner/src/native-self-resume.ts +202 -0
- package/runner/src/outbox.ts +342 -0
- package/runner/src/process-meta.ts +9 -0
- package/runner/src/profile-home.ts +260 -0
- package/runner/src/profile-projection.ts +305 -0
- package/runner/src/providers.ts +20 -0
- package/runner/src/provisioning.ts +314 -0
- package/runner/src/quota.ts +40 -0
- package/runner/src/rate-limit.ts +189 -0
- package/runner/src/relay-injection-events.ts +102 -0
- package/runner/src/relay-instructions.ts +60 -0
- package/runner/src/relay-mcp-proxy.ts +383 -0
- package/runner/src/relay-mcp.ts +156 -0
- package/runner/src/reply-obligation-cache.ts +136 -0
- package/runner/src/response-capture-report.ts +63 -0
- package/runner/src/runner-core.ts +2409 -0
- package/runner/src/runner-helpers.ts +435 -0
- package/runner/src/runner-insights.ts +105 -0
- package/runner/src/runner.ts +14 -0
- package/runner/src/session-destroy.ts +22 -0
- package/runner/src/session-insights.ts +118 -0
- package/runner/src/session-scratch.ts +271 -0
- package/runner/src/template-resolver.ts +29 -0
- package/runner/src/version.ts +43 -0
- package/src/agent-card-projection.ts +5 -1
- package/src/agent-command-activity.ts +39 -0
- package/src/automations/reconcile.ts +2 -2
- package/src/cli/workspace.ts +3 -1
- package/src/config.ts +10 -1
- package/src/db/agents.ts +118 -37
- package/src/db/message-reads.ts +50 -0
- package/src/db/messages.ts +48 -9
- package/src/db/migrations.ts +54 -0
- package/src/db/provider-quota-burn.ts +96 -67
- package/src/db/provider-quotas.ts +29 -7
- package/src/db/scheduled-timers.ts +21 -0
- package/src/db/schema.ts +1 -0
- package/src/db/tasks.ts +33 -3
- package/src/db/work-evidence.ts +10 -7
- package/src/event-loop-lag.ts +15 -0
- package/src/gate-resolver.ts +19 -1
- package/src/maintenance/jobs.ts +23 -6
- package/src/managed-policy.ts +5 -12
- package/src/mcp-plan-tools.ts +13 -1
- package/src/project-instructions.ts +36 -2
- package/src/quota-advisory.ts +6 -4
- package/src/quota-band-transition.ts +23 -14
- package/src/routes/agent-sessions.ts +39 -7
- package/src/routes/agents.ts +3 -2
- package/src/routes/tasks.ts +16 -4
- package/src/routes/workspaces.ts +35 -10
- package/src/services/plan-graph.ts +27 -9
- package/src/services/register-agent.ts +12 -12
- package/src/services/send-message.ts +43 -15
- package/src/services/spawn-agent.ts +3 -0
- package/src/spawn-command.ts +8 -0
- package/src/spawn-targets.ts +19 -34
- package/src/sse.ts +25 -3
- package/src/workspace-merge.ts +25 -5
- package/src/workspace-orphans.ts +13 -6
- package/public/assets/display-exQDWum3.js +0 -3
- package/public/assets/display-exQDWum3.js.map +0 -1
- package/public/assets/index-D9OogaB5.js +0 -25
- package/public/assets/projects-DdPNPFJI.js +0 -2
- package/public/assets/prompt-settings-BpZse7-X.js +0 -2
- package/public/assets/store-DKTLubBT.js.map +0 -1
- package/public/assets/teams-B8f2pGJe.js +0 -2
- package/public/assets/types-uVOXgvMT.js.map +0 -1
|
@@ -0,0 +1,2409 @@
|
|
|
1
|
+
import { hostname } from "node:os";
|
|
2
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import type { AgentLifecycle, AgentProfile, ContextState, Message, MessageSessionMeta, ProviderState, SendMessageInput, TaskStatusInput, WorkspaceMetadata } from "agent-relay-sdk";
|
|
6
|
+
import { errMessage, RelayBusClient, RelayHttpClient } from "agent-relay-sdk";
|
|
7
|
+
import { contextStateFromProbeMetrics, quotaStateFromProbeMetrics } from "agent-relay-sdk/context-probe";
|
|
8
|
+
import { isPidAlive } from "agent-relay-sdk/process-utils";
|
|
9
|
+
import { providerAttachmentText, providerMessageText } from "./adapter";
|
|
10
|
+
import { computeLivenessSignal, type LivenessInputs } from "./liveness";
|
|
11
|
+
import type { ManagedProcess, ProviderAdapter, ProviderConfig, ProviderPermissionDecision, ProviderPermissionDecisionInput, ProviderSessionEvent, ProviderStatusUpdate, RunnerSpawnConfig, SemanticStatus, TerminalAttachSpec } from "./adapter";
|
|
12
|
+
import { messagesWithCachedAttachments } from "./attachment-cache";
|
|
13
|
+
import { ClaimTracker } from "./claim-tracker";
|
|
14
|
+
import { ClaudeQuotaHarvest } from "./adapters/claude-quota-harvest";
|
|
15
|
+
import { readRunnerContextProbeState } from "./context-probe-state";
|
|
16
|
+
import { startControlServer, type ControlServer, type ResolvedPermissionPrompt } from "./control-server";
|
|
17
|
+
import { ReplyObligationCache, obligationRequiresExplicitReply, responseCaptureReplyRoute } from "./reply-obligation-cache";
|
|
18
|
+
import { Outbox, type OutboxRecord } from "./outbox";
|
|
19
|
+
import { extractLastAssistantTurn, extractLastAssistantTurnAfterEntry, countTranscriptEntries, extractFinalAssistantMessage, extractFinalAssistantMessageAfterEntry, extractHookAssistantMessage, extractLatestTurnSteps, stepDedupKeys, transcriptLooksComplete } from "./adapters/claude-transcript";
|
|
20
|
+
import { IncrementalClaudeTranscriptTail } from "./adapters/claude-transcript-tail";
|
|
21
|
+
import { getManifest } from "agent-relay-providers";
|
|
22
|
+
import { profileUsesProviderHostGlobals } from "./profile-home";
|
|
23
|
+
import { RELAY_MCP_TOKEN_ENV, relayMcpEndpoint, resolveSharedMcpUrl } from "./relay-mcp";
|
|
24
|
+
import { RelayMcpProxy } from "./relay-mcp-proxy";
|
|
25
|
+
import { runtimeMetadata } from "./version";
|
|
26
|
+
import { logger, parseLogLevel } from "./logger";
|
|
27
|
+
import { ensureSessionScratch, reapSessionLauncher, reapSessionScratch, sweepStaleLaunchers, sweepStaleSessions, type SessionScratchLayout } from "./session-scratch";
|
|
28
|
+
import { deliverBufferedMcpCall as deliverBufferedMcpOutboxCall, deliverRunnerOutboxEvent } from "./mcp-outbox";
|
|
29
|
+
import { RunnerInsights } from "./runner-insights";
|
|
30
|
+
import { BusyReconciler } from "./busy-reconciler";
|
|
31
|
+
import { publishCapturedResponse } from "./response-capture-report";
|
|
32
|
+
import { runnerLaunchInjectionSessionEvents, type RunnerRelayInjectionEvent } from "./relay-injection-events";
|
|
33
|
+
import { providerTerminalSession, providerTerminalSocket } from "./process-meta";
|
|
34
|
+
import { capsFromEnv, logLevelFromEnv, mcpProxyEnabledFromEnv, orchestratorBaseDirFromEnv, orchestratorUrlFromEnv, registrationTimeoutMsFromEnv, runnerInfoFileFromEnv, runnerLogFileFromEnv, runnerOutboxDirWithInfoFallback, sessionDebugEnabled, tagsFromEnv, workspaceModeFromEnv } from "./config";
|
|
35
|
+
import { boundaryReasonForCommand, PRE_DESTROY_TIMEOUT_MS, reasonExitsRunner, type LifecycleAction, type SessionDestroyReason } from "./session-destroy";
|
|
36
|
+
import { isNativeSelfResumeCompact, NativeSelfResumeTracker, recoverNativeSelfResumeExit } from "./native-self-resume";
|
|
37
|
+
import { initializeRunnerTemplates } from "./template-resolver";
|
|
38
|
+
import {
|
|
39
|
+
appliedAgentProfileMetadata,
|
|
40
|
+
commandTimeoutMs,
|
|
41
|
+
csvTags,
|
|
42
|
+
httpErrorKey,
|
|
43
|
+
isContextState,
|
|
44
|
+
isHttpAuthError,
|
|
45
|
+
isHttpStatusError,
|
|
46
|
+
lifecycleCapabilities,
|
|
47
|
+
providerExitDiagnostics,
|
|
48
|
+
providerStateFromActiveWork,
|
|
49
|
+
registerWithinDeadline,
|
|
50
|
+
relayBusUrl,
|
|
51
|
+
runnerAgentStatus,
|
|
52
|
+
runnerBusErrorAction,
|
|
53
|
+
runnerMessageMatches,
|
|
54
|
+
runnerMessageShouldDirtyObligations,
|
|
55
|
+
runnerShouldResolveProviderExit,
|
|
56
|
+
runnerShouldRestartUnexpectedProviderExit,
|
|
57
|
+
runtimeProviderCapabilities,
|
|
58
|
+
runtimeTokenRenewDelayMs,
|
|
59
|
+
runtimeTokenRenewTimerSchedule,
|
|
60
|
+
shouldLogDeliveryFailure,
|
|
61
|
+
stripRunnerClaimedGuidance,
|
|
62
|
+
taskIdFromMessage,
|
|
63
|
+
type ProbeModelInfo,
|
|
64
|
+
} from "./runner-helpers";
|
|
65
|
+
|
|
66
|
+
// Pre-destroy work is best-effort and must never hang teardown. Capping it keeps a slow
|
|
67
|
+
// transcript read or a wedged provider from stalling a shutdown the operator asked for.
|
|
68
|
+
// Bounded window to deliver the durable outbox before an exit-bound teardown (#183). Kept
|
|
69
|
+
// short so a wedged/down server can't stall an operator-requested shutdown for long; a
|
|
70
|
+
// row that still can't land is logged, not silently dropped.
|
|
71
|
+
const OUTBOX_FLUSH_TIMEOUT_MS = 3_000;
|
|
72
|
+
|
|
73
|
+
interface RunnerOptions {
|
|
74
|
+
provider: string;
|
|
75
|
+
model?: string;
|
|
76
|
+
effort?: string;
|
|
77
|
+
runnerId: string;
|
|
78
|
+
instanceId: string;
|
|
79
|
+
agentId?: string;
|
|
80
|
+
relayUrl: string;
|
|
81
|
+
token?: string;
|
|
82
|
+
tokenJti?: string;
|
|
83
|
+
tokenProfileId?: string;
|
|
84
|
+
tokenExpiresAt?: number;
|
|
85
|
+
rootTokenFallback?: boolean;
|
|
86
|
+
cwd: string;
|
|
87
|
+
headless: boolean;
|
|
88
|
+
approvalMode: string;
|
|
89
|
+
label?: string;
|
|
90
|
+
rig?: string;
|
|
91
|
+
profile?: string;
|
|
92
|
+
agentProfile?: AgentProfile;
|
|
93
|
+
prompt?: string;
|
|
94
|
+
systemPromptAppend?: string;
|
|
95
|
+
relayInjectionEvents?: RunnerRelayInjectionEvent[];
|
|
96
|
+
tags: string[];
|
|
97
|
+
capabilities: string[];
|
|
98
|
+
providerArgs: string[];
|
|
99
|
+
policyName?: string;
|
|
100
|
+
spawnRequestId?: string;
|
|
101
|
+
taskId?: number;
|
|
102
|
+
automationId?: string; automationRunId?: string;
|
|
103
|
+
workspace?: WorkspaceMetadata; lifecycle?: AgentLifecycle;
|
|
104
|
+
tmuxSession?: string;
|
|
105
|
+
startedAt: number;
|
|
106
|
+
providerConfig: ProviderConfig;
|
|
107
|
+
adapter: ProviderAdapter;
|
|
108
|
+
onProviderExit?: (code: number | null) => void;
|
|
109
|
+
exitProcessOnShutdown?: boolean;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
interface ActiveTaskClaim {
|
|
113
|
+
messageId: number;
|
|
114
|
+
taskId: number;
|
|
115
|
+
observedProviderBusy: boolean;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
|
|
119
|
+
const HTTP_LIVENESS_INTERVAL_MS = 20_000;
|
|
120
|
+
// #650: how often to reconcile pid-bearing busy work against actual process liveness.
|
|
121
|
+
const BUSY_LIVENESS_RECONCILE_MS = 5_000;
|
|
122
|
+
const HTTP_LIVENESS_LOG_INTERVAL_MS = 5 * 60 * 1000;
|
|
123
|
+
const TOKEN_RENEW_RETRY_MS = 60_000;
|
|
124
|
+
// Debounce reactive token recovery so a burst of 401-ing calls in the same window
|
|
125
|
+
// triggers a single re-mint attempt, not one per failing request.
|
|
126
|
+
const REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS = 10_000;
|
|
127
|
+
const UNEXPECTED_EXIT_WINDOW_MS = 2 * 60 * 1000;
|
|
128
|
+
const RAPID_EXIT_MS = 30 * 1000;
|
|
129
|
+
const MAX_RAPID_UNEXPECTED_EXITS = 3;
|
|
130
|
+
const UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS = 1_500;
|
|
131
|
+
const PROVIDER_EXIT_CONFIRM_POLL_MS = 250;
|
|
132
|
+
// A UserPromptSubmit echo matching a runner-injected prompt within this window is
|
|
133
|
+
// the same prompt arriving back from the provider — drop it to avoid a duplicate.
|
|
134
|
+
const PROMPT_ECHO_DEDUP_MS = 30_000;
|
|
135
|
+
// Relay-injected content (delivered messages, memory context) is wrapped with
|
|
136
|
+
// these markers; a UserPromptSubmit echo starting with one is a runner injection,
|
|
137
|
+
// not a human typing into the terminal, so it must not be mirrored as a prompt.
|
|
138
|
+
// `<task-notification>` is the harness wrapper for async monitor/task wake-ups
|
|
139
|
+
// (which embed a `[relay message #…]` deeper inside, defeating a plain prefix
|
|
140
|
+
// match) — it is emitted only by the harness, never typed by a human, so any
|
|
141
|
+
// echo starting with it is a system injection, not a user prompt (#289).
|
|
142
|
+
const RELAY_INJECTION_MARKERS = ["[relay message #", "[agent-relay", "<task-notification>"];
|
|
143
|
+
// #329: a cold isolated profile (first-run trust/onboarding/install gates) can take far longer
|
|
144
|
+
// than the warm-restart window to render an input-ready TUI, so the first prompt delivery gets a
|
|
145
|
+
// generous timeout. If it still misses, the prompt is re-attempted on the next provider ready
|
|
146
|
+
// signal (short timeout, since it just went idle), up to a bounded number of attempts.
|
|
147
|
+
const INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS = 30_000;
|
|
148
|
+
const INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS = 10_000;
|
|
149
|
+
const MAX_INITIAL_PROMPT_ATTEMPTS = 6;
|
|
150
|
+
// Reasoning tailer poll cadence (item 5). Coarse on purpose — reasoning is a
|
|
151
|
+
// discreet progress signal, not a token stream; each tick reads only appended
|
|
152
|
+
// transcript bytes.
|
|
153
|
+
const REASONING_POLL_MS = 1_200;
|
|
154
|
+
const SESSION_MIRROR_BATCH_MAX = 100;
|
|
155
|
+
interface RunnerTimelineEvent {
|
|
156
|
+
status: string;
|
|
157
|
+
id?: string;
|
|
158
|
+
timestamp: number;
|
|
159
|
+
title?: string;
|
|
160
|
+
body?: string;
|
|
161
|
+
icon?: string;
|
|
162
|
+
metadata?: Record<string, unknown>;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
interface ReasoningTailState {
|
|
166
|
+
timer: ReturnType<typeof setInterval>; emittedNarrationKeys: Set<string>;
|
|
167
|
+
poll(): Promise<void>;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export class AgentRunner {
|
|
171
|
+
private readonly agentId: string;
|
|
172
|
+
private readonly claims = new ClaimTracker();
|
|
173
|
+
private readonly http: RelayHttpClient;
|
|
174
|
+
private readonly bus: RelayBusClient;
|
|
175
|
+
// Phase 2 (#196): the Stop hook reads reply obligations from this local snapshot, never
|
|
176
|
+
// from the server — so a slow server can no longer wedge a turn (the crux fix).
|
|
177
|
+
private readonly obligationCache: ReplyObligationCache;
|
|
178
|
+
// Phase 2 (#196): Runner→server append-log events (session turns, reasoning, prompts,
|
|
179
|
+
// insights, hook-fatal) go through this durable, disk-backed, timestamped queue instead of
|
|
180
|
+
// direct fire-and-forget HTTP — so nothing is lost across a server/Runner restart.
|
|
181
|
+
private readonly outbox: Outbox;
|
|
182
|
+
// #332: a fast-lane outbox for display-only session-mirror trace events. Routing them through
|
|
183
|
+
// their own FIFO queue stops one transient trace failure from head-of-line blocking real-message
|
|
184
|
+
// delivery (and vice versa); its backoff is capped low since a stale live-mirror frame is cheap.
|
|
185
|
+
private readonly sessionOutbox: Outbox;
|
|
186
|
+
private sessionBatchSeq = 0;
|
|
187
|
+
private readonly insights: RunnerInsights;
|
|
188
|
+
private readonly busyReconciler: BusyReconciler;
|
|
189
|
+
private readonly claudeQuotaHarvest: ClaudeQuotaHarvest;
|
|
190
|
+
private currentToken?: string;
|
|
191
|
+
private currentTokenJti?: string;
|
|
192
|
+
private currentTokenProfileId?: string;
|
|
193
|
+
private currentTokenExpiresAt?: number;
|
|
194
|
+
private control?: ControlServer;
|
|
195
|
+
// Stage 2 (#215): the local MCP endpoint the agent connects to, fronting the relay so the
|
|
196
|
+
// Runner owns reconnect/backoff + a durable buffer. Disabled via AGENT_RELAY_MCP_PROXY=0
|
|
197
|
+
// (then the agent connects to the relay directly, Stage-1 behaviour). The proxy secret is the
|
|
198
|
+
// bearer the agent presents to the localhost proxy — it decouples the agent from the rotating
|
|
199
|
+
// relay token (the proxy injects the live token relay-side).
|
|
200
|
+
private proxy?: RelayMcpProxy;
|
|
201
|
+
private mcpProxyEndpoint?: string;
|
|
202
|
+
private readonly mcpProxyEnabled: boolean;
|
|
203
|
+
private readonly mcpProxySecret: string;
|
|
204
|
+
private process?: ManagedProcess;
|
|
205
|
+
private stopped = false;
|
|
206
|
+
// #633 — set when the provider terminally exits but the runner stays alive (Claude native
|
|
207
|
+
// auto-compaction / manual-resume hold). Rides on the offline status frame so the server fires a
|
|
208
|
+
// terminal `agent.exited` event with a #636 diagnosis instead of the death going silent. Cleared
|
|
209
|
+
// on any genuine restart back to busy/idle.
|
|
210
|
+
private terminalProviderExit?: Record<string, unknown>;
|
|
211
|
+
private nativeSelfResume = new NativeSelfResumeTracker();
|
|
212
|
+
private exitCommandInProgress = false;
|
|
213
|
+
private restartInProgress = false;
|
|
214
|
+
// Set for the whole unexpected-exit restart, including the pre-restart backoff
|
|
215
|
+
// sleep, so a second exit during that window can't launch a parallel restart.
|
|
216
|
+
private restartPending = false;
|
|
217
|
+
private delivering = false;
|
|
218
|
+
private drainTimer?: Timer;
|
|
219
|
+
private claimRenewTimer?: Timer;
|
|
220
|
+
private httpLivenessTimer?: Timer;
|
|
221
|
+
private livenessReconcileTimer?: Timer;
|
|
222
|
+
private httpLivenessInFlight = false;
|
|
223
|
+
private httpLivenessAuthFailed = false;
|
|
224
|
+
private httpLivenessLastLog?: { key: string; at: number };
|
|
225
|
+
private tokenRenewTimer?: Timer;
|
|
226
|
+
private tokenRenewInFlight = false;
|
|
227
|
+
private tokenRenewLastLog?: { key: string; at: number };
|
|
228
|
+
private reactiveTokenRecoveryAt?: number;
|
|
229
|
+
private processStartedAt = 0;
|
|
230
|
+
private providerSessionId = crypto.randomUUID();
|
|
231
|
+
// Last transcript path seen this session — used by end-of-session Insights (#184)
|
|
232
|
+
// when the SessionEnd hook payload omits it.
|
|
233
|
+
private lastTranscriptPath?: string;
|
|
234
|
+
private lifecycleAction?: LifecycleAction;
|
|
235
|
+
// Coalesces concurrent pre-session-destroy runs (e.g. the shutdown bus command and the
|
|
236
|
+
// SessionEnd hook both fire for the same teardown) so the cursor isn't raced.
|
|
237
|
+
private preDestroyPromise?: Promise<void>;
|
|
238
|
+
private readonly unexpectedExitTimes: number[] = [];
|
|
239
|
+
private readonly pendingMessages = new Map<number, Message>();
|
|
240
|
+
private readonly activeTaskClaims = new Map<number, ActiveTaskClaim>();
|
|
241
|
+
private pendingTimelineEvent?: RunnerTimelineEvent;
|
|
242
|
+
private lastProviderEventAt?: number;
|
|
243
|
+
private pendingPromptMessageId?: number;
|
|
244
|
+
// #329: a spawn-time initial prompt whose first delivery timed out before the TUI was
|
|
245
|
+
// ready. Held (not dropped) for a ready-signal-driven re-attempt; cleared on success.
|
|
246
|
+
private pendingInitialPrompt?: string;
|
|
247
|
+
private deliveringInitialPrompt = false;
|
|
248
|
+
private initialPromptAttempts = 0;
|
|
249
|
+
// Session-mirror: a synthesized id grouping a turn's reasoning/tool steps and
|
|
250
|
+
// its final response. Set when a provider-turn starts, cleared when it ends.
|
|
251
|
+
private currentTurnId?: string;
|
|
252
|
+
private currentTurnStartedAt?: number;
|
|
253
|
+
private completedProviderTurns = 0;
|
|
254
|
+
// #435: high-water-mark for the pre-turn narrative flush. Counts how many
|
|
255
|
+
// valid JSONL entries were in the transcript at last pre-flush time so the
|
|
256
|
+
// Stop-hook capture (full mode) can skip what was already emitted.
|
|
257
|
+
private narrativeFlushEntryCount = 0;
|
|
258
|
+
// True while a turn that was already in flight is being compacted. Claude's PostCompact
|
|
259
|
+
// hook posts a single `idle`, but a mid-turn compaction RESUMES the turn afterward — so
|
|
260
|
+
// treating that idle as a turn end (flip idle, kill the reasoning tail) makes chat go dark
|
|
261
|
+
// until the next prompt. Set when `compacting` arrives with a turn running; cleared on the
|
|
262
|
+
// genuine end (a plain idle with no compaction timeline).
|
|
263
|
+
private compactionMidTurn = false;
|
|
264
|
+
// Prompt-echo dedup: a short, time-bounded queue of prompts the runner itself
|
|
265
|
+
// injected (chat box or initial prompt) that are still awaiting their matching
|
|
266
|
+
// UserPromptSubmit echo. A single slot dropped earlier entries when several prompts
|
|
267
|
+
// were injected before their echoes returned (rapid sends while the provider is busy
|
|
268
|
+
// and queues them) — the evicted ones then double-posted. Match consumes one entry.
|
|
269
|
+
private injectedPrompts: Array<{ text: string; at: number }> = [];
|
|
270
|
+
// Verbose session-mirror diagnostics (turn lifecycle, reconciler probes, tail
|
|
271
|
+
// emits) → a dedicated clean log. Always on for key transitions; AGENT_RELAY_SESSION_DEBUG=1 adds the high-frequency probe/emit lines.
|
|
272
|
+
private readonly sessionDebugVerbose = sessionDebugEnabled();
|
|
273
|
+
// Tracks whether the provider is in a legitimate blocked/approval state, so the
|
|
274
|
+
// busy reconciler doesn't mistake a permission prompt for a stuck-busy turn.
|
|
275
|
+
private providerBlocked = false;
|
|
276
|
+
private terminalFailure?: { reason: string; message: string; providerState?: ProviderState };
|
|
277
|
+
// #286: a usage/rate-limit hold. The turn truly ended (idle), but the agent is
|
|
278
|
+
// held until the limit resets, so this can't ride an active-work claim like the
|
|
279
|
+
// permission-blocked state does. publishStatus surfaces it as the agent's
|
|
280
|
+
// providerState; it clears when a new turn starts (the relay's resume message
|
|
281
|
+
// wakes one) so the blocked badge lifts on its own.
|
|
282
|
+
private rateLimitHold?: ProviderState;
|
|
283
|
+
private pendingRateLimitHold?: ProviderState;
|
|
284
|
+
// Reasoning tailer (item 5): streams the in-flight turn's reasoning/tool steps
|
|
285
|
+
// from the Claude transcript into chat as discreet session events.
|
|
286
|
+
private reasoningTail?: ReasoningTailState;
|
|
287
|
+
private scratch?: SessionScratchLayout;
|
|
288
|
+
// #417: for non-claude providers, tracks how many times each spawner obligation has
|
|
289
|
+
// been re-injected so a non-responsive agent can't spin in a re-injection loop.
|
|
290
|
+
private readonly pendingObligationInjections = new Map<number, number>();
|
|
291
|
+
|
|
292
|
+
constructor(private readonly options: RunnerOptions) {
|
|
293
|
+
this.agentId = options.agentId ?? options.runnerId;
|
|
294
|
+
// Bind the process-global logger to this agent. AGENT_RELAY_SESSION_DEBUG=1 is
|
|
295
|
+
// kept as a back-compat alias for the verbose probe/emit lines, now expressed
|
|
296
|
+
// as log level "debug" (AGENT_RELAY_LOG_LEVEL still wins when both are set).
|
|
297
|
+
logger.configure({
|
|
298
|
+
agentId: this.agentId,
|
|
299
|
+
headless: options.headless,
|
|
300
|
+
...(this.sessionDebugVerbose && !parseLogLevel(logLevelFromEnv()) ? { level: "debug" as const } : {}),
|
|
301
|
+
});
|
|
302
|
+
this.currentToken = options.token;
|
|
303
|
+
this.currentTokenJti = options.tokenJti;
|
|
304
|
+
this.currentTokenProfileId = options.tokenProfileId;
|
|
305
|
+
this.currentTokenExpiresAt = options.tokenExpiresAt;
|
|
306
|
+
this.mcpProxyEnabled = mcpProxyEnabledFromEnv();
|
|
307
|
+
this.mcpProxySecret = crypto.randomUUID();
|
|
308
|
+
const runtime = runtimeMetadata(options.provider);
|
|
309
|
+
this.http = new RelayHttpClient({ baseUrl: options.relayUrl, token: this.currentToken });
|
|
310
|
+
this.claudeQuotaHarvest = new ClaudeQuotaHarvest({ agentId: this.agentId, provider: options.provider, http: () => this.http });
|
|
311
|
+
this.obligationCache = new ReplyObligationCache({ fetch: () => this.http.listReplyObligations(this.agentId) });
|
|
312
|
+
// Co-locate the durable outbox with the runner's runtime state (survives reboot) when the
|
|
313
|
+
// orchestrator told us where that is; otherwise the Outbox falls back to a temp dir.
|
|
314
|
+
const outboxDir = runnerOutboxDirWithInfoFallback();
|
|
315
|
+
this.outbox = new Outbox({ agentId: this.agentId, dir: outboxDir, send: (record) => this.deliverOutboxEvent(record) });
|
|
316
|
+
this.sessionOutbox = new Outbox({
|
|
317
|
+
agentId: `${this.agentId}-session`,
|
|
318
|
+
dir: outboxDir,
|
|
319
|
+
send: (record) => this.deliverOutboxEvent(record),
|
|
320
|
+
// Ephemeral trace frames: cap the per-row stall low and poison sooner than the main queue.
|
|
321
|
+
maxBackoffMs: 5_000,
|
|
322
|
+
maxAttempts: 6,
|
|
323
|
+
});
|
|
324
|
+
this.insights = new RunnerInsights({
|
|
325
|
+
agentId: this.agentId,
|
|
326
|
+
cwd: options.cwd,
|
|
327
|
+
workspace: options.workspace,
|
|
328
|
+
getOutbox: () => this.outbox,
|
|
329
|
+
getProviderSessionId: () => this.providerSessionId,
|
|
330
|
+
getProcess: () => this.process,
|
|
331
|
+
getAdapter: () => this.options.adapter,
|
|
332
|
+
getLastTranscriptPath: () => this.lastTranscriptPath,
|
|
333
|
+
sessionLog: (message) => this.sessionLog(message),
|
|
334
|
+
});
|
|
335
|
+
this.busyReconciler = new BusyReconciler({
|
|
336
|
+
isStopped: () => this.stopped,
|
|
337
|
+
hasProcess: () => Boolean(this.process),
|
|
338
|
+
currentStatus: () => this.claims.currentStatus(),
|
|
339
|
+
isProviderBlocked: () => this.providerBlocked,
|
|
340
|
+
hasProviderTurn: () => this.claims.activeWork().some((w) => w.kind === "provider-turn"),
|
|
341
|
+
canProbeActivity: () => Boolean(this.options.adapter.probeActivity),
|
|
342
|
+
probeActivity: () => this.process && this.options.adapter.probeActivity
|
|
343
|
+
? this.options.adapter.probeActivity(this.process)
|
|
344
|
+
: undefined,
|
|
345
|
+
probePaneBusy: () => this.process && this.options.adapter.probePaneBusy
|
|
346
|
+
? this.options.adapter.probePaneBusy(this.process)
|
|
347
|
+
: undefined,
|
|
348
|
+
clearProviderTurn: (reason) => this.forceClearProviderTurn(reason),
|
|
349
|
+
sessionDebug: (message) => this.sessionDebug(message),
|
|
350
|
+
});
|
|
351
|
+
this.bus = new RelayBusClient({
|
|
352
|
+
url: relayBusUrl(options.relayUrl),
|
|
353
|
+
role: "provider",
|
|
354
|
+
componentId: options.runnerId,
|
|
355
|
+
agentId: this.agentId,
|
|
356
|
+
instanceId: options.instanceId,
|
|
357
|
+
token: this.currentToken,
|
|
358
|
+
machine: hostname(),
|
|
359
|
+
capabilities: [
|
|
360
|
+
...new Set([
|
|
361
|
+
...options.providerConfig.defaultCapabilities,
|
|
362
|
+
...options.capabilities,
|
|
363
|
+
...csvTags(capsFromEnv()),
|
|
364
|
+
"lifecycle.shutdown.hard",
|
|
365
|
+
"lifecycle.restart.hard",
|
|
366
|
+
"lifecycle.status.semantic",
|
|
367
|
+
"lifecycle.heartbeat",
|
|
368
|
+
"lifecycle.reconnect.transport",
|
|
369
|
+
"commands.lifecycle",
|
|
370
|
+
"capabilities.report",
|
|
371
|
+
]),
|
|
372
|
+
],
|
|
373
|
+
tags: [...new Set([options.provider, ...csvTags(tagsFromEnv()), ...options.tags, ...options.providerConfig.defaultTags, ...(options.headless ? ["headless"] : [])])],
|
|
374
|
+
meta: {
|
|
375
|
+
...runtime,
|
|
376
|
+
version: runtime.package.version,
|
|
377
|
+
provider: options.provider,
|
|
378
|
+
model: options.model ?? null,
|
|
379
|
+
effort: options.effort ?? null,
|
|
380
|
+
profile: options.profile ?? null,
|
|
381
|
+
// #557 — pass a spawn-shaped config so the projection describes the SAME assembled
|
|
382
|
+
// launch the adapter builds (env/controlPort are placeholders the pure assembler ignores).
|
|
383
|
+
agentProfile: options.agentProfile ? appliedAgentProfileMetadata(options.provider, options.agentProfile, { ...options, agentId: this.agentId, env: {}, controlPort: 0 } as RunnerSpawnConfig, options.providerConfig) : null,
|
|
384
|
+
runnerId: options.runnerId,
|
|
385
|
+
startedAt: options.startedAt,
|
|
386
|
+
tmuxSession: providerTerminalSession(this.process) ?? options.tmuxSession ?? null,
|
|
387
|
+
tmuxSocket: providerTerminalSocket(this.process) ?? null,
|
|
388
|
+
policyName: options.policyName ?? null,
|
|
389
|
+
spawnRequestId: options.spawnRequestId ?? null,
|
|
390
|
+
taskId: options.taskId ?? null,
|
|
391
|
+
automationId: options.automationId ?? null,
|
|
392
|
+
automationRunId: options.automationRunId ?? null,
|
|
393
|
+
workspace: options.workspace ?? null,
|
|
394
|
+
lifecycle: options.lifecycle ?? "persistent",
|
|
395
|
+
workspaceMode: options.workspace?.requestedMode ?? options.workspace?.mode ?? workspaceModeFromEnv() ?? null,
|
|
396
|
+
runnerManaged: true,
|
|
397
|
+
cwd: options.cwd,
|
|
398
|
+
approvalMode: options.approvalMode,
|
|
399
|
+
label: options.label ?? null,
|
|
400
|
+
auth: {
|
|
401
|
+
profileId: this.currentTokenProfileId ?? null,
|
|
402
|
+
jti: this.currentTokenJti ?? null,
|
|
403
|
+
expiresAt: this.currentTokenExpiresAt ?? null,
|
|
404
|
+
rootTokenFallback: options.rootTokenFallback === true,
|
|
405
|
+
},
|
|
406
|
+
lifecycleCapabilities: lifecycleCapabilities(),
|
|
407
|
+
providerCapabilities: runtimeProviderCapabilities(options),
|
|
408
|
+
},
|
|
409
|
+
heartbeatMeta: () => this.heartbeatRuntimeMeta(),
|
|
410
|
+
heartbeatIntervalMs: 30_000,
|
|
411
|
+
initialStatus: "idle",
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
get id(): string {
|
|
416
|
+
return this.agentId;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
async run(): Promise<void> {
|
|
420
|
+
this.control = startControlServer({
|
|
421
|
+
onStatus: (status) => this.setProviderStatus(status),
|
|
422
|
+
onTerminalAttachSpec: () => this.terminalAttachSpec(),
|
|
423
|
+
// Hot-path-safe: answered instantly from the local snapshot, never a server
|
|
424
|
+
// round-trip. The snapshot is kept warm by the background refresh below (#196).
|
|
425
|
+
onReplyObligations: () => Promise.resolve(this.obligationCache.get()),
|
|
426
|
+
onSessionTurn: (input) => this.publishSessionTurn(input), onPermissionResolved: (input) => this.publishSessionEvent({ from: this.agentId, to: "user", body: input.body, session: { type: "prompt-resolution", origin: "provider", label: input.decisionLabel, ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}), stepId: `prompt-resolution:${input.approvalId}`, promptResolution: input } }),
|
|
427
|
+
onUserPrompt: (input) => this.handleUserPrompt(input),
|
|
428
|
+
onSessionBoundary: (input) => this.handleSessionBoundary(input),
|
|
429
|
+
onHookFatal: (report) => this.reportHookFatal(report),
|
|
430
|
+
});
|
|
431
|
+
this.startMcpProxy();
|
|
432
|
+
this.writeRunnerInfoFile();
|
|
433
|
+
this.options.adapter.onStatusChange((status) => {
|
|
434
|
+
if (this.restartInProgress || this.restartPending) return;
|
|
435
|
+
const semanticStatus = typeof status === "string" ? status : status.status;
|
|
436
|
+
if (this.shouldRestartUnexpectedProviderExit(semanticStatus)) {
|
|
437
|
+
void this.restartUnexpectedProviderExit(semanticStatus);
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
this.setProviderStatus(status);
|
|
441
|
+
if (runnerShouldResolveProviderExit(semanticStatus, this.exitCommandInProgress)) this.options.onProviderExit?.(semanticStatus === "offline" ? 0 : 1);
|
|
442
|
+
});
|
|
443
|
+
this.options.adapter.onSessionEvent?.((event) => { void this.publishProviderSessionEvent(event); });
|
|
444
|
+
this.bus.on("message.new", (message) => {
|
|
445
|
+
// A delivered message may create a new reply obligation — warm the snapshot so the
|
|
446
|
+
// next turn-end sees it without a hot-path server read.
|
|
447
|
+
const msg = message as Message;
|
|
448
|
+
if (runnerMessageShouldDirtyObligations(msg, this.agentId)) this.obligationCache.markDirty();
|
|
449
|
+
this.enqueueMessage(msg);
|
|
450
|
+
});
|
|
451
|
+
this.bus.on("command", (type, params, commandId, command) => {
|
|
452
|
+
void this.handleCommand(type, params, commandId, command);
|
|
453
|
+
});
|
|
454
|
+
this.bus.on("error", (code, message) => this.handleBusError(String(code), String(message)));
|
|
455
|
+
await registerWithinDeadline(this.bus, registrationTimeoutMsFromEnv(), this.options.relayUrl, (reason) => logger.fatal("register", reason)); await initializeRunnerTemplates(this.http);
|
|
456
|
+
this.obligationCache.start();
|
|
457
|
+
this.outbox.start();
|
|
458
|
+
this.sessionOutbox.start();
|
|
459
|
+
this.ensureScratch();
|
|
460
|
+
void this.sweepStaleScratch();
|
|
461
|
+
this.process = await this.spawnProvider();
|
|
462
|
+
this.writeRunnerInfoFile();
|
|
463
|
+
this.processStartedAt = Date.now();
|
|
464
|
+
this.publishStatus();
|
|
465
|
+
await this.deliverInitialPrompt();
|
|
466
|
+
await this.bootstrapUnreadMessages();
|
|
467
|
+
this.startClaimRenewer();
|
|
468
|
+
this.startHttpLiveness();
|
|
469
|
+
this.scheduleRuntimeTokenRenewal();
|
|
470
|
+
this.scheduleDrain();
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
async stop(): Promise<void> {
|
|
474
|
+
const alreadyStopped = this.stopped;
|
|
475
|
+
this.stopped = true;
|
|
476
|
+
const launcherScript = typeof this.process?.meta?.launcherScript === "string" ? this.process.meta.launcherScript : undefined; const tmuxSession = typeof this.process?.meta?.tmuxSession === "string" ? this.process.meta.tmuxSession : undefined;
|
|
477
|
+
reapSessionScratch({ agentId: this.agentId, cwd: this.options.cwd, fallbackBaseDir: orchestratorBaseDirFromEnv() });
|
|
478
|
+
reapSessionLauncher({ launcherScript, tmuxSession });
|
|
479
|
+
if (!alreadyStopped) await this.bus.statusAsync({ agentStatus: "offline", ready: false });
|
|
480
|
+
if (this.process && !alreadyStopped) {
|
|
481
|
+
await this.options.adapter.shutdown(this.process, {
|
|
482
|
+
graceful: true,
|
|
483
|
+
timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
if (this.drainTimer) clearTimeout(this.drainTimer);
|
|
487
|
+
this.drainTimer = undefined;
|
|
488
|
+
if (this.claimRenewTimer) clearInterval(this.claimRenewTimer);
|
|
489
|
+
this.claimRenewTimer = undefined;
|
|
490
|
+
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
491
|
+
this.httpLivenessTimer = undefined;
|
|
492
|
+
if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
|
|
493
|
+
this.tokenRenewTimer = undefined;
|
|
494
|
+
this.disarmBusyLivenessReconcile();
|
|
495
|
+
this.busyReconciler.disarm();
|
|
496
|
+
this.stopReasoningTail();
|
|
497
|
+
this.obligationCache.stop();
|
|
498
|
+
this.outbox.close(); this.sessionOutbox.close();
|
|
499
|
+
this.proxy?.stop();
|
|
500
|
+
this.control?.stop();
|
|
501
|
+
await this.bus.close();
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Start the local MCP proxy the agent connects to (Stage 2, #215). Forwards tool calls to the
|
|
505
|
+
// relay with the runner's LIVE token, buffers bufferable writes durably during a relay outage,
|
|
506
|
+
// and narrows the tool list to this agent's workspace context. Best-effort: if it can't bind,
|
|
507
|
+
// we fall back to a direct relay MCP connection (the agent env still works, no resilience).
|
|
508
|
+
private startMcpProxy(): void {
|
|
509
|
+
if (!this.mcpProxyEnabled) return;
|
|
510
|
+
try {
|
|
511
|
+
this.proxy = new RelayMcpProxy({
|
|
512
|
+
relayMcpEndpoint: relayMcpEndpoint(this.options.relayUrl),
|
|
513
|
+
getToken: () => this.currentToken,
|
|
514
|
+
authSecret: this.mcpProxySecret,
|
|
515
|
+
enqueueBuffered: (call) => {
|
|
516
|
+
this.outbox.enqueue({
|
|
517
|
+
kind: "mcp-tool-call",
|
|
518
|
+
payload: { tool: call.tool, arguments: call.arguments },
|
|
519
|
+
idempotencyKey: call.idempotencyKey,
|
|
520
|
+
});
|
|
521
|
+
},
|
|
522
|
+
initialContext: { isolatedWorktree: this.ownsIsolatedWorktree() },
|
|
523
|
+
});
|
|
524
|
+
this.mcpProxyEndpoint = this.proxy.start().url;
|
|
525
|
+
logger.info("mcp-proxy", `runner MCP proxy listening at ${this.mcpProxyEndpoint} (worktree=${this.ownsIsolatedWorktree()})`);
|
|
526
|
+
} catch (error) {
|
|
527
|
+
this.proxy = undefined;
|
|
528
|
+
this.mcpProxyEndpoint = undefined;
|
|
529
|
+
logger.warn("mcp-proxy", `failed to start MCP proxy; agent will connect to the relay directly: ${errMessage(error)}`);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
private ownsIsolatedWorktree(): boolean {
|
|
534
|
+
const mode = this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? workspaceModeFromEnv();
|
|
535
|
+
return mode === "isolated";
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
private async spawnProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<ManagedProcess> {
|
|
539
|
+
this.providerSessionId = crypto.randomUUID();
|
|
540
|
+
this.lastTranscriptPath = undefined;
|
|
541
|
+
const includeProviderGlobals = profileUsesProviderHostGlobals(this.options);
|
|
542
|
+
const env = {
|
|
543
|
+
...process.env as Record<string, string>,
|
|
544
|
+
...(includeProviderGlobals ? this.options.providerConfig.env : {}),
|
|
545
|
+
AGENT_RELAY_RUNNER_PORT: String(this.control!.port),
|
|
546
|
+
AGENT_RELAY_RUNNER_ID: this.options.runnerId,
|
|
547
|
+
AGENT_RELAY_PROVIDER_SESSION_ID: this.providerSessionId,
|
|
548
|
+
AGENT_RELAY_ID: this.agentId,
|
|
549
|
+
// The repo-name project id the runner uses for insight observations. Exposed so
|
|
550
|
+
// agent-side signals (e.g. /introspect) tag the SAME project and aggregate together,
|
|
551
|
+
// instead of the agent's worktree cwd splitting one repo into many "projects".
|
|
552
|
+
AGENT_RELAY_PROJECT: this.insightProject(),
|
|
553
|
+
...(this.scratch ? { AGENT_RELAY_SCRATCH_DIR: this.scratch.tmpDir } : {}),
|
|
554
|
+
AGENT_RELAY_URL: this.options.relayUrl,
|
|
555
|
+
AGENT_RELAY_APPROVAL: this.options.approvalMode,
|
|
556
|
+
...(this.currentToken ? { AGENT_RELAY_TOKEN: this.currentToken } : {}),
|
|
557
|
+
// Dedicated, un-clobberable credential for the injected MCP endpoint. A rig's
|
|
558
|
+
// settings.json `env.AGENT_RELAY_TOKEN` would override the scoped token above at
|
|
559
|
+
// MCP-parse time → server-actor auth, no identity (#233). The MCP config references
|
|
560
|
+
// ${AGENT_RELAY_SESSION_TOKEN}, which rigs never set. See runner/src/relay-mcp.ts.
|
|
561
|
+
//
|
|
562
|
+
// Stage 2 (#215): when the proxy is active the agent connects to the LOCAL proxy, so this
|
|
563
|
+
// holds the per-session PROXY SECRET (not the relay token). The proxy injects the live
|
|
564
|
+
// relay token itself — the agent never holds it, and token rotation is invisible. With the
|
|
565
|
+
// proxy disabled this stays the scoped relay token (Stage-1 direct connection).
|
|
566
|
+
...(this.proxy
|
|
567
|
+
? { [RELAY_MCP_TOKEN_ENV]: this.mcpProxySecret }
|
|
568
|
+
: (this.currentToken ? { [RELAY_MCP_TOKEN_ENV]: this.currentToken } : {})),
|
|
569
|
+
...(this.currentTokenJti ? { AGENT_RELAY_TOKEN_JTI: this.currentTokenJti } : {}),
|
|
570
|
+
...(this.currentTokenProfileId ? { AGENT_RELAY_TOKEN_PROFILE: this.currentTokenProfileId } : {}),
|
|
571
|
+
...(this.currentTokenExpiresAt ? { AGENT_RELAY_TOKEN_EXPIRES_AT: String(this.currentTokenExpiresAt) } : {}),
|
|
572
|
+
};
|
|
573
|
+
const config: RunnerSpawnConfig = {
|
|
574
|
+
provider: this.options.provider,
|
|
575
|
+
model: this.options.model,
|
|
576
|
+
effort: this.options.effort,
|
|
577
|
+
runnerId: this.options.runnerId,
|
|
578
|
+
instanceId: this.options.instanceId,
|
|
579
|
+
agentId: this.agentId,
|
|
580
|
+
relayUrl: this.options.relayUrl,
|
|
581
|
+
cwd: this.options.cwd,
|
|
582
|
+
headless: this.options.headless,
|
|
583
|
+
approvalMode: this.options.approvalMode,
|
|
584
|
+
...(this.options.label ? { label: this.options.label } : {}),
|
|
585
|
+
...(this.options.rig ? { rig: this.options.rig } : {}),
|
|
586
|
+
...(this.options.profile ? { profile: this.options.profile } : {}),
|
|
587
|
+
...(this.options.agentProfile ? { agentProfile: this.options.agentProfile } : {}),
|
|
588
|
+
...(this.options.prompt && !opts.resumeId && !opts.suppressPrompt ? { prompt: this.options.prompt } : {}),
|
|
589
|
+
...(this.options.systemPromptAppend ? { systemPromptAppend: this.options.systemPromptAppend } : {}),
|
|
590
|
+
...(this.options.tmuxSession ? { tmuxSession: this.options.tmuxSession } : {}),
|
|
591
|
+
providerArgs: opts.resumeId ? [...this.options.providerArgs, "--resume", opts.resumeId] : this.options.providerArgs,
|
|
592
|
+
providerConfig: this.options.providerConfig,
|
|
593
|
+
env,
|
|
594
|
+
controlPort: this.control!.port,
|
|
595
|
+
// Stage 2 (#215): the MCP endpoint the agent's client should target — the runner-local
|
|
596
|
+
// proxy when active, undefined when disabled (adapters fall back to the direct relay URL).
|
|
597
|
+
...(this.mcpProxyEndpoint ? { relayMcpEndpoint: this.mcpProxyEndpoint } : {}),
|
|
598
|
+
// #672: the shared host-listener URL the `callmux bridge` connects to (env → default).
|
|
599
|
+
// Always resolved; the assembler only emits the bridge when `agentProfile.sharedMcp` is on.
|
|
600
|
+
sharedMcpUrl: resolveSharedMcpUrl(),
|
|
601
|
+
monitor: {
|
|
602
|
+
deliver: (messages) => this.control!.deliverToMonitor(messages),
|
|
603
|
+
},
|
|
604
|
+
};
|
|
605
|
+
const launchSeededPrompt = this.options.adapter.seedsInitialPromptAtLaunch && !opts.resumeId && !opts.suppressPrompt ? this.options.prompt?.trim() : ""; if (launchSeededPrompt) this.recordInjectedPrompt(launchSeededPrompt);
|
|
606
|
+
const managedProcess = await this.options.adapter.spawn(config);
|
|
607
|
+
const injectionEvents = runnerLaunchInjectionSessionEvents({ carried: this.options.relayInjectionEvents, provider: this.options.provider, config, providerConfig: this.options.providerConfig, agentId: this.agentId, providerSessionId: this.providerSessionId });
|
|
608
|
+
for (const event of injectionEvents) {
|
|
609
|
+
this.publishSessionEvent(event);
|
|
610
|
+
}
|
|
611
|
+
return managedProcess;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
private async terminalAttachSpec(): Promise<TerminalAttachSpec> {
|
|
615
|
+
if (!this.process) throw new Error("provider process is unavailable");
|
|
616
|
+
if (!this.options.adapter.terminalAttachSpec) throw new Error("provider does not support terminal attach");
|
|
617
|
+
return this.options.adapter.terminalAttachSpec(this.process);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
private writeRunnerInfoFile(): void {
|
|
621
|
+
const file = runnerInfoFileFromEnv();
|
|
622
|
+
if (!file || !this.control) return;
|
|
623
|
+
try {
|
|
624
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
625
|
+
writeFileSync(file, JSON.stringify({
|
|
626
|
+
agentId: this.agentId,
|
|
627
|
+
runnerId: this.options.runnerId,
|
|
628
|
+
provider: this.options.provider,
|
|
629
|
+
controlUrl: this.control.url,
|
|
630
|
+
pid: process.pid,
|
|
631
|
+
...(providerTerminalSession(this.process) ? { tmuxSession: providerTerminalSession(this.process) } : {}),
|
|
632
|
+
...(providerTerminalSocket(this.process) ? { tmuxSocket: providerTerminalSocket(this.process) } : {}),
|
|
633
|
+
startedAt: this.options.startedAt,
|
|
634
|
+
}, null, 2) + "\n", { mode: 0o600 });
|
|
635
|
+
} catch (error) {
|
|
636
|
+
logger.error("runner", `failed to write runner info file: ${error}`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
private enqueueMessage(message: Message): void {
|
|
641
|
+
if (!this.matchesMessage(message)) return;
|
|
642
|
+
if (message.kind === "session") return;
|
|
643
|
+
this.pendingMessages.set(message.id, message);
|
|
644
|
+
this.scheduleDrain();
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
private async bootstrapUnreadMessages(): Promise<void> {
|
|
648
|
+
try {
|
|
649
|
+
const messages = await this.http.pollMessages({ for: this.agentId, unread: true, limit: 100 });
|
|
650
|
+
for (const message of messages) this.enqueueMessage(message);
|
|
651
|
+
} catch (error) {
|
|
652
|
+
logger.error("runner", `inbox bootstrap failed: ${error}`);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
private async deliverInitialPrompt(): Promise<void> {
|
|
657
|
+
const prompt = this.options.prompt?.trim();
|
|
658
|
+
if (!prompt || this.options.adapter.seedsInitialPromptAtLaunch) return;
|
|
659
|
+
await this.attemptInitialPromptDelivery(prompt, INITIAL_PROMPT_FIRST_READY_TIMEOUT_MS);
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// Deliver the spawn-time first prompt, surviving a cold-start TUI that isn't input-ready yet
|
|
663
|
+
// (#329). Timeout stashes it for ready-signal retry and surfaces the stall on the timeline.
|
|
664
|
+
private async attemptInitialPromptDelivery(prompt: string, readyTimeoutMs: number): Promise<void> {
|
|
665
|
+
if (this.deliveringInitialPrompt || this.stopped || !this.process || !this.options.adapter.deliverInitialPrompt) return;
|
|
666
|
+
this.deliveringInitialPrompt = true;
|
|
667
|
+
const attempt = (this.initialPromptAttempts += 1);
|
|
668
|
+
try {
|
|
669
|
+
this.recordInjectedPrompt(prompt); await this.options.adapter.deliverInitialPrompt(this.process, prompt, { readyTimeoutMs });
|
|
670
|
+
this.pendingInitialPrompt = undefined;
|
|
671
|
+
} catch (error) {
|
|
672
|
+
const giveUp = attempt >= MAX_INITIAL_PROMPT_ATTEMPTS;
|
|
673
|
+
this.pendingInitialPrompt = giveUp ? undefined : prompt;
|
|
674
|
+
logger.error("runner", `initial prompt attempt ${attempt} failed${giveUp ? " (giving up)" : "; will retry on next ready signal"}: ${errMessage(error)}`);
|
|
675
|
+
this.publishRunnerTimelineEvent({ status: "prompt_failed", timestamp: Date.now(), title: "Initial prompt not delivered", body: errMessage(error), metadata: { attempts: attempt, terminal: giveUp } });
|
|
676
|
+
} finally {
|
|
677
|
+
this.deliveringInitialPrompt = false;
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
private scheduleDrain(delayMs = 0): void {
|
|
682
|
+
if (this.stopped || this.drainTimer) return;
|
|
683
|
+
this.drainTimer = setTimeout(() => {
|
|
684
|
+
this.drainTimer = undefined;
|
|
685
|
+
void this.drainMessages();
|
|
686
|
+
}, delayMs);
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
private async drainMessages(): Promise<void> {
|
|
690
|
+
if (this.stopped || this.delivering || this.pendingMessages.size === 0) return;
|
|
691
|
+
if (!this.process) {
|
|
692
|
+
this.scheduleDrain(500);
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
this.delivering = true;
|
|
696
|
+
const messages = [...this.pendingMessages.values()].sort((a, b) => a.id - b.id);
|
|
697
|
+
this.pendingMessages.clear();
|
|
698
|
+
const deliverable: Message[] = [];
|
|
699
|
+
const providerAlreadyBusy = this.claims.reasons().includes("provider-turn");
|
|
700
|
+
for (const message of messages) {
|
|
701
|
+
let toDeliver = message;
|
|
702
|
+
if (message.claimable) {
|
|
703
|
+
const claimed = await this.http.claimMessageResult(message.id, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
|
|
704
|
+
if (!claimed.ok) continue;
|
|
705
|
+
this.claims.startClaim("message", String(message.id), claimed.claimExpiresAt);
|
|
706
|
+
const taskId = taskIdFromMessage(message);
|
|
707
|
+
if (taskId) {
|
|
708
|
+
this.activeTaskClaims.set(message.id, { messageId: message.id, taskId, observedProviderBusy: providerAlreadyBusy });
|
|
709
|
+
this.claims.startClaim("task", String(taskId), claimed.claimExpiresAt);
|
|
710
|
+
await this.updateTaskStatus(taskId, {
|
|
711
|
+
status: "in_progress",
|
|
712
|
+
agentId: this.agentId,
|
|
713
|
+
metadata: { messageId: message.id, completedBy: "runner" },
|
|
714
|
+
}).catch((error) => logger.error("task", `task ${taskId} in_progress update failed: ${error}`));
|
|
715
|
+
// Runner owns claim + status here; drop the server's self-claim instruction
|
|
716
|
+
// so the agent doesn't improvise a stray claim send (see stripRunnerClaimedGuidance).
|
|
717
|
+
toDeliver = { ...message, body: stripRunnerClaimedGuidance(message.body) };
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
deliverable.push(toDeliver);
|
|
721
|
+
}
|
|
722
|
+
if (deliverable.length === 0) {
|
|
723
|
+
this.delivering = false;
|
|
724
|
+
this.publishStatus();
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
this.publishStatus();
|
|
728
|
+
let failed = false;
|
|
729
|
+
try {
|
|
730
|
+
const prepared = await messagesWithCachedAttachments(deliverable, this.http, {
|
|
731
|
+
agentId: this.agentId,
|
|
732
|
+
onError: (message) => logger.error("runner", message),
|
|
733
|
+
});
|
|
734
|
+
await this.options.adapter.deliver(this.process, prepared);
|
|
735
|
+
for (const message of deliverable) {
|
|
736
|
+
await this.http.recordMessageDeliveryAttempt(message.id, { agentId: this.agentId, status: "delivered" }).catch(() => {});
|
|
737
|
+
await this.http.markRead(message.id, this.agentId).catch(() => {});
|
|
738
|
+
if (!taskIdFromMessage(message)) this.claims.finishClaim("message", String(message.id));
|
|
739
|
+
}
|
|
740
|
+
} catch (error) {
|
|
741
|
+
failed = true;
|
|
742
|
+
if (shouldLogDeliveryFailure(error)) logger.warn("delivery", `message delivery failed: ${error}`);
|
|
743
|
+
for (const message of deliverable) {
|
|
744
|
+
this.clearActiveClaim(message);
|
|
745
|
+
this.pendingMessages.set(message.id, message);
|
|
746
|
+
}
|
|
747
|
+
} finally {
|
|
748
|
+
this.delivering = false;
|
|
749
|
+
this.publishStatus();
|
|
750
|
+
if (this.pendingMessages.size > 0) this.scheduleDrain(failed ? 1000 : 0);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
private async handleCommand(type: string, params: Record<string, unknown>, commandId: string, command?: Record<string, unknown>): Promise<void> {
|
|
755
|
+
const target = typeof command?.target === "string" ? command.target : this.agentId;
|
|
756
|
+
if (target !== this.agentId && target !== this.options.runnerId) return;
|
|
757
|
+
if (type !== "agent.shutdown" && type !== "agent.restart" && type !== "agent.reconnect" && type !== "agent.kill" && type !== "agent.compact" && type !== "agent.clearContext" && type !== "agent.injectContext" && type !== "agent.permissionDecision" && type !== "agent.interrupt" && type !== "prompt.inject") return;
|
|
758
|
+
|
|
759
|
+
const exitAfterCommand = type === "agent.shutdown" || type === "agent.kill";
|
|
760
|
+
if (exitAfterCommand) this.exitCommandInProgress = true;
|
|
761
|
+
const nativeSelfResume = isNativeSelfResumeCompact(type, params)
|
|
762
|
+
? this.nativeSelfResume.begin(commandId, params)
|
|
763
|
+
: undefined;
|
|
764
|
+
this.claims.startClaim("command", commandId);
|
|
765
|
+
try {
|
|
766
|
+
await this.updateCommand(commandId, "accepted");
|
|
767
|
+
await this.updateCommand(commandId, "running");
|
|
768
|
+
// Pre-session-destroy seam (#183): for destructive transitions, run end-of-session
|
|
769
|
+
// work (Insights capture, #184) BEFORE the invasive operation, surfaced as a
|
|
770
|
+
// non-addressable "finalizing" state so the agent isn't mistaken for merely busy.
|
|
771
|
+
const destroyReason = boundaryReasonForCommand(type);
|
|
772
|
+
if (destroyReason) await this.runPreSessionDestroy(destroyReason);
|
|
773
|
+
// Move from the transient finalizing window to the executing teardown state (or drop
|
|
774
|
+
// it entirely for compact/clear, which complete promptly once capture is done).
|
|
775
|
+
if (exitAfterCommand) this.lifecycleAction = type === "agent.kill" ? "killing" : "shutting-down";
|
|
776
|
+
else if (type === "agent.restart") this.lifecycleAction = "restarting";
|
|
777
|
+
else this.lifecycleAction = undefined;
|
|
778
|
+
this.publishStatus();
|
|
779
|
+
let providerResult: Record<string, unknown> | void = undefined;
|
|
780
|
+
if (type === "agent.restart") await this.restartProvider();
|
|
781
|
+
else if (type === "agent.reconnect") this.publishStatus();
|
|
782
|
+
else if (type === "agent.compact") {
|
|
783
|
+
if (!this.options.adapter.compact || !this.process) throw new Error("provider does not support compact");
|
|
784
|
+
providerResult = await this.options.adapter.compact(this.process, {
|
|
785
|
+
instructions: typeof params.instructions === "string" ? params.instructions : undefined,
|
|
786
|
+
});
|
|
787
|
+
if (nativeSelfResume) {
|
|
788
|
+
const handoff = await nativeSelfResume.promise;
|
|
789
|
+
providerResult = { ...(providerResult ?? {}), nativeSelfResume: handoff };
|
|
790
|
+
}
|
|
791
|
+
} else if (type === "agent.clearContext") {
|
|
792
|
+
if (!this.options.adapter.clearContext || !this.process) throw new Error("provider does not support clearContext");
|
|
793
|
+
providerResult = await this.options.adapter.clearContext(this.process);
|
|
794
|
+
} else if (type === "agent.interrupt") {
|
|
795
|
+
if (!this.options.adapter.interrupt || !this.process) throw new Error("provider does not support interrupt");
|
|
796
|
+
this.sessionLog("interrupt requested from dashboard");
|
|
797
|
+
providerResult = await this.options.adapter.interrupt(this.process);
|
|
798
|
+
this.busyReconciler.scheduleInterruptReconcile();
|
|
799
|
+
} else if (type === "agent.injectContext") {
|
|
800
|
+
if (!this.process) throw new Error("provider process is unavailable");
|
|
801
|
+
providerResult = await this.injectContext(params);
|
|
802
|
+
} else if (type === "agent.permissionDecision") {
|
|
803
|
+
providerResult = await this.respondToPermissionDecision(params);
|
|
804
|
+
} else if (type === "prompt.inject") {
|
|
805
|
+
providerResult = await this.injectPrompt(params);
|
|
806
|
+
} else await this.shutdownProvider(type === "agent.kill", commandTimeoutMs(params));
|
|
807
|
+
await this.updateCommand(commandId, "succeeded", {
|
|
808
|
+
action: type,
|
|
809
|
+
agentId: this.agentId,
|
|
810
|
+
runnerId: this.options.runnerId,
|
|
811
|
+
policyName: this.options.policyName,
|
|
812
|
+
spawnRequestId: this.options.spawnRequestId,
|
|
813
|
+
reason: typeof params.reason === "string" ? params.reason : undefined,
|
|
814
|
+
...(providerResult ? { providerResult } : {}),
|
|
815
|
+
});
|
|
816
|
+
} catch (error) {
|
|
817
|
+
if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
|
|
818
|
+
await this.updateCommand(commandId, "failed", undefined, errMessage(error)).catch(() => {});
|
|
819
|
+
} finally {
|
|
820
|
+
this.claims.finishClaim("command", commandId);
|
|
821
|
+
if (exitAfterCommand) {
|
|
822
|
+
if (params.preserveRegistration === true) {
|
|
823
|
+
await this.http.setStatus(this.agentId, "offline", this.options.instanceId).catch(() => {});
|
|
824
|
+
} else {
|
|
825
|
+
await this.http.deleteAgent(this.agentId).catch(() => {});
|
|
826
|
+
}
|
|
827
|
+
if (this.options.exitProcessOnShutdown !== false) {
|
|
828
|
+
setTimeout(() => void this.stop().catch((error) => {
|
|
829
|
+
logger.error("lifecycle", `stop after command failed: ${error}`);
|
|
830
|
+
}).finally(() => process.exit(0)), 10);
|
|
831
|
+
}
|
|
832
|
+
} else if (!this.stopped) {
|
|
833
|
+
if (nativeSelfResume) this.nativeSelfResume.finish(nativeSelfResume);
|
|
834
|
+
this.lifecycleAction = undefined;
|
|
835
|
+
this.publishStatus();
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
private async injectContext(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
841
|
+
const content = typeof params.content === "string" ? params.content.trim() : "";
|
|
842
|
+
if (!content) throw new Error("content required");
|
|
843
|
+
const memoryIds = Array.isArray(params.memoryIds) ? params.memoryIds.filter((id): id is string => typeof id === "string") : [];
|
|
844
|
+
await this.options.adapter.deliver(this.process!, [{
|
|
845
|
+
id: 0,
|
|
846
|
+
from: "system",
|
|
847
|
+
to: this.agentId,
|
|
848
|
+
kind: "system",
|
|
849
|
+
subject: "Agent Relay memory context",
|
|
850
|
+
body: content,
|
|
851
|
+
payload: {
|
|
852
|
+
memoryInjection: true,
|
|
853
|
+
reason: typeof params.reason === "string" ? params.reason : undefined,
|
|
854
|
+
memoryIds,
|
|
855
|
+
},
|
|
856
|
+
readBy: [],
|
|
857
|
+
createdAt: Date.now(),
|
|
858
|
+
}]);
|
|
859
|
+
return { memoryIds, injectedMemoryCount: memoryIds.length };
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
private async respondToPermissionDecision(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
863
|
+
const approvalId = typeof params.approvalId === "string" ? params.approvalId : "";
|
|
864
|
+
const decision = typeof params.decision === "string" ? params.decision : "";
|
|
865
|
+
if (!approvalId) throw new Error("approvalId required");
|
|
866
|
+
if (decision !== "approve" && decision !== "approve-session" && decision !== "deny" && decision !== "abort" && decision !== "answer") {
|
|
867
|
+
throw new Error("decision must be approve, approve-session, deny, abort, or answer");
|
|
868
|
+
}
|
|
869
|
+
const answersRaw = params.answers;
|
|
870
|
+
const answers = answersRaw && typeof answersRaw === "object" && !Array.isArray(answersRaw)
|
|
871
|
+
? Object.fromEntries(
|
|
872
|
+
Object.entries(answersRaw as Record<string, unknown>).filter(([, v]) => typeof v === "string") as [string, string][],
|
|
873
|
+
)
|
|
874
|
+
: undefined;
|
|
875
|
+
if (decision === "answer" && (!answers || Object.keys(answers).length === 0)) {
|
|
876
|
+
throw new Error("answers required for answer decision");
|
|
877
|
+
}
|
|
878
|
+
const input: ProviderPermissionDecisionInput = {
|
|
879
|
+
approvalId,
|
|
880
|
+
decision: decision as ProviderPermissionDecision,
|
|
881
|
+
...(typeof params.reason === "string" ? { reason: params.reason } : {}),
|
|
882
|
+
...(answers ? { answers } : {}),
|
|
883
|
+
};
|
|
884
|
+
if (this.control?.resolvePermissionDecision(input)) {
|
|
885
|
+
return { approvalId, decision, provider: this.options.provider };
|
|
886
|
+
}
|
|
887
|
+
if (!this.process) throw new Error("provider process is unavailable");
|
|
888
|
+
if (!this.options.adapter.respondToPermissionDecision) throw new Error("provider does not support permission decisions");
|
|
889
|
+
const result = await this.options.adapter.respondToPermissionDecision(this.process, input);
|
|
890
|
+
return { approvalId, decision, ...(result ? { providerResult: result } : {}) };
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
private async injectPrompt(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
894
|
+
const body = typeof params.body === "string" ? params.body : "";
|
|
895
|
+
if (!body) throw new Error("body required");
|
|
896
|
+
if (!this.process) throw new Error("provider process is unavailable");
|
|
897
|
+
if (!this.options.adapter.deliverInitialPrompt) throw new Error("provider does not support prompt injection");
|
|
898
|
+
const messageId = typeof params.messageId === "number" ? params.messageId : undefined;
|
|
899
|
+
const attachments = Array.isArray(params.attachments)
|
|
900
|
+
? params.attachments.filter((item): item is Record<string, unknown> => item !== null && typeof item === "object" && !Array.isArray(item))
|
|
901
|
+
: [];
|
|
902
|
+
if (messageId) this.pendingPromptMessageId = messageId;
|
|
903
|
+
const prompt = await this.promptInjectionText(body, messageId, attachments);
|
|
904
|
+
// Mark so the matching UserPromptSubmit echo isn't double-posted: a chat-box
|
|
905
|
+
// prompt already created its own session message shown in the dashboard.
|
|
906
|
+
this.recordInjectedPrompt(prompt.trim());
|
|
907
|
+
await this.options.adapter.deliverInitialPrompt(this.process, prompt);
|
|
908
|
+
return { injected: true, messageId, attachmentCount: attachments.length };
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
private async promptInjectionText(body: string, messageId: number | undefined, attachments: Record<string, unknown>[]): Promise<string> {
|
|
912
|
+
if (!attachments.length) return body;
|
|
913
|
+
const messages = await messagesWithCachedAttachments([{
|
|
914
|
+
id: messageId ?? 0,
|
|
915
|
+
from: "user",
|
|
916
|
+
to: this.agentId,
|
|
917
|
+
kind: "session",
|
|
918
|
+
body,
|
|
919
|
+
payload: { attachments },
|
|
920
|
+
readBy: [],
|
|
921
|
+
createdAt: Date.now(),
|
|
922
|
+
}], this.http, {
|
|
923
|
+
agentId: this.agentId,
|
|
924
|
+
onError: (message) => logger.warn("attachments", message),
|
|
925
|
+
});
|
|
926
|
+
const attachmentText = providerAttachmentText(messages[0]!);
|
|
927
|
+
return attachmentText ? `${body}\n\n${attachmentText}` : body;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
private async restartProvider(opts: { resumeId?: string; suppressPrompt?: boolean } = {}): Promise<void> {
|
|
931
|
+
this.restartInProgress = true;
|
|
932
|
+
try {
|
|
933
|
+
if (this.process) {
|
|
934
|
+
await this.options.adapter.shutdown(this.process, {
|
|
935
|
+
graceful: true,
|
|
936
|
+
timeoutMs: this.options.providerConfig.headless.shutdownTimeoutMs,
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
this.claims.clearTerminalStatus();
|
|
940
|
+
this.claims.clearWorkKind("provider-turn");
|
|
941
|
+
this.claims.clearWorkKind("subagent");
|
|
942
|
+
this.claims.clearWorkKind("background-script");
|
|
943
|
+
if (this.stopped) return;
|
|
944
|
+
this.process = await this.spawnProvider(opts);
|
|
945
|
+
this.processStartedAt = Date.now();
|
|
946
|
+
} finally {
|
|
947
|
+
this.restartInProgress = false;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
private shouldRestartUnexpectedProviderExit(status: SemanticStatus): boolean {
|
|
952
|
+
return runnerShouldRestartUnexpectedProviderExit(status, {
|
|
953
|
+
exitCommandInProgress: this.exitCommandInProgress,
|
|
954
|
+
stopped: this.stopped,
|
|
955
|
+
restartInProgress: this.restartInProgress,
|
|
956
|
+
provider: this.options.provider,
|
|
957
|
+
headless: this.options.headless,
|
|
958
|
+
hasTerminalSession: typeof this.process?.meta?.tmuxSession === "string"
|
|
959
|
+
|| (this.options.adapter.supportsUnexpectedExitRestart?.() ?? false),
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
private async restartUnexpectedProviderExit(status: SemanticStatus): Promise<void> {
|
|
964
|
+
if (this.restartPending) return;
|
|
965
|
+
this.restartPending = true;
|
|
966
|
+
try {
|
|
967
|
+
// Best-effort Insights capture for the segment that just ended in a crash (#183). This
|
|
968
|
+
// path has no controlled teardown, so without it crashed sessions silently drop their
|
|
969
|
+
// context-ratio datapoint. The process handle is still set (cleared later), so the
|
|
970
|
+
// Claude transcript is readable; the runner stays alive here (restart or offline), so the
|
|
971
|
+
// durable outbox drains normally — no flush needed.
|
|
972
|
+
await Promise.race([
|
|
973
|
+
this.captureContextRatio("crash"),
|
|
974
|
+
new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
|
|
975
|
+
]).catch((error) => this.sessionLog(`insights: crash capture failed: ${errMessage(error)}`));
|
|
976
|
+
|
|
977
|
+
const now = Date.now();
|
|
978
|
+
const runtimeMs = this.processStartedAt ? now - this.processStartedAt : Number.POSITIVE_INFINITY;
|
|
979
|
+
const recent = this.unexpectedExitTimes.filter((time) => now - time <= UNEXPECTED_EXIT_WINDOW_MS);
|
|
980
|
+
recent.push(now);
|
|
981
|
+
this.unexpectedExitTimes.splice(0, this.unexpectedExitTimes.length, ...recent);
|
|
982
|
+
const diagnostics = providerExitDiagnostics({ status, runtimeMs, processMeta: this.process?.meta, hasProcess: Boolean(this.process?.process), exitCommandInProgress: this.exitCommandInProgress, stopped: this.stopped, restartInProgress: this.restartInProgress, restartPending: this.restartPending, headless: this.options.headless, provider: this.options.provider, logFile: runnerLogFileFromEnv() });
|
|
983
|
+
|
|
984
|
+
this.publishRunnerTimelineEvent({
|
|
985
|
+
status: "provider.exit_detected",
|
|
986
|
+
id: `provider-exit-${this.providerSessionId}-${now}`,
|
|
987
|
+
timestamp: now,
|
|
988
|
+
title: "Provider exited",
|
|
989
|
+
body: `${this.options.provider} reported ${status} after ${Math.round(runtimeMs / 1000)}s`,
|
|
990
|
+
icon: "ti-plug-off",
|
|
991
|
+
metadata: {
|
|
992
|
+
eventType: "provider.exit_detected",
|
|
993
|
+
...diagnostics,
|
|
994
|
+
},
|
|
995
|
+
});
|
|
996
|
+
|
|
997
|
+
const nativeSelfResume = this.nativeSelfResume.active;
|
|
998
|
+
if (nativeSelfResume && getManifest(this.options.provider)?.resume?.supported) {
|
|
999
|
+
await recoverNativeSelfResumeExit({ handoff: nativeSelfResume, tracker: this.nativeSelfResume, providerSessionId: this.providerSessionId, status, diagnostics, now, publishTimeline: (event) => this.publishRunnerTimelineEvent(event), setTerminalProviderExit: (marker) => { this.terminalProviderExit = marker; }, setProviderStatus: (update) => this.setProviderStatus(update), restartProvider: (resumeId) => this.restartProvider(resumeId ? { resumeId } : { suppressPrompt: true }), publishStatus: () => this.publishStatus(), scheduleDrain: () => this.scheduleDrain() });
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
if (this.shouldStopUnexpectedProviderExit(diagnostics)) {
|
|
1004
|
+
const hasResumeId = typeof diagnostics.claudeResumeId === "string" && diagnostics.claudeResumeId.length > 0;
|
|
1005
|
+
if (!hasResumeId) {
|
|
1006
|
+
const recoveredStatus = await this.recoveredStatusAfterUndeterminedProviderExit();
|
|
1007
|
+
if (recoveredStatus) {
|
|
1008
|
+
logger.warn("lifecycle", `${this.options.provider} exit signal recovered as ${recoveredStatus}; suppressing terminal exit`);
|
|
1009
|
+
this.publishRunnerTimelineEvent({
|
|
1010
|
+
status: "provider.restart_decision",
|
|
1011
|
+
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
1012
|
+
timestamp: Date.now(),
|
|
1013
|
+
title: "Provider exit suppressed",
|
|
1014
|
+
body: `${this.options.provider} reported ${recoveredStatus} during exit confirmation; runner kept the provider live.`,
|
|
1015
|
+
icon: "ti-player-play",
|
|
1016
|
+
metadata: {
|
|
1017
|
+
eventType: "provider.restart_decision",
|
|
1018
|
+
decision: "suppress-recovered",
|
|
1019
|
+
reason: "undetermined-provider-exit-recovered",
|
|
1020
|
+
recoveredStatus,
|
|
1021
|
+
confirmWindowMs: UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS,
|
|
1022
|
+
...diagnostics,
|
|
1023
|
+
},
|
|
1024
|
+
});
|
|
1025
|
+
this.setProviderStatus(recoveredStatus);
|
|
1026
|
+
return;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
if (!hasResumeId && this.shouldRestartRecoverableProviderExit(diagnostics)) {
|
|
1030
|
+
logger.warn("lifecycle", `${this.options.provider} exit signal is early/recoverable; allowing fresh restart`);
|
|
1031
|
+
} else {
|
|
1032
|
+
logger.warn("lifecycle", `${this.options.provider} exited; leaving agent offline for manual recovery`);
|
|
1033
|
+
this.publishRunnerTimelineEvent({
|
|
1034
|
+
status: "provider.restart_decision",
|
|
1035
|
+
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
1036
|
+
timestamp: Date.now(),
|
|
1037
|
+
title: "Provider restart skipped",
|
|
1038
|
+
body: hasResumeId
|
|
1039
|
+
? "Claude exited; runner will not auto-resume. Resume id captured for manual recovery."
|
|
1040
|
+
: "Claude exited; runner will not restart automatically.",
|
|
1041
|
+
icon: "ti-player-stop",
|
|
1042
|
+
metadata: {
|
|
1043
|
+
eventType: "provider.restart_decision",
|
|
1044
|
+
decision: "stop-surface",
|
|
1045
|
+
reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
|
|
1046
|
+
...diagnostics,
|
|
1047
|
+
},
|
|
1048
|
+
});
|
|
1049
|
+
this.process = undefined;
|
|
1050
|
+
// #633: the runner stays alive (manual-resume hold), so neither the orchestrator tmux-gone
|
|
1051
|
+
// path nor the heartbeat reaper will ever fire. Stamp the terminal marker so the SAME offline
|
|
1052
|
+
// status frame (published by setProviderStatus below) carries it; the server's bus status
|
|
1053
|
+
// handler turns that edge into a terminal `agent.exited` event with a #636 diagnosis.
|
|
1054
|
+
this.terminalProviderExit = {
|
|
1055
|
+
reason: hasResumeId ? "claude-exit-manual-resume-available" : "claude-exit-manual-intervention-required",
|
|
1056
|
+
...diagnostics,
|
|
1057
|
+
};
|
|
1058
|
+
this.setProviderStatus({
|
|
1059
|
+
status,
|
|
1060
|
+
reason: "provider-turn",
|
|
1061
|
+
id: `provider-exit-${this.providerSessionId}`,
|
|
1062
|
+
clear: ["provider-turn", "subagent", "background-script"],
|
|
1063
|
+
});
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
if (runtimeMs < RAPID_EXIT_MS && recent.length > MAX_RAPID_UNEXPECTED_EXITS) {
|
|
1069
|
+
logger.error("lifecycle", `provider session exited ${recent.length} times within ${Math.round(UNEXPECTED_EXIT_WINDOW_MS / 1000)}s; giving up`);
|
|
1070
|
+
this.publishRunnerTimelineEvent({
|
|
1071
|
+
status: "provider.restart_decision",
|
|
1072
|
+
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
1073
|
+
timestamp: Date.now(),
|
|
1074
|
+
title: "Provider restart skipped",
|
|
1075
|
+
body: `rapid unexpected exits exceeded ${MAX_RAPID_UNEXPECTED_EXITS}`,
|
|
1076
|
+
icon: "ti-alert-triangle",
|
|
1077
|
+
metadata: {
|
|
1078
|
+
eventType: "provider.restart_decision",
|
|
1079
|
+
decision: "give-up",
|
|
1080
|
+
reason: "rapid-unexpected-provider-exits",
|
|
1081
|
+
rapidExitCount: recent.length,
|
|
1082
|
+
rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
|
|
1083
|
+
maxRapidUnexpectedExits: MAX_RAPID_UNEXPECTED_EXITS,
|
|
1084
|
+
...diagnostics,
|
|
1085
|
+
},
|
|
1086
|
+
});
|
|
1087
|
+
this.setProviderStatus(status);
|
|
1088
|
+
this.options.onProviderExit?.(0);
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
const delayMs = Math.min(10_000, Math.max(500, 500 * recent.length));
|
|
1093
|
+
logger.warn("lifecycle", `provider session exited unexpectedly after ${Math.round(runtimeMs / 1000)}s; restarting in ${delayMs}ms`);
|
|
1094
|
+
this.publishRunnerTimelineEvent({
|
|
1095
|
+
status: "provider.restart_decision",
|
|
1096
|
+
id: `provider-restart-decision-${this.providerSessionId}-${now}`,
|
|
1097
|
+
timestamp: Date.now(),
|
|
1098
|
+
title: "Provider restart scheduled",
|
|
1099
|
+
body: `runner will start a fresh ${this.options.provider} provider in ${delayMs}ms`,
|
|
1100
|
+
icon: "ti-refresh",
|
|
1101
|
+
metadata: {
|
|
1102
|
+
eventType: "provider.restart_decision",
|
|
1103
|
+
decision: "restart-fresh",
|
|
1104
|
+
reason: "unexpected-headless-terminal-exit",
|
|
1105
|
+
delayMs,
|
|
1106
|
+
rapidExitCount: recent.length,
|
|
1107
|
+
rapidExitWindowMs: UNEXPECTED_EXIT_WINDOW_MS,
|
|
1108
|
+
...diagnostics,
|
|
1109
|
+
},
|
|
1110
|
+
});
|
|
1111
|
+
await Bun.sleep(delayMs);
|
|
1112
|
+
if (this.stopped || this.exitCommandInProgress) return;
|
|
1113
|
+
try {
|
|
1114
|
+
await this.restartProvider();
|
|
1115
|
+
this.publishStatus();
|
|
1116
|
+
this.scheduleDrain();
|
|
1117
|
+
} catch (error) {
|
|
1118
|
+
logger.error("lifecycle", `provider restart after unexpected exit failed: ${error}`);
|
|
1119
|
+
this.setProviderStatus("error");
|
|
1120
|
+
this.options.onProviderExit?.(1);
|
|
1121
|
+
}
|
|
1122
|
+
} finally {
|
|
1123
|
+
this.restartPending = false;
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
private shouldStopUnexpectedProviderExit(diagnostics: Record<string, unknown>): boolean {
|
|
1128
|
+
return getManifest(this.options.provider)?.launch?.managedPolicyPrompt?.alwaysOn === "append-system-prompt" && diagnostics.exitCommandInProgress !== true;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
private shouldRestartRecoverableProviderExit(diagnostics: Record<string, unknown>): boolean {
|
|
1132
|
+
return diagnostics.exitSource === "tmux-session-ended"
|
|
1133
|
+
&& typeof diagnostics.runtimeMs === "number"
|
|
1134
|
+
&& diagnostics.runtimeMs >= 0
|
|
1135
|
+
&& diagnostics.runtimeMs < RAPID_EXIT_MS;
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
private async recoveredStatusAfterUndeterminedProviderExit(): Promise<"busy" | "idle" | undefined> {
|
|
1139
|
+
if (!this.process || !this.options.adapter.probeActivity) return undefined;
|
|
1140
|
+
const deadline = Date.now() + UNDETERMINED_PROVIDER_EXIT_CONFIRM_MS;
|
|
1141
|
+
do {
|
|
1142
|
+
const activity = await this.options.adapter.probeActivity(this.process).catch((error) => {
|
|
1143
|
+
this.sessionLog(`provider-exit confirmation probe failed: ${errMessage(error)}`);
|
|
1144
|
+
return "unknown" as const;
|
|
1145
|
+
});
|
|
1146
|
+
if (activity === "busy" || activity === "idle") return activity;
|
|
1147
|
+
if (Date.now() >= deadline) return undefined;
|
|
1148
|
+
await Bun.sleep(PROVIDER_EXIT_CONFIRM_POLL_MS);
|
|
1149
|
+
} while (!this.stopped && !this.exitCommandInProgress);
|
|
1150
|
+
return undefined;
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
private async shutdownProvider(hard: boolean, timeoutMs = this.options.providerConfig.headless.shutdownTimeoutMs): Promise<void> {
|
|
1154
|
+
this.lifecycleAction = hard ? "killing" : "shutting-down";
|
|
1155
|
+
this.publishStatus();
|
|
1156
|
+
await this.bus.statusAsync({ agentStatus: "idle", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
|
|
1157
|
+
if (this.process) {
|
|
1158
|
+
await this.options.adapter.shutdown(this.process, {
|
|
1159
|
+
graceful: !hard,
|
|
1160
|
+
timeoutMs,
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
this.claims.setTerminalStatus("offline");
|
|
1164
|
+
this.publishStatus();
|
|
1165
|
+
this.stopped = true;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
private publishRunnerTimelineEvent(event: RunnerTimelineEvent): void {
|
|
1169
|
+
this.pendingTimelineEvent = {
|
|
1170
|
+
...event,
|
|
1171
|
+
metadata: {
|
|
1172
|
+
source: "runner",
|
|
1173
|
+
provider: this.options.provider,
|
|
1174
|
+
runnerId: this.options.runnerId,
|
|
1175
|
+
agentId: this.agentId,
|
|
1176
|
+
policyName: this.options.policyName ?? null,
|
|
1177
|
+
spawnRequestId: this.options.spawnRequestId ?? null,
|
|
1178
|
+
label: this.options.label ?? null,
|
|
1179
|
+
providerSessionId: this.providerSessionId,
|
|
1180
|
+
...(event.metadata ?? {}),
|
|
1181
|
+
},
|
|
1182
|
+
};
|
|
1183
|
+
this.publishStatus();
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
private async updateCommand(commandId: string, status: string, result?: Record<string, unknown>, error?: string): Promise<void> {
|
|
1187
|
+
await this.bus.updateCommand(commandId, { status, ...(result ? { result } : {}), ...(error ? { error } : {}) });
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
private handleBusError(code: string, message: string): void {
|
|
1191
|
+
const action = runnerBusErrorAction(code, this.stopped);
|
|
1192
|
+
if (action === "ignore") return;
|
|
1193
|
+
logger.error("bus", `bus error ${code}: ${message}`);
|
|
1194
|
+
if (action === "stop") {
|
|
1195
|
+
void this.stop().catch((error) => {
|
|
1196
|
+
logger.error("bus", `stop after bus error failed: ${error}`);
|
|
1197
|
+
}).finally(() => process.exit(0));
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
private setProviderStatus(update: ProviderStatusUpdate): void {
|
|
1202
|
+
const status = typeof update === "string" ? update : update.status;
|
|
1203
|
+
this.lastProviderEventAt = Date.now();
|
|
1204
|
+
const providerSessionId = typeof update === "string" ? undefined : update.providerSessionId;
|
|
1205
|
+
if (providerSessionId && providerSessionId !== this.providerSessionId) return;
|
|
1206
|
+
const reason = typeof update === "string" ? "provider-turn" : update.reason ?? "provider-turn";
|
|
1207
|
+
const id = typeof update === "string" ? reason : update.id ?? reason;
|
|
1208
|
+
if (typeof update !== "string" && update.timeline) {
|
|
1209
|
+
this.pendingTimelineEvent = {
|
|
1210
|
+
status: update.timeline.status,
|
|
1211
|
+
...(update.timeline.id ? { id: update.timeline.id } : {}),
|
|
1212
|
+
timestamp: update.timeline.timestamp ?? Date.now(),
|
|
1213
|
+
...(update.timeline.title ? { title: update.timeline.title } : {}),
|
|
1214
|
+
...(update.timeline.body ? { body: update.timeline.body } : {}),
|
|
1215
|
+
...(update.timeline.icon ? { icon: update.timeline.icon } : {}),
|
|
1216
|
+
...(update.timeline.metadata ? { metadata: update.timeline.metadata } : {}),
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
let deferredRateLimitHold = false;
|
|
1220
|
+
if (typeof update !== "string" && update.providerState) {
|
|
1221
|
+
const ps = update.providerState;
|
|
1222
|
+
// A rate-limit hold persists across the idle the turn ends on; any other
|
|
1223
|
+
// providerState supersedes it (clears a stale hold).
|
|
1224
|
+
if (ps.state === "blocked" && ps.reason === "rate_limit") {
|
|
1225
|
+
const paneRateLimitHold = typeof ps.source === "string" && ps.source.startsWith("claude-pane");
|
|
1226
|
+
if (status === "idle" && reason === "provider-turn" && paneRateLimitHold && this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
|
|
1227
|
+
this.pendingRateLimitHold = update.providerState;
|
|
1228
|
+
this.providerBlocked = false;
|
|
1229
|
+
deferredRateLimitHold = true;
|
|
1230
|
+
this.sessionLog(`rate-limit hold deferred until active turn finishes (turn ${this.currentTurnId ?? "?"})`);
|
|
1231
|
+
} else {
|
|
1232
|
+
this.providerBlocked = true;
|
|
1233
|
+
this.pendingRateLimitHold = undefined;
|
|
1234
|
+
const fresh = !this.rateLimitHold;
|
|
1235
|
+
this.rateLimitHold = update.providerState;
|
|
1236
|
+
if (fresh) this.publishRateLimitNotice(update.providerState);
|
|
1237
|
+
}
|
|
1238
|
+
} else {
|
|
1239
|
+
this.providerBlocked = ps.state === "blocked";
|
|
1240
|
+
this.pendingRateLimitHold = undefined;
|
|
1241
|
+
this.rateLimitHold = undefined;
|
|
1242
|
+
}
|
|
1243
|
+
} else if (status === "idle") {
|
|
1244
|
+
this.providerBlocked = false;
|
|
1245
|
+
}
|
|
1246
|
+
// Forward progress (a real turn) lifts the hold so the blocked badge clears.
|
|
1247
|
+
if (status === "busy") this.rateLimitHold = undefined;
|
|
1248
|
+
if (deferredRateLimitHold) {
|
|
1249
|
+
if (typeof update !== "string") {
|
|
1250
|
+
for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
|
|
1251
|
+
}
|
|
1252
|
+
this.publishStatus();
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
if (typeof update !== "string" && status === "error") {
|
|
1256
|
+
const terminalReason = typeof update.metadata?.terminalFailureReason === "string"
|
|
1257
|
+
? update.metadata.terminalFailureReason
|
|
1258
|
+
: typeof update.providerState?.reason === "string"
|
|
1259
|
+
? update.providerState.reason
|
|
1260
|
+
: "provider-error";
|
|
1261
|
+
const terminalMessage = typeof update.metadata?.terminalFailureMessage === "string"
|
|
1262
|
+
? update.metadata.terminalFailureMessage
|
|
1263
|
+
: typeof update.providerState?.message === "string"
|
|
1264
|
+
? update.providerState.message
|
|
1265
|
+
: "Provider reported an unrecoverable error.";
|
|
1266
|
+
this.terminalFailure = {
|
|
1267
|
+
reason: terminalReason,
|
|
1268
|
+
message: terminalMessage,
|
|
1269
|
+
...(update.providerState ? { providerState: update.providerState } : {}),
|
|
1270
|
+
};
|
|
1271
|
+
} else if (status === "idle" || status === "busy") {
|
|
1272
|
+
this.terminalFailure = undefined;
|
|
1273
|
+
}
|
|
1274
|
+
// Compaction lifecycle (Claude PreCompact→`compacting` busy / PostCompact→`compacted`
|
|
1275
|
+
// idle). A `compacted` idle for a turn that predated compaction is a hook artifact — the
|
|
1276
|
+
// turn resumes — so it must NOT end the turn. Discriminate on whether a turn was running,
|
|
1277
|
+
// captured BEFORE the busy logic below mints a currentTurnId (it does for /compact at idle).
|
|
1278
|
+
const timelineStatus = typeof update !== "string" ? update.timeline?.status : undefined;
|
|
1279
|
+
if (timelineStatus === "compacting") {
|
|
1280
|
+
this.compactionMidTurn = this.currentTurnId !== undefined;
|
|
1281
|
+
} else if (timelineStatus === "compacted") {
|
|
1282
|
+
this.nativeSelfResume.noteCompacted(typeof update !== "string" && typeof update.timeline?.timestamp === "number" ? update.timeline.timestamp : Date.now());
|
|
1283
|
+
this.publishCompactionNotice();
|
|
1284
|
+
if (this.compactionMidTurn) {
|
|
1285
|
+
// Keep the turn (and its live reasoning tail) alive; the genuine Stop hook ends it.
|
|
1286
|
+
// pendingTimelineEvent (the marker) was set above and is consumed by publishStatus.
|
|
1287
|
+
this.sessionLog(`compaction completed mid-turn (turn ${this.currentTurnId ?? "?"}) — staying busy`);
|
|
1288
|
+
this.publishStatus();
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
if (status === "busy" && reason === "provider-turn") {
|
|
1293
|
+
if (!this.currentTurnId) {
|
|
1294
|
+
this.currentTurnId = typeof update !== "string" && update.id ? update.id : crypto.randomUUID();
|
|
1295
|
+
this.currentTurnStartedAt = Date.now();
|
|
1296
|
+
this.compactionMidTurn = false;
|
|
1297
|
+
this.sessionLog(`turn started (turn ${this.currentTurnId})`);
|
|
1298
|
+
}
|
|
1299
|
+
this.busyReconciler.arm();
|
|
1300
|
+
} else if (status === "idle" && reason === "provider-turn") {
|
|
1301
|
+
if (this.currentTurnId) { this.completedProviderTurns += 1; this.sessionLog(`turn ended via provider idle (turn ${this.currentTurnId})`); }
|
|
1302
|
+
this.currentTurnId = undefined;
|
|
1303
|
+
this.currentTurnStartedAt = undefined;
|
|
1304
|
+
this.compactionMidTurn = false;
|
|
1305
|
+
this.busyReconciler.disarm();
|
|
1306
|
+
this.stopReasoningTail();
|
|
1307
|
+
// Providers with append-system-prompt policy use lifecycle hooks for obligation handling;
|
|
1308
|
+
// others need the runner to re-inject pending obligations on idle.
|
|
1309
|
+
if (getManifest(this.options.provider)?.launch?.managedPolicyPrompt?.alwaysOn !== "append-system-prompt") void this.reInjectPendingObligation();
|
|
1310
|
+
}
|
|
1311
|
+
// A genuine restart back to forward progress clears the terminal-exit marker so a later offline
|
|
1312
|
+
// doesn't re-announce a stale provider exit.
|
|
1313
|
+
if (status === "busy" || status === "idle") this.terminalProviderExit = undefined;
|
|
1314
|
+
if (status === "busy") {
|
|
1315
|
+
this.claims.clearTerminalStatus();
|
|
1316
|
+
// #650: liveness-track the pids a reporter explicitly supplied; otherwise, for a
|
|
1317
|
+
// provider-turn, fall back to the managed provider process itself. If that process
|
|
1318
|
+
// dies out-of-band with no exit event reaching the runner (crash/orphan/external
|
|
1319
|
+
// kill), the reconcile clears the stuck turn → idle. A dead pid means the turn is
|
|
1320
|
+
// definitively over, so this can never falsely clear a live turn (#653's hazard).
|
|
1321
|
+
const reportedPids = typeof update === "string" ? undefined : update.pids;
|
|
1322
|
+
const livenessPids = reportedPids?.length
|
|
1323
|
+
? reportedPids
|
|
1324
|
+
: reason === "provider-turn" && typeof this.process?.pid === "number" && this.process.pid > 0
|
|
1325
|
+
? [this.process.pid]
|
|
1326
|
+
: undefined;
|
|
1327
|
+
this.claims.startWork(reason, id, typeof update === "string" ? (livenessPids ? { pids: livenessPids } : {}) : {
|
|
1328
|
+
label: update.label,
|
|
1329
|
+
role: update.role,
|
|
1330
|
+
parentId: update.parentId,
|
|
1331
|
+
...(livenessPids ? { pids: livenessPids } : {}),
|
|
1332
|
+
metadata: {
|
|
1333
|
+
...(update.metadata ?? {}),
|
|
1334
|
+
...(update.providerState ? { providerState: update.providerState } : {}),
|
|
1335
|
+
},
|
|
1336
|
+
});
|
|
1337
|
+
// A busy report with liveness-trackable pids arms the periodic reconcile (self-disarms
|
|
1338
|
+
// once no pid-bearing work remains, so idle/untracked agents pay nothing).
|
|
1339
|
+
if (livenessPids?.length) this.armBusyLivenessReconcile();
|
|
1340
|
+
if (reason === "provider-turn") {
|
|
1341
|
+
for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
|
|
1342
|
+
}
|
|
1343
|
+
} else if (status === "idle") {
|
|
1344
|
+
this.claims.clearTerminalStatus();
|
|
1345
|
+
this.claims.finishWork(reason, id);
|
|
1346
|
+
if (reason === "provider-turn") void this.completeObservedTaskClaims();
|
|
1347
|
+
}
|
|
1348
|
+
else if (status === "offline" || status === "error") this.claims.setTerminalStatus(status);
|
|
1349
|
+
if (typeof update !== "string") {
|
|
1350
|
+
for (const kind of update.clear ?? []) this.claims.clearWorkKind(kind);
|
|
1351
|
+
}
|
|
1352
|
+
// #329: the provider just went idle (ready). If a first initial-prompt delivery timed out on
|
|
1353
|
+
// a cold-start TUI, re-attempt now — the ready wait should resolve almost immediately.
|
|
1354
|
+
if (status === "idle" && this.pendingInitialPrompt && !this.deliveringInitialPrompt) {
|
|
1355
|
+
void this.attemptInitialPromptDelivery(this.pendingInitialPrompt, INITIAL_PROMPT_RETRY_READY_TIMEOUT_MS);
|
|
1356
|
+
}
|
|
1357
|
+
if (status === "idle" && this.pendingRateLimitHold && !this.claims.activeWork().some((work) => work.kind === "provider-turn")) {
|
|
1358
|
+
const hold = this.pendingRateLimitHold;
|
|
1359
|
+
this.pendingRateLimitHold = undefined;
|
|
1360
|
+
this.providerBlocked = true;
|
|
1361
|
+
const fresh = !this.rateLimitHold;
|
|
1362
|
+
this.rateLimitHold = hold;
|
|
1363
|
+
if (fresh) this.publishRateLimitNotice(hold);
|
|
1364
|
+
}
|
|
1365
|
+
if (status === "idle" && this.nativeSelfResume.active) {
|
|
1366
|
+
this.nativeSelfResume.resolve(this.nativeSelfResume.active, { resumed: true, via: "provider-idle" });
|
|
1367
|
+
}
|
|
1368
|
+
this.publishStatus();
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
// #417: when a non-claude provider finishes a turn with a pending spawner obligation
|
|
1372
|
+
// still unresolved, re-inject the obligation as a synthetic message so providerMessageText
|
|
1373
|
+
// surfaces the specific `agent-relay /reply <id>` command. Claude handles this via the
|
|
1374
|
+
// stop.sh hook; every other provider needs this in-process nudge.
|
|
1375
|
+
private async reInjectPendingObligation(): Promise<void> {
|
|
1376
|
+
if (!this.process) return;
|
|
1377
|
+
// Filter to spawner obligations only — user obligations go through the existing codex
|
|
1378
|
+
// prompt-capture path (publishProviderSessionEvent) and the session mirror (#418), never
|
|
1379
|
+
// an explicit /reply. Shared predicate so this rule stays in lockstep with the Claude
|
|
1380
|
+
// Stop-hook nag exemption (control-server.replyObligationStopDecision).
|
|
1381
|
+
const obligations = this.obligationCache.get().filter(obligationRequiresExplicitReply);
|
|
1382
|
+
if (!obligations.length) return;
|
|
1383
|
+
const MAX_REINJECTIONS = 3;
|
|
1384
|
+
for (const obligation of obligations) {
|
|
1385
|
+
const count = this.pendingObligationInjections.get(obligation.messageId) ?? 0;
|
|
1386
|
+
if (count >= MAX_REINJECTIONS) continue;
|
|
1387
|
+
this.pendingObligationInjections.set(obligation.messageId, count + 1);
|
|
1388
|
+
this.sessionLog(`re-injecting pending reply obligation #${obligation.messageId} from ${obligation.from} (attempt ${count + 1}/${MAX_REINJECTIONS})`);
|
|
1389
|
+
const replayMessage = {
|
|
1390
|
+
id: obligation.messageId,
|
|
1391
|
+
from: obligation.from,
|
|
1392
|
+
to: this.agentId,
|
|
1393
|
+
kind: obligation.kind,
|
|
1394
|
+
...(obligation.subject ? { subject: obligation.subject } : {}),
|
|
1395
|
+
body: obligation.bodyPreview,
|
|
1396
|
+
payload: {},
|
|
1397
|
+
readBy: [] as string[],
|
|
1398
|
+
createdAt: obligation.createdAt,
|
|
1399
|
+
};
|
|
1400
|
+
// Suppress the userMessage echo that providers (e.g. Codex) reflect back when
|
|
1401
|
+
// a turn is started with relay-formatted content — the echo body is exactly what
|
|
1402
|
+
// providerMessageText produces, which starts with "[relay message #" and is caught
|
|
1403
|
+
// by the RELAY_INJECTION_MARKERS prefix check, but record it explicitly as
|
|
1404
|
+
// defense-in-depth for adapters that might echo only the unformatted body text.
|
|
1405
|
+
this.recordInjectedPrompt(providerMessageText([replayMessage]));
|
|
1406
|
+
await this.options.adapter.deliver(this.process, [replayMessage]);
|
|
1407
|
+
// Kick off a cache refresh so a cleared obligation (agent replied) is detected
|
|
1408
|
+
// before the next idle fires — without this, the stale snapshot re-injects up to
|
|
1409
|
+
// MAX_REINJECTIONS times even after the agent has already /reply-ed.
|
|
1410
|
+
this.obligationCache.markDirty();
|
|
1411
|
+
// One obligation at a time — let the provider reply before nudging again.
|
|
1412
|
+
break;
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
// Session-mirror lane: capture the assistant turn from the Claude transcript and
|
|
1417
|
+
// post it as a "session" message so it shows in the dashboard chat with zero
|
|
1418
|
+
// agent tokens. Capture is UNCONDITIONAL — it no longer depends on a triggering
|
|
1419
|
+
// relay message existing, so turns started from the web terminal (which create
|
|
1420
|
+
// no relay message) are mirrored too. A reply obligation, when present, is still
|
|
1421
|
+
// used as replyTo so the Stop hook stops nagging the agent to /reply.
|
|
1422
|
+
//
|
|
1423
|
+
// isPreFlush: true (#435/#499) — mid-turn PreToolUse boundary. Drain the live
|
|
1424
|
+
// tail and force-flush session messages before showing the blocking control;
|
|
1425
|
+
// the entry cursor lets Stop capture skip already-emitted text.
|
|
1426
|
+
private async publishSessionTurn(input: { transcriptPath: string; lastAssistantMessage?: unknown; isPreFlush?: boolean }): Promise<void> {
|
|
1427
|
+
if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
|
|
1428
|
+
|
|
1429
|
+
if (input.isPreFlush) {
|
|
1430
|
+
if (!input.transcriptPath) return;
|
|
1431
|
+
let jsonl: string | undefined;
|
|
1432
|
+
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { return; }
|
|
1433
|
+
if (!jsonl) return;
|
|
1434
|
+
const body = extractLastAssistantTurnAfterEntry(jsonl, this.narrativeFlushEntryCount);
|
|
1435
|
+
await this.drainReasoningTail();
|
|
1436
|
+
this.narrativeFlushEntryCount = countTranscriptEntries(jsonl);
|
|
1437
|
+
if (body && !this.preFlushBodyAlreadyMirroredByReasoningTail(jsonl, body)) {
|
|
1438
|
+
const turnId = this.currentTurnId;
|
|
1439
|
+
this.sessionLog(`pre-flush narrative for turn ${turnId ?? "?"} (${body.length} chars)`);
|
|
1440
|
+
await this.publishSessionEvent({ from: this.agentId, to: "user", body, session: { type: "response", origin: "provider", ...(turnId ? { turnId } : {}) } });
|
|
1441
|
+
}
|
|
1442
|
+
await this.sessionOutbox.flush(2_000).then((ok) => { if (!ok) this.sessionLog(`pre-flush session outbox incomplete before blocking control (${this.sessionOutbox.pendingCount()} pending)`); }, (error) => this.sessionLog(`pre-flush session outbox failed before blocking control: ${errMessage(error)}`));
|
|
1443
|
+
return;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
const turnId = this.currentTurnId;
|
|
1447
|
+
this.stopReasoningTail();
|
|
1448
|
+
// Optional correlation for threading + obligation clearing — never a capture gate.
|
|
1449
|
+
let replyToMessageId: number | undefined, replyTarget = "user";
|
|
1450
|
+
const pendingPrompt = this.pendingPromptMessageId;
|
|
1451
|
+
if (pendingPrompt) {
|
|
1452
|
+
replyToMessageId = pendingPrompt;
|
|
1453
|
+
this.pendingPromptMessageId = undefined;
|
|
1454
|
+
} else {
|
|
1455
|
+
// Correlation-only (threading + obligation clearing) — the local snapshot is fresh
|
|
1456
|
+
// enough and never blocks the response-capture path (#196).
|
|
1457
|
+
const route = responseCaptureReplyRoute(this.obligationCache.get(), this.currentTurnStartedAt);
|
|
1458
|
+
replyToMessageId = route?.replyToMessageId; replyTarget = route?.to ?? "user";
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
// Grab and reset the pre-flush cursor before extraction: the cursor may be
|
|
1462
|
+
// non-zero when a PreToolUse hook flushed narrative earlier this turn (#435).
|
|
1463
|
+
// In "full" mode, skip those already-emitted entries so they don't duplicate.
|
|
1464
|
+
const flushCount = this.narrativeFlushEntryCount;
|
|
1465
|
+
this.narrativeFlushEntryCount = 0;
|
|
1466
|
+
|
|
1467
|
+
// The Stop hook can fire before the final assistant entry is flushed to disk.
|
|
1468
|
+
// Claude writes thinking and text as separate entries (both with end_turn), so
|
|
1469
|
+
// the transcript can "look complete" while the text entry is still pending.
|
|
1470
|
+
// Retry until both the transcript has an end_turn AND extraction yields text.
|
|
1471
|
+
let body = "";
|
|
1472
|
+
let jsonl: string | undefined;
|
|
1473
|
+
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { jsonl = undefined; }
|
|
1474
|
+
if (jsonl !== undefined) {
|
|
1475
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
1476
|
+
if (attempt > 0) {
|
|
1477
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
1478
|
+
try { jsonl = await readFile(input.transcriptPath, "utf8"); } catch { break; }
|
|
1479
|
+
}
|
|
1480
|
+
if (!transcriptLooksComplete(jsonl)) continue;
|
|
1481
|
+
// If a pre-flush already emitted entries 1..flushCount, resume after that mark to avoid double-emit (#435/#499).
|
|
1482
|
+
const extract = this.options.providerConfig.chatCaptureMode === "full"
|
|
1483
|
+
? (j: string) => (flushCount > 0 ? extractLastAssistantTurnAfterEntry(j, flushCount) : extractLastAssistantTurn(j))
|
|
1484
|
+
: (j: string) => (flushCount > 0 ? extractFinalAssistantMessageAfterEntry(j, flushCount) : extractFinalAssistantMessage(j));
|
|
1485
|
+
body = extract(jsonl);
|
|
1486
|
+
if (body) break;
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
// Fallback: last_assistant_message from the Stop hook payload, which bypasses
|
|
1490
|
+
// the transcript file race entirely.
|
|
1491
|
+
if (!body && input.lastAssistantMessage) {
|
|
1492
|
+
body = extractHookAssistantMessage(input.lastAssistantMessage);
|
|
1493
|
+
}
|
|
1494
|
+
// A pure tool-use turn with no closing text is fine to skip — its reasoning and
|
|
1495
|
+
// tool steps already carried the visibility into chat.
|
|
1496
|
+
if (!body) {
|
|
1497
|
+
this.sessionLog(`response capture: no closing text for turn ${turnId ?? "?"} (skipped)`);
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
this.sessionLog(`response captured for turn ${turnId ?? "?"} (${body.length} chars${replyToMessageId ? `, replyTo #${replyToMessageId}` : ", no replyTo"})`);
|
|
1502
|
+
await publishCapturedResponse({ publishSessionEvent: (event) => this.publishSessionEvent(event), outbox: this.sessionOutbox, provider: this.options.provider, from: this.agentId, to: replyTarget, body, replyTo: replyToMessageId, session: { type: "response", origin: "provider", final: true, ...(turnId ? { turnId } : {}) } });
|
|
1503
|
+
// The agent's reply may have cleared an obligation — refresh the snapshot so the next
|
|
1504
|
+
// turn-end doesn't re-prompt for a message already answered (#196).
|
|
1505
|
+
if (replyToMessageId) this.obligationCache.markDirty();
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
// Post one session-mirror event (prompt echo, assistant response, reasoning or
|
|
1509
|
+
// tool step) as a `kind: "session"` relay message tagged with payload.session so
|
|
1510
|
+
// the dashboard can render the live provider session faithfully. Display-only:
|
|
1511
|
+
// session messages are never delivered back into a provider.
|
|
1512
|
+
private publishSessionEvent(input: {
|
|
1513
|
+
from: string;
|
|
1514
|
+
to: string;
|
|
1515
|
+
body: string;
|
|
1516
|
+
session: MessageSessionMeta;
|
|
1517
|
+
replyTo?: number;
|
|
1518
|
+
}): void {
|
|
1519
|
+
// Durable, ordered, timestamped (#196): the actual POST happens in deliverOutboxEvent,
|
|
1520
|
+
// retried until it lands. occurredAt is stamped now so a queued event reports when it
|
|
1521
|
+
// truly happened, not when the server finally accepted it. Routed through the fast-lane
|
|
1522
|
+
// sessionOutbox (#332) so a transient trace failure can't head-of-line block real messages.
|
|
1523
|
+
// A stepId-bearing step (Codex tool running→completed, streamed reasoning/response) uses a
|
|
1524
|
+
// STABLE idempotency key so the server upserts the row in place instead of appending a dup.
|
|
1525
|
+
const stepId = input.session.stepId;
|
|
1526
|
+
this.sessionOutbox.enqueue({
|
|
1527
|
+
kind: "session-message",
|
|
1528
|
+
...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
|
|
1529
|
+
payload: {
|
|
1530
|
+
from: input.from,
|
|
1531
|
+
to: input.to,
|
|
1532
|
+
...(input.replyTo ? { replyTo: input.replyTo } : {}),
|
|
1533
|
+
kind: "session",
|
|
1534
|
+
body: input.body,
|
|
1535
|
+
payload: { session: { provider: this.options.provider, ...input.session } },
|
|
1536
|
+
} satisfies SendMessageInput,
|
|
1537
|
+
});
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
private publishSessionEventsBatch(inputs: Array<{
|
|
1541
|
+
from: string;
|
|
1542
|
+
to: string;
|
|
1543
|
+
body: string;
|
|
1544
|
+
session: MessageSessionMeta;
|
|
1545
|
+
replyTo?: number;
|
|
1546
|
+
}>): void {
|
|
1547
|
+
for (let i = 0; i < inputs.length; i += SESSION_MIRROR_BATCH_MAX) {
|
|
1548
|
+
const chunk = inputs.slice(i, i + SESSION_MIRROR_BATCH_MAX);
|
|
1549
|
+
if (chunk.length === 1) {
|
|
1550
|
+
this.publishSessionEvent(chunk[0]!);
|
|
1551
|
+
continue;
|
|
1552
|
+
}
|
|
1553
|
+
const messages = chunk.map((input) => {
|
|
1554
|
+
const stepId = input.session.stepId;
|
|
1555
|
+
return {
|
|
1556
|
+
from: input.from,
|
|
1557
|
+
to: input.to,
|
|
1558
|
+
...(input.replyTo ? { replyTo: input.replyTo } : {}),
|
|
1559
|
+
kind: "session",
|
|
1560
|
+
body: input.body,
|
|
1561
|
+
...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
|
|
1562
|
+
payload: { session: { provider: this.options.provider, ...input.session } },
|
|
1563
|
+
} satisfies SendMessageInput;
|
|
1564
|
+
});
|
|
1565
|
+
this.sessionOutbox.enqueue({
|
|
1566
|
+
kind: "session-message-batch",
|
|
1567
|
+
idempotencyKey: `session-batch:${this.agentId}:${++this.sessionBatchSeq}:${messages.length}`,
|
|
1568
|
+
payload: { messages },
|
|
1569
|
+
});
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
// Map queued records to HTTP calls. Throw to retry, return to ack/delete.
|
|
1574
|
+
private async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
|
|
1575
|
+
await deliverRunnerOutboxEvent({
|
|
1576
|
+
record,
|
|
1577
|
+
http: this.http,
|
|
1578
|
+
relayUrl: this.options.relayUrl,
|
|
1579
|
+
token: this.currentToken,
|
|
1580
|
+
updatePayload: (seq, payload) => this.outbox.updatePayload(seq, payload),
|
|
1581
|
+
sessionLog: (message) => this.sessionLog(message),
|
|
1582
|
+
recoverRuntimeTokenAfterAuthFailure: (source) => this.recoverRuntimeTokenAfterAuthFailure(source),
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
// Replay a buffered MCP tool call (Stage 2, #215) that the proxy queued while the relay was
|
|
1587
|
+
// unreachable. Kept as a method wrapper for tests; the concern lives in mcp-outbox.ts.
|
|
1588
|
+
private async deliverBufferedMcpCall(record: OutboxRecord): Promise<void> {
|
|
1589
|
+
await deliverBufferedMcpOutboxCall({
|
|
1590
|
+
record,
|
|
1591
|
+
relayUrl: this.options.relayUrl,
|
|
1592
|
+
token: this.currentToken,
|
|
1593
|
+
recoverRuntimeTokenAfterAuthFailure: (source) => this.recoverRuntimeTokenAfterAuthFailure(source),
|
|
1594
|
+
});
|
|
1595
|
+
}
|
|
1596
|
+
|
|
1597
|
+
// A hook reported an unhandled failure (#198 seam). Already logged FATAL by the control
|
|
1598
|
+
// server; here we additionally surface it durably to the server as a generic insight so
|
|
1599
|
+
// it shows up in observability rather than only in the per-agent log (#196).
|
|
1600
|
+
private reportHookFatal(report: { hook: string; error: string }): void {
|
|
1601
|
+
try {
|
|
1602
|
+
this.insights.queueHookFatal(report);
|
|
1603
|
+
} catch (error) {
|
|
1604
|
+
logger.error("outbox", `failed to queue hook-fatal report: ${errMessage(error)}`);
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
// A human typed a prompt directly into the provider (web terminal / TUI). Mirror
|
|
1609
|
+
// it into the dashboard chat so both surfaces stay in sync, and kick off reasoning
|
|
1610
|
+
// tailing for the turn. Skips prompts the runner itself injected (chat box, relay
|
|
1611
|
+
// deliveries) so those aren't double-posted.
|
|
1612
|
+
private async handleUserPrompt(input: { prompt: string; transcriptPath?: string }): Promise<void> {
|
|
1613
|
+
if (input.transcriptPath) this.lastTranscriptPath = input.transcriptPath;
|
|
1614
|
+
if (!this.currentTurnId) {
|
|
1615
|
+
this.currentTurnId = crypto.randomUUID();
|
|
1616
|
+
this.currentTurnStartedAt = Date.now();
|
|
1617
|
+
}
|
|
1618
|
+
const text = input.prompt.trim();
|
|
1619
|
+
if (text && !this.isRunnerInjectedPrompt(text)) {
|
|
1620
|
+
this.sessionLog(`prompt echoed from terminal (${text.length} chars)`);
|
|
1621
|
+
await this.publishSessionEvent({
|
|
1622
|
+
from: "user",
|
|
1623
|
+
to: this.agentId,
|
|
1624
|
+
body: text,
|
|
1625
|
+
session: { type: "prompt", origin: "terminal", turnId: this.currentTurnId },
|
|
1626
|
+
});
|
|
1627
|
+
} else if (text) {
|
|
1628
|
+
this.sessionDebug("user-prompt hook: skipped echo (runner-injected)");
|
|
1629
|
+
}
|
|
1630
|
+
if (input.transcriptPath) this.startReasoningTail(input.transcriptPath);
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// A provider lifecycle hook reported a session boundary (Claude PreCompact / SessionEnd
|
|
1634
|
+
// → control server). Normalize the raw provider reason to a SessionDestroyReason and run
|
|
1635
|
+
// the same pre-destroy seam the bus commands use. `clear`/`compact` continue the session;
|
|
1636
|
+
// anything else (logout, prompt_input_exit, other) is a real termination.
|
|
1637
|
+
private async handleSessionBoundary(input: { reason?: string; transcriptPath?: string }): Promise<void> {
|
|
1638
|
+
// Reason mapping is fail-safe-toward-termination: only the two known session-
|
|
1639
|
+
// CONTINUING reasons are special-cased; everything else (logout, prompt_input_exit,
|
|
1640
|
+
// other, AND any future reason) maps to "shutdown" → full pre-destroy capture.
|
|
1641
|
+
// ⚠ If Claude Code adds a new BENIGN/continuing boundary reason, add it here — until
|
|
1642
|
+
// then it will trigger a (harmless but wasteful) full context capture on a session
|
|
1643
|
+
// that isn't actually ending.
|
|
1644
|
+
const reason = input.reason === "compact" ? "compact"
|
|
1645
|
+
: input.reason === "clear" ? "clear"
|
|
1646
|
+
: "shutdown";
|
|
1647
|
+
await this.runPreSessionDestroy(reason, { transcriptPath: input.transcriptPath });
|
|
1648
|
+
// clear/compact CONTINUE the session — the finalizing window is transient. The bus-command
|
|
1649
|
+
// path (handleCommand) restores the addressable status in its finally; the hook path has no
|
|
1650
|
+
// such teardown, so without this the dashboard stays stuck on "wrapping up — messaging
|
|
1651
|
+
// paused" with the composer disabled forever. Only restore if we're still in the exact
|
|
1652
|
+
// finalizing state we published — a concurrent bus command (kill/restart/shutdown) may have
|
|
1653
|
+
// transitioned lifecycleAction since, and that must win. "shutdown" reasons end the session,
|
|
1654
|
+
// so their finalizing state is superseded by going offline; leave it be.
|
|
1655
|
+
if ((reason === "clear" || reason === "compact") && this.lifecycleAction === `finalizing-${reason}` && !this.stopped) {
|
|
1656
|
+
this.lifecycleAction = undefined;
|
|
1657
|
+
this.publishStatus();
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
// The pre-session-destroy seam (#183): the single place end-of-session work runs before
|
|
1662
|
+
// an invasive transition (compact/clear/restart/shutdown/kill). Best-effort and
|
|
1663
|
+
// time-boxed so it never hangs teardown; concurrent calls for the same teardown coalesce
|
|
1664
|
+
// (a shutdown bus command and the SessionEnd hook can both fire). During the window the
|
|
1665
|
+
// agent is published non-addressable so the operator sees "wrapping up", not "busy".
|
|
1666
|
+
private runPreSessionDestroy(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1667
|
+
if (this.preDestroyPromise) return this.preDestroyPromise;
|
|
1668
|
+
const run = (async () => {
|
|
1669
|
+
this.publishFinalizing(reason);
|
|
1670
|
+
try {
|
|
1671
|
+
await Promise.race([
|
|
1672
|
+
Promise.all([
|
|
1673
|
+
this.captureContextRatio(reason, opts),
|
|
1674
|
+
this.captureContinuationArchive(reason, opts),
|
|
1675
|
+
]).then(() => undefined),
|
|
1676
|
+
new Promise<void>((resolve) => setTimeout(resolve, PRE_DESTROY_TIMEOUT_MS)),
|
|
1677
|
+
]);
|
|
1678
|
+
} catch (error) {
|
|
1679
|
+
this.sessionLog(`pre-destroy capture failed: ${errMessage(error)}`);
|
|
1680
|
+
}
|
|
1681
|
+
// For exit-bound transitions the runner won't be alive afterward to drain the durable
|
|
1682
|
+
// outbox, so block (bounded) on delivering what capture just enqueued. This runs before
|
|
1683
|
+
// handleCommand's finally deletes the agent, so the runtime token is still valid here.
|
|
1684
|
+
if (reasonExitsRunner(reason)) {
|
|
1685
|
+
try {
|
|
1686
|
+
const delivered = await this.outbox.flush(OUTBOX_FLUSH_TIMEOUT_MS);
|
|
1687
|
+
if (!delivered) this.sessionLog(`insights: outbox flush incomplete before ${reason} (${this.outbox.pendingCount()} pending)`);
|
|
1688
|
+
} catch (error) {
|
|
1689
|
+
this.sessionLog(`insights: outbox flush failed: ${errMessage(error)}`);
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
})();
|
|
1693
|
+
this.preDestroyPromise = run;
|
|
1694
|
+
void run.finally(() => { this.preDestroyPromise = undefined; });
|
|
1695
|
+
return run;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
// Publish the transient pre-destroy state: a non-offline status with ready:false (so the
|
|
1699
|
+
// agent drops out of isAgentOnline fan-out targeting without going "offline") plus a
|
|
1700
|
+
// finalizing-<reason> lifecycleAction the dashboard renders as "wrapping up" with the
|
|
1701
|
+
// composer disabled.
|
|
1702
|
+
private publishFinalizing(reason: SessionDestroyReason): void {
|
|
1703
|
+
this.lifecycleAction = `finalizing-${reason}`;
|
|
1704
|
+
void this.bus.statusAsync({ agentStatus: "busy", ready: false, meta: { lifecycleAction: this.lifecycleAction, lifecycleActionAt: Date.now() } });
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
// Repo-name project id for insight observations, resolved once by the insights collaborator.
|
|
1708
|
+
private insightProject(): string {
|
|
1709
|
+
return this.insights.project();
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
private async captureContinuationArchive(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1713
|
+
await this.insights.captureContinuationArchive(reason, opts);
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
private async captureContextRatio(reason: SessionDestroyReason, opts?: { transcriptPath?: string }): Promise<void> {
|
|
1717
|
+
await this.insights.captureContextRatio(reason, opts);
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
// Route a provider-emitted session event (Codex app-server) into the chat mirror.
|
|
1721
|
+
// Mirrors the same semantics as the Claude lane: prompts are echoed with dedup,
|
|
1722
|
+
// and a response is only auto-captured when the agent won't separately reply to a
|
|
1723
|
+
// relay obligation (so relay-triggered turns aren't double-posted).
|
|
1724
|
+
private async publishProviderSessionEvent(event: ProviderSessionEvent): Promise<void> {
|
|
1725
|
+
const body = event.body.trim();
|
|
1726
|
+
if (!body) return;
|
|
1727
|
+
const turnId = event.turnId ?? this.currentTurnId;
|
|
1728
|
+
if (event.type === "prompt") {
|
|
1729
|
+
if (this.isRunnerInjectedPrompt(body)) return;
|
|
1730
|
+
await this.publishSessionEvent({
|
|
1731
|
+
from: "user",
|
|
1732
|
+
to: this.agentId,
|
|
1733
|
+
body,
|
|
1734
|
+
session: { type: "prompt", origin: event.origin ?? "terminal", ...(turnId ? { turnId } : {}) },
|
|
1735
|
+
});
|
|
1736
|
+
return;
|
|
1737
|
+
}
|
|
1738
|
+
if (event.type === "response") {
|
|
1739
|
+
// Dashboard prompt injection is already answered by this captured App Server
|
|
1740
|
+
// response. User obligations still belong to the session mirror; spawner
|
|
1741
|
+
// obligations (#413) route to the spawn-parent with replyTo to clear the debt.
|
|
1742
|
+
let replyToMessageId: number | undefined, replyTarget = "user";
|
|
1743
|
+
const pendingPrompt = this.pendingPromptMessageId;
|
|
1744
|
+
if (pendingPrompt) {
|
|
1745
|
+
replyToMessageId = pendingPrompt;
|
|
1746
|
+
this.pendingPromptMessageId = undefined;
|
|
1747
|
+
} else {
|
|
1748
|
+
const route = responseCaptureReplyRoute(this.obligationCache.get(), this.currentTurnStartedAt, { suppressUser: true });
|
|
1749
|
+
if (!route) return; // The session mirror will satisfy the user obligation; don't double-post (#196).
|
|
1750
|
+
replyToMessageId = route.replyToMessageId; replyTarget = route.to;
|
|
1751
|
+
}
|
|
1752
|
+
await publishCapturedResponse({ publishSessionEvent: (sessionEvent) => this.publishSessionEvent(sessionEvent), outbox: this.sessionOutbox, provider: this.options.provider, from: this.agentId, to: replyTarget, body, replyTo: replyToMessageId, session: { type: "response", origin: event.origin ?? "provider", ...(turnId ? { turnId } : {}) } });
|
|
1753
|
+
if (replyToMessageId) this.obligationCache.markDirty();
|
|
1754
|
+
return;
|
|
1755
|
+
}
|
|
1756
|
+
if (this.options.providerConfig.reasoningCapture === false) return;
|
|
1757
|
+
await this.publishSessionEvent({
|
|
1758
|
+
from: this.agentId,
|
|
1759
|
+
to: "user",
|
|
1760
|
+
body,
|
|
1761
|
+
session: {
|
|
1762
|
+
type: event.type,
|
|
1763
|
+
origin: event.origin ?? "provider",
|
|
1764
|
+
...(turnId ? { turnId } : {}),
|
|
1765
|
+
...(event.label ? { label: event.label } : {}),
|
|
1766
|
+
...(event.status ? { status: event.status } : {}),
|
|
1767
|
+
...(event.streaming !== undefined ? { streaming: event.streaming } : {}),
|
|
1768
|
+
...(event.stepId ? { stepId: event.stepId } : {}),
|
|
1769
|
+
...(event.final ? { final: true } : {}),
|
|
1770
|
+
},
|
|
1771
|
+
});
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
// Remember an injected prompt so its UserPromptSubmit echo can be suppressed. Prunes
|
|
1775
|
+
// expired entries first; a defensive length cap guards against echoes that never
|
|
1776
|
+
// arrive (e.g. the provider drops a queued prompt) so the queue can't grow unbounded.
|
|
1777
|
+
private recordInjectedPrompt(text: string): void {
|
|
1778
|
+
const now = Date.now();
|
|
1779
|
+
this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
|
|
1780
|
+
this.injectedPrompts.push({ text, at: now });
|
|
1781
|
+
if (this.injectedPrompts.length > 50) this.injectedPrompts.shift();
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
private isRunnerInjectedPrompt(text: string): boolean {
|
|
1785
|
+
if (RELAY_INJECTION_MARKERS.some((marker) => text.startsWith(marker))) return true;
|
|
1786
|
+
const now = Date.now();
|
|
1787
|
+
this.injectedPrompts = this.injectedPrompts.filter((p) => now - p.at < PROMPT_ECHO_DEDUP_MS);
|
|
1788
|
+
const idx = this.injectedPrompts.findIndex((p) => p.text === text);
|
|
1789
|
+
if (idx !== -1) {
|
|
1790
|
+
this.injectedPrompts.splice(idx, 1); // consume one — identical repeats each match once
|
|
1791
|
+
return true;
|
|
1792
|
+
}
|
|
1793
|
+
return false;
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
// Force-clear a stuck provider-turn claim directly. Unlike the idle status path
|
|
1797
|
+
// it does NOT depend on a matching claim id (the Stop hook keys busy as
|
|
1798
|
+
// provider-turn:provider-turn, but reconciliation has no specific id), and it
|
|
1799
|
+
// deliberately leaves the reasoning tail alone so a late clear can't truncate
|
|
1800
|
+
// a turn's activity stream.
|
|
1801
|
+
private forceClearProviderTurn(reason: string): void {
|
|
1802
|
+
if (!this.claims.activeWork().some((w) => w.kind === "provider-turn")) return;
|
|
1803
|
+
this.sessionLog(`force-clearing stuck provider-turn (${reason})`);
|
|
1804
|
+
this.claims.clearWorkKind("provider-turn");
|
|
1805
|
+
this.currentTurnId = undefined;
|
|
1806
|
+
this.compactionMidTurn = false;
|
|
1807
|
+
this.publishStatus();
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// #650: reconcile pid-bearing busy work against actual process liveness. A tracked
|
|
1811
|
+
// command/shell that dies out-of-band (killed by the operator, crashes, orphaned)
|
|
1812
|
+
// sends no exit event, so its busy marker would otherwise never clear and the agent
|
|
1813
|
+
// shows `busy` forever — blocking transient auto-reap and (for Claude) holding quota.
|
|
1814
|
+
// The timer self-disarms once no pid-bearing work remains; it is armed lazily when a
|
|
1815
|
+
// busy report carrying pids arrives, so idle/non-tracked agents pay nothing.
|
|
1816
|
+
private armBusyLivenessReconcile(): void {
|
|
1817
|
+
if (this.livenessReconcileTimer || this.stopped) return;
|
|
1818
|
+
this.livenessReconcileTimer = setInterval(() => this.runBusyLivenessReconcile(), BUSY_LIVENESS_RECONCILE_MS);
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
private disarmBusyLivenessReconcile(): void {
|
|
1822
|
+
if (this.livenessReconcileTimer) clearInterval(this.livenessReconcileTimer);
|
|
1823
|
+
this.livenessReconcileTimer = undefined;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
private runBusyLivenessReconcile(): void {
|
|
1827
|
+
if (this.stopped) {
|
|
1828
|
+
this.disarmBusyLivenessReconcile();
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
const removed = this.claims.reconcileLiveness(isPidAlive);
|
|
1832
|
+
if (removed.length > 0) {
|
|
1833
|
+
this.sessionLog(`cleared ${removed.length} stuck busy work item(s) whose tracked process died out-of-band: ${removed.map((w) => `${w.kind}:${w.id}`).join(", ")}`);
|
|
1834
|
+
// A cleared provider-turn must also reset the turn bookkeeping so a later idle
|
|
1835
|
+
// doesn't double-end it and the reasoning tail stops.
|
|
1836
|
+
if (removed.some((w) => w.kind === "provider-turn")) {
|
|
1837
|
+
this.currentTurnId = undefined;
|
|
1838
|
+
this.currentTurnStartedAt = undefined;
|
|
1839
|
+
this.compactionMidTurn = false;
|
|
1840
|
+
this.busyReconciler.disarm();
|
|
1841
|
+
this.stopReasoningTail();
|
|
1842
|
+
}
|
|
1843
|
+
this.publishStatus();
|
|
1844
|
+
}
|
|
1845
|
+
if (!this.claims.hasLivenessTrackedWork()) this.disarmBusyLivenessReconcile();
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
// --- Turn-step tailer (item 5) ------------------------------------------------------
|
|
1849
|
+
// Tail the in-flight turn's Claude transcript and surface new narration/reasoning/tool
|
|
1850
|
+
// steps as session events, in transcript order. `narration` (the agent's intermediate
|
|
1851
|
+
// text) is the primary visible content; reasoning visibility is a client-side toggle.
|
|
1852
|
+
// Coalesced and coarse; the final response still comes through publishSessionTurn.
|
|
1853
|
+
// `reasoningCapture: false` disables the whole live trace (server-side kill switch).
|
|
1854
|
+
private startReasoningTail(transcriptPath: string): void {
|
|
1855
|
+
if (this.options.providerConfig.reasoningCapture === false) return;
|
|
1856
|
+
this.stopReasoningTail();
|
|
1857
|
+
// Track emitted steps by content signature, not by index/count: the "latest
|
|
1858
|
+
// turn" window in the transcript can shrink/reset (a tool_result entry, a
|
|
1859
|
+
// mid-turn user line), and an index cursor would then either re-emit or stall
|
|
1860
|
+
// and drop the rest of the turn. A seen-set is idempotent under any reshuffle.
|
|
1861
|
+
// The signature is salted with each step's occurrence-within-window (stepDedupKeys)
|
|
1862
|
+
// so two identical steps in one turn — same tool, same input — both surface (#265).
|
|
1863
|
+
const seen = new Set<string>();
|
|
1864
|
+
const emittedNarrationKeys = new Set<string>();
|
|
1865
|
+
const transcriptTail = new IncrementalClaudeTranscriptTail();
|
|
1866
|
+
const turnIdAtStart = this.currentTurnId;
|
|
1867
|
+
// On the first poll the new prompt usually hasn't landed in the transcript yet,
|
|
1868
|
+
// so extractLatestTurnSteps still returns the PRIOR (completed) turn. Seed those
|
|
1869
|
+
// signatures as already-seen so we don't replay last turn's reasoning/tools as
|
|
1870
|
+
// this turn's activity. Once our prompt lands the window resets at the new user
|
|
1871
|
+
// boundary and genuinely-new steps emit normally. Only seed when the transcript
|
|
1872
|
+
// is complete (last entry is an end_turn assistant) — otherwise we're already
|
|
1873
|
+
// inside the new turn and those steps are legitimately ours.
|
|
1874
|
+
let seeded = false;
|
|
1875
|
+
let pendingPoll = Promise.resolve();
|
|
1876
|
+
const pollBody = async (): Promise<void> => {
|
|
1877
|
+
const result = await transcriptTail.poll(transcriptPath);
|
|
1878
|
+
if (!result.available) return;
|
|
1879
|
+
if (result.reset) { seen.clear(); emittedNarrationKeys.clear(); seeded = false; }
|
|
1880
|
+
if (!result.changed && (seeded || (!result.steps.length && !result.transcriptLooksComplete))) return;
|
|
1881
|
+
const keyed = stepDedupKeys(result.steps).map((sig, i) => ({ sig, step: result.steps[i]! }));
|
|
1882
|
+
if (!seeded) {
|
|
1883
|
+
seeded = true;
|
|
1884
|
+
if (result.transcriptLooksComplete) {
|
|
1885
|
+
for (const { sig } of keyed) seen.add(sig);
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
const turnId = this.currentTurnId ?? turnIdAtStart;
|
|
1889
|
+
let emitted = 0;
|
|
1890
|
+
const batch: Array<{ from: string; to: string; body: string; session: MessageSessionMeta }> = [];
|
|
1891
|
+
for (const { sig, step } of keyed) {
|
|
1892
|
+
if (seen.has(sig)) continue;
|
|
1893
|
+
seen.add(sig);
|
|
1894
|
+
emitted += 1;
|
|
1895
|
+
batch.push({
|
|
1896
|
+
from: this.agentId,
|
|
1897
|
+
to: "user",
|
|
1898
|
+
body: step.text,
|
|
1899
|
+
session: { type: step.type, origin: "provider", ...(turnId ? { turnId } : {}), ...(step.label ? { label: step.label } : {}) },
|
|
1900
|
+
});
|
|
1901
|
+
if (step.type === "narration") emittedNarrationKeys.add(sig);
|
|
1902
|
+
}
|
|
1903
|
+
this.publishSessionEventsBatch(batch);
|
|
1904
|
+
if (emitted) this.sessionDebug(`reasoning tail emitted ${emitted} step(s) (turn ${turnId ?? "?"}, ${seen.size} total)`);
|
|
1905
|
+
};
|
|
1906
|
+
const poll = (): Promise<void> => {
|
|
1907
|
+
const next = pendingPoll.then(pollBody, pollBody);
|
|
1908
|
+
pendingPoll = next.catch(() => {});
|
|
1909
|
+
return next;
|
|
1910
|
+
};
|
|
1911
|
+
this.reasoningTail = { emittedNarrationKeys, poll, timer: setInterval(() => { void poll(); }, REASONING_POLL_MS) };
|
|
1912
|
+
this.sessionLog(`reasoning tail started (turn ${turnIdAtStart ?? "?"})`);
|
|
1913
|
+
void poll();
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
private async drainReasoningTail(): Promise<void> {
|
|
1917
|
+
if (this.options.providerConfig.reasoningCapture === false) return;
|
|
1918
|
+
await this.reasoningTail?.poll();
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
private preFlushBodyAlreadyMirroredByReasoningTail(jsonl: string, body: string): boolean {
|
|
1922
|
+
const tail = this.reasoningTail;
|
|
1923
|
+
if (!tail || !tail.emittedNarrationKeys.size) return false;
|
|
1924
|
+
const steps = extractLatestTurnSteps(jsonl);
|
|
1925
|
+
const keys = stepDedupKeys(steps);
|
|
1926
|
+
const mirrored = steps
|
|
1927
|
+
.map((step, index) => ({ step, key: keys[index]! }))
|
|
1928
|
+
.filter(({ step, key }) => step.type === "narration" && tail.emittedNarrationKeys.has(key))
|
|
1929
|
+
.map(({ step }) => step.text)
|
|
1930
|
+
.join("\n\n")
|
|
1931
|
+
.trim();
|
|
1932
|
+
const trimmed = body.trim();
|
|
1933
|
+
return mirrored === trimmed || mirrored.endsWith(`\n\n${trimmed}`);
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
private stopReasoningTail(): void {
|
|
1937
|
+
if (this.reasoningTail) clearInterval(this.reasoningTail.timer);
|
|
1938
|
+
this.reasoningTail = undefined;
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
// Mirror a discreet, durable "context compacted" marker into chat via the existing
|
|
1942
|
+
// session-mirror lane (not a parallel channel). `notice` renders as an inline timeline
|
|
1943
|
+
// marker (never a bubble) and survives reload, unlike the ephemeral timeline-status one.
|
|
1944
|
+
private publishCompactionNotice(): void {
|
|
1945
|
+
this.publishSessionEvent({
|
|
1946
|
+
from: this.agentId,
|
|
1947
|
+
to: "user",
|
|
1948
|
+
body: "🗜 Context compacted",
|
|
1949
|
+
session: { type: "notice", origin: "provider", label: "compacted", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
// #286: a discreet, durable chat marker when a usage/rate-limit hold begins, via
|
|
1954
|
+
// the same session-mirror lane as the compaction notice. Outbound session event
|
|
1955
|
+
// (NOT an inbound message) so it shows in the dashboard chat WITHOUT waking a turn
|
|
1956
|
+
// — waking a still-limited agent would just re-fail. The relay sends the waking
|
|
1957
|
+
// resume message later, once the window has reset.
|
|
1958
|
+
private publishRateLimitNotice(providerState: ProviderState): void {
|
|
1959
|
+
const label = providerState.label || "usage limit reached";
|
|
1960
|
+
this.publishSessionEvent({
|
|
1961
|
+
from: this.agentId,
|
|
1962
|
+
to: "user",
|
|
1963
|
+
body: `⏳ ${label} — holding; agent-relay will auto-resume at reset.`,
|
|
1964
|
+
session: { type: "notice", origin: "provider", label: "rate-limit", ...(this.currentTurnId ? { turnId: this.currentTurnId } : {}) },
|
|
1965
|
+
});
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
private publishStatus(): void {
|
|
1969
|
+
this.claims.expire();
|
|
1970
|
+
const status = this.claims.currentStatus();
|
|
1971
|
+
const agentStatus = runnerAgentStatus(status);
|
|
1972
|
+
const activeWork = this.claims.activeWork();
|
|
1973
|
+
const activeSubagents = activeWork.filter((item) => item.kind === "subagent");
|
|
1974
|
+
const terminalFailure = this.terminalFailure;
|
|
1975
|
+
const providerState = terminalFailure?.providerState ?? this.rateLimitHold ?? providerStateFromActiveWork(activeWork);
|
|
1976
|
+
this.bus.setSemanticStatus(status === "offline" || status === "error" ? "idle" : status);
|
|
1977
|
+
const timelineEvent = this.pendingTimelineEvent;
|
|
1978
|
+
this.pendingTimelineEvent = undefined;
|
|
1979
|
+
// §4.1 Runner↔provider liveness contract (#725 / #746): fold the raw transport/timeline/
|
|
1980
|
+
// token/work-kind signals into ONE normalized verdict the server reads instead of six raw
|
|
1981
|
+
// `meta` fields. The adapter owns what counts as progress for its provider (#713/#724);
|
|
1982
|
+
// mock adapters fall back to the shared fold.
|
|
1983
|
+
const livenessInputs: LivenessInputs = {
|
|
1984
|
+
transport: this.bus.transportState,
|
|
1985
|
+
...(this.lastProviderEventAt !== undefined ? { lastProviderEventAt: this.lastProviderEventAt } : {}),
|
|
1986
|
+
...(timelineEvent ? { timelineEvent } : {}),
|
|
1987
|
+
activeWork,
|
|
1988
|
+
};
|
|
1989
|
+
const liveness = this.options.adapter.liveness
|
|
1990
|
+
? this.options.adapter.liveness(livenessInputs)
|
|
1991
|
+
: computeLivenessSignal(livenessInputs);
|
|
1992
|
+
this.bus.status({
|
|
1993
|
+
agentStatus,
|
|
1994
|
+
ready: agentStatus !== "offline" && !this.stopped,
|
|
1995
|
+
meta: {
|
|
1996
|
+
runnerId: this.options.runnerId,
|
|
1997
|
+
startedAt: this.options.startedAt,
|
|
1998
|
+
tmuxSession: providerTerminalSession(this.process) ?? this.options.tmuxSession ?? null,
|
|
1999
|
+
tmuxSocket: providerTerminalSocket(this.process) ?? null,
|
|
2000
|
+
policyName: this.options.policyName ?? null,
|
|
2001
|
+
spawnRequestId: this.options.spawnRequestId ?? null,
|
|
2002
|
+
taskId: this.options.taskId ?? null,
|
|
2003
|
+
automationId: this.options.automationId ?? null,
|
|
2004
|
+
automationRunId: this.options.automationRunId ?? null,
|
|
2005
|
+
workspace: this.options.workspace ?? null,
|
|
2006
|
+
lifecycle: this.options.lifecycle ?? "persistent", workspaceMode: this.options.workspace?.requestedMode ?? this.options.workspace?.mode ?? workspaceModeFromEnv() ?? null,
|
|
2007
|
+
lifecycleAction: this.lifecycleAction ?? null,
|
|
2008
|
+
profile: this.options.profile ?? null,
|
|
2009
|
+
...(status === "error" ? { terminalStatus: "error" } : {}),
|
|
2010
|
+
...(terminalFailure ? {
|
|
2011
|
+
lastError: terminalFailure.message,
|
|
2012
|
+
terminalFailureReason: terminalFailure.reason,
|
|
2013
|
+
terminalFailureMessage: terminalFailure.message,
|
|
2014
|
+
} : {}),
|
|
2015
|
+
// #633 — terminal provider-exit marker (runner alive, provider gone). Drives the server's
|
|
2016
|
+
// silent-death terminal event + #636 post-mortem.
|
|
2017
|
+
...(this.terminalProviderExit ? {
|
|
2018
|
+
terminalProviderExit: this.terminalProviderExit,
|
|
2019
|
+
...(typeof this.terminalProviderExit.claudeResumeId === "string"
|
|
2020
|
+
? { lastError: `Claude provider exited; resume id ${this.terminalProviderExit.claudeResumeId} captured for manual recovery` }
|
|
2021
|
+
: { lastError: "Claude provider exited; manual intervention required" }),
|
|
2022
|
+
} : {}),
|
|
2023
|
+
busyReasons: this.claims.reasons(),
|
|
2024
|
+
completedProviderTurns: this.completedProviderTurns,
|
|
2025
|
+
activeWork,
|
|
2026
|
+
activeSubagents,
|
|
2027
|
+
activeSubagentCount: activeSubagents.length,
|
|
2028
|
+
...(this.lastProviderEventAt !== undefined ? { lastProviderEventAt: this.lastProviderEventAt } : {}),
|
|
2029
|
+
providerState,
|
|
2030
|
+
auth: {
|
|
2031
|
+
profileId: this.currentTokenProfileId ?? null,
|
|
2032
|
+
jti: this.currentTokenJti ?? null,
|
|
2033
|
+
expiresAt: this.currentTokenExpiresAt ?? null,
|
|
2034
|
+
rootTokenFallback: this.options.rootTokenFallback === true,
|
|
2035
|
+
},
|
|
2036
|
+
...(timelineEvent ? { timelineEvent } : {}),
|
|
2037
|
+
transport: this.bus.transportState,
|
|
2038
|
+
// #746 — normalized liveness verdict; the server reads ONLY this for stuck-busy.
|
|
2039
|
+
// The raw fields above stay for other consumers (timeline rendering, diagnostics).
|
|
2040
|
+
liveness,
|
|
2041
|
+
},
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
private startHttpLiveness(): void {
|
|
2046
|
+
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
2047
|
+
void this.publishHttpLiveness();
|
|
2048
|
+
this.httpLivenessTimer = setInterval(() => {
|
|
2049
|
+
void this.publishHttpLiveness();
|
|
2050
|
+
}, HTTP_LIVENESS_INTERVAL_MS);
|
|
2051
|
+
}
|
|
2052
|
+
|
|
2053
|
+
private async publishHttpLiveness(): Promise<void> {
|
|
2054
|
+
if (this.stopped || this.httpLivenessInFlight || this.httpLivenessAuthFailed) return;
|
|
2055
|
+
this.httpLivenessInFlight = true;
|
|
2056
|
+
const status = this.claims.currentStatus();
|
|
2057
|
+
const agentStatus = runnerAgentStatus(status);
|
|
2058
|
+
try {
|
|
2059
|
+
await this.http.setStatus(this.agentId, agentStatus, this.options.instanceId);
|
|
2060
|
+
await this.http.setReady(this.agentId, agentStatus !== "offline", this.options.instanceId);
|
|
2061
|
+
await this.http.heartbeat(this.agentId, this.options.instanceId);
|
|
2062
|
+
await this.claudeQuotaHarvest.publish().catch((error) => this.logRunnerDiagnostic(`passive Claude quota publish failed: ${errMessage(error)}`));
|
|
2063
|
+
await this.bootstrapUnreadMessages();
|
|
2064
|
+
} catch (error) {
|
|
2065
|
+
if (!this.stopped) this.handleHttpLivenessFailure(error);
|
|
2066
|
+
} finally {
|
|
2067
|
+
this.httpLivenessInFlight = false;
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
|
|
2071
|
+
private handleHttpLivenessFailure(error: unknown): void {
|
|
2072
|
+
const authFailed = isHttpAuthError(error);
|
|
2073
|
+
this.logHttpLivenessFailure(error, authFailed);
|
|
2074
|
+
if (!authFailed) return;
|
|
2075
|
+
this.httpLivenessAuthFailed = true;
|
|
2076
|
+
if (this.httpLivenessTimer) clearInterval(this.httpLivenessTimer);
|
|
2077
|
+
this.httpLivenessTimer = undefined;
|
|
2078
|
+
// A 401/403 here is the only timely signal that the token died — stopping the
|
|
2079
|
+
// liveness timer means there is no second chance, so recover from THIS failure.
|
|
2080
|
+
this.recoverRuntimeTokenAfterAuthFailure("http-liveness");
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
// A definitive relay auth failure (401/403) means the runtime token is dead right
|
|
2084
|
+
// now — expired, or (the common case) revoked when the relay marked this agent
|
|
2085
|
+
// stale across its own restart/reconnect. The proactive renew timer is keyed to
|
|
2086
|
+
// TTL and structurally cannot catch a revocation, so the auth failure itself must
|
|
2087
|
+
// drive recovery. renewRuntimeToken() prefers an orchestrator re-mint, which heals
|
|
2088
|
+
// an expired token AND a token revoked-as-"stale" (the restart strand, #484) —
|
|
2089
|
+
// but NOT a hard kill (admin revoke / rotation / agent removal), which stays dead
|
|
2090
|
+
// by design. Debounced so a burst of failing calls re-mints once.
|
|
2091
|
+
private recoverRuntimeTokenAfterAuthFailure(source: string): void {
|
|
2092
|
+
if (this.stopped || this.tokenRenewInFlight) return;
|
|
2093
|
+
if (!this.isRuntimeTokenRenewable() && !this.canRemintViaOrchestrator()) return;
|
|
2094
|
+
const now = Date.now();
|
|
2095
|
+
if (this.reactiveTokenRecoveryAt && now - this.reactiveTokenRecoveryAt < REACTIVE_TOKEN_RECOVERY_DEBOUNCE_MS) return;
|
|
2096
|
+
this.reactiveTokenRecoveryAt = now;
|
|
2097
|
+
this.logRunnerDiagnostic(`[runner] relay auth failure on ${source}; recovering runtime token`);
|
|
2098
|
+
void this.renewRuntimeToken();
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
private logHttpLivenessFailure(error: unknown, authFailed: boolean): void {
|
|
2102
|
+
const key = httpErrorKey(error);
|
|
2103
|
+
const now = Date.now();
|
|
2104
|
+
if (
|
|
2105
|
+
this.httpLivenessLastLog?.key === key &&
|
|
2106
|
+
now - this.httpLivenessLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
|
|
2107
|
+
) return;
|
|
2108
|
+
this.httpLivenessLastLog = { key, at: now };
|
|
2109
|
+
const suffix = authFailed ? "auth failed; stopping HTTP liveness retries until restart" : String(error);
|
|
2110
|
+
this.logRunnerDiagnostic(`[runner] HTTP liveness update failed: ${suffix}`);
|
|
2111
|
+
}
|
|
2112
|
+
|
|
2113
|
+
// Runner operational diagnostics (HTTP liveness, token renewal failures). Routed
|
|
2114
|
+
// through the leveled logger at warn — see logger.ts. Kept as a thin wrapper so
|
|
2115
|
+
// the existing call sites and their `[runner]` framing stay put.
|
|
2116
|
+
private logRunnerDiagnostic(message: string): void {
|
|
2117
|
+
logger.warn("runner", message.replace(/^\[runner\]\s*/, ""));
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
// Session-mirror diagnostics → the leveled logger (component "mirror"), written
|
|
2121
|
+
// to the dashboard-surfaced session-mirror-<agent>.log. Key transitions log at
|
|
2122
|
+
// info; the single place to look when chat/terminal sync misbehaves.
|
|
2123
|
+
private sessionLog(message: string): void {
|
|
2124
|
+
logger.info("mirror", message);
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
// Verbose, high-frequency lines (per-probe, per-emit) — surfaced only at log
|
|
2128
|
+
// level "debug" (AGENT_RELAY_LOG_LEVEL=debug, or flip live via /log-level).
|
|
2129
|
+
private sessionDebug(message: string): void {
|
|
2130
|
+
logger.debug("mirror", message);
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
private ensureScratch(): void {
|
|
2134
|
+
try {
|
|
2135
|
+
this.scratch = ensureSessionScratch({
|
|
2136
|
+
agentId: this.agentId,
|
|
2137
|
+
cwd: this.options.cwd,
|
|
2138
|
+
fallbackBaseDir: orchestratorBaseDirFromEnv(),
|
|
2139
|
+
});
|
|
2140
|
+
} catch (error) {
|
|
2141
|
+
this.logRunnerDiagnostic(`session scratch setup failed: ${errMessage(error)}`);
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
private async sweepStaleScratch(): Promise<void> {
|
|
2146
|
+
try {
|
|
2147
|
+
const agents = await this.http.listAgents().catch(() => [] as Awaited<ReturnType<RelayHttpClient["listAgents"]>>);
|
|
2148
|
+
const keep = new Set(agents.map((a) => a.id)); keep.add(this.agentId); const now = Date.now();
|
|
2149
|
+
const removed = sweepStaleSessions({
|
|
2150
|
+
cwd: this.options.cwd,
|
|
2151
|
+
fallbackBaseDir: orchestratorBaseDirFromEnv(),
|
|
2152
|
+
keepAgentIds: keep,
|
|
2153
|
+
now,
|
|
2154
|
+
});
|
|
2155
|
+
if (removed.length) this.logRunnerDiagnostic(`swept ${removed.length} stale session scratch dir(s)`);
|
|
2156
|
+
const removedLaunchers = sweepStaleLaunchers({ keepAgentIds: keep, now }); if (removedLaunchers.length) this.logRunnerDiagnostic(`swept ${removedLaunchers.length} stale launcher script(s)`);
|
|
2157
|
+
} catch {}
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
private scheduleRuntimeTokenRenewal(delayMs?: number): void {
|
|
2161
|
+
if (this.tokenRenewTimer) clearTimeout(this.tokenRenewTimer);
|
|
2162
|
+
this.tokenRenewTimer = undefined;
|
|
2163
|
+
if (this.stopped) return;
|
|
2164
|
+
const canSelfRenew = this.isRuntimeTokenRenewable();
|
|
2165
|
+
const canRemint = this.canRemintViaOrchestrator();
|
|
2166
|
+
// Keep the renewal clock ticking as long as the session can recover its token
|
|
2167
|
+
// by EITHER path. Without the re-mint fallback an expired token would stop the
|
|
2168
|
+
// timer forever (the old deadlock that stranded live agents off the bus).
|
|
2169
|
+
if (!canSelfRenew && !canRemint) return;
|
|
2170
|
+
let computedDelay = delayMs;
|
|
2171
|
+
if (computedDelay === undefined) {
|
|
2172
|
+
computedDelay = canSelfRenew
|
|
2173
|
+
? runtimeTokenRenewDelayMs(this.currentTokenExpiresAt!, Date.now())
|
|
2174
|
+
: TOKEN_RENEW_RETRY_MS; // expired but re-mintable → retry via orchestrator soon
|
|
2175
|
+
if (computedDelay === undefined) computedDelay = TOKEN_RENEW_RETRY_MS;
|
|
2176
|
+
}
|
|
2177
|
+
const schedule = runtimeTokenRenewTimerSchedule(computedDelay);
|
|
2178
|
+
if (!schedule) return;
|
|
2179
|
+
this.tokenRenewTimer = setTimeout(() => {
|
|
2180
|
+
this.tokenRenewTimer = undefined;
|
|
2181
|
+
if (!schedule.renew) {
|
|
2182
|
+
this.scheduleRuntimeTokenRenewal();
|
|
2183
|
+
return;
|
|
2184
|
+
}
|
|
2185
|
+
void this.renewRuntimeToken();
|
|
2186
|
+
}, schedule.delayMs);
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
// Can the runner self-renew right now? Requires a non-expired runner-profile token
|
|
2190
|
+
// (the relay rejects renewal of an expired token).
|
|
2191
|
+
private isRuntimeTokenRenewable(): boolean {
|
|
2192
|
+
return Boolean(
|
|
2193
|
+
this.currentToken &&
|
|
2194
|
+
this.currentTokenExpiresAt &&
|
|
2195
|
+
this.currentTokenExpiresAt * 1000 > Date.now() &&
|
|
2196
|
+
(this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
|
|
2197
|
+
);
|
|
2198
|
+
}
|
|
2199
|
+
|
|
2200
|
+
// Can the runner recover its token via the orchestrator? Works even when the token
|
|
2201
|
+
// is already expired — the orchestrator's standing credential is the authority.
|
|
2202
|
+
private canRemintViaOrchestrator(): boolean {
|
|
2203
|
+
return Boolean(
|
|
2204
|
+
orchestratorUrlFromEnv() &&
|
|
2205
|
+
this.currentToken &&
|
|
2206
|
+
(this.currentTokenProfileId === "provider-agent" || this.currentTokenProfileId === "provider-interactive"),
|
|
2207
|
+
);
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
private async renewRuntimeToken(): Promise<void> {
|
|
2211
|
+
if (this.stopped || this.tokenRenewInFlight || !this.currentToken) return;
|
|
2212
|
+
this.tokenRenewInFlight = true;
|
|
2213
|
+
try {
|
|
2214
|
+
// Preferred path: self-renew directly against the relay while the token is
|
|
2215
|
+
// still valid. Cheapest and needs no orchestrator round-trip.
|
|
2216
|
+
if (this.isRuntimeTokenRenewable()) {
|
|
2217
|
+
try {
|
|
2218
|
+
const renewed = await this.http.renewRuntimeToken();
|
|
2219
|
+
this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-renewed");
|
|
2220
|
+
return;
|
|
2221
|
+
} catch (error) {
|
|
2222
|
+
this.logRuntimeTokenRenewalFailure(error);
|
|
2223
|
+
// Relay unreachable or token rejected — fall through to orchestrator re-mint.
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
// Recovery path: token expired, or self-renew failed. Ask the orchestrator —
|
|
2227
|
+
// it holds a long-lived credential and can mint a fresh runner token, so a
|
|
2228
|
+
// live session heals instead of being stranded off the bus.
|
|
2229
|
+
if (this.canRemintViaOrchestrator() && await this.remintViaOrchestrator()) return;
|
|
2230
|
+
this.pendingTimelineEvent = {
|
|
2231
|
+
status: "runtime-token-renewal-failed",
|
|
2232
|
+
timestamp: Date.now(),
|
|
2233
|
+
};
|
|
2234
|
+
this.publishStatus();
|
|
2235
|
+
this.scheduleRuntimeTokenRenewal(TOKEN_RENEW_RETRY_MS);
|
|
2236
|
+
} finally {
|
|
2237
|
+
this.tokenRenewInFlight = false;
|
|
2238
|
+
}
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
// Apply a freshly issued token across every live surface — runner state, the
|
|
2242
|
+
// RunnerOptions bag (re-injected into the provider on respawn), the HTTP client,
|
|
2243
|
+
// the bus client — then force a bus handshake with the new token and reschedule.
|
|
2244
|
+
private applyRenewedToken(
|
|
2245
|
+
token: string,
|
|
2246
|
+
record: { jti: string; profileId?: string; expiresAt?: number },
|
|
2247
|
+
status: "runtime-token-renewed" | "runtime-token-reminted",
|
|
2248
|
+
): void {
|
|
2249
|
+
this.currentToken = token;
|
|
2250
|
+
this.currentTokenJti = record.jti;
|
|
2251
|
+
this.currentTokenProfileId = record.profileId ?? this.currentTokenProfileId;
|
|
2252
|
+
this.currentTokenExpiresAt = record.expiresAt;
|
|
2253
|
+
this.options.token = token;
|
|
2254
|
+
this.options.tokenJti = record.jti;
|
|
2255
|
+
this.options.tokenProfileId = this.currentTokenProfileId;
|
|
2256
|
+
this.options.tokenExpiresAt = this.currentTokenExpiresAt;
|
|
2257
|
+
this.http.setToken(token);
|
|
2258
|
+
this.bus.setToken(token);
|
|
2259
|
+
// The proxy reads the token live via getToken(), so forwarding already uses the new one.
|
|
2260
|
+
// A re-mint can change scope (e.g. a profile change), so refresh the relay tool list and
|
|
2261
|
+
// emit tools/list_changed if the visible set changed (#215 — token-scope transition).
|
|
2262
|
+
void this.proxy?.refreshTools().catch(() => {});
|
|
2263
|
+
this.httpLivenessAuthFailed = false;
|
|
2264
|
+
this.reactiveTokenRecoveryAt = undefined;
|
|
2265
|
+
// An earlier auth failure may have stopped the liveness loop; restart it so the
|
|
2266
|
+
// agent reports live again on the fresh token. startHttpLiveness clears any
|
|
2267
|
+
// existing timer first, so this is safe on the normal (proactive) renew path too.
|
|
2268
|
+
this.startHttpLiveness();
|
|
2269
|
+
this.pendingTimelineEvent = { status, id: record.jti, timestamp: Date.now() };
|
|
2270
|
+
this.bus.reconnectTransport(status === "runtime-token-reminted" ? "runtime token re-minted" : "runtime token renewed");
|
|
2271
|
+
this.publishStatus();
|
|
2272
|
+
this.scheduleRuntimeTokenRenewal();
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
// Recover the runtime token through the orchestrator. The runner proxies its own
|
|
2276
|
+
// (possibly expired) token; the orchestrator re-mints it via the relay using its
|
|
2277
|
+
// standing credential. Returns true on success.
|
|
2278
|
+
private async remintViaOrchestrator(): Promise<boolean> {
|
|
2279
|
+
const orchUrl = orchestratorUrlFromEnv();
|
|
2280
|
+
if (!orchUrl || !this.currentToken) return false;
|
|
2281
|
+
try {
|
|
2282
|
+
const res = await fetch(`${orchUrl.replace(/\/+$/, "")}/api/runtime-tokens/runner-renew`, {
|
|
2283
|
+
method: "POST",
|
|
2284
|
+
headers: { "Content-Type": "application/json" },
|
|
2285
|
+
body: JSON.stringify({ token: this.currentToken }),
|
|
2286
|
+
signal: AbortSignal.timeout(10_000),
|
|
2287
|
+
});
|
|
2288
|
+
if (!res.ok) return false;
|
|
2289
|
+
const renewed = await res.json() as { token?: string; record?: { jti: string; profileId?: string; expiresAt?: number } };
|
|
2290
|
+
if (!renewed?.token || !renewed.record) return false;
|
|
2291
|
+
this.applyRenewedToken(renewed.token, renewed.record, "runtime-token-reminted");
|
|
2292
|
+
this.logRunnerDiagnostic(`[runner] runtime token re-minted via orchestrator (jti ${renewed.record.jti})`);
|
|
2293
|
+
return true;
|
|
2294
|
+
} catch (error) {
|
|
2295
|
+
this.logRuntimeTokenRenewalFailure(error);
|
|
2296
|
+
return false;
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
|
|
2300
|
+
private logRuntimeTokenRenewalFailure(error: unknown): void {
|
|
2301
|
+
const key = httpErrorKey(error);
|
|
2302
|
+
const now = Date.now();
|
|
2303
|
+
if (
|
|
2304
|
+
this.tokenRenewLastLog?.key === key &&
|
|
2305
|
+
now - this.tokenRenewLastLog.at < HTTP_LIVENESS_LOG_INTERVAL_MS
|
|
2306
|
+
) return;
|
|
2307
|
+
this.tokenRenewLastLog = { key, at: now };
|
|
2308
|
+
this.logRunnerDiagnostic(`[runner] runtime token renewal failed: ${String(error)}`);
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
private heartbeatRuntimeMeta(): Record<string, unknown> | undefined {
|
|
2312
|
+
const processContext = this.latestProcessContext();
|
|
2313
|
+
const probeMetrics = this.latestProbeMetrics();
|
|
2314
|
+
const probeContext = probeMetrics ? contextStateFromProbeMetrics(probeMetrics) : undefined;
|
|
2315
|
+
const context = processContext ?? probeContext;
|
|
2316
|
+
const quota = probeMetrics ? quotaStateFromProbeMetrics(probeMetrics) : undefined;
|
|
2317
|
+
const terminalSession = providerTerminalSession(this.process);
|
|
2318
|
+
const terminalSocket = providerTerminalSocket(this.process);
|
|
2319
|
+
const probeModel: ProbeModelInfo | undefined = probeMetrics?.model || probeMetrics?.effort
|
|
2320
|
+
? { model: probeMetrics.model, effort: probeMetrics.effort }
|
|
2321
|
+
: undefined;
|
|
2322
|
+
const meta: Record<string, unknown> = {
|
|
2323
|
+
providerCapabilities: runtimeProviderCapabilities(this.options, context, probeModel, quota),
|
|
2324
|
+
...(terminalSession ? { tmuxSession: terminalSession } : {}),
|
|
2325
|
+
...(terminalSocket ? { tmuxSocket: terminalSocket } : {}),
|
|
2326
|
+
};
|
|
2327
|
+
if (context) meta.context = context;
|
|
2328
|
+
return meta;
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
private latestProcessContext(): ContextState | undefined {
|
|
2332
|
+
const getContext = this.process?.meta?.getContext;
|
|
2333
|
+
if (typeof getContext === "function") {
|
|
2334
|
+
const context = getContext();
|
|
2335
|
+
if (isContextState(context)) return context;
|
|
2336
|
+
}
|
|
2337
|
+
const context = this.process?.meta?.context;
|
|
2338
|
+
return isContextState(context) ? context : undefined;
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
private latestProbeMetrics() {
|
|
2342
|
+
return readRunnerContextProbeState(this.agentId);
|
|
2343
|
+
}
|
|
2344
|
+
|
|
2345
|
+
private matchesMessage(message: Message): boolean {
|
|
2346
|
+
return runnerMessageMatches(message, {
|
|
2347
|
+
agentId: this.agentId,
|
|
2348
|
+
label: this.options.label,
|
|
2349
|
+
provider: this.options.provider,
|
|
2350
|
+
tags: this.options.tags,
|
|
2351
|
+
capabilities: this.options.capabilities,
|
|
2352
|
+
defaultTags: this.options.providerConfig.defaultTags,
|
|
2353
|
+
defaultCapabilities: this.options.providerConfig.defaultCapabilities,
|
|
2354
|
+
allowBroadTargets: this.options.agentProfile?.relay?.context !== false,
|
|
2355
|
+
});
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
private startClaimRenewer(): void {
|
|
2359
|
+
if (this.claimRenewTimer) return;
|
|
2360
|
+
this.claimRenewTimer = setInterval(() => {
|
|
2361
|
+
void this.renewActiveTaskClaims();
|
|
2362
|
+
}, CLAIM_RENEW_INTERVAL_MS);
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
private async renewActiveTaskClaims(): Promise<void> {
|
|
2366
|
+
for (const claim of this.activeTaskClaims.values()) {
|
|
2367
|
+
const renewed = await this.http.renewMessageClaim(claim.messageId, this.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
|
|
2368
|
+
if (renewed.ok) {
|
|
2369
|
+
this.claims.startClaim("message", String(claim.messageId), renewed.claimExpiresAt);
|
|
2370
|
+
this.claims.startClaim("task", String(claim.taskId), renewed.claimExpiresAt);
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
this.publishStatus();
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
private async completeObservedTaskClaims(): Promise<void> {
|
|
2377
|
+
const completed = [...this.activeTaskClaims.values()].filter((claim) => claim.observedProviderBusy);
|
|
2378
|
+
for (const claim of completed) {
|
|
2379
|
+
const ok = await this.updateTaskStatus(claim.taskId, {
|
|
2380
|
+
status: "done",
|
|
2381
|
+
agentId: this.agentId,
|
|
2382
|
+
body: `Runner observed provider completion for message #${claim.messageId}`,
|
|
2383
|
+
metadata: { messageId: claim.messageId, completedBy: "runner" },
|
|
2384
|
+
})
|
|
2385
|
+
.then(() => true)
|
|
2386
|
+
.catch((error) => {
|
|
2387
|
+
logger.error("task", `task ${claim.taskId} completion update failed: ${error}`);
|
|
2388
|
+
return false;
|
|
2389
|
+
});
|
|
2390
|
+
if (!ok) continue;
|
|
2391
|
+
this.activeTaskClaims.delete(claim.messageId);
|
|
2392
|
+
this.claims.finishClaim("message", String(claim.messageId));
|
|
2393
|
+
this.claims.finishClaim("task", String(claim.taskId));
|
|
2394
|
+
}
|
|
2395
|
+
if (completed.length > 0) this.publishStatus();
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
private async updateTaskStatus(taskId: number, input: TaskStatusInput): Promise<void> {
|
|
2399
|
+
await this.http.updateTaskStatus(taskId, input);
|
|
2400
|
+
}
|
|
2401
|
+
|
|
2402
|
+
private clearActiveClaim(message: Message): void {
|
|
2403
|
+
const taskId = taskIdFromMessage(message);
|
|
2404
|
+
this.claims.finishClaim("message", String(message.id));
|
|
2405
|
+
if (!taskId) return;
|
|
2406
|
+
this.activeTaskClaims.delete(message.id);
|
|
2407
|
+
this.claims.finishClaim("task", String(taskId));
|
|
2408
|
+
}
|
|
2409
|
+
}
|