@runuai/host 0.9.14 → 0.9.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -5
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +80 -30
- package/images/standard/Dockerfile +36 -10
- package/images/standard/README.md +63 -18
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +308 -38
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/lib/agent-cli.ts +33 -2
- package/lib/agent.ts +46 -7
- package/lib/agents/claude.ts +13 -8
- package/lib/agents/codex.ts +11 -6
- package/lib/agents/cursor.ts +39 -29
- package/lib/agents/durable-proc.ts +20 -27
- package/lib/agents/factory.ts +9 -25
- package/lib/agents/grok.ts +43 -30
- package/lib/agents/kimi.ts +44 -29
- package/lib/agents/opencode.ts +43 -31
- package/lib/agents/proc.ts +149 -114
- package/lib/agents/transport.ts +62 -50
- package/lib/agents/types.ts +6 -4
- package/lib/apple-runtime-recycle.ts +236 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/browser-testing.ts +233 -93
- package/lib/codex-auth.ts +40 -6
- package/lib/command-db.ts +20 -0
- package/lib/container-runtime.ts +1338 -0
- package/lib/db.ts +1 -0
- package/lib/docker-exec.ts +87 -5
- package/lib/engine-accounts.ts +68 -5
- package/lib/engine-login.ts +1952 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +155 -0
- package/lib/env.ts +4 -0
- package/lib/git-diff.ts +98 -32
- package/lib/git-identity.ts +199 -87
- package/lib/github-tokens.ts +202 -91
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-logs.ts +962 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +49 -0
- package/lib/managed-runtime.ts +3644 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-gateway.ts +450 -23
- package/lib/orchestrator.ts +3051 -200
- package/lib/preview-sidecar.ts +68 -14
- package/lib/release-manifest.ts +708 -0
- package/lib/release-trust.ts +28 -0
- package/lib/runtime-activation-tail.ts +232 -0
- package/lib/runtime-archive.ts +1086 -0
- package/lib/runtime-authority.ts +79 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-state.ts +232 -12
- package/lib/skills.ts +24 -3
- package/lib/ssh.ts +18 -0
- package/lib/standard-image.ts +1104 -141
- package/lib/stopped-task-status-queue.ts +44 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-diff.ts +66 -46
- package/lib/task-environment/apple-container.ts +757 -0
- package/lib/task-environment/docker.ts +956 -0
- package/lib/task-environment/index.ts +364 -0
- package/lib/task-environment/legacy-adoption.ts +459 -0
- package/lib/task-environment/registry.ts +58 -0
- package/lib/task-environment/types.ts +408 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +585 -0
- package/lib/tunnel-registry.ts +135 -19
- package/lib/tunnel-runtime.ts +235 -0
- package/package.json +1 -1
- package/scripts/agent/_common.sh +123 -3
- package/scripts/agent/task-down.sh +146 -38
- package/scripts/agent/task-status.sh +19 -3
- package/scripts/agent/task-up.sh +1405 -107
- package/scripts/install/darwin.ts +848 -50
- package/scripts/install/linux.ts +838 -35
- package/scripts/install/types.ts +43 -0
- package/scripts/install/util.ts +215 -8
- package/scripts/install/win.ts +12 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/cli.ts +1464 -72
- package/src/event-outbox.ts +83 -4
- package/src/index.ts +766 -42
- package/src/main.ts +1398 -255
- package/src/paths.ts +17 -1
- package/src/protocol.ts +695 -1
- package/src/runtime-bootstrap.ts +165 -0
- package/src/ui/server.ts +46 -10
- package/src/ui/types.ts +37 -0
package/src/main.ts
CHANGED
|
@@ -22,8 +22,21 @@ import {
|
|
|
22
22
|
markDisconnected,
|
|
23
23
|
markReconnecting,
|
|
24
24
|
} from "../lib/cloud-state";
|
|
25
|
-
import {
|
|
26
|
-
|
|
25
|
+
import {
|
|
26
|
+
applyTerminalHistoryRetention,
|
|
27
|
+
getHostTask,
|
|
28
|
+
hasInFlightHostTaskLifecycle,
|
|
29
|
+
} from "../lib/runtime-state";
|
|
30
|
+
import { StoppedTaskStatusQueue } from "../lib/stopped-task-status-queue";
|
|
31
|
+
import {
|
|
32
|
+
getOrchestrator,
|
|
33
|
+
quarantineWritableRuntimeContainers,
|
|
34
|
+
recoveryComplete,
|
|
35
|
+
requestRuntimeRecovery,
|
|
36
|
+
resumeRuntimeDeferredAgentWork,
|
|
37
|
+
runtimeRecoveryVerdict,
|
|
38
|
+
waitForRuntimeRecovery,
|
|
39
|
+
} from "../lib/orchestrator";
|
|
27
40
|
import {
|
|
28
41
|
claimGithubCredentialGeneration,
|
|
29
42
|
connectedUserIds,
|
|
@@ -58,7 +71,9 @@ import { handleMcpOp } from "../lib/mcp-connections";
|
|
|
58
71
|
import { startMcpGateway } from "../lib/mcp-gateway";
|
|
59
72
|
import { addHostBreadcrumb, initHostObs, setHostObsTag } from "../lib/obs";
|
|
60
73
|
import {
|
|
74
|
+
hostIdFilePath,
|
|
61
75
|
packageVersion,
|
|
76
|
+
readPersistedHostId,
|
|
62
77
|
serviceLogPath,
|
|
63
78
|
uiAssetsDir,
|
|
64
79
|
uiPortFilePath,
|
|
@@ -66,10 +81,45 @@ import {
|
|
|
66
81
|
import { dockerMemoryBytes, startUiServer } from "./ui/server";
|
|
67
82
|
import { parsePreviewPortRuntimes } from "../lib/preview-ports";
|
|
68
83
|
import { dockerCli } from "../lib/docker-exec";
|
|
84
|
+
import {
|
|
85
|
+
hostTaskInventorySnapshots,
|
|
86
|
+
isSafeHostTaskId,
|
|
87
|
+
isSafeInventoryScanId,
|
|
88
|
+
} from "../lib/task-inventory";
|
|
89
|
+
import { setContainerRuntimeOperationalCheck } from "../lib/runtime-guard";
|
|
90
|
+
import {
|
|
91
|
+
RuntimeActivationTail,
|
|
92
|
+
runMaintainedRuntimeActivation,
|
|
93
|
+
type RuntimeActivationAttemptResult,
|
|
94
|
+
} from "../lib/runtime-activation-tail";
|
|
95
|
+
import {
|
|
96
|
+
containerRuntimeActivationRequired,
|
|
97
|
+
containerRuntimeBecameReady,
|
|
98
|
+
containerRuntimeCapabilityState,
|
|
99
|
+
containerRuntimeEpochIsCurrent,
|
|
100
|
+
containerRuntimeProblem,
|
|
101
|
+
containerRuntimeReadinessEpoch,
|
|
102
|
+
containerRuntimeState,
|
|
103
|
+
forceContainerRuntimeRecheck,
|
|
104
|
+
initializeContainerRuntime,
|
|
105
|
+
markContainerRuntimeOperational,
|
|
106
|
+
onContainerRuntimeChange,
|
|
107
|
+
startContainerRuntimeAutoRecheck,
|
|
108
|
+
suspendContainerRuntime,
|
|
109
|
+
containerRuntimeMachineIdentity,
|
|
110
|
+
publishActivationPhase,
|
|
111
|
+
} from "../lib/container-runtime";
|
|
112
|
+
import { reconstructPersistedTaskEnvironment } from "../lib/task-environment";
|
|
113
|
+
import { resolveAppleTunnelRoute } from "./apple-tunnel-route";
|
|
114
|
+
import {
|
|
115
|
+
configureBundledContainerRuntime,
|
|
116
|
+
selectedRuntimeMatchesPersistedTaskEnvironments,
|
|
117
|
+
} from "./runtime-bootstrap";
|
|
69
118
|
import {
|
|
70
119
|
ensurePreviewSidecar,
|
|
71
120
|
invalidatePreviewSidecar,
|
|
72
121
|
} from "../lib/preview-sidecar";
|
|
122
|
+
import { inspectTunnelContainer } from "../lib/tunnel-runtime";
|
|
73
123
|
import { TunnelRegistry } from "../lib/tunnel-registry";
|
|
74
124
|
import { newId } from "../lib/ulid";
|
|
75
125
|
import {
|
|
@@ -77,18 +127,44 @@ import {
|
|
|
77
127
|
onChange as onRegistryChange,
|
|
78
128
|
} from "../lib/agents/registry";
|
|
79
129
|
import { canAdvertiseTypedSecretaryDispatch } from "../lib/agents/mode";
|
|
130
|
+
import { MANAGED_UPDATE_RESTART_EXIT_CODE } from "../lib/managed-runtime";
|
|
131
|
+
import { ManagedOperationDrain } from "../lib/managed-operation-drain";
|
|
132
|
+
import {
|
|
133
|
+
createHostLogManager,
|
|
134
|
+
HostLogStartError,
|
|
135
|
+
type HostLogsEmit,
|
|
136
|
+
} from "../lib/host-logs";
|
|
137
|
+
import {
|
|
138
|
+
getHostConfig,
|
|
139
|
+
patchHostConfig,
|
|
140
|
+
type HostConfigResult,
|
|
141
|
+
} from "../lib/host-config";
|
|
142
|
+
import { createEngineLoginManager } from "../lib/engine-login";
|
|
80
143
|
// Importing the real factory triggers the built-in adapters' register()
|
|
81
144
|
// calls (claude, codex), so the registry is populated before we advertise.
|
|
82
145
|
import "../lib/agents/factory";
|
|
83
|
-
import {
|
|
146
|
+
import {
|
|
147
|
+
areAgentClisReady,
|
|
148
|
+
configuredOptionalEngines,
|
|
149
|
+
ensureStandardImage,
|
|
150
|
+
onAgentClisReady,
|
|
151
|
+
standardRuntimes,
|
|
152
|
+
} from "../lib/standard-image";
|
|
84
153
|
import { hostCommands, hostEvents } from "./index";
|
|
85
|
-
import { EventOutbox } from "./event-outbox";
|
|
154
|
+
import { EventOutbox, transmitEventOutbox } from "./event-outbox";
|
|
86
155
|
import {
|
|
87
156
|
HostErrorCode,
|
|
88
157
|
EVENT_REPLAY_PROTOCOL_FEATURE,
|
|
89
158
|
GITHUB_CREDENTIAL_GENERATION_PROTOCOL_FEATURE,
|
|
90
159
|
GITHUB_INSTALLATION_VERIFICATION_PROTOCOL_FEATURE,
|
|
91
160
|
GITHUB_REPOSITORY_ACCESS_PROTOCOL_FEATURE,
|
|
161
|
+
HOST_MAINTENANCE_READINESS_PROTOCOL_FEATURE,
|
|
162
|
+
HOST_CONFIG_PROTOCOL_FEATURE,
|
|
163
|
+
HOST_ENGINE_LOGIN_PROTOCOL_FEATURE,
|
|
164
|
+
HOST_LOGS_PROTOCOL_FEATURE,
|
|
165
|
+
HOST_TASK_INVENTORY_PROTOCOL_FEATURE,
|
|
166
|
+
MAX_HOST_TASK_INVENTORY_PAGE_SIZE,
|
|
167
|
+
MCP_GATEWAY_HEALTH_PROTOCOL_FEATURE,
|
|
92
168
|
SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
|
|
93
169
|
TRANSCRIPT_TARGETS_PROTOCOL_FEATURE,
|
|
94
170
|
type CloudToHost,
|
|
@@ -103,6 +179,14 @@ import {
|
|
|
103
179
|
parseGitHubCredentialGeneration,
|
|
104
180
|
parseChannelMode,
|
|
105
181
|
parseTranscriptTargets,
|
|
182
|
+
isEngineLoginCallbackFrame,
|
|
183
|
+
isEngineLoginInputFrame,
|
|
184
|
+
isEngineLoginStartFrame,
|
|
185
|
+
isEngineLoginStopFrame,
|
|
186
|
+
isHostConfigGetFrame,
|
|
187
|
+
isHostConfigSetFrame,
|
|
188
|
+
isHostLogsStartFrame,
|
|
189
|
+
isHostLogsStopFrame,
|
|
106
190
|
type TaskAgent,
|
|
107
191
|
type TaskCommandProject,
|
|
108
192
|
type TaskCommandTask,
|
|
@@ -112,6 +196,8 @@ import {
|
|
|
112
196
|
type FilesOpInput,
|
|
113
197
|
} from "./protocol";
|
|
114
198
|
|
|
199
|
+
setContainerRuntimeOperationalCheck(() => containerRuntimeProblem() === null);
|
|
200
|
+
|
|
115
201
|
// Crash reporting (ADR-071) — default-ON for installed hosts via the baked
|
|
116
202
|
// DSN (lib/obs.ts); repo checkouts and dev/test never default-activate.
|
|
117
203
|
// UAI_TELEMETRY_DISABLED=1 turns it off; UAI_SENTRY_DSN overrides.
|
|
@@ -146,9 +232,70 @@ let ws: WebSocket | null = null;
|
|
|
146
232
|
let stopping = false;
|
|
147
233
|
let fatal = false;
|
|
148
234
|
let reconnectAttempt = 0;
|
|
235
|
+
let connectionRequestGeneration = 0;
|
|
149
236
|
let inFlight = 0;
|
|
150
237
|
let shutdownRequested = false;
|
|
238
|
+
let shutdownExitCode = 0;
|
|
239
|
+
const managedMutationDrain = new ManagedOperationDrain(maybeExit);
|
|
240
|
+
const hostLogManager = createHostLogManager();
|
|
241
|
+
const pendingRemoteOperationCleanups = new Set<Promise<void>>();
|
|
242
|
+
let stopActiveRemoteOperations = (): Promise<void> => Promise.resolve();
|
|
243
|
+
|
|
244
|
+
function trackRemoteOperationCleanup(cleanup: Promise<void>): Promise<void> {
|
|
245
|
+
const tracked = cleanup.catch(() => {
|
|
246
|
+
// A replacement generation must fail closed if an unforeseen manager bug
|
|
247
|
+
// escapes its best-effort cleanup contract. Shutdown may still exit after
|
|
248
|
+
// the failed cleanup settles so the service supervisor can recover.
|
|
249
|
+
fatal = true;
|
|
250
|
+
console.warn("[host-agent] remote operation cleanup failed");
|
|
251
|
+
});
|
|
252
|
+
pendingRemoteOperationCleanups.add(tracked);
|
|
253
|
+
void tracked.finally(() => {
|
|
254
|
+
pendingRemoteOperationCleanups.delete(tracked);
|
|
255
|
+
maybeExit();
|
|
256
|
+
});
|
|
257
|
+
return tracked;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function stopRemoteOperationManagers(): Promise<void> {
|
|
261
|
+
hostLogManager.stopAll();
|
|
262
|
+
return trackRemoteOperationCleanup(
|
|
263
|
+
Promise.all([
|
|
264
|
+
engineLoginManager.stopAll(),
|
|
265
|
+
// Tunnel upstreams are generation-scoped resources just like login
|
|
266
|
+
// containers; a bridge replacement or shutdown must join their
|
|
267
|
+
// asynchronous release before a new socket/process generation is
|
|
268
|
+
// admitted.
|
|
269
|
+
tunnels.abortAll(),
|
|
270
|
+
]).then(() => undefined),
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
async function waitForRemoteOperationCleanups(): Promise<void> {
|
|
275
|
+
// A stop admitted just before disconnect may have detached its operation
|
|
276
|
+
// from the manager map already. Join the globally tracked stop as well as
|
|
277
|
+
// stopAll(), and loop in case settling one cleanup queues its exact sweep.
|
|
278
|
+
while (pendingRemoteOperationCleanups.size > 0) {
|
|
279
|
+
await Promise.all([...pendingRemoteOperationCleanups]);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
151
283
|
let pendingBinaryTunnelId: string | null = null;
|
|
284
|
+
const pendingRuntimeIsolationStatuses = new StoppedTaskStatusQueue(
|
|
285
|
+
(taskId) => getHostTask(taskId)?.statusMirror,
|
|
286
|
+
);
|
|
287
|
+
let stopRuntimeAutoRecheck = (): void => {};
|
|
288
|
+
const runtimeActivationTail = new RuntimeActivationTail({
|
|
289
|
+
isCurrent: runtimeActivationIsCurrent,
|
|
290
|
+
onError: (error) =>
|
|
291
|
+
console.warn(
|
|
292
|
+
`[host-agent] container runtime activation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
293
|
+
),
|
|
294
|
+
onExhausted: (_epoch, attempts) =>
|
|
295
|
+
console.warn(
|
|
296
|
+
`[host-agent] container runtime recovery remained unresolved after ${attempts} follow-up attempt(s); leaving the host in checking until the next runtime recheck`,
|
|
297
|
+
),
|
|
298
|
+
});
|
|
152
299
|
const tunnels = new TunnelRegistry();
|
|
153
300
|
|
|
154
301
|
// ADR-103: event outbox. BOOT_ID scopes seqs to this process lifetime; the
|
|
@@ -162,7 +309,37 @@ const eventOutbox = new EventOutbox();
|
|
|
162
309
|
// replay cursor (or the fallback timer concedes the cloud predates ADR-103).
|
|
163
310
|
let eventFlushEnabled = false;
|
|
164
311
|
let resumeFallbackTimer: NodeJS.Timeout | null = null;
|
|
312
|
+
let eventFlushBackpressureTimer: NodeJS.Timeout | null = null;
|
|
165
313
|
const RESUME_FALLBACK_MS = 3_000;
|
|
314
|
+
const EVENT_FLUSH_POLL_MS = 5;
|
|
315
|
+
|
|
316
|
+
function cancelEventFlushBackpressureTimer(): void {
|
|
317
|
+
if (!eventFlushBackpressureTimer) return;
|
|
318
|
+
clearTimeout(eventFlushBackpressureTimer);
|
|
319
|
+
eventFlushBackpressureTimer = null;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function scheduleEventFlushAfterBackpressure(socket: WebSocket): void {
|
|
323
|
+
if (eventFlushBackpressureTimer) return;
|
|
324
|
+
const poll = (): void => {
|
|
325
|
+
eventFlushBackpressureTimer = null;
|
|
326
|
+
if (
|
|
327
|
+
!eventFlushEnabled ||
|
|
328
|
+
ws !== socket ||
|
|
329
|
+
socket.readyState !== WebSocket.OPEN
|
|
330
|
+
) {
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
if (socket.bufferedAmount > MAX_WS_BUFFERED_BYTES / 2) {
|
|
334
|
+
eventFlushBackpressureTimer = setTimeout(poll, EVENT_FLUSH_POLL_MS);
|
|
335
|
+
eventFlushBackpressureTimer.unref?.();
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
flushEvents();
|
|
339
|
+
};
|
|
340
|
+
eventFlushBackpressureTimer = setTimeout(poll, EVENT_FLUSH_POLL_MS);
|
|
341
|
+
eventFlushBackpressureTimer.unref?.();
|
|
342
|
+
}
|
|
166
343
|
|
|
167
344
|
function flushEvents(): void {
|
|
168
345
|
const socket = ws;
|
|
@@ -175,7 +352,16 @@ function flushEvents(): void {
|
|
|
175
352
|
`[host-agent] event outbox overflowed: ${dropped} unsent event(s) lost`,
|
|
176
353
|
);
|
|
177
354
|
}
|
|
178
|
-
|
|
355
|
+
const result = transmitEventOutbox(
|
|
356
|
+
eventOutbox,
|
|
357
|
+
socket,
|
|
358
|
+
MAX_WS_BUFFERED_BYTES,
|
|
359
|
+
);
|
|
360
|
+
if (result === "backpressured") {
|
|
361
|
+
scheduleEventFlushAfterBackpressure(socket);
|
|
362
|
+
} else if (result === "send_failed") {
|
|
363
|
+
socket.close(1011, "event transmission failed");
|
|
364
|
+
}
|
|
179
365
|
}
|
|
180
366
|
|
|
181
367
|
interface PausableSource {
|
|
@@ -186,6 +372,38 @@ interface PausableSource {
|
|
|
186
372
|
|
|
187
373
|
console.log(`[host-agent] starting host ${hostId}`);
|
|
188
374
|
migrateHostDb();
|
|
375
|
+
// Reconcile a durable retention-off setting before recovery/connect can make
|
|
376
|
+
// crash-left owner-facing terminal history observable again. Correctness
|
|
377
|
+
// tombstones remain intact inside applyTerminalHistoryRetention.
|
|
378
|
+
applyTerminalHistoryRetention();
|
|
379
|
+
try {
|
|
380
|
+
await configureBundledContainerRuntime();
|
|
381
|
+
} catch {
|
|
382
|
+
// A malformed/tampered active generation must never fall back to PATH or an
|
|
383
|
+
// inferred asset layout. Keep Docker-first operation available and surface
|
|
384
|
+
// the failure through the normal no-runtime capability instead.
|
|
385
|
+
console.warn(
|
|
386
|
+
"[host-agent] signed bundled runtime validation failed; the bundled runtime is disabled",
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
const initialRuntime = await initializeContainerRuntime();
|
|
390
|
+
// Do not touch the selected daemon merely because detection succeeded. The
|
|
391
|
+
// activation gate below must first prove that every durable TaskEnvironment
|
|
392
|
+
// belongs to this exact provider/endpoint. The manager's first admitted login
|
|
393
|
+
// performs the same bounded reconciliation after operational publication.
|
|
394
|
+
const engineLoginManager = createEngineLoginManager({
|
|
395
|
+
refreshEngineAccountAgents: (engine) =>
|
|
396
|
+
getOrchestrator().refreshEngineAccountAgents(engine),
|
|
397
|
+
tempRoot: join(env.uaiHome, "engine-login-bind"),
|
|
398
|
+
reconcileOnCreate: false,
|
|
399
|
+
});
|
|
400
|
+
if (initialRuntime.status === "ready") {
|
|
401
|
+
console.log(
|
|
402
|
+
`[host-agent] container runtime ready (${initialRuntime.provider}, ${initialRuntime.preference})`,
|
|
403
|
+
);
|
|
404
|
+
} else if (initialRuntime.status === "no-runtime") {
|
|
405
|
+
console.warn(`[host-agent] ${initialRuntime.message}`);
|
|
406
|
+
}
|
|
189
407
|
// Surface a task's GitHub setup failure in its channel (ADR-027). Classify the
|
|
190
408
|
// reason first: a GitHub-side 5xx / network blip (e.g. `gh auth login`
|
|
191
409
|
// validating the token during a github.com outage) is transient and self-heals
|
|
@@ -201,9 +419,6 @@ setAuthExpiredHandler((taskId, _userId, reason) => {
|
|
|
201
419
|
`Account, then run /retry-gh in this task to restore.`;
|
|
202
420
|
getOrchestrator().emitSystemNote(taskId, note);
|
|
203
421
|
});
|
|
204
|
-
// Best-effort: build the standard image + asdf volume if missing. Logs and
|
|
205
|
-
// continues on failure (e.g. docker unavailable) so the host still boots.
|
|
206
|
-
void ensureStandardImage();
|
|
207
422
|
// Codex creds are docker-cp'd into containers at task-up. A re-login (revoked
|
|
208
423
|
// token → `codex login`) otherwise reaches only NEW tasks — re-copy into
|
|
209
424
|
// running tasks on start (covers the desktop "Connect Codex", which restarts
|
|
@@ -214,8 +429,12 @@ void ensureStandardImage();
|
|
|
214
429
|
// files owned by the host uid (macOS 501) that the container's node (1000)
|
|
215
430
|
// can't read, and Codex died at startup. The post-recovery sweep also
|
|
216
431
|
// self-heals such containers: it re-copies and chowns every running task.
|
|
217
|
-
|
|
218
|
-
|
|
432
|
+
if (initialRuntime.status === "ready") {
|
|
433
|
+
void requestContainerRuntimeActivation(
|
|
434
|
+
recoveryComplete,
|
|
435
|
+
containerRuntimeReadinessEpoch(),
|
|
436
|
+
);
|
|
437
|
+
}
|
|
219
438
|
// ADR-103: subscribe ONCE, for the process — not per connection. Every event
|
|
220
439
|
// lands in the outbox regardless of socket state; flushEvents is a no-op
|
|
221
440
|
// while disconnected and the backlog drains after the resume handshake.
|
|
@@ -223,12 +442,21 @@ hostEvents.subscribe((event) => {
|
|
|
223
442
|
eventOutbox.enqueue(event);
|
|
224
443
|
flushEvents();
|
|
225
444
|
});
|
|
226
|
-
|
|
445
|
+
// Start local health-bearing services before the bridge. The first
|
|
446
|
+
// authenticated capability frame must describe the gateway even when its
|
|
447
|
+
// initial bind is still pending, and both one-way readiness edges re-advertise
|
|
448
|
+
// on the exact live socket generation.
|
|
449
|
+
const mcpGateway = startMcpGateway();
|
|
450
|
+
const stopMcpGatewaySubscription = mcpGateway.subscribe(() =>
|
|
451
|
+
sendCapabilities(),
|
|
452
|
+
);
|
|
453
|
+
const stopAgentClisReadySubscription = onAgentClisReady(() =>
|
|
454
|
+
sendCapabilities(),
|
|
455
|
+
);
|
|
456
|
+
void connect();
|
|
227
457
|
// Local browser UI (ADR-028) — same single process, alongside the WSS client.
|
|
228
458
|
// Best-effort: a UI bind failure must not take the host service down.
|
|
229
459
|
void startLocalUi();
|
|
230
|
-
// ADR-057: the MCP gateway task containers reach via host.docker.internal.
|
|
231
|
-
startMcpGateway();
|
|
232
460
|
|
|
233
461
|
async function startLocalUi(): Promise<void> {
|
|
234
462
|
try {
|
|
@@ -241,7 +469,16 @@ async function startLocalUi(): Promise<void> {
|
|
|
241
469
|
cloudUrl: bridgeUrl,
|
|
242
470
|
hostId,
|
|
243
471
|
logPath: serviceLogPath(),
|
|
244
|
-
taskMemory:
|
|
472
|
+
taskMemory: async (composeProject) =>
|
|
473
|
+
containerRuntimeProblem()
|
|
474
|
+
? null
|
|
475
|
+
: dockerMemoryBytes(composeProject),
|
|
476
|
+
runtimeState: containerRuntimeCapabilityState,
|
|
477
|
+
recheckRuntime: async () => {
|
|
478
|
+
await requestForcedContainerRuntimeRecheck();
|
|
479
|
+
return containerRuntimeCapabilityState();
|
|
480
|
+
},
|
|
481
|
+
containersOperational: () => containerRuntimeProblem() === null,
|
|
245
482
|
// Engine connect/disconnect in the local UI re-advertises capabilities so
|
|
246
483
|
// the cloud's task picker reflects a newly-configured engine promptly.
|
|
247
484
|
readvertise: sendCapabilities,
|
|
@@ -249,10 +486,10 @@ async function startLocalUi(): Promise<void> {
|
|
|
249
486
|
// On success, push the status up so the cloud mirrors `stopped` (it has no
|
|
250
487
|
// auto-reconcile trigger); otherwise it would keep showing `running`.
|
|
251
488
|
stopTask: async (taskId) => {
|
|
489
|
+
const runtimeProblem = containerRuntimeProblem();
|
|
490
|
+
if (runtimeProblem) return { ok: false, error: runtimeProblem };
|
|
252
491
|
const result = await getOrchestrator().stopTask(taskId);
|
|
253
|
-
if (result.ok
|
|
254
|
-
send(ws, { kind: "task.status", taskId, status: "stopped" });
|
|
255
|
-
}
|
|
492
|
+
if (result.ok) queueStoppedTaskStatus(taskId);
|
|
256
493
|
return result;
|
|
257
494
|
},
|
|
258
495
|
});
|
|
@@ -266,6 +503,7 @@ async function startLocalUi(): Promise<void> {
|
|
|
266
503
|
|
|
267
504
|
/** Build the current host capability advertisement (ADR-021). */
|
|
268
505
|
function buildCapabilities(): HostCapabilities {
|
|
506
|
+
const runtime = containerRuntimeCapabilityState();
|
|
269
507
|
return {
|
|
270
508
|
version: packageVersion(),
|
|
271
509
|
protocolFeatures: [
|
|
@@ -274,6 +512,12 @@ function buildCapabilities(): HostCapabilities {
|
|
|
274
512
|
GITHUB_CREDENTIAL_GENERATION_PROTOCOL_FEATURE,
|
|
275
513
|
GITHUB_INSTALLATION_VERIFICATION_PROTOCOL_FEATURE,
|
|
276
514
|
GITHUB_REPOSITORY_ACCESS_PROTOCOL_FEATURE,
|
|
515
|
+
HOST_TASK_INVENTORY_PROTOCOL_FEATURE,
|
|
516
|
+
HOST_MAINTENANCE_READINESS_PROTOCOL_FEATURE,
|
|
517
|
+
MCP_GATEWAY_HEALTH_PROTOCOL_FEATURE,
|
|
518
|
+
HOST_LOGS_PROTOCOL_FEATURE,
|
|
519
|
+
HOST_ENGINE_LOGIN_PROTOCOL_FEATURE,
|
|
520
|
+
HOST_CONFIG_PROTOCOL_FEATURE,
|
|
277
521
|
// The echo adapter cannot execute the in-task CLI. Advertising typed
|
|
278
522
|
// dispatch in mock mode would let the composer create a Secretary that
|
|
279
523
|
// has no way to wake crew.
|
|
@@ -282,7 +526,13 @@ function buildCapabilities(): HostCapabilities {
|
|
|
282
526
|
: []),
|
|
283
527
|
],
|
|
284
528
|
agentKinds: agentKindCapabilities(),
|
|
285
|
-
runtimes
|
|
529
|
+
// Language runtimes live in the standard container image. Do not
|
|
530
|
+
// advertise them as usable while the machine cannot run a container.
|
|
531
|
+
runtimes: runtime.status === "ready" ? standardRuntimes() : [],
|
|
532
|
+
containerRuntime: runtime,
|
|
533
|
+
maintenanceReady: areAgentClisReady(),
|
|
534
|
+
mcpGateway: mcpGateway.state(),
|
|
535
|
+
engineLogins: engineLoginManager.capabilities(),
|
|
286
536
|
githubUsers: connectedUserIds(),
|
|
287
537
|
};
|
|
288
538
|
}
|
|
@@ -298,14 +548,291 @@ function sendCapabilities(): void {
|
|
|
298
548
|
}
|
|
299
549
|
}
|
|
300
550
|
|
|
551
|
+
/** Keep a rolling-isolation/local-UI stop from remaining cloud-ghost-running
|
|
552
|
+
* merely because the bridge was between connections at the transition. */
|
|
553
|
+
function queueStoppedTaskStatus(taskId: string): void {
|
|
554
|
+
pendingRuntimeIsolationStatuses.enqueue(taskId);
|
|
555
|
+
flushStoppedTaskStatuses();
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
function flushStoppedTaskStatuses(socket: WebSocket | null = ws): void {
|
|
559
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) return;
|
|
560
|
+
pendingRuntimeIsolationStatuses.flush((taskId) => {
|
|
561
|
+
send(socket, { kind: "task.status", taskId, status: "stopped" });
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
|
|
301
565
|
onRegistryChange(() => sendCapabilities());
|
|
566
|
+
onContainerRuntimeChange((next, previous) => {
|
|
567
|
+
if (stopping) return;
|
|
568
|
+
sendCapabilities();
|
|
569
|
+
if (next.status === "ready") {
|
|
570
|
+
stopRuntimeAutoRecheck();
|
|
571
|
+
stopRuntimeAutoRecheck = () => {};
|
|
572
|
+
} else if (previous.status === "ready") {
|
|
573
|
+
stopRuntimeAutoRecheck();
|
|
574
|
+
stopRuntimeAutoRecheck = startContainerRuntimeAutoRecheck();
|
|
575
|
+
}
|
|
576
|
+
if (
|
|
577
|
+
containerRuntimeBecameReady(next, previous) ||
|
|
578
|
+
containerRuntimeActivationRequired()
|
|
579
|
+
) {
|
|
580
|
+
void requestContainerRuntimeActivation(
|
|
581
|
+
requestRuntimeRecovery,
|
|
582
|
+
containerRuntimeReadinessEpoch(),
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
});
|
|
586
|
+
stopRuntimeAutoRecheck = startContainerRuntimeAutoRecheck();
|
|
302
587
|
// Re-advertise when a gh token is added/removed (ADR-033) so the host page's
|
|
303
588
|
// per-host connected state updates promptly.
|
|
304
589
|
onGithubChange(() => sendCapabilities());
|
|
305
590
|
|
|
306
|
-
|
|
591
|
+
type RuntimeRecoveryRequest = (
|
|
592
|
+
isCurrent: () => boolean,
|
|
593
|
+
) => Promise<boolean>;
|
|
594
|
+
/** Serialize activation generations. The promise tail avoids a late-enqueue
|
|
595
|
+
* gap between a pump's last queue read and its cleanup callback. */
|
|
596
|
+
function requestContainerRuntimeActivation(
|
|
597
|
+
recover: RuntimeRecoveryRequest,
|
|
598
|
+
epoch: number,
|
|
599
|
+
): Promise<void> {
|
|
600
|
+
return runtimeActivationTail.request(epoch, () =>
|
|
601
|
+
activateContainerRuntime(recover, epoch),
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function scheduleContainerRuntimeActivationRetry(
|
|
606
|
+
recover: RuntimeRecoveryRequest,
|
|
607
|
+
epoch: number,
|
|
608
|
+
): void {
|
|
609
|
+
runtimeActivationTail.scheduleRetry(epoch, () =>
|
|
610
|
+
activateContainerRuntime(recover, epoch),
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** Turn a detected daemon into an operational task backend. Until every step
|
|
615
|
+
* completes, capabilities stay `checking` and Docker-dependent commands fail
|
|
616
|
+
* retryably instead of racing recovery or mistaking unknown state for death. */
|
|
617
|
+
async function activateContainerRuntime(
|
|
618
|
+
recover: RuntimeRecoveryRequest,
|
|
619
|
+
epoch: number,
|
|
620
|
+
): Promise<RuntimeActivationAttemptResult> {
|
|
621
|
+
publishActivationPhase("joining in-flight recovery");
|
|
622
|
+
await waitForRuntimeRecovery();
|
|
623
|
+
return runMaintainedRuntimeActivation({
|
|
624
|
+
isCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
625
|
+
beforeMaintenance: async () => {
|
|
626
|
+
// This is the first selected-daemon operation in activation. Durable
|
|
627
|
+
// TaskEnvironment locators must match the exact process pin before
|
|
628
|
+
// quarantine, image/shared-volume writers, or recovery can touch it.
|
|
629
|
+
publishActivationPhase("fencing persisted task environments");
|
|
630
|
+
if (!(await selectedRuntimeMatchesPersistedTaskEnvironments())) {
|
|
631
|
+
publishActivationPhase("fence rejected the selected runtime");
|
|
632
|
+
return "stale";
|
|
633
|
+
}
|
|
634
|
+
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
635
|
+
publishActivationPhase("quarantining writable runtime containers");
|
|
636
|
+
const isolated = await quarantineWritableRuntimeContainers({
|
|
637
|
+
lifecycle: getOrchestrator(),
|
|
638
|
+
isCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
639
|
+
});
|
|
640
|
+
for (const taskId of isolated.stoppedTaskIds) {
|
|
641
|
+
queueStoppedTaskStatus(taskId);
|
|
642
|
+
}
|
|
643
|
+
publishActivationPhase(
|
|
644
|
+
`writable-container quarantine: ${isolated.verdict}`,
|
|
645
|
+
);
|
|
646
|
+
return isolated.verdict;
|
|
647
|
+
},
|
|
648
|
+
maintenance: () => {
|
|
649
|
+
publishActivationPhase(
|
|
650
|
+
"maintenance: standard image + shared runtime volume",
|
|
651
|
+
);
|
|
652
|
+
return reconcileStableStandardImage(epoch);
|
|
653
|
+
},
|
|
654
|
+
recover,
|
|
655
|
+
// A bounded recovery pass can finish with active tasks still unknowable.
|
|
656
|
+
// Keep the public runtime in `checking`. A real outage becomes
|
|
657
|
+
// `no-runtime`; a still-ready daemon requests a bounded follow-up only
|
|
658
|
+
// after this activation promise has left the tail.
|
|
659
|
+
runtimeReady: async () =>
|
|
660
|
+
(await initializeContainerRuntime()).status === "ready",
|
|
661
|
+
afterRecoveryBeforeFinalMaintenance: async () => {
|
|
662
|
+
try {
|
|
663
|
+
await reinjectCodexRunningTasks();
|
|
664
|
+
} catch (error) {
|
|
665
|
+
console.warn(
|
|
666
|
+
`[codex] post-recovery reinject failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
667
|
+
);
|
|
668
|
+
}
|
|
669
|
+
},
|
|
670
|
+
// Credentials can be edited from the local UI during a long recovery. A
|
|
671
|
+
// final stable pass ensures a newly-enabled optional engine is in the
|
|
672
|
+
// image before the ready frame reattaches persisted agent sessions.
|
|
673
|
+
publishOperational: async () => {
|
|
674
|
+
if (
|
|
675
|
+
runtimeActivationIsCurrent(epoch) &&
|
|
676
|
+
markContainerRuntimeOperational(epoch)
|
|
677
|
+
) {
|
|
678
|
+
watchCodexAuth(() => containerRuntimeProblem() === null);
|
|
679
|
+
resumeRuntimeDeferredAgentWork();
|
|
680
|
+
sendCapabilities();
|
|
681
|
+
}
|
|
682
|
+
},
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
function runtimeActivationIsCurrent(epoch: number): boolean {
|
|
687
|
+
return !stopping && containerRuntimeEpochIsCurrent(epoch);
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
async function reconcileStableStandardImage(
|
|
691
|
+
epoch: number,
|
|
692
|
+
): Promise<"ready" | "retry" | "stale"> {
|
|
693
|
+
while (runtimeActivationIsCurrent(epoch)) {
|
|
694
|
+
const before = JSON.stringify(configuredOptionalEngines());
|
|
695
|
+
const result = await ensureStandardImage();
|
|
696
|
+
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
697
|
+
if (!result.ok && result.remedy === "stop-volume-holders") {
|
|
698
|
+
// The shared volume needs a repair the running holder tasks block and
|
|
699
|
+
// cannot safely use (their agent CLIs fail the probe). Lifecycle-stop
|
|
700
|
+
// them (resumable) so the retry can attach writable — the non-writer
|
|
701
|
+
// escape from the review's activation deadlock (2026-08-18 round 2).
|
|
702
|
+
console.warn(
|
|
703
|
+
`[host-agent] ${result.error ?? "shared runtime volume needs repair"} — stopping holder tasks (resumable)`,
|
|
704
|
+
);
|
|
705
|
+
const swept = await quarantineWritableRuntimeContainers({
|
|
706
|
+
lifecycle: getOrchestrator(),
|
|
707
|
+
isCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
708
|
+
scope: "volume-holders",
|
|
709
|
+
});
|
|
710
|
+
for (const taskId of swept.stoppedTaskIds) {
|
|
711
|
+
queueStoppedTaskStatus(taskId);
|
|
712
|
+
}
|
|
713
|
+
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
714
|
+
return "retry";
|
|
715
|
+
}
|
|
716
|
+
if (!result.ok) {
|
|
717
|
+
// A stale image remains usable only when ensureStandardImage also
|
|
718
|
+
// completed the shared-volume transaction successfully. A failed
|
|
719
|
+
// default-Node/CLI/Corepack proof must never release boot recovery or
|
|
720
|
+
// operational publication: a stopped legacy app could otherwise start
|
|
721
|
+
// with a partially removed CLI-owning candidate. Re-probe Docker to
|
|
722
|
+
// choose between this epoch's bounded activation retry and the normal
|
|
723
|
+
// daemon-loss generation transition.
|
|
724
|
+
const runtime = await initializeContainerRuntime();
|
|
725
|
+
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
726
|
+
return runtime.status === "ready" ? "retry" : "stale";
|
|
727
|
+
}
|
|
728
|
+
if (before === JSON.stringify(configuredOptionalEngines())) {
|
|
729
|
+
return "ready";
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
return "stale";
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/** The bridge may have been offline while Docker stopped. Gate commands while
|
|
736
|
+
* every successful socket open performs a cheap daemon proof. A healthy
|
|
737
|
+
* ready-to-ready proof restores operation without a full recovery sweep;
|
|
738
|
+
* actual loss/restoration creates a readiness edge and takes the full path. */
|
|
739
|
+
function revalidateContainerRuntimeOnConnect(): void {
|
|
740
|
+
const runtime = containerRuntimeState();
|
|
741
|
+
const suspendedEpoch =
|
|
742
|
+
runtime.status === "ready" ? suspendContainerRuntime() : null;
|
|
743
|
+
void (async () => {
|
|
744
|
+
const next = await initializeContainerRuntime();
|
|
745
|
+
if (
|
|
746
|
+
suspendedEpoch === null ||
|
|
747
|
+
next.status !== "ready" ||
|
|
748
|
+
!containerRuntimeEpochIsCurrent(suspendedEpoch)
|
|
749
|
+
) {
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
if (!(await selectedRuntimeMatchesPersistedTaskEnvironments())) return;
|
|
753
|
+
if (!runtimeActivationIsCurrent(suspendedEpoch)) return;
|
|
754
|
+
// A healthy daemon can still have restarted while the bridge was offline.
|
|
755
|
+
// One label scan keeps ordinary cloud/network flaps cheap; only a missing,
|
|
756
|
+
// exited, or unknowable active app takes the full per-task recovery path.
|
|
757
|
+
const recovery = await runtimeRecoveryVerdict();
|
|
758
|
+
let recovered = recovery === "clean";
|
|
759
|
+
if (!recovered) {
|
|
760
|
+
recovered = await requestRuntimeRecovery(() =>
|
|
761
|
+
runtimeActivationIsCurrent(suspendedEpoch),
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
if (!runtimeActivationIsCurrent(suspendedEpoch)) return;
|
|
765
|
+
// Prove the daemon once more after recovery so a mid-sweep loss cannot
|
|
766
|
+
// reopen commands and tunnels on an unresolved backend.
|
|
767
|
+
const final = await initializeContainerRuntime();
|
|
768
|
+
if (
|
|
769
|
+
final.status !== "ready" ||
|
|
770
|
+
!containerRuntimeEpochIsCurrent(suspendedEpoch)
|
|
771
|
+
) {
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
if (!recovered) {
|
|
775
|
+
scheduleContainerRuntimeActivationRetry(
|
|
776
|
+
requestRuntimeRecovery,
|
|
777
|
+
suspendedEpoch,
|
|
778
|
+
);
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (markContainerRuntimeOperational(suspendedEpoch)) {
|
|
782
|
+
runtimeActivationTail.resolve(suspendedEpoch);
|
|
783
|
+
resumeRuntimeDeferredAgentWork();
|
|
784
|
+
sendCapabilities();
|
|
785
|
+
}
|
|
786
|
+
})().catch((error) => {
|
|
787
|
+
console.warn(
|
|
788
|
+
`[host-agent] container runtime reconnect validation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
789
|
+
);
|
|
790
|
+
if (
|
|
791
|
+
suspendedEpoch !== null &&
|
|
792
|
+
runtimeActivationIsCurrent(suspendedEpoch)
|
|
793
|
+
) {
|
|
794
|
+
scheduleContainerRuntimeActivationRetry(
|
|
795
|
+
requestRuntimeRecovery,
|
|
796
|
+
suspendedEpoch,
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/** An operator recheck is an explicit recovery request. Invalidate even a
|
|
803
|
+
* stale ready verdict so an unseen daemon restart gets image maintenance,
|
|
804
|
+
* persisted-task recovery, credential reinjection, and channel reattach. */
|
|
805
|
+
async function requestForcedContainerRuntimeRecheck(): Promise<void> {
|
|
806
|
+
const result = await forceContainerRuntimeRecheck();
|
|
807
|
+
if (result.epoch === null || !containerRuntimeEpochIsCurrent(result.epoch)) {
|
|
808
|
+
return;
|
|
809
|
+
}
|
|
810
|
+
void requestContainerRuntimeActivation(
|
|
811
|
+
requestRuntimeRecovery,
|
|
812
|
+
result.epoch,
|
|
813
|
+
);
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
async function remoteOperationsReadyForConnect(
|
|
817
|
+
requestGeneration: number,
|
|
818
|
+
): Promise<boolean> {
|
|
819
|
+
await stopActiveRemoteOperations();
|
|
820
|
+
await waitForRemoteOperationCleanups();
|
|
821
|
+
return (
|
|
822
|
+
!stopping &&
|
|
823
|
+
!fatal &&
|
|
824
|
+
requestGeneration === connectionRequestGeneration
|
|
825
|
+
);
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
async function connect(): Promise<void> {
|
|
307
829
|
if (stopping || fatal) return;
|
|
830
|
+
const requestGeneration = ++connectionRequestGeneration;
|
|
308
831
|
|
|
832
|
+
// A connection owns every interactive remote operation it starts. Tear the
|
|
833
|
+
// previous generation down before constructing its replacement so a late
|
|
834
|
+
// file read/child-process event can never escape onto the new socket.
|
|
835
|
+
if (!(await remoteOperationsReadyForConnect(requestGeneration))) return;
|
|
309
836
|
console.log(`[host-agent] connecting ${bridgeUrl}`);
|
|
310
837
|
const socket = new WebSocket(bridgeUrl, { perMessageDeflate: WS_DEFLATE });
|
|
311
838
|
ws = socket;
|
|
@@ -314,19 +841,83 @@ function connect(): void {
|
|
|
314
841
|
let ready = false;
|
|
315
842
|
let pingTimer: NodeJS.Timeout | null = null;
|
|
316
843
|
let deadTimer: NodeJS.Timeout | null = null;
|
|
844
|
+
let remoteOperationsActive = true;
|
|
845
|
+
let remoteOperationCleanup: Promise<void> | null = null;
|
|
846
|
+
|
|
847
|
+
const stopRemoteOperations = (): Promise<void> => {
|
|
848
|
+
if (remoteOperationCleanup) return remoteOperationCleanup;
|
|
849
|
+
remoteOperationsActive = false;
|
|
850
|
+
remoteOperationCleanup = stopRemoteOperationManagers();
|
|
851
|
+
return remoteOperationCleanup;
|
|
852
|
+
};
|
|
853
|
+
stopActiveRemoteOperations = stopRemoteOperations;
|
|
854
|
+
|
|
855
|
+
// This emitter closes over exactly this socket generation. Returning false
|
|
856
|
+
// applies the log manager's bounded retry/drop policy while the WS upload
|
|
857
|
+
// queue is congested; terminal frames remain small and are always attempted.
|
|
858
|
+
const emitHostLogFrame: HostLogsEmit = (frame) => {
|
|
859
|
+
if (!remoteOperationsActive || socket.readyState !== WebSocket.OPEN) {
|
|
860
|
+
return false;
|
|
861
|
+
}
|
|
862
|
+
if (
|
|
863
|
+
frame.kind === "host.logs.chunk" &&
|
|
864
|
+
socket.bufferedAmount >= MAX_WS_BUFFERED_BYTES
|
|
865
|
+
) {
|
|
866
|
+
return false;
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
send(socket, frame);
|
|
870
|
+
return true;
|
|
871
|
+
} catch {
|
|
872
|
+
return false;
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
|
|
876
|
+
const sendRemoteOperationFrame = (frame: HostToCloud): boolean => {
|
|
877
|
+
if (!remoteOperationsActive || socket.readyState !== WebSocket.OPEN) {
|
|
878
|
+
return false;
|
|
879
|
+
}
|
|
880
|
+
try {
|
|
881
|
+
send(socket, frame);
|
|
882
|
+
return true;
|
|
883
|
+
} catch {
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
};
|
|
887
|
+
|
|
888
|
+
const emitEngineLoginEvent = (
|
|
889
|
+
event: Extract<HostToCloud, { kind: "engine.login.event" }>,
|
|
890
|
+
): void => {
|
|
891
|
+
if (!sendRemoteOperationFrame(event)) return;
|
|
892
|
+
if (event.phase === "succeeded") {
|
|
893
|
+
sendRemoteOperationFrame({
|
|
894
|
+
kind: "host.capabilities",
|
|
895
|
+
capabilities: buildCapabilities(),
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
};
|
|
317
899
|
|
|
318
900
|
const cleanup = (): void => {
|
|
901
|
+
void stopRemoteOperations();
|
|
319
902
|
if (pingTimer) clearInterval(pingTimer);
|
|
320
903
|
if (deadTimer) clearInterval(deadTimer);
|
|
321
904
|
eventFlushEnabled = false;
|
|
905
|
+
cancelEventFlushBackpressureTimer();
|
|
322
906
|
if (resumeFallbackTimer) {
|
|
323
907
|
clearTimeout(resumeFallbackTimer);
|
|
324
908
|
resumeFallbackTimer = null;
|
|
325
909
|
}
|
|
326
|
-
if (ws === socket)
|
|
910
|
+
if (ws === socket) {
|
|
911
|
+
hostTaskInventorySnapshots.clear();
|
|
912
|
+
ws = null;
|
|
913
|
+
}
|
|
327
914
|
};
|
|
328
915
|
|
|
329
916
|
socket.on("open", () => {
|
|
917
|
+
// Suspend synchronously before auth: cloud activation immediately starts
|
|
918
|
+
// task reconciliation, which must observe `checking` until this proof
|
|
919
|
+
// finishes instead of treating a dead daemon as an empty one.
|
|
920
|
+
revalidateContainerRuntimeOnConnect();
|
|
330
921
|
send(socket, { kind: "auth", token, hostId, bootId: BOOT_ID });
|
|
331
922
|
// Advertise capabilities immediately after auth (ADR-021). The bridge
|
|
332
923
|
// rejects with close-code 4001 if auth fails, so sending here is harmless
|
|
@@ -336,6 +927,7 @@ function connect(): void {
|
|
|
336
927
|
kind: "host.capabilities",
|
|
337
928
|
capabilities: buildCapabilities(),
|
|
338
929
|
});
|
|
930
|
+
flushStoppedTaskStatuses(socket);
|
|
339
931
|
ready = true;
|
|
340
932
|
reconnectAttempt = 0;
|
|
341
933
|
markConnected();
|
|
@@ -368,6 +960,10 @@ function connect(): void {
|
|
|
368
960
|
|
|
369
961
|
socket.on("message", (data, isBinary) => {
|
|
370
962
|
lastTraffic = Date.now();
|
|
963
|
+
// A superseded socket is read-dead even if the transport has not emitted
|
|
964
|
+
// close yet. Never let a late frame recreate work in shared managers or
|
|
965
|
+
// attach binary data to the replacement connection's pending tunnel.
|
|
966
|
+
if (!remoteOperationsActive) return;
|
|
371
967
|
if (isBinary) {
|
|
372
968
|
const tunnelId = pendingBinaryTunnelId;
|
|
373
969
|
pendingBinaryTunnelId = null;
|
|
@@ -379,6 +975,7 @@ function connect(): void {
|
|
|
379
975
|
|
|
380
976
|
const frame = parseCloudFrame(data);
|
|
381
977
|
if (!frame) return;
|
|
978
|
+
if (stopping && rejectFrameDuringShutdown(socket, frame)) return;
|
|
382
979
|
|
|
383
980
|
switch (frame.kind) {
|
|
384
981
|
case "pong":
|
|
@@ -396,10 +993,109 @@ function connect(): void {
|
|
|
396
993
|
break;
|
|
397
994
|
case "event.ack":
|
|
398
995
|
eventOutbox.ack(frame.seq);
|
|
996
|
+
flushEvents();
|
|
399
997
|
break;
|
|
400
998
|
case "command":
|
|
401
999
|
void handleCommand(socket, frame);
|
|
402
1000
|
break;
|
|
1001
|
+
case "host.inventory.request":
|
|
1002
|
+
void handleHostInventoryRequest(socket, frame);
|
|
1003
|
+
break;
|
|
1004
|
+
case "host.logs.start":
|
|
1005
|
+
try {
|
|
1006
|
+
hostLogManager.start(
|
|
1007
|
+
frame.opId,
|
|
1008
|
+
frame.lines,
|
|
1009
|
+
frame.follow,
|
|
1010
|
+
emitHostLogFrame,
|
|
1011
|
+
);
|
|
1012
|
+
} catch (error) {
|
|
1013
|
+
const message =
|
|
1014
|
+
error instanceof HostLogStartError
|
|
1015
|
+
? hostLogStartErrorMessage(error)
|
|
1016
|
+
: "host log operation could not be started";
|
|
1017
|
+
sendRemoteOperationFrame({
|
|
1018
|
+
kind: "host.logs.end",
|
|
1019
|
+
opId: frame.opId,
|
|
1020
|
+
reason: "error",
|
|
1021
|
+
message,
|
|
1022
|
+
});
|
|
1023
|
+
}
|
|
1024
|
+
break;
|
|
1025
|
+
case "host.logs.stop":
|
|
1026
|
+
if (!hostLogManager.stop(frame.opId)) {
|
|
1027
|
+
// Stop is idempotent from the cloud's perspective. A terminal reply
|
|
1028
|
+
// also resolves a stop racing natural completion or reconnect.
|
|
1029
|
+
sendRemoteOperationFrame({
|
|
1030
|
+
kind: "host.logs.end",
|
|
1031
|
+
opId: frame.opId,
|
|
1032
|
+
reason: "stopped",
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
break;
|
|
1036
|
+
case "engine.login.start":
|
|
1037
|
+
if (containerRuntimeProblem()) {
|
|
1038
|
+
sendRemoteOperationFrame({
|
|
1039
|
+
kind: "engine.login.event",
|
|
1040
|
+
opId: frame.opId,
|
|
1041
|
+
engine: frame.engine,
|
|
1042
|
+
phase: "failed",
|
|
1043
|
+
errorCode: "unavailable",
|
|
1044
|
+
message: "The container runtime is not ready on this host.",
|
|
1045
|
+
});
|
|
1046
|
+
break;
|
|
1047
|
+
}
|
|
1048
|
+
engineLoginManager.start(
|
|
1049
|
+
frame.opId,
|
|
1050
|
+
frame.engine,
|
|
1051
|
+
emitEngineLoginEvent,
|
|
1052
|
+
);
|
|
1053
|
+
break;
|
|
1054
|
+
case "engine.login.input":
|
|
1055
|
+
engineLoginManager.input(frame.opId, frame.text);
|
|
1056
|
+
break;
|
|
1057
|
+
case "engine.login.callback":
|
|
1058
|
+
void engineLoginManager.callback(frame.opId, frame.query);
|
|
1059
|
+
break;
|
|
1060
|
+
case "engine.login.stop":
|
|
1061
|
+
void trackRemoteOperationCleanup(
|
|
1062
|
+
engineLoginManager.stop(frame.opId).then(() => {}),
|
|
1063
|
+
);
|
|
1064
|
+
break;
|
|
1065
|
+
case "host.config.get":
|
|
1066
|
+
sendHostConfigAck(
|
|
1067
|
+
sendRemoteOperationFrame,
|
|
1068
|
+
frame.opId,
|
|
1069
|
+
getHostConfig(),
|
|
1070
|
+
);
|
|
1071
|
+
break;
|
|
1072
|
+
case "host.config.set":
|
|
1073
|
+
// Count only patches admitted before shutdown. A late set is rejected
|
|
1074
|
+
// synchronously above, so it can never extend the managed drain.
|
|
1075
|
+
inFlight += 1;
|
|
1076
|
+
void Promise.resolve()
|
|
1077
|
+
.then(() => patchHostConfig(frame.patch))
|
|
1078
|
+
.then((result) => {
|
|
1079
|
+
sendHostConfigAck(sendRemoteOperationFrame, frame.opId, result);
|
|
1080
|
+
if (result.ok) {
|
|
1081
|
+
sendRemoteOperationFrame({
|
|
1082
|
+
kind: "host.capabilities",
|
|
1083
|
+
capabilities: buildCapabilities(),
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
})
|
|
1087
|
+
.catch(() => {
|
|
1088
|
+
sendHostConfigAck(sendRemoteOperationFrame, frame.opId, {
|
|
1089
|
+
ok: false,
|
|
1090
|
+
errorCode: "persistence_failed",
|
|
1091
|
+
message: "host config could not be persisted",
|
|
1092
|
+
});
|
|
1093
|
+
})
|
|
1094
|
+
.finally(() => {
|
|
1095
|
+
inFlight -= 1;
|
|
1096
|
+
maybeExit();
|
|
1097
|
+
});
|
|
1098
|
+
break;
|
|
403
1099
|
case "tunnel.open":
|
|
404
1100
|
// Synchronously, BEFORE handleTunnelOpen's first await: the frames that
|
|
405
1101
|
// follow this one (a bodyless GET's `requestEnd` arrives on the next
|
|
@@ -421,83 +1117,103 @@ function connect(): void {
|
|
|
421
1117
|
closeTunnel(socket, frame.tunnelId, frame.reason);
|
|
422
1118
|
break;
|
|
423
1119
|
case "gh.connect.set": {
|
|
1120
|
+
const runtimeProblem = containerRuntimeProblem();
|
|
1121
|
+
if (runtimeProblem) {
|
|
1122
|
+
send(socket, {
|
|
1123
|
+
kind: "gh.connect.ack",
|
|
1124
|
+
opId: frame.opId,
|
|
1125
|
+
userId: frame.userId,
|
|
1126
|
+
ok: false,
|
|
1127
|
+
error: runtimeProblem,
|
|
1128
|
+
});
|
|
1129
|
+
break;
|
|
1130
|
+
}
|
|
424
1131
|
// Account switches fence every host-side Git operation using the old
|
|
425
1132
|
// credential before the replacement grant is stored or reinjected.
|
|
426
|
-
void
|
|
427
|
-
|
|
1133
|
+
void managedMutationDrain.track(() =>
|
|
1134
|
+
runGithubConnectionTransition(
|
|
428
1135
|
frame.userId,
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
1136
|
+
async () => {
|
|
1137
|
+
const claim = claimGithubCredentialGeneration(
|
|
1138
|
+
frame.userId,
|
|
1139
|
+
frame.generation,
|
|
1140
|
+
"set",
|
|
1141
|
+
);
|
|
1142
|
+
if (!claim) {
|
|
1143
|
+
return {
|
|
1144
|
+
ok: false as const,
|
|
1145
|
+
code: "stale_generation" as const,
|
|
1146
|
+
error: "stale GitHub credential generation",
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
const transitionDeps = {
|
|
1150
|
+
invalidateCredentials: invalidateTaskGithubGitCredentials,
|
|
1151
|
+
reconcile: (taskId: string, userId: string) => {
|
|
1152
|
+
requireContainerRuntime();
|
|
1153
|
+
return getOrchestrator().runTaskLifecycle(taskId, () => {
|
|
1154
|
+
requireContainerRuntime();
|
|
1155
|
+
return reconcileTaskGitAuth(taskId, userId);
|
|
1156
|
+
});
|
|
1157
|
+
},
|
|
1158
|
+
};
|
|
1159
|
+
let result: Awaited<ReturnType<typeof onConnectSet>>;
|
|
1160
|
+
try {
|
|
1161
|
+
result = await onConnectSet(
|
|
1162
|
+
{ ...frame, generation: claim.generation },
|
|
1163
|
+
transitionDeps,
|
|
1164
|
+
);
|
|
1165
|
+
} catch (err) {
|
|
1166
|
+
result = {
|
|
1167
|
+
ok: false,
|
|
1168
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1169
|
+
};
|
|
1170
|
+
}
|
|
1171
|
+
if (!result.ok) {
|
|
1172
|
+
// Storage can succeed before live-task reconciliation fails. Never
|
|
1173
|
+
// emit a negative set ack while that untrusted credential remains
|
|
1174
|
+
// active: persist the same-generation clear tombstone first, then
|
|
1175
|
+
// best-effort scrub/revoke under the same serialized transition.
|
|
1176
|
+
await rollbackClaimedGitHubCredentialSet(
|
|
1177
|
+
frame.userId,
|
|
1178
|
+
claim.generation,
|
|
1179
|
+
transitionDeps,
|
|
1180
|
+
);
|
|
1181
|
+
}
|
|
1182
|
+
return result;
|
|
1183
|
+
},
|
|
1184
|
+
requireContainerRuntime,
|
|
1185
|
+
).then(
|
|
1186
|
+
(result) =>
|
|
1187
|
+
send(
|
|
1188
|
+
socket,
|
|
1189
|
+
result.ok
|
|
1190
|
+
? {
|
|
1191
|
+
kind: "gh.connect.ack",
|
|
1192
|
+
opId: frame.opId,
|
|
1193
|
+
userId: frame.userId,
|
|
1194
|
+
ok: true,
|
|
1195
|
+
}
|
|
1196
|
+
: {
|
|
1197
|
+
kind: "gh.connect.ack",
|
|
1198
|
+
opId: frame.opId,
|
|
1199
|
+
userId: frame.userId,
|
|
1200
|
+
ok: false,
|
|
1201
|
+
code: "code" in result ? result.code : undefined,
|
|
1202
|
+
error: result.error ?? "store failed",
|
|
1203
|
+
},
|
|
444
1204
|
),
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
}
|
|
458
|
-
if (!result.ok) {
|
|
459
|
-
// Storage can succeed before live-task reconciliation fails. Never
|
|
460
|
-
// emit a negative set ack while that untrusted credential remains
|
|
461
|
-
// active: persist the same-generation clear tombstone first, then
|
|
462
|
-
// best-effort scrub/revoke under the same serialized transition.
|
|
463
|
-
await rollbackClaimedGitHubCredentialSet(
|
|
464
|
-
frame.userId,
|
|
465
|
-
claim.generation,
|
|
466
|
-
transitionDeps,
|
|
467
|
-
);
|
|
468
|
-
}
|
|
469
|
-
return result;
|
|
470
|
-
}).then(
|
|
471
|
-
(result) =>
|
|
472
|
-
send(
|
|
473
|
-
socket,
|
|
474
|
-
result.ok
|
|
475
|
-
? {
|
|
476
|
-
kind: "gh.connect.ack",
|
|
477
|
-
opId: frame.opId,
|
|
478
|
-
userId: frame.userId,
|
|
479
|
-
ok: true,
|
|
480
|
-
}
|
|
481
|
-
: {
|
|
482
|
-
kind: "gh.connect.ack",
|
|
483
|
-
opId: frame.opId,
|
|
484
|
-
userId: frame.userId,
|
|
485
|
-
ok: false,
|
|
486
|
-
code: "code" in result ? result.code : undefined,
|
|
487
|
-
error: result.error ?? "store failed",
|
|
488
|
-
},
|
|
489
|
-
),
|
|
490
|
-
(err) => {
|
|
491
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
492
|
-
console.warn(`[github] connect.set failed: ${error}`);
|
|
493
|
-
send(socket, {
|
|
494
|
-
kind: "gh.connect.ack",
|
|
495
|
-
opId: frame.opId,
|
|
496
|
-
userId: frame.userId,
|
|
497
|
-
ok: false,
|
|
498
|
-
error,
|
|
499
|
-
});
|
|
500
|
-
},
|
|
1205
|
+
(err) => {
|
|
1206
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
1207
|
+
console.warn(`[github] connect.set failed: ${error}`);
|
|
1208
|
+
send(socket, {
|
|
1209
|
+
kind: "gh.connect.ack",
|
|
1210
|
+
opId: frame.opId,
|
|
1211
|
+
userId: frame.userId,
|
|
1212
|
+
ok: false,
|
|
1213
|
+
error,
|
|
1214
|
+
});
|
|
1215
|
+
},
|
|
1216
|
+
),
|
|
501
1217
|
);
|
|
502
1218
|
break;
|
|
503
1219
|
}
|
|
@@ -575,61 +1291,81 @@ function connect(): void {
|
|
|
575
1291
|
break;
|
|
576
1292
|
}
|
|
577
1293
|
case "gh.connect.clear": {
|
|
1294
|
+
const runtimeProblem = containerRuntimeProblem();
|
|
1295
|
+
if (runtimeProblem) {
|
|
1296
|
+
send(socket, {
|
|
1297
|
+
kind: "gh.connect.ack",
|
|
1298
|
+
opId: frame.opId,
|
|
1299
|
+
userId: frame.userId,
|
|
1300
|
+
ok: false,
|
|
1301
|
+
error: runtimeProblem,
|
|
1302
|
+
});
|
|
1303
|
+
break;
|
|
1304
|
+
}
|
|
578
1305
|
// The serialized transition removes the local credential, waits for
|
|
579
1306
|
// best-effort remote revocation, and scrubs live containers before it
|
|
580
1307
|
// acknowledges. That keeps a delayed revoke from racing a later set.
|
|
581
|
-
void
|
|
582
|
-
|
|
1308
|
+
void managedMutationDrain.track(() =>
|
|
1309
|
+
runGithubConnectionTransition(
|
|
583
1310
|
frame.userId,
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
1311
|
+
async () => {
|
|
1312
|
+
const claim = claimGithubCredentialGeneration(
|
|
1313
|
+
frame.userId,
|
|
1314
|
+
frame.generation,
|
|
1315
|
+
"clear",
|
|
1316
|
+
);
|
|
1317
|
+
if (!claim) {
|
|
1318
|
+
return {
|
|
1319
|
+
ok: false as const,
|
|
1320
|
+
code: "stale_generation" as const,
|
|
1321
|
+
error: "stale GitHub credential generation",
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1324
|
+
await onConnectClear(frame.userId, {
|
|
1325
|
+
invalidateCredentials: invalidateTaskGithubGitCredentials,
|
|
1326
|
+
reconcile: (taskId, userId) => {
|
|
1327
|
+
requireContainerRuntime();
|
|
1328
|
+
return getOrchestrator().runTaskLifecycle(taskId, () => {
|
|
1329
|
+
requireContainerRuntime();
|
|
1330
|
+
return reconcileTaskGitAuth(taskId, userId);
|
|
1331
|
+
});
|
|
1332
|
+
},
|
|
1333
|
+
});
|
|
1334
|
+
return { ok: true as const };
|
|
1335
|
+
},
|
|
1336
|
+
requireContainerRuntime,
|
|
1337
|
+
).then(
|
|
1338
|
+
(result) =>
|
|
1339
|
+
send(
|
|
1340
|
+
socket,
|
|
1341
|
+
result.ok
|
|
1342
|
+
? {
|
|
1343
|
+
kind: "gh.connect.ack",
|
|
1344
|
+
opId: frame.opId,
|
|
1345
|
+
userId: frame.userId,
|
|
1346
|
+
ok: true,
|
|
1347
|
+
}
|
|
1348
|
+
: {
|
|
1349
|
+
kind: "gh.connect.ack",
|
|
1350
|
+
opId: frame.opId,
|
|
1351
|
+
userId: frame.userId,
|
|
1352
|
+
ok: false,
|
|
1353
|
+
code: result.code,
|
|
1354
|
+
error: result.error,
|
|
1355
|
+
},
|
|
599
1356
|
),
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
}
|
|
613
|
-
: {
|
|
614
|
-
kind: "gh.connect.ack",
|
|
615
|
-
opId: frame.opId,
|
|
616
|
-
userId: frame.userId,
|
|
617
|
-
ok: false,
|
|
618
|
-
code: result.code,
|
|
619
|
-
error: result.error,
|
|
620
|
-
},
|
|
621
|
-
),
|
|
622
|
-
(err) => {
|
|
623
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
624
|
-
console.warn(`[github] connect.clear failed: ${error}`);
|
|
625
|
-
send(socket, {
|
|
626
|
-
kind: "gh.connect.ack",
|
|
627
|
-
opId: frame.opId,
|
|
628
|
-
userId: frame.userId,
|
|
629
|
-
ok: false,
|
|
630
|
-
error,
|
|
631
|
-
});
|
|
632
|
-
},
|
|
1357
|
+
(err) => {
|
|
1358
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
1359
|
+
console.warn(`[github] connect.clear failed: ${error}`);
|
|
1360
|
+
send(socket, {
|
|
1361
|
+
kind: "gh.connect.ack",
|
|
1362
|
+
opId: frame.opId,
|
|
1363
|
+
userId: frame.userId,
|
|
1364
|
+
ok: false,
|
|
1365
|
+
error,
|
|
1366
|
+
});
|
|
1367
|
+
},
|
|
1368
|
+
),
|
|
633
1369
|
);
|
|
634
1370
|
break;
|
|
635
1371
|
}
|
|
@@ -637,27 +1373,36 @@ function connect(): void {
|
|
|
637
1373
|
case "ssh.key.ensure":
|
|
638
1374
|
case "ssh.key.delete": {
|
|
639
1375
|
if (frame.kind === "ssh.key.delete") {
|
|
640
|
-
void
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
}
|
|
653
|
-
(
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
1376
|
+
void managedMutationDrain.track(() =>
|
|
1377
|
+
deleteUserSshIdentity(frame.userId, {
|
|
1378
|
+
// Deletion has no durable cleanup outbox. Do not revoke the
|
|
1379
|
+
// host-side authority unless credential-bearing containers can be
|
|
1380
|
+
// scrubbed in the same attempt.
|
|
1381
|
+
preflight: requireContainerRuntime,
|
|
1382
|
+
removeFromContainer: (taskId) => {
|
|
1383
|
+
requireContainerRuntime();
|
|
1384
|
+
return getOrchestrator().runTaskLifecycle(taskId, () => {
|
|
1385
|
+
requireContainerRuntime();
|
|
1386
|
+
return removeTaskSshIdentityFromContainer(taskId);
|
|
1387
|
+
});
|
|
1388
|
+
},
|
|
1389
|
+
}).then(
|
|
1390
|
+
() =>
|
|
1391
|
+
send(socket, {
|
|
1392
|
+
kind: "ssh.key.ack",
|
|
1393
|
+
userId: frame.userId,
|
|
1394
|
+
ok: true,
|
|
1395
|
+
publicKey: null,
|
|
1396
|
+
}),
|
|
1397
|
+
(err) =>
|
|
1398
|
+
send(socket, {
|
|
1399
|
+
kind: "ssh.key.ack",
|
|
1400
|
+
userId: frame.userId,
|
|
1401
|
+
ok: false,
|
|
1402
|
+
error:
|
|
1403
|
+
err instanceof Error ? err.message : "ssh key delete failed",
|
|
1404
|
+
}),
|
|
1405
|
+
),
|
|
661
1406
|
);
|
|
662
1407
|
break;
|
|
663
1408
|
}
|
|
@@ -679,18 +1424,25 @@ function connect(): void {
|
|
|
679
1424
|
}
|
|
680
1425
|
case "mcp.op": {
|
|
681
1426
|
// Async network work (discovery, DCR, exchange) — ack when done.
|
|
682
|
-
void
|
|
683
|
-
.
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1427
|
+
void managedMutationDrain.track(() =>
|
|
1428
|
+
handleMcpOp(frame.op)
|
|
1429
|
+
.then((ack) =>
|
|
1430
|
+
send(socket, {
|
|
1431
|
+
kind: "mcp.ack",
|
|
1432
|
+
opId: frame.opId,
|
|
1433
|
+
ok: true,
|
|
1434
|
+
...ack,
|
|
1435
|
+
}),
|
|
1436
|
+
)
|
|
1437
|
+
.catch((err: unknown) =>
|
|
1438
|
+
send(socket, {
|
|
1439
|
+
kind: "mcp.ack",
|
|
1440
|
+
opId: frame.opId,
|
|
1441
|
+
ok: false,
|
|
1442
|
+
error: err instanceof Error ? err.message : "mcp op failed",
|
|
1443
|
+
}),
|
|
1444
|
+
),
|
|
1445
|
+
);
|
|
694
1446
|
break;
|
|
695
1447
|
}
|
|
696
1448
|
case "env.var.list":
|
|
@@ -733,6 +1485,11 @@ function connect(): void {
|
|
|
733
1485
|
socket.on("close", (code, reason) => {
|
|
734
1486
|
cleanup();
|
|
735
1487
|
const text = reason.toString("utf8");
|
|
1488
|
+
if (stopping) {
|
|
1489
|
+
markDisconnected();
|
|
1490
|
+
maybeExit();
|
|
1491
|
+
return;
|
|
1492
|
+
}
|
|
736
1493
|
if (code === 4001) {
|
|
737
1494
|
fatal = true;
|
|
738
1495
|
markDisconnected(text || "auth rejected by bridge");
|
|
@@ -742,11 +1499,6 @@ function connect(): void {
|
|
|
742
1499
|
process.exitCode = 1;
|
|
743
1500
|
return;
|
|
744
1501
|
}
|
|
745
|
-
if (stopping) {
|
|
746
|
-
markDisconnected();
|
|
747
|
-
maybeExit();
|
|
748
|
-
return;
|
|
749
|
-
}
|
|
750
1502
|
const delay = reconnectDelay();
|
|
751
1503
|
markReconnecting(text || `disconnected${ready ? "" : " before auth"}`);
|
|
752
1504
|
console.warn(
|
|
@@ -757,7 +1509,9 @@ function connect(): void {
|
|
|
757
1509
|
authed: ready,
|
|
758
1510
|
retryInMs: Math.round(delay),
|
|
759
1511
|
});
|
|
760
|
-
setTimeout(
|
|
1512
|
+
setTimeout(() => {
|
|
1513
|
+
void connect();
|
|
1514
|
+
}, delay);
|
|
761
1515
|
});
|
|
762
1516
|
}
|
|
763
1517
|
|
|
@@ -772,6 +1526,17 @@ async function handleCommand(
|
|
|
772
1526
|
if (socket.readyState === WebSocket.OPEN) {
|
|
773
1527
|
send(socket, { kind: "result", commandId: frame.commandId, result });
|
|
774
1528
|
}
|
|
1529
|
+
// Publish after the command result so a retryable taskUp failure cannot
|
|
1530
|
+
// overwrite the physical stopped truth established while quarantining a
|
|
1531
|
+
// legacy runtime. A closed bridge retains the queue until reconnect.
|
|
1532
|
+
if (
|
|
1533
|
+
pendingRuntimeIsolationStatuses.enqueueStoppedTaskUp(
|
|
1534
|
+
frame.command,
|
|
1535
|
+
frame.args,
|
|
1536
|
+
)
|
|
1537
|
+
) {
|
|
1538
|
+
flushStoppedTaskStatuses();
|
|
1539
|
+
}
|
|
775
1540
|
} catch (err) {
|
|
776
1541
|
if (socket.readyState === WebSocket.OPEN) {
|
|
777
1542
|
send(socket, {
|
|
@@ -790,11 +1555,57 @@ async function handleCommand(
|
|
|
790
1555
|
}
|
|
791
1556
|
}
|
|
792
1557
|
|
|
1558
|
+
async function handleHostInventoryRequest(
|
|
1559
|
+
socket: WebSocket,
|
|
1560
|
+
frame: Extract<CloudToHost, { kind: "host.inventory.request" }>,
|
|
1561
|
+
): Promise<void> {
|
|
1562
|
+
try {
|
|
1563
|
+
const page = await hostTaskInventorySnapshots.page(frame);
|
|
1564
|
+
// A scan belongs to the authenticated socket generation that requested
|
|
1565
|
+
// it. Never leak a late page onto a replacement connection.
|
|
1566
|
+
if (ws !== socket || socket.readyState !== WebSocket.OPEN) return;
|
|
1567
|
+
send(socket, {
|
|
1568
|
+
kind: "host.inventory.page",
|
|
1569
|
+
scanId: frame.scanId,
|
|
1570
|
+
cursor: frame.cursor,
|
|
1571
|
+
...page,
|
|
1572
|
+
});
|
|
1573
|
+
} catch (error) {
|
|
1574
|
+
console.warn(
|
|
1575
|
+
`[host-agent] inventory request rejected: ${
|
|
1576
|
+
error instanceof Error ? error.message : String(error)
|
|
1577
|
+
}`,
|
|
1578
|
+
);
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
|
|
793
1582
|
async function handleTunnelOpen(
|
|
794
1583
|
wsSocket: WebSocket,
|
|
795
1584
|
frame: Extract<CloudToHost, { kind: "tunnel.open" }>,
|
|
796
1585
|
): Promise<void> {
|
|
797
|
-
|
|
1586
|
+
let target: UpstreamAddr | null;
|
|
1587
|
+
try {
|
|
1588
|
+
target = await resolveTunnelTarget(frame);
|
|
1589
|
+
} catch (error) {
|
|
1590
|
+
// A thrown resolution (daemon flap mid-inspect) must settle the
|
|
1591
|
+
// opening entry: abortAll() joins it during bridge replacement, and the
|
|
1592
|
+
// rejection must not escape the `void handleTunnelOpen(...)` call as a
|
|
1593
|
+
// process-fatal unhandled rejection.
|
|
1594
|
+
tunnels.abandon(frame.tunnelId);
|
|
1595
|
+
console.warn(
|
|
1596
|
+
`[tunnel] ${frame.tunnelId}: target resolution failed: ${
|
|
1597
|
+
error instanceof Error ? error.message : String(error)
|
|
1598
|
+
}`,
|
|
1599
|
+
);
|
|
1600
|
+
send(wsSocket, {
|
|
1601
|
+
kind: "tunnel.ack",
|
|
1602
|
+
tunnelId: frame.tunnelId,
|
|
1603
|
+
ok: false,
|
|
1604
|
+
status: 503,
|
|
1605
|
+
message: "target port is not available on this host",
|
|
1606
|
+
});
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
798
1609
|
if (!target) {
|
|
799
1610
|
// Nothing will ever service what queued behind this open.
|
|
800
1611
|
tunnels.abandon(frame.tunnelId);
|
|
@@ -822,14 +1633,18 @@ function handleHttpTunnelOpen(
|
|
|
822
1633
|
target: UpstreamAddr,
|
|
823
1634
|
): void {
|
|
824
1635
|
let acked = false;
|
|
1636
|
+
const requestOptions = {
|
|
1637
|
+
host: target.host,
|
|
1638
|
+
port: target.port,
|
|
1639
|
+
method: frame.reqLine.method,
|
|
1640
|
+
path: frame.reqLine.url,
|
|
1641
|
+
headers: requestHeaders(frame.reqLine.headers),
|
|
1642
|
+
// Reuse the connection proven inside the task lifecycle slot; http will
|
|
1643
|
+
// manage (and eventually close) the socket it is handed.
|
|
1644
|
+
...(target.socket ? { createConnection: () => target.socket! } : {}),
|
|
1645
|
+
};
|
|
825
1646
|
const upstream = httpRequest(
|
|
826
|
-
|
|
827
|
-
host: target.host,
|
|
828
|
-
port: target.port,
|
|
829
|
-
method: frame.reqLine.method,
|
|
830
|
-
path: frame.reqLine.url,
|
|
831
|
-
headers: requestHeaders(frame.reqLine.headers),
|
|
832
|
-
},
|
|
1647
|
+
requestOptions,
|
|
833
1648
|
(response) => {
|
|
834
1649
|
const headers = responseHeaders(response);
|
|
835
1650
|
send(wsSocket, {
|
|
@@ -862,10 +1677,6 @@ function handleHttpTunnelOpen(
|
|
|
862
1677
|
|
|
863
1678
|
upstream.on("error", (err) => {
|
|
864
1679
|
tunnels.delete(frame.tunnelId);
|
|
865
|
-
// A connect failure to a cached container IP likely means the container was
|
|
866
|
-
// recreated (resume) and got a new IP — drop the cache so the next request
|
|
867
|
-
// re-resolves (ADR-036).
|
|
868
|
-
invalidateContainerIpByAddr(target.host);
|
|
869
1680
|
// ADR-043: if this was an ad-hoc preview sidecar, drop its cached port so
|
|
870
1681
|
// the next request recreates it (no-op for published / container-IP targets).
|
|
871
1682
|
invalidatePreviewSidecar(target.port);
|
|
@@ -897,14 +1708,16 @@ function handleRawTunnelOpen(
|
|
|
897
1708
|
frame: Extract<CloudToHost, { kind: "tunnel.open" }>,
|
|
898
1709
|
target: UpstreamAddr,
|
|
899
1710
|
): void {
|
|
900
|
-
const upstream = new Socket();
|
|
1711
|
+
const upstream = target.socket ?? new Socket();
|
|
901
1712
|
|
|
902
1713
|
let acked = false;
|
|
903
1714
|
let responseBuffer = Buffer.alloc(0);
|
|
904
1715
|
|
|
905
|
-
|
|
906
|
-
upstream.
|
|
907
|
-
|
|
1716
|
+
if (!target.socket) {
|
|
1717
|
+
upstream.on("connect", () => {
|
|
1718
|
+
upstream.write(serializeRequest(frame.reqLine));
|
|
1719
|
+
});
|
|
1720
|
+
}
|
|
908
1721
|
|
|
909
1722
|
upstream.on("data", (chunk) => {
|
|
910
1723
|
if (!acked) {
|
|
@@ -930,7 +1743,6 @@ function handleRawTunnelOpen(
|
|
|
930
1743
|
});
|
|
931
1744
|
|
|
932
1745
|
upstream.on("error", (err) => {
|
|
933
|
-
invalidateContainerIpByAddr(target.host);
|
|
934
1746
|
// ADR-043: if this was an ad-hoc preview sidecar, drop its cached port so
|
|
935
1747
|
// the next request recreates it (no-op for published / container-IP targets).
|
|
936
1748
|
invalidatePreviewSidecar(target.port);
|
|
@@ -961,12 +1773,18 @@ function handleRawTunnelOpen(
|
|
|
961
1773
|
});
|
|
962
1774
|
});
|
|
963
1775
|
|
|
964
|
-
upstream.connect(target.port, target.host);
|
|
1776
|
+
if (!target.socket) upstream.connect(target.port, target.host);
|
|
965
1777
|
// AFTER `connect`, not before: replaying a queued write onto a socket that
|
|
966
1778
|
// has not started connecting errors, while node buffers writes made once it
|
|
967
1779
|
// is connecting. An upgrade's own request goes out from the `connect`
|
|
968
1780
|
// handler, so only client frames that overtook the open replay here.
|
|
969
|
-
tunnels.register(frame.tunnelId, upstream);
|
|
1781
|
+
const owned = tunnels.register(frame.tunnelId, upstream);
|
|
1782
|
+
// A preconnected socket writes only once registration has accepted
|
|
1783
|
+
// ownership: a tunnel canceled during resolution is destroyed inside
|
|
1784
|
+
// register(), and an aborted/stale generation must never emit a request.
|
|
1785
|
+
if (target.socket && owned) {
|
|
1786
|
+
upstream.write(serializeRequest(frame.reqLine));
|
|
1787
|
+
}
|
|
970
1788
|
}
|
|
971
1789
|
|
|
972
1790
|
function closeTunnel(
|
|
@@ -985,51 +1803,78 @@ function closeTunnel(
|
|
|
985
1803
|
interface UpstreamAddr {
|
|
986
1804
|
host: string;
|
|
987
1805
|
port: number;
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
// Short TTL + invalidate-on-connect-error so a resumed container's new IP heals.
|
|
993
|
-
const CONTAINER_IP_TTL_MS = 60_000;
|
|
994
|
-
const containerIpCache = new Map<string, { ip: string; ts: number }>();
|
|
995
|
-
|
|
996
|
-
async function resolveContainerIp(composeProject: string): Promise<string | null> {
|
|
997
|
-
const container = `${composeProject}-app-1`;
|
|
998
|
-
const cached = containerIpCache.get(container);
|
|
999
|
-
if (cached && Date.now() - cached.ts < CONTAINER_IP_TTL_MS) return cached.ip;
|
|
1000
|
-
const res = await dockerCli(
|
|
1001
|
-
[
|
|
1002
|
-
"inspect",
|
|
1003
|
-
"-f",
|
|
1004
|
-
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
|
1005
|
-
container,
|
|
1006
|
-
],
|
|
1007
|
-
{ timeoutMs: 5_000 },
|
|
1008
|
-
);
|
|
1009
|
-
if (res.status !== 0) return null;
|
|
1010
|
-
const ip = res.stdout.trim();
|
|
1011
|
-
if (!ip) return null;
|
|
1012
|
-
containerIpCache.set(container, { ip, ts: Date.now() });
|
|
1013
|
-
return ip;
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
/** Drop any cached container IP equal to `addr` (no-op for `127.0.0.1`). */
|
|
1017
|
-
function invalidateContainerIpByAddr(addr: string): void {
|
|
1018
|
-
if (addr === "127.0.0.1") return;
|
|
1019
|
-
for (const [key, val] of containerIpCache) {
|
|
1020
|
-
if (val.ip === addr) containerIpCache.delete(key);
|
|
1021
|
-
}
|
|
1806
|
+
/** Connection already established while the identity proof was still held
|
|
1807
|
+
* (ADR-106 apple routing). An established TCP peer cannot be retargeted by
|
|
1808
|
+
* later IP reuse, so consumers must use this socket instead of redialing. */
|
|
1809
|
+
socket?: Socket;
|
|
1022
1810
|
}
|
|
1023
1811
|
|
|
1024
1812
|
async function resolveTunnelTarget(
|
|
1025
1813
|
frame: Extract<CloudToHost, { kind: "tunnel.open" }>,
|
|
1026
1814
|
): Promise<UpstreamAddr | null> {
|
|
1815
|
+
// Persisted ports are meaningful only while the selected runtime is known
|
|
1816
|
+
// operational. After a daemon restart an unrelated local process could bind
|
|
1817
|
+
// a stale port before recovery refreshes the task's mappings.
|
|
1818
|
+
if (containerRuntimeProblem()) return null;
|
|
1027
1819
|
const task = getHostTask(frame.taskId);
|
|
1028
1820
|
if (!task) return null;
|
|
1821
|
+
if (containerRuntimeMachineIdentity()?.backend === "apple-container") {
|
|
1822
|
+
// Apple vmnet IPs are host-routable, so editor and every preview resolve
|
|
1823
|
+
// to the same direct route: the label-proven container IP plus the
|
|
1824
|
+
// requested container port. Nothing is published and no sidecar exists.
|
|
1825
|
+
// The proof/connect/epoch discipline lives in apple-tunnel-route.ts,
|
|
1826
|
+
// where it is directly testable.
|
|
1827
|
+
const appleRoute = await resolveAppleTunnelRoute(
|
|
1828
|
+
{
|
|
1829
|
+
taskId: frame.taskId,
|
|
1830
|
+
target: frame.target,
|
|
1831
|
+
...(frame.containerPort !== undefined
|
|
1832
|
+
? { containerPort: frame.containerPort }
|
|
1833
|
+
: {}),
|
|
1834
|
+
},
|
|
1835
|
+
{
|
|
1836
|
+
getTask: (taskId) => getHostTask(taskId) ?? undefined,
|
|
1837
|
+
reconstruct: (current) =>
|
|
1838
|
+
reconstructPersistedTaskEnvironment(current as never),
|
|
1839
|
+
runTaskLifecycle: (taskId, work) =>
|
|
1840
|
+
getOrchestrator().runTaskLifecycle(taskId, work),
|
|
1841
|
+
runtimeProblem: () => containerRuntimeProblem(),
|
|
1842
|
+
readinessEpoch: () => containerRuntimeReadinessEpoch(),
|
|
1843
|
+
epochIsCurrent: (epoch) => containerRuntimeEpochIsCurrent(epoch),
|
|
1844
|
+
poison: (detail) => poisonContainerRuntimeFromTunnel(detail),
|
|
1845
|
+
createSocket: () => new Socket(),
|
|
1846
|
+
},
|
|
1847
|
+
);
|
|
1848
|
+
if (appleRoute === null) return null;
|
|
1849
|
+
return {
|
|
1850
|
+
host: appleRoute.host,
|
|
1851
|
+
port: appleRoute.port,
|
|
1852
|
+
socket: appleRoute.socket as Socket,
|
|
1853
|
+
};
|
|
1854
|
+
}
|
|
1855
|
+
const appContainer = task.composeProject
|
|
1856
|
+
? `${task.composeProject}-app-1`
|
|
1857
|
+
: null;
|
|
1858
|
+
let appProof: Awaited<ReturnType<typeof inspectTunnelContainer>> | null = null;
|
|
1859
|
+
const inspectApp = async (containerPort: number) => {
|
|
1860
|
+
// One frame resolves one target port. Keep a local memo only so a failed
|
|
1861
|
+
// sidecar path can reuse the exact live app proof; no proof survives the
|
|
1862
|
+
// tunnel open that requested it.
|
|
1863
|
+
if (!appContainer) return null;
|
|
1864
|
+
appProof ??= await inspectTunnelContainer(appContainer, containerPort);
|
|
1865
|
+
if (appProof.kind === "daemon-unavailable") {
|
|
1866
|
+
poisonContainerRuntimeFromTunnel(appProof.detail);
|
|
1867
|
+
return null;
|
|
1868
|
+
}
|
|
1869
|
+
// A forced recheck or a different Docker caller can invalidate the runtime
|
|
1870
|
+
// while this inspect is in flight. Do not publish even a successful proof
|
|
1871
|
+
// from an epoch that is no longer operational.
|
|
1872
|
+
if (containerRuntimeProblem()) return null;
|
|
1873
|
+
return appProof.kind === "running" ? appProof : null;
|
|
1874
|
+
};
|
|
1029
1875
|
if (frame.target === "editor") {
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
: null;
|
|
1876
|
+
if (!task.codeServerPort) return null;
|
|
1877
|
+
return (await inspectApp(8080))?.published ?? null;
|
|
1033
1878
|
}
|
|
1034
1879
|
// Preview. Prefer the PUBLISHED host port for declared previews (published at
|
|
1035
1880
|
// task-up): a 127.0.0.1 port that's reachable on every backend, incl.
|
|
@@ -1038,13 +1883,25 @@ async function resolveTunnelTarget(
|
|
|
1038
1883
|
const declared = parsePreviewPortRuntimes(task.previewPorts).find(
|
|
1039
1884
|
(port) => port.name === frame.name,
|
|
1040
1885
|
);
|
|
1041
|
-
if (declared)
|
|
1886
|
+
if (declared) {
|
|
1887
|
+
// The stored hostPort is authorization/display state, not routing truth.
|
|
1888
|
+
// Docker may remap it after a daemon restart. The cloud supplies the
|
|
1889
|
+
// declaration's containerPort so we can resolve the exact current route.
|
|
1890
|
+
if (!frame.containerPort) return null;
|
|
1891
|
+
const proof = await inspectApp(frame.containerPort);
|
|
1892
|
+
if (proof?.published) return proof.published;
|
|
1893
|
+
if (proof?.containerIp) {
|
|
1894
|
+
return { host: proof.containerIp, port: frame.containerPort };
|
|
1895
|
+
}
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1042
1898
|
// Ad-hoc port (not published): proxy via a node-proxy sidecar that publishes
|
|
1043
1899
|
// a 127.0.0.1 port forwarding to <app>:<containerPort> (ADR-043).
|
|
1044
1900
|
if (frame.containerPort && task.composeProject) {
|
|
1045
1901
|
const hostPort = await getOrchestrator().runTaskLifecycle(
|
|
1046
1902
|
frame.taskId,
|
|
1047
1903
|
async () => {
|
|
1904
|
+
if (containerRuntimeProblem()) return null;
|
|
1048
1905
|
const current = getHostTask(frame.taskId);
|
|
1049
1906
|
if (
|
|
1050
1907
|
current?.statusMirror !== "running" ||
|
|
@@ -1060,19 +1917,39 @@ async function resolveTunnelTarget(
|
|
|
1060
1917
|
});
|
|
1061
1918
|
},
|
|
1062
1919
|
);
|
|
1063
|
-
if (hostPort)
|
|
1920
|
+
if (hostPort && !containerRuntimeProblem()) {
|
|
1921
|
+
return { host: "127.0.0.1", port: hostPort };
|
|
1922
|
+
}
|
|
1064
1923
|
}
|
|
1065
1924
|
}
|
|
1066
1925
|
// Linux fallback (pre-ADR-043 / non-macOS host where bridge IPs route): proxy
|
|
1067
1926
|
// straight to the container IP. Unreachable on macOS/OrbStack — the
|
|
1068
1927
|
// connect-error path returns 502 there.
|
|
1069
1928
|
if (frame.containerPort && task.composeProject) {
|
|
1070
|
-
const
|
|
1071
|
-
if (
|
|
1929
|
+
const proof = await inspectApp(frame.containerPort);
|
|
1930
|
+
if (proof?.containerIp) {
|
|
1931
|
+
return { host: proof.containerIp, port: frame.containerPort };
|
|
1932
|
+
}
|
|
1072
1933
|
}
|
|
1073
1934
|
return null;
|
|
1074
1935
|
}
|
|
1075
1936
|
|
|
1937
|
+
function poisonContainerRuntimeFromTunnel(detail: string): void {
|
|
1938
|
+
// Fail closed synchronously: another frame in this event-loop turn must see
|
|
1939
|
+
// `checking`, not route through a stale localhost port. The normal runtime
|
|
1940
|
+
// listener owns bounded recheck, recovery, and the eventual ready advert.
|
|
1941
|
+
if (containerRuntimeProblem()) return;
|
|
1942
|
+
console.warn(`[host-agent] tunnel Docker proof failed: ${detail}`);
|
|
1943
|
+
suspendContainerRuntime(true);
|
|
1944
|
+
sendCapabilities();
|
|
1945
|
+
void initializeContainerRuntime();
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
function requireContainerRuntime(): void {
|
|
1949
|
+
const problem = containerRuntimeProblem();
|
|
1950
|
+
if (problem) throw new Error(problem);
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1076
1953
|
function serializeRequest(reqLine: {
|
|
1077
1954
|
method: string;
|
|
1078
1955
|
url: string;
|
|
@@ -1294,6 +2171,33 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
1294
2171
|
if (frame.kind === "event.ack" && typeof frame.seq === "number") {
|
|
1295
2172
|
return { kind: "event.ack", seq: frame.seq };
|
|
1296
2173
|
}
|
|
2174
|
+
if (
|
|
2175
|
+
frame.kind === "host.inventory.request" &&
|
|
2176
|
+
isSafeInventoryScanId(frame.scanId) &&
|
|
2177
|
+
(frame.cursor === null || isSafeHostTaskId(frame.cursor)) &&
|
|
2178
|
+
typeof frame.limit === "number" &&
|
|
2179
|
+
Number.isSafeInteger(frame.limit) &&
|
|
2180
|
+
frame.limit >= 1 &&
|
|
2181
|
+
frame.limit <= MAX_HOST_TASK_INVENTORY_PAGE_SIZE
|
|
2182
|
+
) {
|
|
2183
|
+
return {
|
|
2184
|
+
kind: "host.inventory.request",
|
|
2185
|
+
scanId: frame.scanId,
|
|
2186
|
+
cursor: frame.cursor,
|
|
2187
|
+
limit: frame.limit,
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
// ADR-100 remote operations are exact-shape parsed in the shared protocol
|
|
2191
|
+
// module. Do not reconstruct them here: doing so could accidentally admit a
|
|
2192
|
+
// secret-bearing or forward-version field that the opposite peer rejects.
|
|
2193
|
+
if (isHostLogsStartFrame(frame)) return frame;
|
|
2194
|
+
if (isHostLogsStopFrame(frame)) return frame;
|
|
2195
|
+
if (isEngineLoginStartFrame(frame)) return frame;
|
|
2196
|
+
if (isEngineLoginInputFrame(frame)) return frame;
|
|
2197
|
+
if (isEngineLoginCallbackFrame(frame)) return frame;
|
|
2198
|
+
if (isEngineLoginStopFrame(frame)) return frame;
|
|
2199
|
+
if (isHostConfigGetFrame(frame)) return frame;
|
|
2200
|
+
if (isHostConfigSetFrame(frame)) return frame;
|
|
1297
2201
|
if (
|
|
1298
2202
|
frame.kind === "command" &&
|
|
1299
2203
|
typeof frame.commandId === "string" &&
|
|
@@ -1519,10 +2423,186 @@ function isReqLine(value: unknown): value is {
|
|
|
1519
2423
|
);
|
|
1520
2424
|
}
|
|
1521
2425
|
|
|
2426
|
+
function hostLogStartErrorMessage(error: HostLogStartError): string {
|
|
2427
|
+
switch (error.code) {
|
|
2428
|
+
case "invalid_op_id":
|
|
2429
|
+
return "invalid host log operation id";
|
|
2430
|
+
case "duplicate_op":
|
|
2431
|
+
return "host log operation is already active";
|
|
2432
|
+
case "too_many_streams":
|
|
2433
|
+
return "too many host log streams are active";
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
function sendHostConfigAck(
|
|
2438
|
+
emit: (frame: HostToCloud) => boolean,
|
|
2439
|
+
opId: string,
|
|
2440
|
+
result: HostConfigResult,
|
|
2441
|
+
): void {
|
|
2442
|
+
if (result.ok) {
|
|
2443
|
+
emit({
|
|
2444
|
+
kind: "host.config.ack",
|
|
2445
|
+
opId,
|
|
2446
|
+
ok: true,
|
|
2447
|
+
config: result.config,
|
|
2448
|
+
restartRequired: result.restartRequired,
|
|
2449
|
+
});
|
|
2450
|
+
return;
|
|
2451
|
+
}
|
|
2452
|
+
emit({
|
|
2453
|
+
kind: "host.config.ack",
|
|
2454
|
+
opId,
|
|
2455
|
+
ok: false,
|
|
2456
|
+
errorCode: result.errorCode,
|
|
2457
|
+
message: result.message,
|
|
2458
|
+
});
|
|
2459
|
+
}
|
|
2460
|
+
|
|
1522
2461
|
function send(socket: WebSocket, frame: HostToCloud): void {
|
|
1523
2462
|
socket.send(JSON.stringify(frame));
|
|
1524
2463
|
}
|
|
1525
2464
|
|
|
2465
|
+
/**
|
|
2466
|
+
* Once shutdown begins, keep the authenticated socket open only to flush work
|
|
2467
|
+
* that was already admitted. New requests receive a retryable/negative reply
|
|
2468
|
+
* instead of extending the drain or mutating state after teardown was chosen.
|
|
2469
|
+
* Transport housekeeping and frames belonging to an already-open tunnel are
|
|
2470
|
+
* allowed to finish until the socket closes.
|
|
2471
|
+
*/
|
|
2472
|
+
function rejectFrameDuringShutdown(
|
|
2473
|
+
socket: WebSocket,
|
|
2474
|
+
frame: CloudToHost,
|
|
2475
|
+
): boolean {
|
|
2476
|
+
const error = "managed host is restarting; retry the operation";
|
|
2477
|
+
switch (frame.kind) {
|
|
2478
|
+
case "command":
|
|
2479
|
+
send(socket, {
|
|
2480
|
+
kind: "result",
|
|
2481
|
+
commandId: frame.commandId,
|
|
2482
|
+
result: {
|
|
2483
|
+
ok: false,
|
|
2484
|
+
code: HostErrorCode.HostUnavailable,
|
|
2485
|
+
message: error,
|
|
2486
|
+
retryable: true,
|
|
2487
|
+
retryReason: "managed_restart",
|
|
2488
|
+
},
|
|
2489
|
+
});
|
|
2490
|
+
return true;
|
|
2491
|
+
case "host.logs.start":
|
|
2492
|
+
send(socket, {
|
|
2493
|
+
kind: "host.logs.end",
|
|
2494
|
+
opId: frame.opId,
|
|
2495
|
+
reason: "error",
|
|
2496
|
+
message: error,
|
|
2497
|
+
});
|
|
2498
|
+
return true;
|
|
2499
|
+
case "engine.login.start":
|
|
2500
|
+
send(socket, {
|
|
2501
|
+
kind: "engine.login.event",
|
|
2502
|
+
opId: frame.opId,
|
|
2503
|
+
engine: frame.engine,
|
|
2504
|
+
phase: "failed",
|
|
2505
|
+
errorCode: "unavailable",
|
|
2506
|
+
message: error,
|
|
2507
|
+
});
|
|
2508
|
+
return true;
|
|
2509
|
+
case "host.config.get":
|
|
2510
|
+
case "host.config.set":
|
|
2511
|
+
send(socket, {
|
|
2512
|
+
kind: "host.config.ack",
|
|
2513
|
+
opId: frame.opId,
|
|
2514
|
+
ok: false,
|
|
2515
|
+
errorCode: "persistence_failed",
|
|
2516
|
+
message: error,
|
|
2517
|
+
});
|
|
2518
|
+
return true;
|
|
2519
|
+
case "tunnel.open":
|
|
2520
|
+
send(socket, {
|
|
2521
|
+
kind: "tunnel.close",
|
|
2522
|
+
tunnelId: frame.tunnelId,
|
|
2523
|
+
reason: error,
|
|
2524
|
+
});
|
|
2525
|
+
return true;
|
|
2526
|
+
case "gh.connect.set":
|
|
2527
|
+
case "gh.connect.clear":
|
|
2528
|
+
send(socket, {
|
|
2529
|
+
kind: "gh.connect.ack",
|
|
2530
|
+
opId: frame.opId,
|
|
2531
|
+
userId: frame.userId,
|
|
2532
|
+
ok: false,
|
|
2533
|
+
error,
|
|
2534
|
+
});
|
|
2535
|
+
return true;
|
|
2536
|
+
case "gh.installations.list":
|
|
2537
|
+
send(socket, {
|
|
2538
|
+
kind: "gh.installations.ack",
|
|
2539
|
+
opId: frame.opId,
|
|
2540
|
+
ok: false,
|
|
2541
|
+
code: "github_unavailable",
|
|
2542
|
+
error,
|
|
2543
|
+
});
|
|
2544
|
+
return true;
|
|
2545
|
+
case "gh.repositories.list":
|
|
2546
|
+
send(socket, {
|
|
2547
|
+
kind: "gh.repositories.ack",
|
|
2548
|
+
opId: frame.opId,
|
|
2549
|
+
ok: false,
|
|
2550
|
+
code: "github_unavailable",
|
|
2551
|
+
error,
|
|
2552
|
+
});
|
|
2553
|
+
return true;
|
|
2554
|
+
case "ssh.key.get":
|
|
2555
|
+
case "ssh.key.ensure":
|
|
2556
|
+
case "ssh.key.delete":
|
|
2557
|
+
send(socket, {
|
|
2558
|
+
kind: "ssh.key.ack",
|
|
2559
|
+
userId: frame.userId,
|
|
2560
|
+
ok: false,
|
|
2561
|
+
error,
|
|
2562
|
+
});
|
|
2563
|
+
return true;
|
|
2564
|
+
case "env.var.list":
|
|
2565
|
+
case "env.var.set":
|
|
2566
|
+
case "env.var.delete":
|
|
2567
|
+
send(socket, {
|
|
2568
|
+
kind: "env.var.ack",
|
|
2569
|
+
opId: frame.opId,
|
|
2570
|
+
ok: false,
|
|
2571
|
+
error,
|
|
2572
|
+
});
|
|
2573
|
+
return true;
|
|
2574
|
+
case "mcp.op":
|
|
2575
|
+
send(socket, {
|
|
2576
|
+
kind: "mcp.ack",
|
|
2577
|
+
opId: frame.opId,
|
|
2578
|
+
ok: false,
|
|
2579
|
+
error,
|
|
2580
|
+
});
|
|
2581
|
+
return true;
|
|
2582
|
+
case "host.inventory.request":
|
|
2583
|
+
// Read-only and retryable after the replacement process reconnects. A
|
|
2584
|
+
// partial page during shutdown would be tied to a dying generation.
|
|
2585
|
+
return true;
|
|
2586
|
+
case "host.logs.stop":
|
|
2587
|
+
case "engine.login.stop":
|
|
2588
|
+
// Stops remain idempotent during drain and cannot admit new work.
|
|
2589
|
+
return false;
|
|
2590
|
+
case "engine.login.input":
|
|
2591
|
+
case "engine.login.callback":
|
|
2592
|
+
// All login managers were synchronously cancelled when shutdown began.
|
|
2593
|
+
// These single-use artifacts are dropped instead of being retained or
|
|
2594
|
+
// reflected in an event after their operation ceased to exist.
|
|
2595
|
+
return true;
|
|
2596
|
+
case "pong":
|
|
2597
|
+
case "event.resume":
|
|
2598
|
+
case "event.ack":
|
|
2599
|
+
case "tunnel.data":
|
|
2600
|
+
case "tunnel.requestEnd":
|
|
2601
|
+
case "tunnel.close":
|
|
2602
|
+
return false;
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
|
|
1526
2606
|
function rawDataToBuffer(data: RawData): Buffer {
|
|
1527
2607
|
if (Buffer.isBuffer(data)) return data;
|
|
1528
2608
|
if (data instanceof ArrayBuffer) return Buffer.from(data);
|
|
@@ -1655,11 +2735,24 @@ function expectTaskLaunchInput(
|
|
|
1655
2735
|
|
|
1656
2736
|
function expectTaskDownInput(args: unknown[], index: number): TaskDownInput {
|
|
1657
2737
|
const input = expectRecord(args[index], "task down input");
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
2738
|
+
const taskId = expectStringValue(input.taskId, "taskId");
|
|
2739
|
+
const task = expectTaskCommandTask(input.task);
|
|
2740
|
+
if (!isSafeHostTaskId(taskId) || task.id !== taskId) {
|
|
2741
|
+
throw new Error("invalid task down identity");
|
|
2742
|
+
}
|
|
2743
|
+
const out: TaskDownInput = {
|
|
2744
|
+
taskId,
|
|
2745
|
+
task,
|
|
1661
2746
|
projects: expectTaskCommandProjects(input.projects),
|
|
1662
2747
|
};
|
|
2748
|
+
if (Object.prototype.hasOwnProperty.call(input, "cleanup")) {
|
|
2749
|
+
const cleanup = expectRecord(input.cleanup, "task down cleanup");
|
|
2750
|
+
if (cleanup.kind !== "orphan-gc" || cleanup.pruneLocalState !== true) {
|
|
2751
|
+
throw new Error("invalid task down cleanup marker");
|
|
2752
|
+
}
|
|
2753
|
+
out.cleanup = { kind: "orphan-gc", pruneLocalState: true };
|
|
2754
|
+
}
|
|
2755
|
+
return out;
|
|
1663
2756
|
}
|
|
1664
2757
|
|
|
1665
2758
|
function expectTaskDiffInput(args: unknown[], index: number): TaskDiffInput {
|
|
@@ -1905,11 +2998,9 @@ function expectStringValue(value: unknown, label: string): string {
|
|
|
1905
2998
|
|
|
1906
2999
|
function ensureHostId(): string {
|
|
1907
3000
|
mkdirSync(env.dataDir, { recursive: true, mode: 0o700 });
|
|
1908
|
-
const
|
|
1909
|
-
if (
|
|
1910
|
-
|
|
1911
|
-
if (existing) return existing;
|
|
1912
|
-
}
|
|
3001
|
+
const existing = readPersistedHostId();
|
|
3002
|
+
if (existing) return existing;
|
|
3003
|
+
const path = hostIdFilePath();
|
|
1913
3004
|
const id = newId();
|
|
1914
3005
|
writeFileSync(path, `${id}\n`, { mode: 0o600 });
|
|
1915
3006
|
return id;
|
|
@@ -1936,16 +3027,68 @@ function reconnectDelay(): number {
|
|
|
1936
3027
|
function requestShutdown(signal: NodeJS.Signals): void {
|
|
1937
3028
|
console.log(`[host-agent] ${signal} received; shutting down`);
|
|
1938
3029
|
stopping = true;
|
|
3030
|
+
// Cancel human-paced operations without invalidating this socket's exact
|
|
3031
|
+
// generation: already-admitted one-shot config writes may still ack while
|
|
3032
|
+
// the ordinary in-flight drain completes.
|
|
3033
|
+
void stopRemoteOperationManagers();
|
|
3034
|
+
stopMcpGatewaySubscription();
|
|
3035
|
+
stopAgentClisReadySubscription();
|
|
3036
|
+
void mcpGateway.stop();
|
|
3037
|
+
managedMutationDrain.stopAccepting();
|
|
3038
|
+
stopRuntimeAutoRecheck();
|
|
3039
|
+
runtimeActivationTail.stop();
|
|
1939
3040
|
shutdownRequested = true;
|
|
1940
|
-
if (ws && ws.readyState === WebSocket.OPEN) {
|
|
1941
|
-
ws.close(1001, "host shutting down");
|
|
1942
|
-
}
|
|
1943
3041
|
maybeExit();
|
|
1944
3042
|
}
|
|
1945
3043
|
|
|
1946
3044
|
function maybeExit(): void {
|
|
1947
|
-
if (
|
|
1948
|
-
|
|
3045
|
+
if (
|
|
3046
|
+
!shutdownRequested ||
|
|
3047
|
+
inFlight > 0 ||
|
|
3048
|
+
managedMutationDrain.activeCount > 0 ||
|
|
3049
|
+
pendingRemoteOperationCleanups.size > 0
|
|
3050
|
+
) return;
|
|
3051
|
+
if (ws) {
|
|
3052
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
3053
|
+
// `ws.close()` queues the close frame after already-queued ack/result
|
|
3054
|
+
// frames. Wait for its close event before terminating the process so
|
|
3055
|
+
// those replies are not truncated by `process.exit`.
|
|
3056
|
+
ws.close(1001, "host shutting down");
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
3059
|
+
if (ws.readyState === WebSocket.CONNECTING) {
|
|
3060
|
+
// There is no authenticated transport to drain yet. Termination emits
|
|
3061
|
+
// `close`, whose handler calls maybeExit again with `ws` cleared.
|
|
3062
|
+
ws.terminate();
|
|
3063
|
+
return;
|
|
3064
|
+
}
|
|
3065
|
+
if (ws.readyState === WebSocket.CLOSING) return;
|
|
3066
|
+
}
|
|
3067
|
+
process.exit(shutdownExitCode);
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
/**
|
|
3071
|
+
* Finish any in-flight bridge command, then let the service supervisor exec
|
|
3072
|
+
* the newly activated `runtime/current`. The non-zero status is intentional:
|
|
3073
|
+
* Linux uses Restart=on-failure, while launchd KeepAlive also respawns it.
|
|
3074
|
+
*/
|
|
3075
|
+
export function requestManagedHostRestart(): void {
|
|
3076
|
+
shutdownExitCode = MANAGED_UPDATE_RESTART_EXIT_CODE;
|
|
3077
|
+
requestShutdown("SIGTERM");
|
|
3078
|
+
}
|
|
3079
|
+
|
|
3080
|
+
/** Automatic host updates wait only for in-flight work to drain. Running
|
|
3081
|
+
* tasks are deliberately not counted: durable sessions survive the service
|
|
3082
|
+
* restart, and gating on their existence made a busy host un-updatable
|
|
3083
|
+
* (live 2026-08-17 — the owner's explicit requirement is update-under-load). */
|
|
3084
|
+
export function isIdleForManagedHostUpdate(): boolean {
|
|
3085
|
+
return (
|
|
3086
|
+
!stopping &&
|
|
3087
|
+
!shutdownRequested &&
|
|
3088
|
+
inFlight === 0 &&
|
|
3089
|
+
managedMutationDrain.activeCount === 0 &&
|
|
3090
|
+
!hasInFlightHostTaskLifecycle()
|
|
3091
|
+
);
|
|
1949
3092
|
}
|
|
1950
3093
|
|
|
1951
3094
|
function requireEnv(name: string): string {
|