@runuai/host 0.9.150 → 0.9.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1876 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1830 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +623 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
package/src/main.ts
CHANGED
|
@@ -16,7 +16,7 @@ import { join } from "node:path";
|
|
|
16
16
|
import WebSocket, { type RawData } from "ws";
|
|
17
17
|
|
|
18
18
|
import { env } from "../lib/env";
|
|
19
|
-
import { getDb, migrateHostDb } from "../lib/db";
|
|
19
|
+
import { getDb, migrateHostDb, schema } from "../lib/db";
|
|
20
20
|
import {
|
|
21
21
|
markCloudError,
|
|
22
22
|
markConnected,
|
|
@@ -25,8 +25,11 @@ import {
|
|
|
25
25
|
} from "../lib/cloud-state";
|
|
26
26
|
import {
|
|
27
27
|
applyTerminalHistoryRetention,
|
|
28
|
+
closeTaskRunAssignment,
|
|
28
29
|
getHostTask,
|
|
29
30
|
hasInFlightHostTaskLifecycle,
|
|
31
|
+
readTaskRunEpochState,
|
|
32
|
+
upsertHostTask,
|
|
30
33
|
} from "../lib/runtime-state";
|
|
31
34
|
import {
|
|
32
35
|
StoppedTaskStatusQueue,
|
|
@@ -39,12 +42,14 @@ import {
|
|
|
39
42
|
requestRuntimeRecovery,
|
|
40
43
|
resumeRuntimeDeferredAgentWork,
|
|
41
44
|
runtimeRecoveryVerdict,
|
|
45
|
+
type StoppedTaskStatusTarget,
|
|
42
46
|
waitForRuntimeRecovery,
|
|
43
47
|
} from "../lib/orchestrator";
|
|
44
48
|
import {
|
|
45
49
|
claimGithubCredentialGeneration,
|
|
46
50
|
connectedUserIds,
|
|
47
51
|
GitHubRepositoryAccessError,
|
|
52
|
+
GitHubTokenUnreadableError,
|
|
48
53
|
isTransientGithubError,
|
|
49
54
|
listUserInstallationIds,
|
|
50
55
|
listUserInstallationRepositoryIds,
|
|
@@ -66,9 +71,9 @@ import {
|
|
|
66
71
|
reinjectGrokRunningTasks,
|
|
67
72
|
watchCodexAuth,
|
|
68
73
|
} from "../lib/codex-auth";
|
|
69
|
-
import { startCredentialStorePickup } from "../lib/credential-store-pickup";
|
|
70
|
-
import { ImmutableParameterConflictError } from "../lib/credential-store";
|
|
71
|
-
import { configureTokenBroker, startTokenBrokerLoop } from "../lib/token-broker-local";
|
|
74
|
+
import { startCredentialStorePickup, stopCredentialStorePickup } from "../lib/credential-store-pickup";
|
|
75
|
+
import { configuredCredentialStore, ImmutableParameterConflictError, storeFailureKind } from "../lib/credential-store";
|
|
76
|
+
import { configureTokenBroker, startTokenBrokerLoop, stopTokenBrokerLoop } from "../lib/token-broker-local";
|
|
72
77
|
import {
|
|
73
78
|
ensureKeyForUser as ensureSshKeyForUser,
|
|
74
79
|
getPublicKey as getSshPublicKey,
|
|
@@ -76,12 +81,40 @@ import {
|
|
|
76
81
|
import {
|
|
77
82
|
deleteProjectEnvVar,
|
|
78
83
|
listProjectEnvKeys,
|
|
84
|
+
noteProjectEnvViewChange,
|
|
79
85
|
onProjectEnvChange,
|
|
80
86
|
setProjectEnvVar,
|
|
81
87
|
noteProjectOrg,
|
|
82
88
|
} from "../lib/host-env";
|
|
89
|
+
import { cancelMcpCapture, completeMcpCapture, probeMcpCapture } from "../lib/mcp-capture";
|
|
90
|
+
import { captureGithub, mintSsh, readGithubRevision, revokeGithub } from "../lib/person-capture";
|
|
83
91
|
import { handleMcpOp } from "../lib/mcp-connections";
|
|
84
|
-
import {
|
|
92
|
+
import {
|
|
93
|
+
ENGINE_CREDENTIAL_FILES,
|
|
94
|
+
ENGINE_ENV_CREDENTIAL_KEYS,
|
|
95
|
+
HOST_CLOUD_CREDENTIALS_FEATURE,
|
|
96
|
+
parseCredentialsActivateResultFrame,
|
|
97
|
+
runEpochFromWire,
|
|
98
|
+
SIGNIN_BOX_CAPTURE_FLOWS_FEATURE,
|
|
99
|
+
type CredentialsActivateResultFrame,
|
|
100
|
+
type CredentialsActivationPhase,
|
|
101
|
+
SIGNIN_BOX_MCP_CAPTURE_FEATURE,
|
|
102
|
+
SIGNIN_BOX_PERSON_CREDENTIALS_FEATURE,
|
|
103
|
+
isPersonGithubParameter,
|
|
104
|
+
parseGithubCaptureFrame,
|
|
105
|
+
parseGithubRevokeFrame,
|
|
106
|
+
parseMcpCaptureCancelFrame,
|
|
107
|
+
parseMcpCaptureCompleteFrame,
|
|
108
|
+
parseMcpCaptureProbeFrame,
|
|
109
|
+
parseSshMintFrame,
|
|
110
|
+
parseCredentialChangedFrame,
|
|
111
|
+
parseCredentialRefreshResultFrame,
|
|
112
|
+
parseLeaseGrantV1Frame,
|
|
113
|
+
parseLeaseStaleFrame,
|
|
114
|
+
parseLeaseUnavailableFrame,
|
|
115
|
+
} from "./cloud-credentials";
|
|
116
|
+
import { captureEngineAccount } from "../lib/capture-flow";
|
|
117
|
+
import { registerTaskGatewayTlsHandler, setMcpGatewayLeaseOnly, startMcpGateway } from "../lib/mcp-gateway";
|
|
85
118
|
import { setActiveSecretsProxy, startSecretsProxyHost, type SecretsProxyHost } from "../lib/secrets-proxy/host";
|
|
86
119
|
import {
|
|
87
120
|
createTaskProxyPolicySource,
|
|
@@ -92,7 +125,7 @@ import { quiesceTaskProxyMutations } from "../lib/secrets-proxy/task-trust";
|
|
|
92
125
|
import { resolveWithSystem, resolverIgnoringHostsFile, secretsProxyEnabled } from "../lib/secrets-proxy/runtime";
|
|
93
126
|
import { reinstallTrustInRunningTasks, unpinRunningTasks } from "../lib/secrets-proxy/trust-watch";
|
|
94
127
|
import { currentTaskGatewayRegistration } from "../lib/task-gateway-state";
|
|
95
|
-
import { proxyTaskEnvSnapshot } from "../lib/task-env-source";
|
|
128
|
+
import { followLeavingLegacy, proxyTaskEnvSnapshot } from "../lib/task-env-source";
|
|
96
129
|
import { recoverActiveTaskGateways } from "../lib/task-gateway-recovery";
|
|
97
130
|
import { addHostBreadcrumb, initHostObs, setHostObsTag } from "../lib/obs";
|
|
98
131
|
import {
|
|
@@ -104,6 +137,7 @@ import {
|
|
|
104
137
|
uiPortFilePath,
|
|
105
138
|
} from "./paths";
|
|
106
139
|
import { dockerMemoryBytes, startUiServer } from "./ui/server";
|
|
140
|
+
import { outboundFrameJson } from "./outbound-frame";
|
|
107
141
|
import { parsePreviewPortRuntimes } from "../lib/preview-ports";
|
|
108
142
|
import { dockerCli } from "../lib/docker-exec";
|
|
109
143
|
import {
|
|
@@ -122,6 +156,7 @@ import {
|
|
|
122
156
|
containerRuntimeBecameReady,
|
|
123
157
|
containerRuntimeCapabilityState,
|
|
124
158
|
containerRuntimeEpochIsCurrent,
|
|
159
|
+
containerRuntimeGenerationOperational,
|
|
125
160
|
containerRuntimeProblem,
|
|
126
161
|
containerRuntimeReadinessEpoch,
|
|
127
162
|
containerRuntimeState,
|
|
@@ -129,12 +164,17 @@ import {
|
|
|
129
164
|
initializeContainerRuntime,
|
|
130
165
|
markContainerRuntimeOperational,
|
|
131
166
|
onContainerRuntimeChange,
|
|
167
|
+
containerRuntimeRecheckRequester,
|
|
168
|
+
setContainerRuntimeRecheckRequester,
|
|
132
169
|
startContainerRuntimeAutoRecheck,
|
|
133
170
|
suspendContainerRuntime,
|
|
134
171
|
containerRuntimeMachineIdentity,
|
|
172
|
+
pinnedContainerRuntimeMachineIdentity,
|
|
135
173
|
publishActivationPhase,
|
|
174
|
+
rejectContainerRuntimeSelection,
|
|
175
|
+
reprobeContainerRuntimeMachineIdentity,
|
|
136
176
|
} from "../lib/container-runtime";
|
|
137
|
-
import { reconstructPersistedTaskEnvironment } from "../lib/task-environment";
|
|
177
|
+
import { persistedTaskEnvironmentsMatchMachine, reconstructPersistedTaskEnvironment } from "../lib/task-environment";
|
|
138
178
|
import { logVnodePressure } from "../lib/vnode-pressure";
|
|
139
179
|
import { resolveAppleTunnelRoute } from "./apple-tunnel-route";
|
|
140
180
|
import {
|
|
@@ -154,15 +194,21 @@ import {
|
|
|
154
194
|
import { inspectTunnelContainer } from "../lib/tunnel-runtime";
|
|
155
195
|
import { TunnelRegistry } from "../lib/tunnel-registry";
|
|
156
196
|
import { newId } from "../lib/ulid";
|
|
197
|
+
import { RUNNER_BOOT_ID } from "../lib/runner-boot";
|
|
157
198
|
import {
|
|
158
199
|
capabilities as agentKindCapabilities,
|
|
159
200
|
onChange as onRegistryChange,
|
|
160
201
|
setAdvertisementFallback,
|
|
202
|
+
setLeasedAdvertisement,
|
|
161
203
|
} from "../lib/agents/registry";
|
|
162
204
|
import { canAdvertiseTypedSecretaryDispatch } from "../lib/agents/mode";
|
|
163
205
|
import {
|
|
164
206
|
MANAGED_UPDATE_RESTART_EXIT_CODE,
|
|
207
|
+
claimCredentialsCapabilityFloor,
|
|
165
208
|
claimManagedRuntimeCapabilityFloor,
|
|
209
|
+
credentialsCapabilityFloorExists,
|
|
210
|
+
CREDENTIALS_ACTIVATION_RECORD,
|
|
211
|
+
commitCredentialsSwitchLocally,
|
|
166
212
|
managedRuntimeAdmissionBlocker,
|
|
167
213
|
reclaimAbandonedMaintenanceCheck,
|
|
168
214
|
} from "../lib/managed-runtime";
|
|
@@ -288,6 +334,10 @@ import {
|
|
|
288
334
|
isHostConfigGetFrame,
|
|
289
335
|
isHostConfigSetFrame,
|
|
290
336
|
isHostLogsStartFrame,
|
|
337
|
+
isProjectEnvReplicaV1ApplyFrame,
|
|
338
|
+
isProjectEnvReplicaV1BeginFrame,
|
|
339
|
+
isProjectEnvReplicaV1CommitFrame,
|
|
340
|
+
isProjectEnvReplicaV1PageFrame,
|
|
291
341
|
isHostLogsStopFrame,
|
|
292
342
|
isProjectEnvBindingSetFrame,
|
|
293
343
|
isProjectEnvDeleteFrame,
|
|
@@ -302,14 +352,18 @@ import {
|
|
|
302
352
|
isProjectEnvReplicaPageFrame,
|
|
303
353
|
isProjectEnvSetFrame,
|
|
304
354
|
isProjectEnvStagingPutFrame,
|
|
355
|
+
isProjectEnvLegacyTagFrame,
|
|
356
|
+
PROJECT_ENV_LEGACY_TAGS_PROTOCOL_FEATURE,
|
|
305
357
|
type TaskAgent,
|
|
306
358
|
type TaskCommandProject,
|
|
307
359
|
type TaskCommandTask,
|
|
308
360
|
type TaskDiffInput,
|
|
309
361
|
type TaskDownInput,
|
|
310
362
|
type TaskLaunchInput,
|
|
363
|
+
type TaskStopInput,
|
|
311
364
|
type TaskLease,
|
|
312
365
|
type FilesOpInput,
|
|
366
|
+
type GitHubRepositoryAccessErrorCode,
|
|
313
367
|
engineAccountScopeFromWire,
|
|
314
368
|
isEngineAccountLabel,
|
|
315
369
|
isEngineConnectKind,
|
|
@@ -323,6 +377,38 @@ import {
|
|
|
323
377
|
import { isEnvironmentTransportProvider } from "../lib/task-environment/types";
|
|
324
378
|
import { LeaseManager, leaseScope } from "../lib/lease";
|
|
325
379
|
import { markSidecarReady, sidecarMode, sidecarPoolMode, sidecarTaskId } from "../lib/sidecar";
|
|
380
|
+
import {
|
|
381
|
+
activationPassOnCredentials,
|
|
382
|
+
cloudCredentialsActivated,
|
|
383
|
+
cloudCredentialsRecovered,
|
|
384
|
+
cloudCredentialsSwept,
|
|
385
|
+
credentialsMode,
|
|
386
|
+
leaveLegacy,
|
|
387
|
+
localStopCredentials,
|
|
388
|
+
onCloudCredentialsActivated,
|
|
389
|
+
onLeavingLegacy,
|
|
390
|
+
onOwnCredentials,
|
|
391
|
+
pauseLegacyStarts,
|
|
392
|
+
recoverCloudCredentials,
|
|
393
|
+
setCloudCredentialRuntime,
|
|
394
|
+
setCredentialsModeSource,
|
|
395
|
+
setRecoveredRuntimeSetup,
|
|
396
|
+
} from "../lib/cloud-credential-host";
|
|
397
|
+
import {
|
|
398
|
+
CredentialsActivationAuthority,
|
|
399
|
+
type ActivationAnswer,
|
|
400
|
+
type StoppedReport,
|
|
401
|
+
} from "../lib/credentials-activation-authority";
|
|
402
|
+
import { purgeLocalCredentialStores } from "../lib/credentials-purge";
|
|
403
|
+
import {
|
|
404
|
+
planLegacyTaskRuntimeRetirement,
|
|
405
|
+
retireLegacyTaskRuntimes,
|
|
406
|
+
type RetirementPlan,
|
|
407
|
+
type RetirementPlanDeps,
|
|
408
|
+
} from "../lib/legacy-runtime-retirement";
|
|
409
|
+
import { taskContainerBackend, taskContainerCli } from "../lib/task-container-cli";
|
|
410
|
+
import { CloudCredentialRuntime } from "../lib/cloud-credential-runtime";
|
|
411
|
+
import { ProjectEnvTaskReplicaManager } from "../lib/project-env-task-replica";
|
|
326
412
|
import {
|
|
327
413
|
purgeDefaultEngineSlots,
|
|
328
414
|
SIGN_IN_BOX_TASK_REFUSAL,
|
|
@@ -340,9 +426,11 @@ import {
|
|
|
340
426
|
ProjectEnvStagingInvalidError,
|
|
341
427
|
writeProjectEnvStaging,
|
|
342
428
|
} from "../lib/project-env-staging";
|
|
429
|
+
import { queueProjectEnvLegacyTag } from "../lib/project-env-legacy-tags";
|
|
343
430
|
import { loadGithubTokenIntoCache } from "../lib/github-tokens";
|
|
344
431
|
import { ensureSshKeyStoreFirst } from "../lib/ssh";
|
|
345
432
|
import { classifySidecarTaskView, resumeSidecarTask } from "../lib/sidecar-resume";
|
|
433
|
+
import { revalidateRuntimeOnConnect } from "../lib/runtime-reconnect";
|
|
346
434
|
import { declareAgentClisReady } from "../lib/standard-image";
|
|
347
435
|
|
|
348
436
|
setContainerRuntimeOperationalCheck(() => containerRuntimeProblem() === null);
|
|
@@ -438,32 +526,49 @@ async function waitForRemoteOperationCleanups(): Promise<void> {
|
|
|
438
526
|
|
|
439
527
|
let pendingBinaryTunnelId: string | null = null;
|
|
440
528
|
const pendingRuntimeIsolationStatuses = new StoppedTaskStatusQueue(
|
|
441
|
-
(taskId) =>
|
|
529
|
+
(taskId) => {
|
|
530
|
+
const task = getHostTask(taskId);
|
|
531
|
+
// The assignment floor, not the reusable runtime row, is the generation
|
|
532
|
+
// authority. taskUp advances it before it may rewrite the task row, so a
|
|
533
|
+
// queued N stop cannot be mistaken for N+1 in that admission window.
|
|
534
|
+
const assignment = readTaskRunEpochState(taskId);
|
|
535
|
+
return task?.statusMirror
|
|
536
|
+
? {
|
|
537
|
+
status: task.statusMirror,
|
|
538
|
+
runEpoch: assignment?.highestRunEpoch ?? 0,
|
|
539
|
+
}
|
|
540
|
+
: null;
|
|
541
|
+
},
|
|
442
542
|
);
|
|
443
543
|
let stopRuntimeAutoRecheck = (): void => {};
|
|
444
544
|
const runtimeActivationTail = new RuntimeActivationTail({
|
|
445
545
|
isCurrent: runtimeActivationIsCurrent,
|
|
446
|
-
onError:
|
|
447
|
-
|
|
448
|
-
`[host-agent] container runtime activation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
449
|
-
),
|
|
450
|
-
onExhausted: (_epoch, attempts) =>
|
|
546
|
+
onError: reportRuntimeActivationError,
|
|
547
|
+
onExhausted: (_epoch, attempts) => {
|
|
451
548
|
console.warn(
|
|
452
549
|
`[host-agent] container runtime recovery remained unresolved after ${attempts} follow-up attempt(s); leaving the host in checking until the next runtime recheck`,
|
|
453
|
-
)
|
|
550
|
+
);
|
|
551
|
+
// ADR-154: a runner already on cloud credentials still needs its boot
|
|
552
|
+
// sweep to say so; the activation authority retries it from here.
|
|
553
|
+
if (!stopping) credentialsActivation.sweepItself();
|
|
554
|
+
},
|
|
454
555
|
});
|
|
455
556
|
const tunnels = new TunnelRegistry();
|
|
456
557
|
|
|
457
|
-
// ADR-103: event outbox.
|
|
558
|
+
// ADR-103: event outbox. RUNNER_BOOT_ID scopes seqs to this process lifetime;
|
|
559
|
+
// ADR-154's task replicas use this exact singleton too. The
|
|
458
560
|
// cloud's replay watermark only applies within a matching boot. Events are
|
|
459
561
|
// enqueued by a PROCESS-level subscription (below, before connect()) so a
|
|
460
562
|
// dropped WSS no longer drops events — they drain on reconnect, after the
|
|
461
563
|
// cloud names its watermark via event.resume.
|
|
462
|
-
const BOOT_ID = newId();
|
|
463
564
|
const eventOutbox = new EventOutbox();
|
|
464
565
|
// Transmission is held per-connection until the cloud's event.resume sets the
|
|
465
566
|
// replay cursor (or the fallback timer concedes the cloud predates ADR-103).
|
|
466
567
|
let eventFlushEnabled = false;
|
|
568
|
+
/** ADR-154: the cloud answered this bridge's auth (event.resume), so a
|
|
569
|
+
* credential frame sent now is not lost in its pre-verdict window. Set
|
|
570
|
+
* before anyone asks again, independently of the event flush. */
|
|
571
|
+
let credentialFramesReady = false;
|
|
467
572
|
let resumeFallbackTimer: NodeJS.Timeout | null = null;
|
|
468
573
|
let eventFlushBackpressureTimer: NodeJS.Timeout | null = null;
|
|
469
574
|
const RESUME_FALLBACK_MS = 3_000;
|
|
@@ -554,6 +659,61 @@ try {
|
|
|
554
659
|
);
|
|
555
660
|
}
|
|
556
661
|
migrateHostDb();
|
|
662
|
+
// ADR-154 §13: the activation authority, before anything reads the mode
|
|
663
|
+
// (the refreshers, the v0 lease, the runtime's recovery, the first
|
|
664
|
+
// capability frame). It moves only owned runners in this release: a task
|
|
665
|
+
// machine needs its run's Files session on v1 first, and the sign-in box
|
|
666
|
+
// runs no task.
|
|
667
|
+
const credentialsActivation = new CredentialsActivationAuthority({
|
|
668
|
+
recordPath: join(env.dataDir, CREDENTIALS_ACTIVATION_RECORD),
|
|
669
|
+
floorClaimed: () => credentialsCapabilityFloorExists(),
|
|
670
|
+
eligible: () => !sidecarMode() && !signInBoxMode(),
|
|
671
|
+
ask: askCredentialsActivation,
|
|
672
|
+
pauseLegacyStarts,
|
|
673
|
+
planRetirement: () => planLegacyTaskRuntimeRetirement(retirementPlanDeps()),
|
|
674
|
+
commitLocally: (record) => commitCredentialsSwitchLocally(record),
|
|
675
|
+
retire: retireTaskRuntimesForCloudCredentials,
|
|
676
|
+
reportStopped: reportRetiredTasksStopped,
|
|
677
|
+
claimFloor: claimCredentialsCapabilityFloor,
|
|
678
|
+
leaveLegacy,
|
|
679
|
+
purge: purgeCredentialStores,
|
|
680
|
+
swept: cloudCredentialsSwept,
|
|
681
|
+
sweep: recoverCloudCredentials,
|
|
682
|
+
onActive: cloudCredentialsActive,
|
|
683
|
+
// Pending again: the next capability frame advertises no v1.
|
|
684
|
+
onActiveWithdrawn: () => sendCapabilities(),
|
|
685
|
+
restart: (why) => {
|
|
686
|
+
console.warn(`[credentials] ${why}: restarting`);
|
|
687
|
+
requestManagedHostRestart();
|
|
688
|
+
},
|
|
689
|
+
log: (line) => console.warn(line),
|
|
690
|
+
});
|
|
691
|
+
setCredentialsModeSource(() => credentialsActivation.mode());
|
|
692
|
+
/** ADR-154 §13: how often a runner not yet on cloud credentials asks again,
|
|
693
|
+
* so the operator turning them on reaches it within minutes. */
|
|
694
|
+
const CREDENTIALS_ACTIVATION_RETRY_MS = 5 * 60_000;
|
|
695
|
+
const CREDENTIALS_ACTIVATION_ANSWER_MS = 60_000;
|
|
696
|
+
const LEASED_ENGINE_KINDS: ReadonlySet<string> = new Set([
|
|
697
|
+
...Object.keys(ENGINE_ENV_CREDENTIAL_KEYS),
|
|
698
|
+
...Object.keys(ENGINE_CREDENTIAL_FILES),
|
|
699
|
+
]);
|
|
700
|
+
// Started here, first fired long after the bridge's state exists.
|
|
701
|
+
const credentialsActivationTimer = setInterval(() => {
|
|
702
|
+
if (stopping || !credentialFramesReady) return;
|
|
703
|
+
// Active: the purge reported again. The cloud acknowledges none and keeps
|
|
704
|
+
// the first, so a report its database failed to record is not lost until
|
|
705
|
+
// the next reconnect, and the runner is not refused tasks meanwhile. And
|
|
706
|
+
// in every mode, the stopped reports still owed go again.
|
|
707
|
+
if (credentialsMode() === "active") reportCredentialsPurged();
|
|
708
|
+
void credentialsActivation.advance();
|
|
709
|
+
}, CREDENTIALS_ACTIVATION_RETRY_MS);
|
|
710
|
+
credentialsActivationTimer.unref?.();
|
|
711
|
+
|
|
712
|
+
const credentialsActivationAsks = new Map<
|
|
713
|
+
string,
|
|
714
|
+
{ phase: CredentialsActivationPhase; resolve: (answer: ActivationAnswer) => void; timer: NodeJS.Timeout }
|
|
715
|
+
>();
|
|
716
|
+
|
|
557
717
|
// A task-machine process starts fail-closed. The prior process may have died
|
|
558
718
|
// after opening its replica, and no task loader or proxy may serve that view
|
|
559
719
|
// until this authenticated socket completes a fresh whole-task snapshot.
|
|
@@ -737,9 +897,7 @@ async function startSignInBox(): Promise<void> {
|
|
|
737
897
|
if (signInBoxMode()) {
|
|
738
898
|
await startSignInBox();
|
|
739
899
|
} else if (!sidecarMode()) {
|
|
740
|
-
|
|
741
|
-
startCredentialStorePickup({ repush: repushBrokeredCopies });
|
|
742
|
-
void reconcileStores();
|
|
900
|
+
startOwnedRefreshers();
|
|
743
901
|
}
|
|
744
902
|
/**
|
|
745
903
|
* ADR-133 sidecar: the task-scoped AWS lease. Requested when the bridge is
|
|
@@ -747,6 +905,49 @@ if (signInBoxMode()) {
|
|
|
747
905
|
* reconciles and the S3 file mounts run. A denial means the task is no
|
|
748
906
|
* longer ours to run — exit 3 and let the cloud's sweep reclaim the machine.
|
|
749
907
|
*/
|
|
908
|
+
// ADR-154: the composer for tasks on cloud credentials. Nothing runs on it
|
|
909
|
+
// until the runner's activation authority makes cloud credentials its
|
|
910
|
+
// tasks' only ones (lib/cloud-credential-host.ts).
|
|
911
|
+
const cloudCredentials = new CloudCredentialRuntime({
|
|
912
|
+
runnerKind: sidecarMode() ? "task-machine" : "owned",
|
|
913
|
+
send: sendCredentialFrame,
|
|
914
|
+
quiesceProcesses: (request) => getOrchestrator().quiesceLeasedEngineAgents(request),
|
|
915
|
+
// The task-keyed env replica the cloud streams for each run. A change to
|
|
916
|
+
// it, or to what the composer serves from it, makes the proxy's trust
|
|
917
|
+
// delivery look again; /env reads it afresh on every command.
|
|
918
|
+
envReplica: new ProjectEnvTaskReplicaManager({ onChange: () => noteProjectEnvViewChange() }),
|
|
919
|
+
envServedChanged: () => noteProjectEnvViewChange(),
|
|
920
|
+
});
|
|
921
|
+
// Leaving its own credentials ends every task's env from the runner's
|
|
922
|
+
// stores at once, and makes the proxy's trust delivery look again.
|
|
923
|
+
followLeavingLegacy();
|
|
924
|
+
// ADR-154 on a task machine: once its recovered task's grant opens the
|
|
925
|
+
// gate, the task's loader, proxy and editor again (its shared files come
|
|
926
|
+
// from its own run's Files session, never the runner's v0 lease); and the
|
|
927
|
+
// proxy's upkeep from activation on, with no v0 first grant to start it.
|
|
928
|
+
if (sidecarMode()) {
|
|
929
|
+
setRecoveredRuntimeSetup(async (_run, signal) => {
|
|
930
|
+
const resumed = await resumeSidecarTask({ remountFiles: false, signal });
|
|
931
|
+
if (resumed.outcome === "resumed") {
|
|
932
|
+
console.log(
|
|
933
|
+
`[sidecar] task ${sidecarTaskId()} resumed on cloud credentials: loader, proxy and editor re-asserted` +
|
|
934
|
+
(resumed.warnings.length > 0 ? ` (${resumed.warnings.join("; ")})` : ""),
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
});
|
|
938
|
+
onCloudCredentialsActivated(() => {
|
|
939
|
+
if (!stopping) startSecretsProxyUpkeep();
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
setCloudCredentialRuntime(cloudCredentials);
|
|
943
|
+
// A runner that was already on cloud credentials follows them from here:
|
|
944
|
+
// once its boot recovery sweep (the container runtime's own pass) is done.
|
|
945
|
+
credentialsActivation.booted();
|
|
946
|
+
void cloudCredentialsRecovered().then(() => {
|
|
947
|
+
if (!stopping) void credentialsActivation.advance();
|
|
948
|
+
});
|
|
949
|
+
let cloudCredentialsStopped = false;
|
|
950
|
+
|
|
750
951
|
// A POOL-mode sidecar requests no lease: it has no task to scope one to,
|
|
751
952
|
// and the cloud would refuse it anyway (docs/machine-warm-pool.md).
|
|
752
953
|
const leaseManager: LeaseManager | null = sidecarTaskId()
|
|
@@ -758,33 +959,11 @@ const leaseManager: LeaseManager | null = sidecarTaskId()
|
|
|
758
959
|
return true;
|
|
759
960
|
},
|
|
760
961
|
onFirstGrant: () => {
|
|
962
|
+
// ADR-154: none of this once the runner is leaving its own
|
|
963
|
+
// credentials (the manager is stopped then, and this drained).
|
|
964
|
+
if (!onOwnCredentials()) return;
|
|
761
965
|
console.log("[lease] granted — reconciling the credential store");
|
|
762
|
-
|
|
763
|
-
// Task-up waits for this: only now are engine accounts, GitHub
|
|
764
|
-
// tokens, and project env on the machine (lib/sidecar.ts).
|
|
765
|
-
markSidecarReady();
|
|
766
|
-
console.log("[lease] credential store reconciled — the task may start");
|
|
767
|
-
// A runner that came back on a machine whose task is already up
|
|
768
|
-
// re-asserts the mounts and the editor (lib/sidecar-resume.ts).
|
|
769
|
-
void resumeSidecarTask()
|
|
770
|
-
.then((resumed) => {
|
|
771
|
-
if (resumed.outcome !== "resumed") return;
|
|
772
|
-
console.log(
|
|
773
|
-
`[sidecar] task ${sidecarTaskId()} resumed after a runner restart: shared files and editor re-asserted` +
|
|
774
|
-
(resumed.warnings.length > 0 ? ` (${resumed.warnings.join("; ")})` : ""),
|
|
775
|
-
);
|
|
776
|
-
})
|
|
777
|
-
.catch((error: unknown) => {
|
|
778
|
-
console.warn(
|
|
779
|
-
`[sidecar] resume failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
780
|
-
);
|
|
781
|
-
})
|
|
782
|
-
// A sidecar runs no container-runtime activation: its task is
|
|
783
|
-
// reconciled by the resume (or by task-up), and kept there from now.
|
|
784
|
-
.finally(() => {
|
|
785
|
-
if (!stopping) startSecretsProxyUpkeep();
|
|
786
|
-
});
|
|
787
|
-
});
|
|
966
|
+
sidecarFirstGrant = sidecarFirstGrantWork();
|
|
788
967
|
startCredentialStorePickup({ repush: repushBrokeredCopies });
|
|
789
968
|
},
|
|
790
969
|
onGrant: (lease) => {
|
|
@@ -798,6 +977,9 @@ const leaseManager: LeaseManager | null = sidecarTaskId()
|
|
|
798
977
|
},
|
|
799
978
|
})
|
|
800
979
|
: null;
|
|
980
|
+
/** The v0 lease's first-grant work under way (sidecarFirstGrantWork). */
|
|
981
|
+
let sidecarFirstGrant: Promise<void> | null = null;
|
|
982
|
+
if (leaseManager) onLeavingLegacy(stopV0Lease);
|
|
801
983
|
if (sidecarMode()) declareAgentClisReady();
|
|
802
984
|
// ADR-153 machines: the task's row is held unless its hosts-view record
|
|
803
985
|
// proves it, before connect() can bring any command for it.
|
|
@@ -860,6 +1042,15 @@ const engineLoginManager = createEngineLoginManager({
|
|
|
860
1042
|
addCapturedConfigDirAccountToStore("codex", label, authJson, scope),
|
|
861
1043
|
addGrokAccount: (label, authJson, scope) =>
|
|
862
1044
|
addCapturedConfigDirAccountToStore("grok", label, authJson, scope),
|
|
1045
|
+
// ADR-154: a capture flow becomes one revision of a cloud credential.
|
|
1046
|
+
captureEngineAccount: (flow, engine, material) =>
|
|
1047
|
+
captureEngineAccount(
|
|
1048
|
+
configuredCredentialStore(),
|
|
1049
|
+
flow,
|
|
1050
|
+
engine,
|
|
1051
|
+
material,
|
|
1052
|
+
engine === "claude" ? "Claude" : engine === "codex" ? "Codex" : "Grok",
|
|
1053
|
+
),
|
|
863
1054
|
},
|
|
864
1055
|
}
|
|
865
1056
|
: {}),
|
|
@@ -875,6 +1066,8 @@ const engineConnectManager = createEngineConnectManager({
|
|
|
875
1066
|
? {
|
|
876
1067
|
addLabeledAccount: (kind, label, secret, scope) =>
|
|
877
1068
|
addEngineAccountToStore(kind, label, { token: secret }, scope),
|
|
1069
|
+
captureAccount: (flow, kind, material) =>
|
|
1070
|
+
captureEngineAccount(configuredCredentialStore(), flow, kind, material, kind),
|
|
878
1071
|
}
|
|
879
1072
|
: {}),
|
|
880
1073
|
});
|
|
@@ -1017,6 +1210,18 @@ const stopMcpGatewaySubscription = mcpGateway.subscribe(() =>
|
|
|
1017
1210
|
const stopAgentClisReadySubscription = onAgentClisReady(() =>
|
|
1018
1211
|
sendCapabilities(),
|
|
1019
1212
|
);
|
|
1213
|
+
// ADR-154 §13: the bridge's writes of the runner's own credentials (an
|
|
1214
|
+
// engine sign-in, a GitHub token or SSH key, an MCP connection, an env value)
|
|
1215
|
+
// are work on them too. Once leaving begins none is admitted
|
|
1216
|
+
// (rejectCredentialFrameOffOwnCredentials), and leaving waits for every one
|
|
1217
|
+
// admitted before, through its acknowledgement, so none lands after the
|
|
1218
|
+
// purge: the sign-in flows stop, and the mutations and queued operations
|
|
1219
|
+
// drain.
|
|
1220
|
+
onLeavingLegacy(async () => {
|
|
1221
|
+
engineConnectManager.stopAll();
|
|
1222
|
+
await Promise.all([engineLoginManager.stopAll(), engineConnectManager.idle()]);
|
|
1223
|
+
await Promise.all([managedMutationDrain.idle(), waitForRemoteOperationCleanups()]);
|
|
1224
|
+
});
|
|
1020
1225
|
void connect();
|
|
1021
1226
|
// Local browser UI (ADR-028) — same single process, alongside the WSS client.
|
|
1022
1227
|
// Best-effort: a UI bind failure must not take the host service down.
|
|
@@ -1070,13 +1275,43 @@ async function startLocalUi(): Promise<void> {
|
|
|
1070
1275
|
// ADR-121: a stopped machine loses its memoised address (same
|
|
1071
1276
|
// before-and-after discipline as the cloud's taskStop command).
|
|
1072
1277
|
invalidateMachineTunnelRoute(taskId);
|
|
1278
|
+
// ADR-154: the start under way ends before Stop queues behind it.
|
|
1279
|
+
const credentials = localStopCredentials(taskId);
|
|
1073
1280
|
try {
|
|
1074
|
-
const
|
|
1281
|
+
const stopped = await getOrchestrator().stopTask(taskId, {
|
|
1282
|
+
...credentials,
|
|
1283
|
+
// A local operator means "whatever this runner currently owns".
|
|
1284
|
+
// Resolve that assignment only after taking the lifecycle slot;
|
|
1285
|
+
// a persisted runtime row may still name the preceding epoch
|
|
1286
|
+
// while a new task-up is being admitted.
|
|
1287
|
+
admitAssignment: () => {
|
|
1288
|
+
const assignment = readTaskRunEpochState(taskId);
|
|
1289
|
+
if (assignment === null) return { kind: "legacy" };
|
|
1290
|
+
const closed = closeTaskRunAssignment(
|
|
1291
|
+
taskId,
|
|
1292
|
+
assignment.highestRunEpoch,
|
|
1293
|
+
);
|
|
1294
|
+
return closed.kind === "closed" ||
|
|
1295
|
+
closed.kind === "already_closed"
|
|
1296
|
+
? { kind: "closed", runEpoch: assignment.highestRunEpoch }
|
|
1297
|
+
: { kind: "stale" };
|
|
1298
|
+
},
|
|
1299
|
+
});
|
|
1300
|
+
const result = stopped.ok
|
|
1301
|
+
? { ok: true as const }
|
|
1302
|
+
: { ok: false as const, error: stopped.error ?? "stop failed" };
|
|
1075
1303
|
// A PERSON pressed Stop on this machine's own UI — ADR-028. Labelled
|
|
1076
1304
|
// as such because the cloud otherwise cannot tell it apart from the
|
|
1077
1305
|
// quarantine sweeps below, which reach the bridge through this same
|
|
1078
1306
|
// frame, and ADR-144 turns that difference into who may undo it.
|
|
1079
|
-
if (result.ok)
|
|
1307
|
+
if (result.ok) {
|
|
1308
|
+
queueStoppedTaskStatus(
|
|
1309
|
+
stopped.stoppedRunEpoch === undefined
|
|
1310
|
+
? { taskId }
|
|
1311
|
+
: { taskId, runEpoch: stopped.stoppedRunEpoch },
|
|
1312
|
+
"manual",
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1080
1315
|
return result;
|
|
1081
1316
|
} finally {
|
|
1082
1317
|
invalidateMachineTunnelRoute(taskId);
|
|
@@ -1128,6 +1363,17 @@ function buildCapabilities(): HostCapabilities {
|
|
|
1128
1363
|
? [SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE]
|
|
1129
1364
|
: []),
|
|
1130
1365
|
...(projectEnvBindings === null ? [] : [projectEnvBindings.feature]),
|
|
1366
|
+
// ADR-154 §13: the runner reads cloud credentials, its stores purged.
|
|
1367
|
+
...(credentialsMode() === "active" ? [HOST_CLOUD_CREDENTIALS_FEATURE] : []),
|
|
1368
|
+
// ADR-154: only the sign-in box writes cloud credentials.
|
|
1369
|
+
...(signInBoxMode()
|
|
1370
|
+
? [
|
|
1371
|
+
SIGNIN_BOX_CAPTURE_FLOWS_FEATURE,
|
|
1372
|
+
SIGNIN_BOX_MCP_CAPTURE_FEATURE,
|
|
1373
|
+
SIGNIN_BOX_PERSON_CREDENTIALS_FEATURE,
|
|
1374
|
+
PROJECT_ENV_LEGACY_TAGS_PROTOCOL_FEATURE,
|
|
1375
|
+
]
|
|
1376
|
+
: []),
|
|
1131
1377
|
],
|
|
1132
1378
|
...(projectEnvBindings === null
|
|
1133
1379
|
? {}
|
|
@@ -1256,26 +1502,360 @@ function sendCapabilities(): void {
|
|
|
1256
1502
|
}
|
|
1257
1503
|
}
|
|
1258
1504
|
|
|
1505
|
+
/**
|
|
1506
|
+
* ADR-133: what the v0 lease's first grant does on a task machine, as one
|
|
1507
|
+
* piece that leaving the runner's own credentials waits for (ADR-154): the
|
|
1508
|
+
* machine's stores reconciled, the task let start, a task already up
|
|
1509
|
+
* resumed, then the proxy's upkeep. After the reconcile, each step only
|
|
1510
|
+
* while the runner is still on its own credentials.
|
|
1511
|
+
*/
|
|
1512
|
+
async function sidecarFirstGrantWork(): Promise<void> {
|
|
1513
|
+
await reconcileStores();
|
|
1514
|
+
// Task-up waits for this: only now are engine accounts, GitHub tokens,
|
|
1515
|
+
// and project env on the machine (lib/sidecar.ts).
|
|
1516
|
+
markSidecarReady();
|
|
1517
|
+
if (!onOwnCredentials()) return;
|
|
1518
|
+
console.log("[lease] credential store reconciled — the task may start");
|
|
1519
|
+
// A runner that came back on a machine whose task is already up
|
|
1520
|
+
// re-asserts the mounts and the editor (lib/sidecar-resume.ts).
|
|
1521
|
+
try {
|
|
1522
|
+
const resumed = await resumeSidecarTask();
|
|
1523
|
+
if (resumed.outcome === "resumed") {
|
|
1524
|
+
console.log(
|
|
1525
|
+
`[sidecar] task ${sidecarTaskId()} resumed after a runner restart: shared files and editor re-asserted` +
|
|
1526
|
+
(resumed.warnings.length > 0 ? ` (${resumed.warnings.join("; ")})` : ""),
|
|
1527
|
+
);
|
|
1528
|
+
}
|
|
1529
|
+
} catch (error) {
|
|
1530
|
+
console.warn(`[sidecar] resume failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1531
|
+
}
|
|
1532
|
+
// A sidecar runs no container-runtime activation: its task is reconciled
|
|
1533
|
+
// by the resume (or by task-up), and kept there from now.
|
|
1534
|
+
if (!stopping && onOwnCredentials()) startSecretsProxyUpkeep();
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
/**
|
|
1538
|
+
* ADR-135 on a runner that is not a sidecar: its refresh loop (owned) or
|
|
1539
|
+
* store pickup (Uai-operated), and the reconcile of its stores. ADR-154 §7:
|
|
1540
|
+
* only while the runner is on its own credentials. Leaving them stops the
|
|
1541
|
+
* loop and the pickup at once, and waits for the cycle, pickup or reconcile
|
|
1542
|
+
* under way: no runner refreshes, and nothing writes a local copy after.
|
|
1543
|
+
*/
|
|
1544
|
+
function startOwnedRefreshers(): void {
|
|
1545
|
+
if (!onOwnCredentials()) return;
|
|
1546
|
+
startTokenBrokerLoop();
|
|
1547
|
+
startCredentialStorePickup({ repush: repushBrokeredCopies });
|
|
1548
|
+
const storesReconciled = reconcileStores();
|
|
1549
|
+
onLeavingLegacy(async () => {
|
|
1550
|
+
const results = await Promise.allSettled([stopTokenBrokerLoop(), stopCredentialStorePickup(), storesReconciled]);
|
|
1551
|
+
const failures = results.flatMap((result) => (result.status === "rejected" ? [result.reason as unknown] : []));
|
|
1552
|
+
if (failures.length > 0) throw new AggregateError(failures, "the runner's own refreshers did not stop");
|
|
1553
|
+
});
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/** ADR-154: leaving its own credentials stops the v0 lease at once (no
|
|
1557
|
+
* request, no grant, no denial acted on after) and its first-grant work:
|
|
1558
|
+
* the store pickup stops, and the reconcile and resume under way drain. */
|
|
1559
|
+
async function stopV0Lease(): Promise<void> {
|
|
1560
|
+
// Each stop attempted at once, in order, one throwing never skipping the
|
|
1561
|
+
// next; then each waited for. Any failure fails the stop, so the runner
|
|
1562
|
+
// stays pending.
|
|
1563
|
+
const attempt = (stop: () => unknown): Promise<unknown> => {
|
|
1564
|
+
try {
|
|
1565
|
+
return Promise.resolve(stop());
|
|
1566
|
+
} catch (error) {
|
|
1567
|
+
return Promise.reject(error);
|
|
1568
|
+
}
|
|
1569
|
+
};
|
|
1570
|
+
const results = await Promise.allSettled([
|
|
1571
|
+
attempt(() => leaseManager?.stop()),
|
|
1572
|
+
attempt(() => stopCredentialStorePickup()),
|
|
1573
|
+
sidecarFirstGrant,
|
|
1574
|
+
]);
|
|
1575
|
+
const failures = results.flatMap((result) => (result.status === "rejected" ? [result.reason as unknown] : []));
|
|
1576
|
+
if (failures.length > 0) throw new AggregateError(failures, "the v0 lease's work did not stop");
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
/** ADR-154: a lease or refresh request, or an env replica ack, to the cloud:
|
|
1580
|
+
* only on a bridge the cloud has answered. False when not sent now; the
|
|
1581
|
+
* composer asks again once the bridge is back. */
|
|
1582
|
+
function sendCredentialFrame(frame: Parameters<typeof send>[1]): boolean {
|
|
1583
|
+
if (!ws || ws.readyState !== WebSocket.OPEN || !credentialFramesReady) return false;
|
|
1584
|
+
send(ws, frame);
|
|
1585
|
+
return true;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
/**
|
|
1589
|
+
* ADR-133/154: the cloud's first frame after a verified auth. A lease
|
|
1590
|
+
* request sent any earlier lands in the bridge's pre-verdict window, where
|
|
1591
|
+
* only auth/ping/result frames survive (live 2026-09-10, hostless smoke 5:
|
|
1592
|
+
* the first request vanished, the machine sat three minutes without
|
|
1593
|
+
* credentials). From here credential frames go, and whoever holds leases
|
|
1594
|
+
* asks again in this same turn: the v0 manager while the runner is on its
|
|
1595
|
+
* own credentials, the composer once it is on cloud ones. Pending asks
|
|
1596
|
+
* nothing: the v0 manager is stopped, and activation asks once it is done.
|
|
1597
|
+
*/
|
|
1598
|
+
function bridgeResumed(): void {
|
|
1599
|
+
credentialFramesReady = true;
|
|
1600
|
+
const mode = credentialsMode();
|
|
1601
|
+
if (mode === "legacy") leaseManager?.reconnected();
|
|
1602
|
+
else if (mode === "active") {
|
|
1603
|
+
cloudCredentials.bridgeAuthenticated();
|
|
1604
|
+
reportCredentialsPurged();
|
|
1605
|
+
}
|
|
1606
|
+
// ADR-154 §13: a runner not yet on cloud credentials asks again on every
|
|
1607
|
+
// connection (and on a timer, CREDENTIALS_ACTIVATION_RETRY_MS).
|
|
1608
|
+
if (!stopping) void credentialsActivation.advance();
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/** One phase of ADR-154 §13's handshake, asked of the cloud. */
|
|
1612
|
+
function askCredentialsActivation(phase: CredentialsActivationPhase): Promise<ActivationAnswer> {
|
|
1613
|
+
return new Promise<ActivationAnswer>((resolve, reject) => {
|
|
1614
|
+
const opId = newId();
|
|
1615
|
+
const timer = setTimeout(() => {
|
|
1616
|
+
credentialsActivationAsks.delete(opId);
|
|
1617
|
+
reject(new Error(`the cloud did not answer ${phase} in time`));
|
|
1618
|
+
}, CREDENTIALS_ACTIVATION_ANSWER_MS);
|
|
1619
|
+
timer.unref?.();
|
|
1620
|
+
credentialsActivationAsks.set(opId, { phase, resolve, timer });
|
|
1621
|
+
if (!sendCredentialFrame({ kind: "credentials.activate", opId, phase })) {
|
|
1622
|
+
clearTimeout(timer);
|
|
1623
|
+
credentialsActivationAsks.delete(opId);
|
|
1624
|
+
reject(new Error("the bridge is not connected"));
|
|
1625
|
+
}
|
|
1626
|
+
});
|
|
1627
|
+
}
|
|
1628
|
+
|
|
1629
|
+
function settleCredentialsActivation(frame: CredentialsActivateResultFrame): void {
|
|
1630
|
+
const asked = credentialsActivationAsks.get(frame.opId);
|
|
1631
|
+
if (asked === undefined || asked.phase !== frame.phase) return;
|
|
1632
|
+
credentialsActivationAsks.delete(frame.opId);
|
|
1633
|
+
clearTimeout(asked.timer);
|
|
1634
|
+
asked.resolve(frame.ok ? { ok: true } : { ok: false, error: frame.error ?? "refused" });
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
/** ADR-154 §13: the purge is done. The cloud acknowledges nothing and keeps
|
|
1638
|
+
* the first report, so it goes on every connection and every activation
|
|
1639
|
+
* tick. */
|
|
1640
|
+
function reportCredentialsPurged(): void {
|
|
1641
|
+
sendCredentialFrame({ kind: "credentials.activated", opId: newId() });
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
/** ADR-154 §13: what the retirement reads (lib/legacy-runtime-retirement.ts):
|
|
1645
|
+
* the task rows once, the identity fence over those same rows, the pinned
|
|
1646
|
+
* engine. */
|
|
1647
|
+
function retirementPlanDeps(): RetirementPlanDeps {
|
|
1648
|
+
return {
|
|
1649
|
+
snapshot: () => {
|
|
1650
|
+
const rows = getDb().select().from(schema.hostTasks).all();
|
|
1651
|
+
const machine = pinnedContainerRuntimeMachineIdentity();
|
|
1652
|
+
return {
|
|
1653
|
+
tasks: rows.map((row) => ({
|
|
1654
|
+
taskId: row.taskId,
|
|
1655
|
+
provider: row.environmentProvider,
|
|
1656
|
+
composeProject: row.composeProject,
|
|
1657
|
+
worktreePath: row.worktreePath,
|
|
1658
|
+
hasRuntime: row.environmentLocator !== null || row.composeProject !== null,
|
|
1659
|
+
})),
|
|
1660
|
+
runtimeProblem: containerRuntimeProblem(),
|
|
1661
|
+
fence: machine === null ? { ok: false, detail: "no container runtime is pinned" } : persistedTaskEnvironmentsMatchMachine(rows, machine),
|
|
1662
|
+
machine,
|
|
1663
|
+
};
|
|
1664
|
+
},
|
|
1665
|
+
apple: taskContainerBackend().apple,
|
|
1666
|
+
docker: dockerCli,
|
|
1667
|
+
appleCli: taskContainerCli,
|
|
1668
|
+
fileExists: existsSync,
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/** Why the live engine, or `taskId`'s row, no longer matches the engine
|
|
1673
|
+
* `plan` was made against; null while it does. The process's pin never
|
|
1674
|
+
* changes, so the daemon behind it is re-probed (a replaced daemon at the
|
|
1675
|
+
* same socket has another engine id), and a mismatch quarantines the
|
|
1676
|
+
* selection as runtime-bootstrap.ts's fence does. A row gone or cleared
|
|
1677
|
+
* since the plan (its task went down) stops the plan: the next one decides. */
|
|
1678
|
+
async function retirementNoLongerCurrent(plan: RetirementPlan, taskId: string | null): Promise<string | null> {
|
|
1679
|
+
const problem = containerRuntimeProblem();
|
|
1680
|
+
if (problem !== null) return `the container runtime is not ready (${problem})`;
|
|
1681
|
+
const machine = pinnedContainerRuntimeMachineIdentity();
|
|
1682
|
+
if (!sameMachine(machine, plan.machine)) return "the container runtime changed since the retirement was planned";
|
|
1683
|
+
const live = await reprobeContainerRuntimeMachineIdentity();
|
|
1684
|
+
if (!sameMachine(live, plan.machine)) {
|
|
1685
|
+
rejectContainerRuntimeSelection("The selected container runtime daemon identity changed while task runtimes were being retired.");
|
|
1686
|
+
return "the container runtime's daemon changed since the retirement was planned";
|
|
1687
|
+
}
|
|
1688
|
+
if (taskId === null) return null;
|
|
1689
|
+
const row = getHostTask(taskId);
|
|
1690
|
+
if (row === null) return `task ${taskId}'s row is gone since the retirement was planned`;
|
|
1691
|
+
if (row.environmentLocator === null && row.composeProject === null) {
|
|
1692
|
+
return `task ${taskId} names no runtime since the retirement was planned`;
|
|
1693
|
+
}
|
|
1694
|
+
const verdict = persistedTaskEnvironmentsMatchMachine([row], plan.machine as NonNullable<typeof machine>);
|
|
1695
|
+
return verdict.ok ? null : (verdict.detail ?? `task ${taskId} no longer matches the container runtime`);
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
function sameMachine(
|
|
1699
|
+
machine: { backend: string; endpoint: string; engineId: string } | null,
|
|
1700
|
+
plan: { backend: string; endpoint: string; engineId: string },
|
|
1701
|
+
): boolean {
|
|
1702
|
+
return (
|
|
1703
|
+
machine !== null &&
|
|
1704
|
+
machine.backend === plan.backend &&
|
|
1705
|
+
machine.endpoint === plan.endpoint &&
|
|
1706
|
+
machine.engineId === plan.engineId
|
|
1707
|
+
);
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
/** ADR-154 §13: each owed stopped report again, at its exact run epoch;
|
|
1711
|
+
* returns those still owed. One is moot once its task's row is gone, or the
|
|
1712
|
+
* task has an assignment newer than the run it reports (a report with no
|
|
1713
|
+
* epoch: any assignment at all), since the cloud has moved on from it. */
|
|
1714
|
+
function reportRetiredTasksStopped(owed: readonly StoppedReport[]): StoppedReport[] {
|
|
1715
|
+
const still: StoppedReport[] = [];
|
|
1716
|
+
for (const report of owed) {
|
|
1717
|
+
if (getHostTask(report.taskId) === null) continue;
|
|
1718
|
+
const assignment = readTaskRunEpochState(report.taskId);
|
|
1719
|
+
if (assignment !== null && (report.runEpoch === null || assignment.highestRunEpoch > report.runEpoch)) continue;
|
|
1720
|
+
queueStoppedTaskStatus(
|
|
1721
|
+
report.runEpoch === null ? { taskId: report.taskId } : { taskId: report.taskId, runEpoch: report.runEpoch },
|
|
1722
|
+
);
|
|
1723
|
+
still.push(report);
|
|
1724
|
+
}
|
|
1725
|
+
return still;
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/** ADR-154 §13, from the local commit on: every task runtime goes, its
|
|
1729
|
+
* worktree kept, each checked in its lifecycle slot against the engine the
|
|
1730
|
+
* plan was made against. Each retired task is owed a stopped report at the
|
|
1731
|
+
* exact run epoch it stopped at (the authority keeps them). */
|
|
1732
|
+
async function retireTaskRuntimesForCloudCredentials(
|
|
1733
|
+
plan: RetirementPlan,
|
|
1734
|
+
): Promise<{ stopped: StoppedReport[]; problem: string | null }> {
|
|
1735
|
+
const report = await retireLegacyTaskRuntimes(plan, {
|
|
1736
|
+
...retirementPlanDeps(),
|
|
1737
|
+
current: (taskId) => retirementNoLongerCurrent(plan, taskId),
|
|
1738
|
+
// In the slot the retirement holds, right after it re-proved the engine.
|
|
1739
|
+
stop: async (taskId) => {
|
|
1740
|
+
invalidateMachineTunnelRoute(taskId);
|
|
1741
|
+
const stopped = await getOrchestrator().stopTaskInHeldLifecycle(taskId, {
|
|
1742
|
+
// The daemon proven again after the channel drained, before previews
|
|
1743
|
+
// or the runtime are touched.
|
|
1744
|
+
beforeRuntimeMutation: () => retirementNoLongerCurrent(plan, taskId),
|
|
1745
|
+
// What this runner owns now, as a local Stop resolves it.
|
|
1746
|
+
admitAssignment: () => {
|
|
1747
|
+
const assignment = readTaskRunEpochState(taskId);
|
|
1748
|
+
if (assignment === null) return { kind: "legacy" };
|
|
1749
|
+
const closed = closeTaskRunAssignment(taskId, assignment.highestRunEpoch);
|
|
1750
|
+
return closed.kind === "closed" || closed.kind === "already_closed"
|
|
1751
|
+
? { kind: "closed", runEpoch: assignment.highestRunEpoch }
|
|
1752
|
+
: { kind: "stale" };
|
|
1753
|
+
},
|
|
1754
|
+
});
|
|
1755
|
+
return stopped.ok ? { ok: true } : { ok: false, error: stopped.error ?? "stop failed" };
|
|
1756
|
+
},
|
|
1757
|
+
inLifecycle: (taskId, work) => getOrchestrator().runTaskLifecycle(taskId, work),
|
|
1758
|
+
log: (line) => console.warn(line),
|
|
1759
|
+
});
|
|
1760
|
+
const stopped: StoppedReport[] = [];
|
|
1761
|
+
for (const taskId of report.retired) {
|
|
1762
|
+
// Its runtime is gone, whatever its Stop said: the mirror says so, as
|
|
1763
|
+
// the queue's flush requires.
|
|
1764
|
+
if (getHostTask(taskId) === null) continue;
|
|
1765
|
+
upsertHostTask(taskId, { statusMirror: "stopped" });
|
|
1766
|
+
stopped.push({ taskId, runEpoch: readTaskRunEpochState(taskId)?.highestRunEpoch ?? null });
|
|
1767
|
+
}
|
|
1768
|
+
console.warn(
|
|
1769
|
+
`[credentials] retired ${report.retired.length} task runtime(s)` +
|
|
1770
|
+
(report.orphans.length > 0 ? ` and ${report.orphans.length} no task named` : ""),
|
|
1771
|
+
);
|
|
1772
|
+
return {
|
|
1773
|
+
stopped,
|
|
1774
|
+
problem:
|
|
1775
|
+
report.problems.length > 0 ? `task runtimes were not all retired: ${report.problems.join("; ").slice(0, 1200)}` : null,
|
|
1776
|
+
};
|
|
1777
|
+
}
|
|
1778
|
+
|
|
1779
|
+
/** ADR-154 §13, after the cloud commit: every runner-local credential store. */
|
|
1780
|
+
async function purgeCredentialStores(): Promise<void> {
|
|
1781
|
+
const report = await purgeLocalCredentialStores();
|
|
1782
|
+
const rows = Object.values(report.rows).reduce((sum, count) => sum + count, 0);
|
|
1783
|
+
console.warn(
|
|
1784
|
+
`[credentials] purged ${rows} credential row(s), ${report.dirs.length} dir(s), ` +
|
|
1785
|
+
`${report.files.length} file(s) and ${report.envKeys.length} env key(s)`,
|
|
1786
|
+
);
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
/**
|
|
1790
|
+
* ADR-154 §13: the runner's tasks run on cloud credentials from here. The
|
|
1791
|
+
* authority calls this once its `active` record is durable and the recovery
|
|
1792
|
+
* sweep is done, so nothing here asks for a grant, or advertises v1, before
|
|
1793
|
+
* that sweep; `mode()` reads `active` while it runs. Each step is idempotent:
|
|
1794
|
+
* should one throw, the authority withdraws (the capabilities sent again,
|
|
1795
|
+
* pending) and calls this again later.
|
|
1796
|
+
*/
|
|
1797
|
+
function cloudCredentialsActive(how: "transition" | "boot"): void {
|
|
1798
|
+
// MCP is served only from leases: never the local store, which is gone.
|
|
1799
|
+
setMcpGatewayLeaseOnly(true);
|
|
1800
|
+
// Every task that survived this process's start asks for its run's
|
|
1801
|
+
// credentials (once per process); its requests go out once the bridge can.
|
|
1802
|
+
cloudCredentialsActivated();
|
|
1803
|
+
// The bridge told now; else its next hello and bridgeResumed tell it.
|
|
1804
|
+
if (credentialFramesReady) {
|
|
1805
|
+
sendCapabilities();
|
|
1806
|
+
reportCredentialsPurged();
|
|
1807
|
+
cloudCredentials.bridgeAuthenticated();
|
|
1808
|
+
}
|
|
1809
|
+
console.warn(`[credentials] the runner is on cloud credentials (${how})`);
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
/**
|
|
1813
|
+
* ADR-133's v0 lease frames, to the v0 manager only while the runner is on
|
|
1814
|
+
* its own credentials. From `pending` on that manager is stopped, and a v0
|
|
1815
|
+
* denial (a runner floored to v1 is refused its epoch-less request) never
|
|
1816
|
+
* reaches its exit. Once active, a denial of the composer's own request is
|
|
1817
|
+
* the composer's.
|
|
1818
|
+
*/
|
|
1819
|
+
function routeV0LeaseFrame(frame: Extract<CloudToHost, { kind: "lease.grant" | "lease.deny" }>): void {
|
|
1820
|
+
const mode = credentialsMode();
|
|
1821
|
+
if (mode === "legacy") leaseManager?.handle(frame);
|
|
1822
|
+
else if (mode === "active" && frame.kind === "lease.deny") void cloudCredentials.handleFrame(frame);
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1259
1825
|
/** Keep a rolling-isolation/local-UI stop from remaining cloud-ghost-running
|
|
1260
1826
|
* merely because the bridge was between connections at the transition. */
|
|
1261
1827
|
function queueStoppedTaskStatus(
|
|
1262
|
-
|
|
1828
|
+
target: StoppedTaskStatusTarget,
|
|
1263
1829
|
reason?: StoppedTaskReason,
|
|
1264
1830
|
): void {
|
|
1265
|
-
pendingRuntimeIsolationStatuses.enqueue(
|
|
1831
|
+
pendingRuntimeIsolationStatuses.enqueue(
|
|
1832
|
+
target.taskId,
|
|
1833
|
+
reason,
|
|
1834
|
+
target.runEpoch,
|
|
1835
|
+
);
|
|
1266
1836
|
flushStoppedTaskStatuses();
|
|
1267
1837
|
}
|
|
1268
1838
|
|
|
1269
1839
|
function flushStoppedTaskStatuses(socket: WebSocket | null = ws): void {
|
|
1270
1840
|
if (!socket || socket.readyState !== WebSocket.OPEN) return;
|
|
1271
|
-
pendingRuntimeIsolationStatuses.flush((taskId, reason) => {
|
|
1272
|
-
send(socket, {
|
|
1841
|
+
pendingRuntimeIsolationStatuses.flush((taskId, reason, runEpoch) => {
|
|
1842
|
+
send(socket, {
|
|
1843
|
+
kind: "task.status",
|
|
1844
|
+
taskId,
|
|
1845
|
+
status: "stopped",
|
|
1846
|
+
reason,
|
|
1847
|
+
...(runEpoch === undefined ? {} : { runEpoch }),
|
|
1848
|
+
});
|
|
1273
1849
|
});
|
|
1274
1850
|
}
|
|
1275
1851
|
|
|
1276
1852
|
// ADR-131 D3: a stored account (any scope) makes a kind advertisable even
|
|
1277
1853
|
// when the host's own credential is absent — see hasStoredEngineAccounts.
|
|
1278
1854
|
setAdvertisementFallback((kind) => hasStoredEngineAccounts(kind));
|
|
1855
|
+
// ADR-154: on cloud credentials the runner holds none of its own, so a kind
|
|
1856
|
+
// is advertised when the cloud can lease an account for it (Kimi, with no
|
|
1857
|
+
// lane yet, is not), whatever this process's env says.
|
|
1858
|
+
setLeasedAdvertisement(() => (credentialsMode() === "active" ? LEASED_ENGINE_KINDS : null));
|
|
1279
1859
|
onRegistryChange(() => sendCapabilities());
|
|
1280
1860
|
onContainerRuntimeChange((next, previous) => {
|
|
1281
1861
|
if (stopping) return;
|
|
@@ -1298,6 +1878,7 @@ onContainerRuntimeChange((next, previous) => {
|
|
|
1298
1878
|
}
|
|
1299
1879
|
});
|
|
1300
1880
|
stopRuntimeAutoRecheck = startContainerRuntimeAutoRecheck();
|
|
1881
|
+
installContainerRuntimeRecheckRequester();
|
|
1301
1882
|
// Re-advertise when a gh token is added/removed (ADR-033) so the host page's
|
|
1302
1883
|
// per-host connected state updates promptly.
|
|
1303
1884
|
onGithubChange(() => sendCapabilities());
|
|
@@ -1334,8 +1915,33 @@ async function activateContainerRuntime(
|
|
|
1334
1915
|
): Promise<RuntimeActivationAttemptResult> {
|
|
1335
1916
|
publishActivationPhase("joining in-flight recovery");
|
|
1336
1917
|
await waitForRuntimeRecovery();
|
|
1918
|
+
// ADR-154: one begun on the runner's own credentials stops the moment
|
|
1919
|
+
// leaving them begins, through publication; the next pass sweeps first,
|
|
1920
|
+
// and recovers afresh.
|
|
1921
|
+
return activationPassOnCredentials({
|
|
1922
|
+
isRuntimeCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
1923
|
+
runtimeOperational: () => containerRuntimeGenerationOperational(epoch),
|
|
1924
|
+
recover,
|
|
1925
|
+
freshRecover: requestRuntimeRecovery,
|
|
1926
|
+
pass: (isCurrent, recoverOnCredentials) => activateRuntimePass(epoch, isCurrent, recoverOnCredentials),
|
|
1927
|
+
onError: reportRuntimeActivationError,
|
|
1928
|
+
});
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
function reportRuntimeActivationError(error: unknown): void {
|
|
1932
|
+
console.warn(
|
|
1933
|
+
`[host-agent] container runtime activation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/** The activation pass itself, under `isCurrent` through publication. */
|
|
1938
|
+
function activateRuntimePass(
|
|
1939
|
+
epoch: number,
|
|
1940
|
+
isCurrent: () => boolean,
|
|
1941
|
+
recoverOnCredentials: RuntimeRecoveryRequest,
|
|
1942
|
+
): Promise<RuntimeActivationAttemptResult> {
|
|
1337
1943
|
return runMaintainedRuntimeActivation({
|
|
1338
|
-
isCurrent
|
|
1944
|
+
isCurrent,
|
|
1339
1945
|
beforeMaintenance: async () => {
|
|
1340
1946
|
// This is the first selected-daemon operation in activation. Durable
|
|
1341
1947
|
// TaskEnvironment locators must match the exact process pin before
|
|
@@ -1351,8 +1957,8 @@ async function activateContainerRuntime(
|
|
|
1351
1957
|
lifecycle: getOrchestrator(),
|
|
1352
1958
|
isCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
1353
1959
|
});
|
|
1354
|
-
for (const
|
|
1355
|
-
queueStoppedTaskStatus(
|
|
1960
|
+
for (const stoppedTask of isolated.stoppedTasks) {
|
|
1961
|
+
queueStoppedTaskStatus(stoppedTask);
|
|
1356
1962
|
}
|
|
1357
1963
|
publishActivationPhase(
|
|
1358
1964
|
`writable-container quarantine: ${isolated.verdict}`,
|
|
@@ -1365,7 +1971,7 @@ async function activateContainerRuntime(
|
|
|
1365
1971
|
);
|
|
1366
1972
|
return reconcileStableStandardImage(epoch);
|
|
1367
1973
|
},
|
|
1368
|
-
recover,
|
|
1974
|
+
recover: recoverOnCredentials,
|
|
1369
1975
|
// A bounded recovery pass can finish with active tasks still unknowable.
|
|
1370
1976
|
// Keep the public runtime in `checking`. A real outage becomes
|
|
1371
1977
|
// `no-runtime`; a still-ready daemon requests a bounded follow-up only
|
|
@@ -1386,13 +1992,15 @@ async function activateContainerRuntime(
|
|
|
1386
1992
|
// image before the ready frame reattaches persisted agent sessions.
|
|
1387
1993
|
publishOperational: async () => {
|
|
1388
1994
|
if (
|
|
1389
|
-
|
|
1995
|
+
isCurrent() &&
|
|
1390
1996
|
markContainerRuntimeOperational(epoch)
|
|
1391
1997
|
) {
|
|
1392
1998
|
watchCodexAuth(() => containerRuntimeProblem() === null);
|
|
1393
1999
|
resumeRuntimeDeferredAgentWork();
|
|
1394
2000
|
startSecretsProxyUpkeep();
|
|
1395
2001
|
sendCapabilities();
|
|
2002
|
+
// ADR-154 §13: a switch waiting for the runtime goes on now.
|
|
2003
|
+
if (!stopping) void credentialsActivation.advance();
|
|
1396
2004
|
}
|
|
1397
2005
|
},
|
|
1398
2006
|
});
|
|
@@ -1407,7 +2015,11 @@ async function reconcileStableStandardImage(
|
|
|
1407
2015
|
): Promise<"ready" | "retry" | "stale"> {
|
|
1408
2016
|
while (runtimeActivationIsCurrent(epoch)) {
|
|
1409
2017
|
const before = JSON.stringify(configuredOptionalEngines());
|
|
1410
|
-
|
|
2018
|
+
// The one pass that may restart a wedged Apple runtime's services, and
|
|
2019
|
+
// only while this activation is current: its recovery follows (#p2jdax).
|
|
2020
|
+
const result = await ensureStandardImage({
|
|
2021
|
+
withinActivation: () => runtimeActivationIsCurrent(epoch),
|
|
2022
|
+
});
|
|
1411
2023
|
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
1412
2024
|
if (!result.ok && result.remedy === "stop-volume-holders") {
|
|
1413
2025
|
// The shared volume needs a repair the running holder tasks block and
|
|
@@ -1422,8 +2034,8 @@ async function reconcileStableStandardImage(
|
|
|
1422
2034
|
isCurrent: () => runtimeActivationIsCurrent(epoch),
|
|
1423
2035
|
scope: "volume-holders",
|
|
1424
2036
|
});
|
|
1425
|
-
for (const
|
|
1426
|
-
queueStoppedTaskStatus(
|
|
2037
|
+
for (const stoppedTask of swept.stoppedTasks) {
|
|
2038
|
+
queueStoppedTaskStatus(stoppedTask);
|
|
1427
2039
|
}
|
|
1428
2040
|
if (!runtimeActivationIsCurrent(epoch)) return "stale";
|
|
1429
2041
|
return "retry";
|
|
@@ -1452,68 +2064,43 @@ async function reconcileStableStandardImage(
|
|
|
1452
2064
|
* ready-to-ready proof restores operation without a full recovery sweep;
|
|
1453
2065
|
* actual loss/restoration creates a readiness edge and takes the full path. */
|
|
1454
2066
|
function revalidateContainerRuntimeOnConnect(): void {
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
)
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
// A healthy daemon can still have restarted while the bridge was offline.
|
|
1470
|
-
// One label scan keeps ordinary cloud/network flaps cheap; only a missing,
|
|
1471
|
-
// exited, or unknowable active app takes the full per-task recovery path.
|
|
1472
|
-
const recovery = await runtimeRecoveryVerdict();
|
|
1473
|
-
let recovered = recovery === "clean";
|
|
1474
|
-
if (!recovered) {
|
|
1475
|
-
recovered = await requestRuntimeRecovery(() =>
|
|
1476
|
-
runtimeActivationIsCurrent(suspendedEpoch),
|
|
1477
|
-
);
|
|
1478
|
-
}
|
|
1479
|
-
if (!runtimeActivationIsCurrent(suspendedEpoch)) return;
|
|
1480
|
-
// Prove the daemon once more after recovery so a mid-sweep loss cannot
|
|
1481
|
-
// reopen commands and tunnels on an unresolved backend.
|
|
1482
|
-
const final = await initializeContainerRuntime();
|
|
1483
|
-
if (
|
|
1484
|
-
final.status !== "ready" ||
|
|
1485
|
-
!containerRuntimeEpochIsCurrent(suspendedEpoch)
|
|
1486
|
-
) {
|
|
1487
|
-
return;
|
|
1488
|
-
}
|
|
1489
|
-
if (!recovered) {
|
|
1490
|
-
scheduleContainerRuntimeActivationRetry(
|
|
1491
|
-
requestRuntimeRecovery,
|
|
1492
|
-
suspendedEpoch,
|
|
1493
|
-
);
|
|
1494
|
-
return;
|
|
1495
|
-
}
|
|
1496
|
-
if (markContainerRuntimeOperational(suspendedEpoch)) {
|
|
1497
|
-
runtimeActivationTail.resolve(suspendedEpoch);
|
|
2067
|
+
// Suspended synchronously, before auth (lib/runtime-reconnect.ts).
|
|
2068
|
+
void revalidateRuntimeOnConnect({
|
|
2069
|
+
suspend: () => (containerRuntimeState().status === "ready" ? suspendContainerRuntime() : null),
|
|
2070
|
+
probe: async () => (await initializeContainerRuntime()).status === "ready",
|
|
2071
|
+
epochIsCurrent: (epoch) => containerRuntimeEpochIsCurrent(epoch),
|
|
2072
|
+
activationIsCurrent: (epoch) => runtimeActivationIsCurrent(epoch),
|
|
2073
|
+
persistedEnvironmentsMatch: () => selectedRuntimeMatchesPersistedTaskEnvironments(),
|
|
2074
|
+
recoveryVerdict: () => runtimeRecoveryVerdict(),
|
|
2075
|
+
recover: (isCurrent) => requestRuntimeRecovery(isCurrent),
|
|
2076
|
+
scheduleRetry: (epoch) => scheduleContainerRuntimeActivationRetry(requestRuntimeRecovery, epoch),
|
|
2077
|
+
requestActivation: (epoch) => void requestContainerRuntimeActivation(requestRuntimeRecovery, epoch),
|
|
2078
|
+
publish: (epoch) => {
|
|
2079
|
+
if (!markContainerRuntimeOperational(epoch)) return false;
|
|
2080
|
+
runtimeActivationTail.resolve(epoch);
|
|
1498
2081
|
resumeRuntimeDeferredAgentWork();
|
|
1499
2082
|
sendCapabilities();
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
console.warn(
|
|
1503
|
-
`[host-agent] container runtime reconnect validation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1504
|
-
);
|
|
1505
|
-
if (
|
|
1506
|
-
suspendedEpoch !== null &&
|
|
1507
|
-
runtimeActivationIsCurrent(suspendedEpoch)
|
|
1508
|
-
) {
|
|
1509
|
-
scheduleContainerRuntimeActivationRetry(
|
|
1510
|
-
requestRuntimeRecovery,
|
|
1511
|
-
suspendedEpoch,
|
|
1512
|
-
);
|
|
1513
|
-
}
|
|
2083
|
+
return true;
|
|
2084
|
+
},
|
|
1514
2085
|
});
|
|
1515
2086
|
}
|
|
1516
2087
|
|
|
2088
|
+
/** Work outside activation that needs the runtime's remedy (a wedged Apple
|
|
2089
|
+
* runtime, or a one-off container that will not go, #p2jdax) asks for the
|
|
2090
|
+
* operator's recheck: its activation alone restarts the runtime's services,
|
|
2091
|
+
* and recovers every task after. An ask while an activation (with its
|
|
2092
|
+
* retries) is under way is asked again once it is done; one while only a
|
|
2093
|
+
* recheck is under way joins it (containerRuntimeRecheckRequester). */
|
|
2094
|
+
function installContainerRuntimeRecheckRequester(): void {
|
|
2095
|
+
setContainerRuntimeRecheckRequester(
|
|
2096
|
+
containerRuntimeRecheckRequester({
|
|
2097
|
+
activationUnderWay: () => runtimeActivationTail.underWay(),
|
|
2098
|
+
activationSettled: () => runtimeActivationTail.whenSettled(),
|
|
2099
|
+
recheck: requestForcedContainerRuntimeRecheck,
|
|
2100
|
+
}),
|
|
2101
|
+
);
|
|
2102
|
+
}
|
|
2103
|
+
|
|
1517
2104
|
/** An operator recheck is an explicit recovery request. Invalidate even a
|
|
1518
2105
|
* stale ready verdict so an unseen daemon restart gets image maintenance,
|
|
1519
2106
|
* persisted-task recovery, credential reinjection, and channel reattach. */
|
|
@@ -2161,6 +2748,7 @@ async function connect(): Promise<void> {
|
|
|
2161
2748
|
if (pingTimer) clearInterval(pingTimer);
|
|
2162
2749
|
if (deadTimer) clearInterval(deadTimer);
|
|
2163
2750
|
eventFlushEnabled = false;
|
|
2751
|
+
credentialFramesReady = false;
|
|
2164
2752
|
cancelEventFlushBackpressureTimer();
|
|
2165
2753
|
if (resumeFallbackTimer) {
|
|
2166
2754
|
clearTimeout(resumeFallbackTimer);
|
|
@@ -2177,7 +2765,7 @@ async function connect(): Promise<void> {
|
|
|
2177
2765
|
// task reconciliation, which must observe `checking` until this proof
|
|
2178
2766
|
// finishes instead of treating a dead daemon as an empty one.
|
|
2179
2767
|
revalidateContainerRuntimeOnConnect();
|
|
2180
|
-
send(socket, { kind: "auth", token, hostId, bootId:
|
|
2768
|
+
send(socket, { kind: "auth", token, hostId, bootId: RUNNER_BOOT_ID });
|
|
2181
2769
|
// Advertise capabilities immediately after auth (ADR-021). The bridge
|
|
2182
2770
|
// rejects with close-code 4001 if auth fails, so sending here is harmless
|
|
2183
2771
|
// on a bad token and saves a round-trip on a good one. Re-sent on every
|
|
@@ -2236,17 +2824,13 @@ async function connect(): Promise<void> {
|
|
|
2236
2824
|
if (!frame) return;
|
|
2237
2825
|
if (stopping && rejectFrameDuringShutdown(socket, frame)) return;
|
|
2238
2826
|
if (signInBoxMode() && rejectFrameOnSignInBox(socket, frame)) return;
|
|
2827
|
+
if (!onOwnCredentials() && rejectCredentialFrameOffOwnCredentials(socket, frame)) return;
|
|
2239
2828
|
|
|
2240
2829
|
switch (frame.kind) {
|
|
2241
2830
|
case "pong":
|
|
2242
2831
|
break;
|
|
2243
2832
|
case "event.resume":
|
|
2244
|
-
|
|
2245
|
-
// request sent any earlier lands in the bridge's pre-verdict window,
|
|
2246
|
-
// where only auth/ping/result frames survive (live 2026-09-10,
|
|
2247
|
-
// hostless smoke 5: the first request vanished, the machine sat
|
|
2248
|
-
// three minutes without credentials).
|
|
2249
|
-
leaseManager?.reconnected();
|
|
2833
|
+
bridgeResumed();
|
|
2250
2834
|
// ADR-103: the cloud named its watermark — rewind the transmit
|
|
2251
2835
|
// cursor to it (null = no state, replay nothing) and start draining.
|
|
2252
2836
|
if (resumeFallbackTimer) {
|
|
@@ -2317,6 +2901,7 @@ async function connect(): Promise<void> {
|
|
|
2317
2901
|
emitEngineLoginEvent,
|
|
2318
2902
|
frame.label,
|
|
2319
2903
|
frame.scope,
|
|
2904
|
+
frame.flow,
|
|
2320
2905
|
);
|
|
2321
2906
|
break;
|
|
2322
2907
|
case "engine.login.input":
|
|
@@ -2338,6 +2923,7 @@ async function connect(): Promise<void> {
|
|
|
2338
2923
|
frame.label,
|
|
2339
2924
|
(event) => sendRemoteOperationFrame(event),
|
|
2340
2925
|
frame.scope,
|
|
2926
|
+
frame.flow,
|
|
2341
2927
|
);
|
|
2342
2928
|
break;
|
|
2343
2929
|
case "engine.connect.stop":
|
|
@@ -2399,7 +2985,21 @@ async function connect(): Promise<void> {
|
|
|
2399
2985
|
break;
|
|
2400
2986
|
case "lease.grant":
|
|
2401
2987
|
case "lease.deny":
|
|
2402
|
-
|
|
2988
|
+
routeV0LeaseFrame(frame);
|
|
2989
|
+
break;
|
|
2990
|
+
case "lease.grant.v1":
|
|
2991
|
+
case "lease.unavailable":
|
|
2992
|
+
case "lease.stale":
|
|
2993
|
+
case "credential.changed":
|
|
2994
|
+
case "credential.refresh.result":
|
|
2995
|
+
case "env.replica.v1.begin":
|
|
2996
|
+
case "env.replica.v1.page":
|
|
2997
|
+
case "env.replica.v1.apply":
|
|
2998
|
+
case "env.replica.v1.commit":
|
|
2999
|
+
if (credentialsMode() === "active") void cloudCredentials.handleFrame(frame);
|
|
3000
|
+
break;
|
|
3001
|
+
case "credentials.activate.result":
|
|
3002
|
+
settleCredentialsActivation(frame);
|
|
2403
3003
|
break;
|
|
2404
3004
|
case "gh.connect.set": {
|
|
2405
3005
|
const runtimeProblem = containerRuntimeProblem();
|
|
@@ -2515,7 +3115,7 @@ async function connect(): Promise<void> {
|
|
|
2515
3115
|
// nowhere".
|
|
2516
3116
|
void listUserInstallationIds(
|
|
2517
3117
|
frame.userId,
|
|
2518
|
-
|
|
3118
|
+
revisionListDeps(frame.userId, frame.parameter),
|
|
2519
3119
|
frame.generation ?? 0,
|
|
2520
3120
|
).then(
|
|
2521
3121
|
(installationIds) =>
|
|
@@ -2526,13 +3126,19 @@ async function connect(): Promise<void> {
|
|
|
2526
3126
|
installationIds,
|
|
2527
3127
|
}),
|
|
2528
3128
|
(err) => {
|
|
2529
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
2530
3129
|
const code =
|
|
2531
3130
|
err instanceof GitHubRepositoryAccessError
|
|
2532
3131
|
? err.code
|
|
2533
3132
|
: "github_unavailable";
|
|
3133
|
+
// From a revision on the box: a fixed message by code.
|
|
3134
|
+
const error =
|
|
3135
|
+
frame.parameter !== undefined
|
|
3136
|
+
? PICKER_FAILURE_TEXT[code]
|
|
3137
|
+
: err instanceof Error
|
|
3138
|
+
? err.message
|
|
3139
|
+
: String(err);
|
|
2534
3140
|
console.warn(
|
|
2535
|
-
`[github] installation list failed for ${frame.userId}: ${error}`,
|
|
3141
|
+
`[github] installation list failed for ${frame.userId}: ${frame.parameter !== undefined ? code : error}`,
|
|
2536
3142
|
);
|
|
2537
3143
|
send(socket, {
|
|
2538
3144
|
kind: "gh.installations.ack",
|
|
@@ -2549,7 +3155,7 @@ async function connect(): Promise<void> {
|
|
|
2549
3155
|
void listUserInstallationRepositoryIds(
|
|
2550
3156
|
frame.userId,
|
|
2551
3157
|
frame.installationId,
|
|
2552
|
-
|
|
3158
|
+
revisionListDeps(frame.userId, frame.parameter),
|
|
2553
3159
|
frame.generation ?? 0,
|
|
2554
3160
|
).then(
|
|
2555
3161
|
({ repositoryIds, truncated }) =>
|
|
@@ -2561,13 +3167,18 @@ async function connect(): Promise<void> {
|
|
|
2561
3167
|
truncated,
|
|
2562
3168
|
}),
|
|
2563
3169
|
(err) => {
|
|
2564
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
2565
3170
|
const code =
|
|
2566
3171
|
err instanceof GitHubRepositoryAccessError
|
|
2567
3172
|
? err.code
|
|
2568
3173
|
: "github_unavailable";
|
|
3174
|
+
const error =
|
|
3175
|
+
frame.parameter !== undefined
|
|
3176
|
+
? PICKER_FAILURE_TEXT[code]
|
|
3177
|
+
: err instanceof Error
|
|
3178
|
+
? err.message
|
|
3179
|
+
: String(err);
|
|
2569
3180
|
console.warn(
|
|
2570
|
-
`[github] repository list failed for ${frame.userId}/${frame.installationId}: ${error}`,
|
|
3181
|
+
`[github] repository list failed for ${frame.userId}/${frame.installationId}: ${frame.parameter !== undefined ? code : error}`,
|
|
2571
3182
|
);
|
|
2572
3183
|
send(socket, {
|
|
2573
3184
|
kind: "gh.repositories.ack",
|
|
@@ -2758,6 +3369,69 @@ async function connect(): Promise<void> {
|
|
|
2758
3369
|
);
|
|
2759
3370
|
break;
|
|
2760
3371
|
}
|
|
3372
|
+
case "mcp.capture.probe":
|
|
3373
|
+
case "mcp.capture.complete": {
|
|
3374
|
+
// ADR-154 §11: only the sign-in box runs an MCP handshake as a
|
|
3375
|
+
// capture flow; anything else leaves it unanswered.
|
|
3376
|
+
if (!signInBoxMode()) break;
|
|
3377
|
+
const store = configuredCredentialStore();
|
|
3378
|
+
const opId = frame.opId;
|
|
3379
|
+
const running = !store
|
|
3380
|
+
? Promise.resolve({
|
|
3381
|
+
outcome: "error" as const,
|
|
3382
|
+
capture: null,
|
|
3383
|
+
authorizeUrl: null,
|
|
3384
|
+
scopes: null,
|
|
3385
|
+
error: "the sign-in box has no credential store",
|
|
3386
|
+
})
|
|
3387
|
+
: frame.kind === "mcp.capture.probe"
|
|
3388
|
+
? probeMcpCapture(frame, { store })
|
|
3389
|
+
: completeMcpCapture(frame, { store });
|
|
3390
|
+
void managedMutationDrain.track(() =>
|
|
3391
|
+
running
|
|
3392
|
+
// Every step answers: an unexpected failure is a fixed refusal,
|
|
3393
|
+
// never a silence the cloud waits out, nor an unhandled rejection.
|
|
3394
|
+
.catch(() => ({
|
|
3395
|
+
outcome: "error" as const,
|
|
3396
|
+
capture: null,
|
|
3397
|
+
authorizeUrl: null,
|
|
3398
|
+
scopes: null,
|
|
3399
|
+
error: "the sign-in box could not complete this step",
|
|
3400
|
+
}))
|
|
3401
|
+
.then((body) => send(socket, { kind: "mcp.capture.ack", opId, ...body })),
|
|
3402
|
+
);
|
|
3403
|
+
break;
|
|
3404
|
+
}
|
|
3405
|
+
case "mcp.capture.cancel":
|
|
3406
|
+
// The cloud ended the sign-in: forget its verifier and secret now.
|
|
3407
|
+
if (signInBoxMode()) cancelMcpCapture(frame);
|
|
3408
|
+
break;
|
|
3409
|
+
case "github.capture":
|
|
3410
|
+
case "ssh.mint": {
|
|
3411
|
+
// ADR-154 §12: only the sign-in box writes a person's GitHub sign-in
|
|
3412
|
+
// or SSH key as a revision; anything else leaves it unanswered.
|
|
3413
|
+
if (!signInBoxMode()) break;
|
|
3414
|
+
const store = configuredCredentialStore();
|
|
3415
|
+
const opId = frame.opId;
|
|
3416
|
+
const running =
|
|
3417
|
+
frame.kind === "github.capture" ? captureGithub(frame, { store }) : mintSsh(frame, { store });
|
|
3418
|
+
void managedMutationDrain.track(() =>
|
|
3419
|
+
running
|
|
3420
|
+
.catch(() => ({ capture: null, publicKey: null, error: "The sign-in box could not complete this; nothing was kept." }))
|
|
3421
|
+
.then((body) => send(socket, { kind: "person.capture.ack", opId, ...body })),
|
|
3422
|
+
);
|
|
3423
|
+
break;
|
|
3424
|
+
}
|
|
3425
|
+
case "github.revoke": {
|
|
3426
|
+
if (!signInBoxMode()) break;
|
|
3427
|
+
const opId = frame.opId;
|
|
3428
|
+
void managedMutationDrain.track(() =>
|
|
3429
|
+
revokeGithub(frame, { store: configuredCredentialStore() })
|
|
3430
|
+
.catch(() => ({ outcome: "retry" as const, error: "the sign-in box could not complete the revocation" }))
|
|
3431
|
+
.then((body) => send(socket, { kind: "github.revoke.ack", opId, ...body })),
|
|
3432
|
+
);
|
|
3433
|
+
break;
|
|
3434
|
+
}
|
|
2761
3435
|
case "env.staging.put": {
|
|
2762
3436
|
// Only the write-only sign-in box owns immutable staging paths. An
|
|
2763
3437
|
// owned runner or task sidecar must never accept plaintext here.
|
|
@@ -2767,6 +3441,17 @@ async function connect(): Promise<void> {
|
|
|
2767
3441
|
);
|
|
2768
3442
|
break;
|
|
2769
3443
|
}
|
|
3444
|
+
case "env.legacy.tag": {
|
|
3445
|
+
// ADR-154 §9: only the sign-in box tags legacy parameters.
|
|
3446
|
+
if (!signInBoxMode()) break;
|
|
3447
|
+
const { opId, keys } = frame;
|
|
3448
|
+
void managedMutationDrain.track(() =>
|
|
3449
|
+
queueProjectEnvLegacyTag(frame, { stopping: () => shutdownRequested })
|
|
3450
|
+
.catch(() => keys.map((key) => ({ key, outcome: "retry" as const })))
|
|
3451
|
+
.then((results) => send(socket, { kind: "env.legacy.tag.ack", opId, results })),
|
|
3452
|
+
);
|
|
3453
|
+
break;
|
|
3454
|
+
}
|
|
2770
3455
|
case "env.replica.begin": {
|
|
2771
3456
|
// Replica authority exists only on a task sidecar. Pool machines,
|
|
2772
3457
|
// owned runners and the sign-in box deliberately leave these frames
|
|
@@ -3605,15 +4290,12 @@ async function dispatchCommand(
|
|
|
3605
4290
|
.finally(() => invalidateMachineTunnelRoute(downInput.taskId));
|
|
3606
4291
|
}
|
|
3607
4292
|
case "taskStop": {
|
|
3608
|
-
const stopTaskId =
|
|
4293
|
+
const stopTaskId = expectString(args, 0);
|
|
4294
|
+
const stopInput = expectTaskStopInput(args, 1);
|
|
3609
4295
|
// A stopped machine loses its address; same before-and-after discipline.
|
|
3610
4296
|
invalidateMachineTunnelRoute(stopTaskId);
|
|
3611
4297
|
return hostCommands
|
|
3612
|
-
.taskStop(
|
|
3613
|
-
ctx,
|
|
3614
|
-
stopTaskId,
|
|
3615
|
-
(args[1] ?? { reason: "manual" }) as { reason: "idle" | "manual" },
|
|
3616
|
-
)
|
|
4298
|
+
.taskStop(ctx, stopTaskId, stopInput)
|
|
3617
4299
|
.finally(() => invalidateMachineTunnelRoute(stopTaskId));
|
|
3618
4300
|
}
|
|
3619
4301
|
case "taskStatus":
|
|
@@ -3836,6 +4518,53 @@ function isTaskLease(value: unknown): value is TaskLease {
|
|
|
3836
4518
|
);
|
|
3837
4519
|
}
|
|
3838
4520
|
|
|
4521
|
+
/**
|
|
4522
|
+
* ADR-154 §12: on the sign-in box the repository picker lists with the exact
|
|
4523
|
+
* GitHub revision the cloud names, read from the store; the cloud's flow
|
|
4524
|
+
* epoch fences it, so the runner-side generation check does not apply. A
|
|
4525
|
+
* store that cannot answer reads as unavailable, never as signed out.
|
|
4526
|
+
* Without a revision, the held token, as before.
|
|
4527
|
+
*/
|
|
4528
|
+
/** One picker request on the sign-in box, the revision's read and GitHub's
|
|
4529
|
+
* listing together: inside the cloud's 30-second answer deadline. */
|
|
4530
|
+
const PICKER_DEADLINE_MS = 25_000;
|
|
4531
|
+
|
|
4532
|
+
function revisionListDeps(
|
|
4533
|
+
userId: string,
|
|
4534
|
+
parameter: string | undefined,
|
|
4535
|
+
): Parameters<typeof listUserInstallationIds>[1] {
|
|
4536
|
+
if (parameter === undefined) return {};
|
|
4537
|
+
const deadlineAt = Date.now() + PICKER_DEADLINE_MS;
|
|
4538
|
+
return {
|
|
4539
|
+
deadlineAt,
|
|
4540
|
+
credentialGenerationMatches: () => true,
|
|
4541
|
+
requestAccessToken: async () => {
|
|
4542
|
+
if (!signInBoxMode()) throw new GitHubTokenUnreadableError();
|
|
4543
|
+
const store = configuredCredentialStore();
|
|
4544
|
+
if (!store) throw new GitHubTokenUnreadableError();
|
|
4545
|
+
let token: string | null;
|
|
4546
|
+
try {
|
|
4547
|
+
token = await readGithubRevision(store, parameter, userId, AbortSignal.timeout(Math.max(1, deadlineAt - Date.now())));
|
|
4548
|
+
} catch (err) {
|
|
4549
|
+
// The kind of failure in the box's log; neither carries its words.
|
|
4550
|
+
console.warn(`[github] the picker could not read ${parameter} (${storeFailureKind(err)})`);
|
|
4551
|
+
throw new GitHubTokenUnreadableError();
|
|
4552
|
+
}
|
|
4553
|
+
return token === null ? null : { accessToken: token, expiresAt: null };
|
|
4554
|
+
},
|
|
4555
|
+
};
|
|
4556
|
+
}
|
|
4557
|
+
|
|
4558
|
+
/** What the cloud hears when the box's picker fails: by its code, never
|
|
4559
|
+
* a dependency's words. */
|
|
4560
|
+
const PICKER_FAILURE_TEXT: Record<GitHubRepositoryAccessErrorCode, string> = {
|
|
4561
|
+
token_missing: "no GitHub sign-in to list with",
|
|
4562
|
+
reauth_required: "the GitHub sign-in needs signing in again",
|
|
4563
|
+
installation_inaccessible: "GitHub no longer shows that installation to this person",
|
|
4564
|
+
github_unavailable: "GitHub could not be asked just now",
|
|
4565
|
+
invalid_response: "GitHub answered with something unexpected",
|
|
4566
|
+
};
|
|
4567
|
+
|
|
3839
4568
|
function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
3840
4569
|
let parsed: unknown;
|
|
3841
4570
|
try {
|
|
@@ -3987,12 +4716,18 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
3987
4716
|
typeof frame.userId === "string" &&
|
|
3988
4717
|
parseGitHubCredentialGeneration(frame.generation) !== null
|
|
3989
4718
|
) {
|
|
4719
|
+
// ADR-154 §12: a revision named is this person's GitHub sign-in, or the
|
|
4720
|
+
// frame is dropped rather than listed with another token.
|
|
4721
|
+
if (frame.parameter !== undefined && !isPersonGithubParameter(frame.parameter, frame.userId)) {
|
|
4722
|
+
return null;
|
|
4723
|
+
}
|
|
3990
4724
|
return {
|
|
3991
4725
|
kind: "gh.installations.list",
|
|
3992
4726
|
opId: frame.opId,
|
|
3993
4727
|
generation:
|
|
3994
4728
|
parseGitHubCredentialGeneration(frame.generation) ?? undefined,
|
|
3995
4729
|
userId: frame.userId,
|
|
4730
|
+
...(frame.parameter !== undefined ? { parameter: frame.parameter } : {}),
|
|
3996
4731
|
};
|
|
3997
4732
|
}
|
|
3998
4733
|
if (
|
|
@@ -4004,6 +4739,9 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
4004
4739
|
frame.installationId > 0 &&
|
|
4005
4740
|
parseGitHubCredentialGeneration(frame.generation) !== null
|
|
4006
4741
|
) {
|
|
4742
|
+
if (frame.parameter !== undefined && !isPersonGithubParameter(frame.parameter, frame.userId)) {
|
|
4743
|
+
return null;
|
|
4744
|
+
}
|
|
4007
4745
|
return {
|
|
4008
4746
|
kind: "gh.repositories.list",
|
|
4009
4747
|
opId: frame.opId,
|
|
@@ -4011,6 +4749,7 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
4011
4749
|
parseGitHubCredentialGeneration(frame.generation) ?? undefined,
|
|
4012
4750
|
userId: frame.userId,
|
|
4013
4751
|
installationId: frame.installationId,
|
|
4752
|
+
...(frame.parameter !== undefined ? { parameter: frame.parameter } : {}),
|
|
4014
4753
|
};
|
|
4015
4754
|
}
|
|
4016
4755
|
if (
|
|
@@ -4045,7 +4784,8 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
4045
4784
|
isProjectEnvReplicaBeginFrame(frame) ||
|
|
4046
4785
|
isProjectEnvReplicaPageFrame(frame) ||
|
|
4047
4786
|
isProjectEnvReplicaApplyFrame(frame) ||
|
|
4048
|
-
isProjectEnvReplicaCommitFrame(frame)
|
|
4787
|
+
isProjectEnvReplicaCommitFrame(frame) ||
|
|
4788
|
+
isProjectEnvLegacyTagFrame(frame)
|
|
4049
4789
|
) {
|
|
4050
4790
|
return frame;
|
|
4051
4791
|
}
|
|
@@ -4056,6 +4796,14 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
4056
4796
|
) {
|
|
4057
4797
|
return { kind: "mcp.op", opId: frame.opId, op: frame.op };
|
|
4058
4798
|
}
|
|
4799
|
+
// ADR-154 §11: an MCP handshake as a capture flow, exact or dropped.
|
|
4800
|
+
if (frame.kind === "mcp.capture.probe") return parseMcpCaptureProbeFrame(frame);
|
|
4801
|
+
// ADR-154 §12: a person's GitHub sign-in and SSH key, exact or dropped.
|
|
4802
|
+
if (frame.kind === "github.capture") return parseGithubCaptureFrame(frame);
|
|
4803
|
+
if (frame.kind === "ssh.mint") return parseSshMintFrame(frame);
|
|
4804
|
+
if (frame.kind === "github.revoke") return parseGithubRevokeFrame(frame);
|
|
4805
|
+
if (frame.kind === "mcp.capture.complete") return parseMcpCaptureCompleteFrame(frame);
|
|
4806
|
+
if (frame.kind === "mcp.capture.cancel") return parseMcpCaptureCancelFrame(frame);
|
|
4059
4807
|
// ADR-133: the cloud's answers to a sidecar's lease request. Without these
|
|
4060
4808
|
// two clauses the sidecar logs "dropping unknown frame kind=lease.grant"
|
|
4061
4809
|
// and the task dies after the 3-minute credentials gate.
|
|
@@ -4073,6 +4821,23 @@ function parseCloudFrame(data: RawData): CloudToHost | null {
|
|
|
4073
4821
|
) {
|
|
4074
4822
|
return { kind: "lease.deny", opId: frame.opId, error: frame.error };
|
|
4075
4823
|
}
|
|
4824
|
+
// ADR-154: the v1 lease, credential and task env replica frames, parsed
|
|
4825
|
+
// to their exact shapes here; only a runner on cloud credentials acts on
|
|
4826
|
+
// them.
|
|
4827
|
+
if (frame.kind === "lease.grant.v1") return parseLeaseGrantV1Frame(frame);
|
|
4828
|
+
if (frame.kind === "lease.unavailable") return parseLeaseUnavailableFrame(frame);
|
|
4829
|
+
if (frame.kind === "lease.stale") return parseLeaseStaleFrame(frame);
|
|
4830
|
+
if (frame.kind === "credential.changed") return parseCredentialChangedFrame(frame);
|
|
4831
|
+
if (frame.kind === "credential.refresh.result") return parseCredentialRefreshResultFrame(frame);
|
|
4832
|
+
if (frame.kind === "credentials.activate.result") return parseCredentialsActivateResultFrame(frame);
|
|
4833
|
+
if (
|
|
4834
|
+
isProjectEnvReplicaV1BeginFrame(frame) ||
|
|
4835
|
+
isProjectEnvReplicaV1PageFrame(frame) ||
|
|
4836
|
+
isProjectEnvReplicaV1ApplyFrame(frame) ||
|
|
4837
|
+
isProjectEnvReplicaV1CommitFrame(frame)
|
|
4838
|
+
) {
|
|
4839
|
+
return frame;
|
|
4840
|
+
}
|
|
4076
4841
|
// Names only — never payloads (frames can carry tokens).
|
|
4077
4842
|
console.warn(
|
|
4078
4843
|
`[host-agent] dropping unknown frame kind=${String(frame.kind)}` +
|
|
@@ -4162,7 +4927,9 @@ function sendHostConfigAck(
|
|
|
4162
4927
|
}
|
|
4163
4928
|
|
|
4164
4929
|
function send(socket: WebSocket, frame: HostToCloud): void {
|
|
4165
|
-
|
|
4930
|
+
// Never past the bridge's ceiling: the cloud would close the connection.
|
|
4931
|
+
const json = outboundFrameJson(frame);
|
|
4932
|
+
if (json !== null) socket.send(json);
|
|
4166
4933
|
}
|
|
4167
4934
|
|
|
4168
4935
|
/**
|
|
@@ -4313,6 +5080,10 @@ function rejectFrameDuringShutdown(
|
|
|
4313
5080
|
case "env.replica.page":
|
|
4314
5081
|
case "env.replica.apply":
|
|
4315
5082
|
case "env.replica.commit":
|
|
5083
|
+
case "env.replica.v1.begin":
|
|
5084
|
+
case "env.replica.v1.page":
|
|
5085
|
+
case "env.replica.v1.apply":
|
|
5086
|
+
case "env.replica.v1.commit":
|
|
4316
5087
|
// These ordered operations need the durable current cursor in an error
|
|
4317
5088
|
// ack. Shutdown has already closed the store admission gate, so leave
|
|
4318
5089
|
// them unanswered for the cloud to retry on the replacement socket.
|
|
@@ -4325,6 +5096,37 @@ function rejectFrameDuringShutdown(
|
|
|
4325
5096
|
error,
|
|
4326
5097
|
});
|
|
4327
5098
|
return true;
|
|
5099
|
+
case "mcp.capture.probe":
|
|
5100
|
+
case "mcp.capture.complete":
|
|
5101
|
+
send(socket, {
|
|
5102
|
+
kind: "mcp.capture.ack",
|
|
5103
|
+
opId: frame.opId,
|
|
5104
|
+
outcome: "error",
|
|
5105
|
+
capture: null,
|
|
5106
|
+
authorizeUrl: null,
|
|
5107
|
+
scopes: null,
|
|
5108
|
+
error,
|
|
5109
|
+
});
|
|
5110
|
+
return true;
|
|
5111
|
+
case "mcp.capture.cancel":
|
|
5112
|
+
// Not new work: it ends work already admitted, which drains on. Handled
|
|
5113
|
+
// as ever (synchronous and idempotent), so a flow the cloud ended while
|
|
5114
|
+
// this process drains stops its HTTP and its writes now.
|
|
5115
|
+
return false;
|
|
5116
|
+
case "github.capture":
|
|
5117
|
+
case "ssh.mint":
|
|
5118
|
+
send(socket, { kind: "person.capture.ack", opId: frame.opId, capture: null, publicKey: null, error });
|
|
5119
|
+
return true;
|
|
5120
|
+
case "github.revoke":
|
|
5121
|
+
send(socket, { kind: "github.revoke.ack", opId: frame.opId, outcome: "retry", error });
|
|
5122
|
+
return true;
|
|
5123
|
+
case "env.legacy.tag":
|
|
5124
|
+
send(socket, {
|
|
5125
|
+
kind: "env.legacy.tag.ack",
|
|
5126
|
+
opId: frame.opId,
|
|
5127
|
+
results: frame.keys.map((key) => ({ key, outcome: "retry" })),
|
|
5128
|
+
});
|
|
5129
|
+
return true;
|
|
4328
5130
|
case "host.inventory.request":
|
|
4329
5131
|
// Read-only and retryable after the replacement process reconnects. A
|
|
4330
5132
|
// partial page during shutdown would be tied to a dying generation.
|
|
@@ -4349,6 +5151,13 @@ function rejectFrameDuringShutdown(
|
|
|
4349
5151
|
// A lease answer is housekeeping the sidecar still wants during a drain.
|
|
4350
5152
|
case "lease.grant":
|
|
4351
5153
|
case "lease.deny":
|
|
5154
|
+
// ADR-154's lease, credential and activation frames, likewise.
|
|
5155
|
+
case "lease.grant.v1":
|
|
5156
|
+
case "lease.unavailable":
|
|
5157
|
+
case "lease.stale":
|
|
5158
|
+
case "credential.changed":
|
|
5159
|
+
case "credential.refresh.result":
|
|
5160
|
+
case "credentials.activate.result":
|
|
4352
5161
|
return false;
|
|
4353
5162
|
}
|
|
4354
5163
|
}
|
|
@@ -4359,6 +5168,88 @@ function rejectFrameDuringShutdown(
|
|
|
4359
5168
|
* nothing here will ever accept them — while the managed lifecycle, engine
|
|
4360
5169
|
* sign-ins and account management pass through untouched.
|
|
4361
5170
|
*/
|
|
5171
|
+
/**
|
|
5172
|
+
* ADR-154 §13: once the runner is leaving its own credentials (or has left
|
|
5173
|
+
* them), no frame that would write or read one of its local credential
|
|
5174
|
+
* stores is admitted: an engine sign-in, a GitHub token, an SSH key, an MCP
|
|
5175
|
+
* connection, a project env value, or a GitHub listing on its token. Each is
|
|
5176
|
+
* answered as shutdown answers it, saying why; ordered env operations, whose
|
|
5177
|
+
* error needs a cursor the closed store cannot give, are left unanswered for
|
|
5178
|
+
* the cloud, which stops sending them at the runner's floor. A GitHub clear
|
|
5179
|
+
* or an SSH key's deletion is refused as well (the purge removes every one,
|
|
5180
|
+
* and none writes its fence after it), and a sign-in's late code or callback
|
|
5181
|
+
* is dropped. Stops go on.
|
|
5182
|
+
*/
|
|
5183
|
+
function rejectCredentialFrameOffOwnCredentials(socket: WebSocket, frame: CloudToHost): boolean {
|
|
5184
|
+
const error = CREDENTIALS_IN_CLOUD;
|
|
5185
|
+
switch (frame.kind) {
|
|
5186
|
+
case "engine.login.start":
|
|
5187
|
+
send(socket, {
|
|
5188
|
+
kind: "engine.login.event",
|
|
5189
|
+
opId: frame.opId,
|
|
5190
|
+
engine: frame.engine,
|
|
5191
|
+
phase: "failed",
|
|
5192
|
+
errorCode: "unavailable",
|
|
5193
|
+
message: error,
|
|
5194
|
+
});
|
|
5195
|
+
return true;
|
|
5196
|
+
case "engine.connect.start":
|
|
5197
|
+
send(socket, {
|
|
5198
|
+
kind: "engine.connect.event",
|
|
5199
|
+
opId: frame.opId,
|
|
5200
|
+
engine: frame.engine,
|
|
5201
|
+
phase: "done",
|
|
5202
|
+
ok: false,
|
|
5203
|
+
message: error,
|
|
5204
|
+
});
|
|
5205
|
+
return true;
|
|
5206
|
+
case "engine.login.input":
|
|
5207
|
+
case "engine.login.callback":
|
|
5208
|
+
// Every sign-in flow stopped when leaving began: a late code or
|
|
5209
|
+
// callback has nothing to finish, and is dropped unread.
|
|
5210
|
+
return true;
|
|
5211
|
+
case "gh.connect.set":
|
|
5212
|
+
case "gh.connect.clear":
|
|
5213
|
+
send(socket, { kind: "gh.connect.ack", opId: frame.opId, userId: frame.userId, ok: false, error });
|
|
5214
|
+
return true;
|
|
5215
|
+
case "gh.installations.list":
|
|
5216
|
+
send(socket, { kind: "gh.installations.ack", opId: frame.opId, ok: false, code: "github_unavailable", error });
|
|
5217
|
+
return true;
|
|
5218
|
+
case "gh.repositories.list":
|
|
5219
|
+
send(socket, { kind: "gh.repositories.ack", opId: frame.opId, ok: false, code: "github_unavailable", error });
|
|
5220
|
+
return true;
|
|
5221
|
+
case "ssh.key.get":
|
|
5222
|
+
case "ssh.key.ensure":
|
|
5223
|
+
case "ssh.key.delete":
|
|
5224
|
+
send(socket, { kind: "ssh.key.ack", userId: frame.userId, ok: false, error });
|
|
5225
|
+
return true;
|
|
5226
|
+
case "mcp.op":
|
|
5227
|
+
send(socket, { kind: "mcp.ack", opId: frame.opId, ok: false, error });
|
|
5228
|
+
return true;
|
|
5229
|
+
case "env.var.list":
|
|
5230
|
+
if ("after" in frame || "reconciliationAttemptId" in frame) return true;
|
|
5231
|
+
send(socket, { kind: "env.var.ack", opId: frame.opId, ok: false, error });
|
|
5232
|
+
return true;
|
|
5233
|
+
case "env.var.set":
|
|
5234
|
+
case "env.var.delete":
|
|
5235
|
+
if ("delivery" in frame) return true;
|
|
5236
|
+
send(socket, { kind: "env.var.ack", opId: frame.opId, ok: false, error });
|
|
5237
|
+
return true;
|
|
5238
|
+
case "env.var.remove":
|
|
5239
|
+
case "env.binding.set":
|
|
5240
|
+
case "env.staging.put":
|
|
5241
|
+
case "env.reconcile.begin":
|
|
5242
|
+
case "env.reconciled":
|
|
5243
|
+
case "env.delivery.skip":
|
|
5244
|
+
return true;
|
|
5245
|
+
default:
|
|
5246
|
+
return false;
|
|
5247
|
+
}
|
|
5248
|
+
}
|
|
5249
|
+
|
|
5250
|
+
/** The cloud's own words for a runner past its floor (lib/credentials-activation.ts). */
|
|
5251
|
+
const CREDENTIALS_IN_CLOUD = "this runner reads its credentials from Uai Cloud; manage them in Connections";
|
|
5252
|
+
|
|
4362
5253
|
function rejectFrameOnSignInBox(socket: WebSocket, frame: CloudToHost): boolean {
|
|
4363
5254
|
switch (frame.kind) {
|
|
4364
5255
|
case "command":
|
|
@@ -4553,6 +5444,21 @@ function expectTaskDownInput(args: unknown[], index: number): TaskDownInput {
|
|
|
4553
5444
|
return out;
|
|
4554
5445
|
}
|
|
4555
5446
|
|
|
5447
|
+
function expectTaskStopInput(args: unknown[], index: number): TaskStopInput {
|
|
5448
|
+
const input = expectRecord(args[index], "task stop input");
|
|
5449
|
+
if (input.reason !== "idle" && input.reason !== "manual") {
|
|
5450
|
+
throw new Error("invalid task stop reason");
|
|
5451
|
+
}
|
|
5452
|
+
const out: TaskStopInput = { reason: input.reason };
|
|
5453
|
+
if (Object.prototype.hasOwnProperty.call(input, "runEpoch")) {
|
|
5454
|
+
if (!Number.isSafeInteger(input.runEpoch) || (input.runEpoch as number) < 1) {
|
|
5455
|
+
throw new Error("invalid task stop run epoch");
|
|
5456
|
+
}
|
|
5457
|
+
out.runEpoch = input.runEpoch as number;
|
|
5458
|
+
}
|
|
5459
|
+
return out;
|
|
5460
|
+
}
|
|
5461
|
+
|
|
4556
5462
|
function expectTaskDiffInput(args: unknown[], index: number): TaskDiffInput {
|
|
4557
5463
|
const input = expectRecord(args[index], "task diff input");
|
|
4558
5464
|
const projectsValue = input.projects;
|
|
@@ -4760,6 +5666,9 @@ function expectTaskCommandTask(value: unknown): TaskCommandTask {
|
|
|
4760
5666
|
"task.engineAccountScope",
|
|
4761
5667
|
);
|
|
4762
5668
|
if (taskScope !== undefined) out.engineAccountScope = taskScope;
|
|
5669
|
+
// ADR-154: the run epoch every v1 grant and replica frame is fenced to.
|
|
5670
|
+
const runEpoch = runEpochFromWire(row.runEpoch, "task.runEpoch");
|
|
5671
|
+
if (runEpoch !== undefined) out.runEpoch = runEpoch;
|
|
4763
5672
|
return out;
|
|
4764
5673
|
}
|
|
4765
5674
|
|
|
@@ -4842,6 +5751,18 @@ function requestShutdown(signal: NodeJS.Signals): void {
|
|
|
4842
5751
|
void stopRemoteOperationManagers();
|
|
4843
5752
|
stopMcpGatewaySubscription();
|
|
4844
5753
|
stopAgentClisReadySubscription();
|
|
5754
|
+
// ADR-154: no credential is served or taken after this; what was accepted
|
|
5755
|
+
// drains, bounded, and no cleanup starts (the next process's recovery runs
|
|
5756
|
+
// it). Exit waits for it.
|
|
5757
|
+
void cloudCredentials
|
|
5758
|
+
.shutdown()
|
|
5759
|
+
.catch((error: unknown) =>
|
|
5760
|
+
console.warn(`[credentials] shutdown failed: ${error instanceof Error ? error.message : String(error)}`),
|
|
5761
|
+
)
|
|
5762
|
+
.finally(() => {
|
|
5763
|
+
cloudCredentialsStopped = true;
|
|
5764
|
+
maybeExit();
|
|
5765
|
+
});
|
|
4845
5766
|
// Before the gateway: unregister its TLS consumer, end proxy connections,
|
|
4846
5767
|
// refuse new root mutations of tasks and wait for those under way. Exit
|
|
4847
5768
|
// waits for all of it (see secretsProxyStopped).
|
|
@@ -4877,7 +5798,8 @@ function maybeExit(): void {
|
|
|
4877
5798
|
inFlight > 0 ||
|
|
4878
5799
|
managedMutationDrain.activeCount > 0 ||
|
|
4879
5800
|
pendingRemoteOperationCleanups.size > 0 ||
|
|
4880
|
-
!secretsProxyStopped
|
|
5801
|
+
!secretsProxyStopped ||
|
|
5802
|
+
!cloudCredentialsStopped
|
|
4881
5803
|
) return;
|
|
4882
5804
|
if (ws) {
|
|
4883
5805
|
if (ws.readyState === WebSocket.OPEN) {
|