@moondeck/station 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +149 -2
- package/bin/uai-host.mjs +130 -0
- package/db/migrations/0000_host_tasks.sql +12 -0
- package/db/migrations/0001_host_ui.sql +11 -0
- package/db/migrations/0002_host_github_tokens.sql +8 -0
- package/db/migrations/0003_host_ssh_keys.sql +8 -0
- package/db/migrations/0004_host_owner_name.sql +1 -0
- package/db/migrations/0005_host_env_vars.sql +5 -0
- package/db/migrations/0006_host_project_env.sql +9 -0
- package/db/migrations/0007_host_github_token_kind.sql +7 -0
- package/db/migrations/0008_host_mcp_connections.sql +18 -0
- package/db/migrations/0009_host_agent_sessions.sql +15 -0
- package/db/migrations/0010_host_engine_accounts.sql +13 -0
- package/db/migrations/0011_codex_thread_id.sql +5 -0
- package/db/migrations/0012_agent_session_attach_key.sql +4 -0
- package/db/migrations/0013_github_credential_generations.sql +11 -0
- package/db/migrations/0014_host_inventory_event_index.sql +1 -0
- package/db/migrations/0015_host_settings.sql +9 -0
- package/db/migrations/0016_task_environment.sql +2 -0
- package/db/migrations/0017_engine_account_scope.sql +7 -0
- package/db/migrations/0018_engine_account_position.sql +3 -0
- package/db/migrations/0019_engine_account_store_ref.sql +3 -0
- package/db/migrations/0020_user_credential_store_ref.sql +6 -0
- package/db/migrations/0021_project_env_store_ref.sql +3 -0
- package/db/migrations/0022_task_project_ids.sql +5 -0
- package/db/migrations/0023_project_env_org.sql +4 -0
- package/db/migrations/0024_task_env_mode.sql +9 -0
- package/db/migrations/0025_project_env_authority.sql +147 -0
- package/db/migrations/0026_project_env_replica.sql +150 -0
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/0029_task_shared_files_mode.sql +5 -0
- package/db/migrations/0030_credential_cleanup_restops.sql +12 -0
- package/db/migrations/meta/_journal.json +223 -0
- package/db/schema.ts +850 -0
- package/images/standard/Dockerfile +347 -0
- package/images/standard/README.md +167 -0
- package/images/standard/container/code-server-settings.json +63 -0
- package/images/standard/container/corepack-version +1 -0
- package/images/standard/container/uai-init +704 -0
- package/images/standard/container/uai-materialize-runtimes +1527 -0
- package/images/standard/container/uai-task-forwarder.mjs +303 -0
- package/images/standard/tool-versions +2 -0
- package/lib/__fixtures__/skill-index/SKILL.md +10 -0
- package/lib/__fixtures__/skill-index/skills/accessibility-checklist.md +7 -0
- package/lib/__fixtures__/skill-index/skills/brand-voice.md +9 -0
- package/lib/__fixtures__/skill-index/skills/design-tokens.md +9 -0
- package/lib/agent-cli.ts +1507 -0
- package/lib/agent-errors.ts +22 -0
- package/lib/agent.ts +672 -0
- package/lib/agents/attach-compatibility.ts +51 -0
- package/lib/agents/claude.ts +918 -0
- package/lib/agents/codex-token-usage.fixture.json +130 -0
- package/lib/agents/codex.ts +1392 -0
- package/lib/agents/cursor-period-usage.ts +253 -0
- package/lib/agents/cursor.ts +562 -0
- package/lib/agents/dispatch.ts +78 -0
- package/lib/agents/durable-proc.ts +334 -0
- package/lib/agents/factory.ts +57 -0
- package/lib/agents/grok.ts +381 -0
- package/lib/agents/kimi.ts +329 -0
- package/lib/agents/machine-durable.ts +604 -0
- package/lib/agents/mock.ts +133 -0
- package/lib/agents/mode.ts +64 -0
- package/lib/agents/opencode.ts +508 -0
- package/lib/agents/output-source.ts +3 -0
- package/lib/agents/proc.ts +221 -0
- package/lib/agents/rate-limit.ts +337 -0
- package/lib/agents/registry.ts +182 -0
- package/lib/agents/session-state.ts +197 -0
- package/lib/agents/skill-index-preamble.fixture.ts +26 -0
- package/lib/agents/transport.ts +741 -0
- package/lib/agents/types.ts +254 -0
- package/lib/agents/uai.ts +106 -0
- package/lib/agents/usage.ts +44 -0
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +237 -0
- package/lib/apple-uninstall-teardown.ts +224 -0
- package/lib/assignment-evidence.ts +94 -0
- package/lib/attachments.ts +48 -0
- package/lib/browser-testing.ts +1537 -0
- package/lib/capture-flow.ts +332 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2587 -0
- package/lib/cloud-credential-runtime.ts +1357 -0
- package/lib/cloud-state.ts +56 -0
- package/lib/codex-auth.ts +456 -0
- package/lib/command-db.ts +310 -0
- package/lib/compose-network-recovery.ts +283 -0
- package/lib/connection-secret-redaction.ts +25 -0
- package/lib/container-runtime.ts +1453 -0
- package/lib/credential-cleanup-inventory.ts +490 -0
- package/lib/credential-consumers/engine-anchor.ts +1880 -0
- package/lib/credential-consumers/engines.ts +797 -0
- package/lib/credential-consumers/files.ts +934 -0
- package/lib/credential-consumers/github.ts +702 -0
- package/lib/credential-consumers/mcp.ts +389 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-consumers/task-secrets.ts +10 -0
- package/lib/credential-store-pickup.ts +149 -0
- package/lib/credential-store-scope.ts +112 -0
- package/lib/credential-store.ts +932 -0
- package/lib/credentials-activation-authority.ts +709 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/custom-connection-capture.ts +395 -0
- package/lib/custom-connection-launch.ts +384 -0
- package/lib/custom-connection-runtime.ts +141 -0
- package/lib/db.ts +80 -0
- package/lib/disk-pressure.ts +42 -0
- package/lib/docker-exec.ts +156 -0
- package/lib/engine-accounts.ts +1970 -0
- package/lib/engine-connect.ts +321 -0
- package/lib/engine-credential-env.ts +52 -0
- package/lib/engine-login.ts +2933 -0
- package/lib/engines.ts +1342 -0
- package/lib/enrollment-state.ts +251 -0
- package/lib/env-file.ts +246 -0
- package/lib/env-schema.ts +891 -0
- package/lib/env.ts +195 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +530 -0
- package/lib/git-identity.ts +715 -0
- package/lib/github-git-auth.ts +207 -0
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +2439 -0
- package/lib/host-cloud-url.ts +62 -0
- package/lib/host-config.ts +279 -0
- package/lib/host-env.ts +424 -0
- package/lib/host-logs.ts +965 -0
- package/lib/internal-connections.ts +5 -0
- package/lib/keyed-promise-tail.ts +23 -0
- package/lib/launch-mode.ts +25 -0
- package/lib/lease.ts +239 -0
- package/lib/legacy-host.ts +769 -0
- package/lib/legacy-runtime-retirement.ts +424 -0
- package/lib/legacy-runtime-v1.fixture.ts +627 -0
- package/lib/machine-exec.ts +238 -0
- package/lib/machine-gateway-tunnel.ts +344 -0
- package/lib/machine-keys.ts +65 -0
- package/lib/machine-project-env-loader.ts +152 -0
- package/lib/machine-provider-aws.ts +923 -0
- package/lib/machine-provider-local.ts +338 -0
- package/lib/machine-provider.ts +90 -0
- package/lib/managed-activation-watcher.ts +72 -0
- package/lib/managed-install-owner-watcher.ts +55 -0
- package/lib/managed-operation-drain.ts +57 -0
- package/lib/managed-runtime.ts +4296 -0
- package/lib/managed-update-scheduler.ts +125 -0
- package/lib/mcp-capture.ts +865 -0
- package/lib/mcp-config-lock.ts +2 -0
- package/lib/mcp-connections.ts +959 -0
- package/lib/mcp-gateway.ts +2594 -0
- package/lib/obs.ts +512 -0
- package/lib/openrouter-budget.ts +247 -0
- package/lib/orchestrator.ts +11409 -0
- package/lib/person-capture.ts +327 -0
- package/lib/preview-ports.ts +85 -0
- package/lib/preview-sidecar.ts +331 -0
- package/lib/project-env-authority.ts +1319 -0
- package/lib/project-env-feature.ts +74 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-loader.ts +242 -0
- package/lib/project-env-replica.ts +2094 -0
- package/lib/project-env-schema-capture.ts +342 -0
- package/lib/project-env-staging.ts +148 -0
- package/lib/project-env-task-policy.ts +176 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/release-feed.ts +61 -0
- package/lib/release-manifest.ts +710 -0
- package/lib/release-trust.ts +46 -0
- package/lib/repo-clone.ts +38 -0
- package/lib/routing/shadow.ts +261 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +279 -0
- package/lib/runtime-archive.ts +1216 -0
- package/lib/runtime-authority.ts +166 -0
- package/lib/runtime-guard.ts +36 -0
- package/lib/runtime-provider-state.ts +169 -0
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +1033 -0
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/address.ts +95 -0
- package/lib/secrets-proxy/audit.ts +30 -0
- package/lib/secrets-proxy/ca-store.ts +200 -0
- package/lib/secrets-proxy/ca.ts +470 -0
- package/lib/secrets-proxy/container-steps.ts +138 -0
- package/lib/secrets-proxy/der.ts +273 -0
- package/lib/secrets-proxy/engine.ts +1633 -0
- package/lib/secrets-proxy/host.ts +93 -0
- package/lib/secrets-proxy/hostname.ts +57 -0
- package/lib/secrets-proxy/placeholders.ts +196 -0
- package/lib/secrets-proxy/policy.ts +150 -0
- package/lib/secrets-proxy/readiness.ts +148 -0
- package/lib/secrets-proxy/runtime.ts +582 -0
- package/lib/secrets-proxy/scrub.ts +559 -0
- package/lib/secrets-proxy/sidecar-view.ts +139 -0
- package/lib/secrets-proxy/task-scripts.ts +262 -0
- package/lib/secrets-proxy/task-trust.ts +605 -0
- package/lib/secrets-proxy/trust-watch.ts +80 -0
- package/lib/secrets.ts +71 -0
- package/lib/shared-files-s3.ts +976 -0
- package/lib/shared-files.ts +156 -0
- package/lib/sidecar-assign.ts +49 -0
- package/lib/sidecar-guard.ts +58 -0
- package/lib/sidecar-resume.ts +178 -0
- package/lib/sidecar.ts +69 -0
- package/lib/signin-box-credentials.ts +465 -0
- package/lib/signin-box.ts +114 -0
- package/lib/skill-index-example.fixture.ts +56 -0
- package/lib/skill-layout.ts +417 -0
- package/lib/skills.ts +489 -0
- package/lib/ssh.ts +401 -0
- package/lib/standard-image-cleanup.ts +117 -0
- package/lib/standard-image-pin.ts +207 -0
- package/lib/standard-image.ts +1697 -0
- package/lib/stopped-task-status-queue.ts +120 -0
- package/lib/task-container-cli.ts +269 -0
- package/lib/task-credential-cleanup.ts +752 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +91 -0
- package/lib/task-env-forwarder.ts +228 -0
- package/lib/task-env-source.ts +249 -0
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +860 -0
- package/lib/task-environment/docker.ts +1344 -0
- package/lib/task-environment/index.ts +466 -0
- package/lib/task-environment/legacy-adoption.ts +467 -0
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +1227 -0
- package/lib/task-environment/machine.ts +974 -0
- package/lib/task-environment/registry.ts +59 -0
- package/lib/task-environment/self-identity.ts +20 -0
- package/lib/task-environment/self.ts +566 -0
- package/lib/task-environment/types.ts +642 -0
- package/lib/task-environment/workspace-files.ts +685 -0
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-files-machine-retirement-outbox.ts +592 -0
- package/lib/task-files-machine-retirement.ts +116 -0
- package/lib/task-files-readiness.ts +127 -0
- package/lib/task-gateway-port.ts +22 -0
- package/lib/task-gateway-protocol.ts +190 -0
- package/lib/task-gateway-recovery.ts +47 -0
- package/lib/task-gateway-state.ts +512 -0
- package/lib/task-identity.ts +19 -0
- package/lib/task-inventory.ts +682 -0
- package/lib/task-machine-credentials-activation.ts +95 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/task-person-credentials.ts +76 -0
- package/lib/task-project-ops-host.ts +147 -0
- package/lib/task-project-ops.ts +1062 -0
- package/lib/task-secret-staging.ts +16 -0
- package/lib/task-status.ts +46 -0
- package/lib/task-token.ts +46 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +105 -0
- package/lib/token-broker-local.ts +392 -0
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +525 -0
- package/lib/token-broker.ts +218 -0
- package/lib/token-lanes/openai-oauth.ts +123 -0
- package/lib/token-lanes/registry.ts +45 -0
- package/lib/token-lanes/types.ts +163 -0
- package/lib/token-lanes/xai-oauth.ts +165 -0
- package/lib/transcript.ts +128 -0
- package/lib/tunnel-registry.ts +220 -0
- package/lib/tunnel-runtime.ts +235 -0
- package/lib/ulid.ts +16 -0
- package/lib/user-bus-env.ts +116 -0
- package/lib/user-credentials-store.ts +435 -0
- package/lib/vnode-pressure.ts +142 -0
- package/package.json +131 -4
- package/runner/runner.mjs +442 -0
- package/scripts/agent/_common.sh +633 -0
- package/scripts/agent/env-hook.sh +626 -0
- package/scripts/agent/moondeck-cli.sh +5 -0
- package/scripts/agent/task-down.sh +291 -0
- package/scripts/agent/task-status.sh +70 -0
- package/scripts/agent/task-up.sh +3761 -0
- package/scripts/agent/uai-env-exec.sh +26 -0
- package/scripts/install/darwin.ts +1048 -0
- package/scripts/install/linux.ts +958 -0
- package/scripts/install/types.ts +78 -0
- package/scripts/install/util.ts +254 -0
- package/scripts/install/win.ts +143 -0
- package/src/activation-respawn.ts +155 -0
- package/src/apple-tunnel-route.ts +104 -0
- package/src/capture-wire.ts +102 -0
- package/src/cli.ts +2583 -0
- package/src/cloud-credentials.ts +2144 -0
- package/src/cloud-origins.ts +23 -0
- package/src/credential-location.ts +248 -0
- package/src/credential-wire-validation.ts +25 -0
- package/src/custom-connections.ts +237 -0
- package/src/engine-account-paths.ts +11 -0
- package/src/engine-models.ts +39 -0
- package/src/engine-usage.ts +104 -0
- package/src/event-outbox.ts +185 -0
- package/src/index.ts +2899 -0
- package/src/load-env.ts +97 -0
- package/src/machine-tunnel-route.ts +156 -0
- package/src/main.ts +6349 -0
- package/src/never-proxied-domains.ts +31 -0
- package/src/outbound-frame.ts +31 -0
- package/src/paths.ts +81 -0
- package/src/project-env-latest.ts +130 -0
- package/src/project-env-policy.ts +396 -0
- package/src/project-env-wording.ts +53 -0
- package/src/protocol.ts +5545 -0
- package/src/runtime-bootstrap.ts +165 -0
- package/src/station-identity.ts +96 -0
- package/src/task-env-names.ts +28 -0
- package/src/task-secrets.ts +75 -0
- package/src/tunnel-dial.ts +102 -0
- package/src/ui/server.ts +852 -0
- package/src/ui/types.ts +173 -0
- package/src/workspace-path-policy.ts +41 -0
- package/ui/app.js +1102 -0
- package/ui/index.html +90 -0
- package/ui/style.css +685 -0
- package/ui/uai-favicon.svg +21 -0
- package/ui/uai-logo-black.svg +9 -0
- package/ui/uai-wheel.svg +9 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,2899 @@
|
|
|
1
|
+
import { assertShareablePath, normalizedWorkspacePath } from "./workspace-path-policy";
|
|
2
|
+
import { readBoundedFile } from "../lib/task-environment/workspace-files";
|
|
3
|
+
import {
|
|
4
|
+
agent,
|
|
5
|
+
AgentError,
|
|
6
|
+
CREDENTIALS_WITHDRAWN_MESSAGE,
|
|
7
|
+
toolStderrDetail,
|
|
8
|
+
type LeasedTaskUp,
|
|
9
|
+
type TaskDownResult,
|
|
10
|
+
} from "../lib/agent";
|
|
11
|
+
import {
|
|
12
|
+
activeCloudCredentials,
|
|
13
|
+
admitLegacyStart,
|
|
14
|
+
cloudCredentialRuntime,
|
|
15
|
+
credentialsMode,
|
|
16
|
+
onOwnCredentials,
|
|
17
|
+
whileOnOwnCredentials,
|
|
18
|
+
type LegacyStartPermit,
|
|
19
|
+
} from "../lib/cloud-credential-host";
|
|
20
|
+
import type { TaskAssignment } from "../lib/cloud-credential-leases";
|
|
21
|
+
import {
|
|
22
|
+
StartInterruptedError,
|
|
23
|
+
type CloudCredentialRuntime,
|
|
24
|
+
type TaskCredentialStart,
|
|
25
|
+
type TaskUpHold,
|
|
26
|
+
} from "../lib/cloud-credential-runtime";
|
|
27
|
+
import { writeTaskPersonCredentials } from "../lib/credential-consumers/github";
|
|
28
|
+
import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
|
|
29
|
+
import { cloneRepo } from "../lib/repo-clone";
|
|
30
|
+
import { handleFilesOp } from "../lib/shared-files";
|
|
31
|
+
import type { SharedFilesMode } from "../lib/shared-files-s3";
|
|
32
|
+
import {
|
|
33
|
+
getOrchestrator,
|
|
34
|
+
inspectTaskRuntimeContainer,
|
|
35
|
+
proveAppleTaskRuntimeContract,
|
|
36
|
+
quarantineTaskRuntimeContainer,
|
|
37
|
+
} from "../lib/orchestrator";
|
|
38
|
+
import {
|
|
39
|
+
removeTaskCliSecretStrict,
|
|
40
|
+
storeTaskCliSecret,
|
|
41
|
+
} from "../lib/agent-cli";
|
|
42
|
+
import { stopMachineGatewayTunnel } from "../lib/machine-gateway-tunnel";
|
|
43
|
+
import {
|
|
44
|
+
clearRefresh,
|
|
45
|
+
injectIntoContainer,
|
|
46
|
+
reconcileTaskGitAuth,
|
|
47
|
+
requestAccessToken,
|
|
48
|
+
} from "../lib/github-tokens";
|
|
49
|
+
import {
|
|
50
|
+
prepareTaskGithubGitCredential,
|
|
51
|
+
type TaskGithubGitCredential,
|
|
52
|
+
} from "../lib/github-git-auth";
|
|
53
|
+
import { readAttachment, writeAttachment } from "../lib/attachments";
|
|
54
|
+
import { readFileSync } from "node:fs";
|
|
55
|
+
import { resolve } from "node:path";
|
|
56
|
+
import { env, taskProjectWorktree, taskWorkspaceDir } from "../lib/env";
|
|
57
|
+
import { getDecryptedForProject } from "../lib/host-env";
|
|
58
|
+
import {
|
|
59
|
+
envSchemaNote,
|
|
60
|
+
missingRequiredKeys,
|
|
61
|
+
parseEnvSchema,
|
|
62
|
+
} from "../lib/env-schema";
|
|
63
|
+
import {
|
|
64
|
+
appendTranscript as writeTranscript,
|
|
65
|
+
appendTranscriptViaEnvironment,
|
|
66
|
+
} from "../lib/transcript";
|
|
67
|
+
import { buildTaskDiff } from "../lib/task-diff";
|
|
68
|
+
import { taskProjectOps } from "../lib/task-project-ops-host";
|
|
69
|
+
import { stoppedTaskDiff } from "../lib/task-diff-sandbox";
|
|
70
|
+
import {
|
|
71
|
+
admitTaskRunEpoch,
|
|
72
|
+
canReuseTaskRuntimeForRun,
|
|
73
|
+
closeTaskRunAssignment,
|
|
74
|
+
closeTaskRunAssignmentForStop,
|
|
75
|
+
deleteHostTask,
|
|
76
|
+
getHostTask,
|
|
77
|
+
hostTaskEnvMode,
|
|
78
|
+
readTaskRunEpochState,
|
|
79
|
+
recordHostEvent,
|
|
80
|
+
recordTaskDown,
|
|
81
|
+
recordTaskEnvironmentPrepared,
|
|
82
|
+
recordTaskError,
|
|
83
|
+
recordTaskOwner,
|
|
84
|
+
recordTaskProjectEnvSpec,
|
|
85
|
+
recordTaskRuntimeEstablished,
|
|
86
|
+
recordTaskSharedFilesMode,
|
|
87
|
+
recordTaskStarting,
|
|
88
|
+
recordTaskUpResult,
|
|
89
|
+
taskProjectEnvSpec,
|
|
90
|
+
purgeHostTaskState,
|
|
91
|
+
upsertHostTask,
|
|
92
|
+
} from "../lib/runtime-state";
|
|
93
|
+
import {
|
|
94
|
+
clearTaskGatewayAcl,
|
|
95
|
+
clearTaskGatewayAclStrict,
|
|
96
|
+
waitForMcpGatewayReady,
|
|
97
|
+
} from "../lib/mcp-gateway";
|
|
98
|
+
import {
|
|
99
|
+
recoverTaskGatewayTask,
|
|
100
|
+
registerTaskGatewayTask,
|
|
101
|
+
reactivateTaskGatewayTask,
|
|
102
|
+
removeTaskGatewayTask,
|
|
103
|
+
suspendTaskGatewayTask,
|
|
104
|
+
taskGatewayForwarderSecretInput,
|
|
105
|
+
} from "../lib/task-gateway-state";
|
|
106
|
+
import { captureTaskEnvSchemas, forgetTaskEnvSchemaObservations } from "../lib/project-env-schema-capture";
|
|
107
|
+
import { forgetTaskEnvSnapshot, taskEnvResolver, taskEnvWithheldNoteNow } from "../lib/task-env-source";
|
|
108
|
+
import { activeSecretsProxy } from "../lib/secrets-proxy/host";
|
|
109
|
+
import { secretsProxyEnabled } from "../lib/secrets-proxy/runtime";
|
|
110
|
+
import { forgetTaskProxyReadiness } from "../lib/secrets-proxy/readiness";
|
|
111
|
+
import { admitFreshTaskView } from "../lib/secrets-proxy/sidecar-view";
|
|
112
|
+
import {
|
|
113
|
+
invalidateTaskWrites,
|
|
114
|
+
reconcileTaskProxyState,
|
|
115
|
+
} from "../lib/secrets-proxy/task-trust";
|
|
116
|
+
import { parsePreviewPortRuntimes } from "../lib/preview-ports";
|
|
117
|
+
import { dockerCli } from "../lib/docker-exec";
|
|
118
|
+
import {
|
|
119
|
+
claimContainerRuntimeForTask,
|
|
120
|
+
containerRuntimeGenerationOperational,
|
|
121
|
+
containerRuntimeProblem,
|
|
122
|
+
containerRuntimeReadinessEpoch,
|
|
123
|
+
containerRuntimeTeardownProblem,
|
|
124
|
+
ensureContainerRuntimeForTask,
|
|
125
|
+
initializeContainerRuntime,
|
|
126
|
+
pinnedContainerRuntimeProvider,
|
|
127
|
+
reprobeContainerRuntimeMachineIdentity,
|
|
128
|
+
suspendContainerRuntime,
|
|
129
|
+
waitForContainerRuntimeOperational,
|
|
130
|
+
} from "../lib/container-runtime";
|
|
131
|
+
import { vnodeAdmissionProblem } from "../lib/vnode-pressure";
|
|
132
|
+
import { reclaimUnusedUaiComposeNetworks } from "../lib/compose-network-recovery";
|
|
133
|
+
import {
|
|
134
|
+
machineTasksEnabled,
|
|
135
|
+
provisionTaskEnvironment,
|
|
136
|
+
reconstructPersistedTaskEnvironment,
|
|
137
|
+
} from "../lib/task-environment";
|
|
138
|
+
import { appleTaskContainerName } from "../lib/task-environment/apple-container";
|
|
139
|
+
import type {
|
|
140
|
+
TaskEnvironmentHandle,
|
|
141
|
+
TaskEnvironmentLocator,
|
|
142
|
+
TaskEnvironmentStatus,
|
|
143
|
+
} from "../lib/task-environment/types";
|
|
144
|
+
import {
|
|
145
|
+
ensurePreviewSidecar,
|
|
146
|
+
stopPreviewSidecars,
|
|
147
|
+
} from "../lib/preview-sidecar";
|
|
148
|
+
import { inspectTunnelContainer } from "../lib/tunnel-runtime";
|
|
149
|
+
import { ContainerRuntimeUnavailableError } from "../lib/runtime-guard";
|
|
150
|
+
import {
|
|
151
|
+
managedRuntimeAdmissionBlocker,
|
|
152
|
+
reclaimAbandonedMaintenanceCheck,
|
|
153
|
+
} from "../lib/managed-runtime";
|
|
154
|
+
import { retireTaskSshIdentityEnsures } from "../lib/git-identity";
|
|
155
|
+
import { removeTaskIdentityStrict } from "../lib/ssh";
|
|
156
|
+
import { proveOrphanTaskResourcesAbsent } from "../lib/task-inventory";
|
|
157
|
+
import {
|
|
158
|
+
HostErrorCode,
|
|
159
|
+
type CommandContext,
|
|
160
|
+
type ChannelEnsureInput,
|
|
161
|
+
type HostCommandResult,
|
|
162
|
+
type HostCommands,
|
|
163
|
+
type HostEventStream,
|
|
164
|
+
type PermissionDecision,
|
|
165
|
+
type TaskDiffInput,
|
|
166
|
+
type TaskDownInput,
|
|
167
|
+
type TaskLaunchInput,
|
|
168
|
+
} from "./protocol";
|
|
169
|
+
|
|
170
|
+
export type {
|
|
171
|
+
TaskDownResult,
|
|
172
|
+
TaskStatusResult,
|
|
173
|
+
TaskUpResult,
|
|
174
|
+
} from "../lib/agent";
|
|
175
|
+
|
|
176
|
+
import {
|
|
177
|
+
removeEngineAccount,
|
|
178
|
+
renameEngineAccount,
|
|
179
|
+
reorderEngineAccounts,
|
|
180
|
+
} from "../lib/engine-accounts";
|
|
181
|
+
import { disconnectEngine } from "../lib/engines";
|
|
182
|
+
|
|
183
|
+
export {
|
|
184
|
+
parseMentions,
|
|
185
|
+
resolveAddressing,
|
|
186
|
+
buildSystemPreamble,
|
|
187
|
+
} from "../lib/orchestrator";
|
|
188
|
+
|
|
189
|
+
export {
|
|
190
|
+
AgentKind,
|
|
191
|
+
Roster,
|
|
192
|
+
RosterAgent,
|
|
193
|
+
parseRoster,
|
|
194
|
+
} from "../lib/agents/types";
|
|
195
|
+
export type {
|
|
196
|
+
AgentEvent,
|
|
197
|
+
AgentEventHandler,
|
|
198
|
+
AgentSession,
|
|
199
|
+
AgentSessionFactory,
|
|
200
|
+
} from "../lib/agents/types";
|
|
201
|
+
|
|
202
|
+
export {
|
|
203
|
+
capabilities as agentCapabilities,
|
|
204
|
+
factoryFor as agentFactoryFor,
|
|
205
|
+
list as listAgentAdapters,
|
|
206
|
+
register as registerAgentAdapter,
|
|
207
|
+
supportsExecutionProfile,
|
|
208
|
+
} from "../lib/agents/registry";
|
|
209
|
+
export type {
|
|
210
|
+
AgentKindCapability,
|
|
211
|
+
RegisteredAdapter,
|
|
212
|
+
} from "../lib/agents/registry";
|
|
213
|
+
|
|
214
|
+
export {
|
|
215
|
+
ASDF_DATA_VOLUME,
|
|
216
|
+
STANDARD_IMAGE_TAG,
|
|
217
|
+
STANDARD_RUNTIMES,
|
|
218
|
+
ensureStandardImage,
|
|
219
|
+
standardRuntimes,
|
|
220
|
+
} from "../lib/standard-image";
|
|
221
|
+
|
|
222
|
+
export { cloneRepo } from "../lib/repo-clone";
|
|
223
|
+
export type { CloneRepoInput, CloneRepoResult } from "../lib/repo-clone";
|
|
224
|
+
|
|
225
|
+
export { parseGitDiff } from "../lib/git-diff";
|
|
226
|
+
export type { DiffFile, DiffHunk, DiffLine, DiffStatus } from "../lib/git-diff";
|
|
227
|
+
|
|
228
|
+
export {
|
|
229
|
+
PREVIEW_PORT_NAME_RE,
|
|
230
|
+
PreviewPortDeclarationSchema,
|
|
231
|
+
PreviewPortDeclarationsSchema,
|
|
232
|
+
PreviewPortRuntimeSchema,
|
|
233
|
+
PreviewPortRuntimesSchema,
|
|
234
|
+
parsePreviewPortDeclarations,
|
|
235
|
+
parsePreviewPortRuntimes,
|
|
236
|
+
} from "../lib/preview-ports";
|
|
237
|
+
export type {
|
|
238
|
+
PreviewPortDeclaration,
|
|
239
|
+
PreviewPortRuntime,
|
|
240
|
+
} from "../lib/preview-ports";
|
|
241
|
+
import { isEnvironmentTransportProvider } from "../lib/task-environment/types";
|
|
242
|
+
import { awaitSidecarReady, sidecarMode } from "../lib/sidecar";
|
|
243
|
+
import { guardSidecarMachine } from "../lib/sidecar-guard";
|
|
244
|
+
import { projectEnvBindingCapability } from "../lib/project-env-feature";
|
|
245
|
+
import { awaitProjectEnvReplicaReady } from "../lib/project-env-replica";
|
|
246
|
+
|
|
247
|
+
export const hostEvents: HostEventStream = {
|
|
248
|
+
subscribe(handler) {
|
|
249
|
+
return getOrchestrator().subscribeHostEvents(handler);
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
/** Set by main.ts at boot: the drain-then-supervisor-respawn used for
|
|
254
|
+
* staged activation. A hook (not an import) because main imports this
|
|
255
|
+
* module. */
|
|
256
|
+
let managedHostRestartHook: (() => void) | null = null;
|
|
257
|
+
export function setManagedHostRestartHook(hook: () => void): void {
|
|
258
|
+
managedHostRestartHook = hook;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* ADR-133: a task machine's runner is pinned by the cloud at launch and its
|
|
263
|
+
* unit's cgroup holds the task itself — the S3 mounts, the display, the
|
|
264
|
+
* editor, the agent sessions. An in-place update or restart replaces
|
|
265
|
+
* nothing (the prepare step reinstalls the pin) and killed all of those
|
|
266
|
+
* (live 2026-09-10, smoke 10). Both verbs are refused here; a new task gets
|
|
267
|
+
* a fresh machine with the current runner.
|
|
268
|
+
*/
|
|
269
|
+
function sidecarRunnerIsPinned(): HostCommandResult<never> | null {
|
|
270
|
+
if (!sidecarMode()) return null;
|
|
271
|
+
return {
|
|
272
|
+
ok: false,
|
|
273
|
+
code: HostErrorCode.Internal,
|
|
274
|
+
message:
|
|
275
|
+
"a task machine's station is pinned for the task's life — it neither updates nor restarts in place; close the task and start another for the current station",
|
|
276
|
+
retryable: false,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
type TaskUpCommand = HostCommands["taskUp"];
|
|
281
|
+
type TaskUpOutcome = Awaited<ReturnType<TaskUpCommand>>;
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* ADR-154: a start on cloud credentials, as task-up carries it. It begins in
|
|
285
|
+
* the short admission slot (the run's startup; every older start of the task
|
|
286
|
+
* interrupted), is awaited outside every slot, holds its installed grant
|
|
287
|
+
* through the long slot (the clone, provisioning and the runtime-ready
|
|
288
|
+
* writes), publishes its success only while it is still the task's current
|
|
289
|
+
* start with its credentials open, and is settled after: ended, or abandoned
|
|
290
|
+
* with its hold let go first.
|
|
291
|
+
*/
|
|
292
|
+
interface CloudCredentialTaskStart {
|
|
293
|
+
readonly runtime: CloudCredentialRuntime;
|
|
294
|
+
readonly assignment: TaskAssignment;
|
|
295
|
+
start: TaskCredentialStart | null;
|
|
296
|
+
hold: TaskUpHold | null;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** The start's lease for task-up: its whole grant's hold, and what the
|
|
300
|
+
* runner writes into the runtime once it exists (the owner's GitHub sign-in
|
|
301
|
+
* and SSH key, recorded for cleanup) before anything in it runs. */
|
|
302
|
+
function leasedTaskUp(
|
|
303
|
+
runtime: CloudCredentialRuntime,
|
|
304
|
+
start: TaskCredentialStart,
|
|
305
|
+
hold: TaskUpHold,
|
|
306
|
+
filesMode: SharedFilesMode,
|
|
307
|
+
): LeasedTaskUp {
|
|
308
|
+
const filesV1 = runtime.taskFilesV1();
|
|
309
|
+
return {
|
|
310
|
+
// The whole grant's signal: its gate closing, a cleanup reaching it or
|
|
311
|
+
// the start being interrupted all end task-up.
|
|
312
|
+
clone: { ...hold.git, signal: hold.signal },
|
|
313
|
+
runtimeReady: async (environment, signal) => {
|
|
314
|
+
if (filesV1) {
|
|
315
|
+
await runtime.prepareTaskFiles(
|
|
316
|
+
start.assignment,
|
|
317
|
+
environment as TaskEnvironmentHandle<unknown>,
|
|
318
|
+
filesMode,
|
|
319
|
+
signal,
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
await writeTaskPersonCredentials(environment, start.assignment, undefined, signal);
|
|
323
|
+
},
|
|
324
|
+
...(filesV1 ? { filesV1: true as const } : {}),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** `wait`, given up (rejecting) as soon as the runner begins leaving its own
|
|
329
|
+
* credentials, when what it waits for may never come. */
|
|
330
|
+
function untilLeavingLegacy<T>(legacy: LegacyStartPermit | null, wait: Promise<T>): Promise<T> {
|
|
331
|
+
if (legacy === null) return wait;
|
|
332
|
+
const signal = legacy.signal;
|
|
333
|
+
return new Promise<T>((resolve, reject) => {
|
|
334
|
+
const onAbort = (): void => reject(signal.reason);
|
|
335
|
+
if (signal.aborted) onAbort();
|
|
336
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
337
|
+
wait.then(
|
|
338
|
+
(value) => {
|
|
339
|
+
signal.removeEventListener("abort", onAbort);
|
|
340
|
+
resolve(value);
|
|
341
|
+
},
|
|
342
|
+
(error: unknown) => {
|
|
343
|
+
signal.removeEventListener("abort", onAbort);
|
|
344
|
+
reject(error);
|
|
345
|
+
},
|
|
346
|
+
);
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** A start the runner does not take while it switches its tasks to cloud
|
|
351
|
+
* credentials: the cloud starts it again shortly. */
|
|
352
|
+
function switchingToCloudCredentials(): HostCommandResult<never> {
|
|
353
|
+
return {
|
|
354
|
+
ok: false,
|
|
355
|
+
code: HostErrorCode.HostUnavailable,
|
|
356
|
+
message: "This host is switching its tasks to cloud credentials; the task starts once it is done.",
|
|
357
|
+
retryable: true,
|
|
358
|
+
retryReason: "managed_restart",
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** What a start reports when Stop, teardown, a newer run or shutdown
|
|
363
|
+
* interrupted it, in place of its own outcome; null when none did. The
|
|
364
|
+
* interruption that won decides: shutdown asks the cloud to start the
|
|
365
|
+
* task again once the runner is back, anything else leaves a run that is
|
|
366
|
+
* no longer current. */
|
|
367
|
+
function interruptedStartResult(cloud: CloudCredentialTaskStart | null): TaskUpOutcome | null {
|
|
368
|
+
if (!cloud?.start?.signal.aborted) return null;
|
|
369
|
+
const reason: unknown = cloud.start.signal.reason;
|
|
370
|
+
if (reason instanceof StartInterruptedError && reason.by === "shutdown") {
|
|
371
|
+
return {
|
|
372
|
+
ok: false,
|
|
373
|
+
code: HostErrorCode.HostUnavailable,
|
|
374
|
+
message: "This host is restarting; the task starts once it is back.",
|
|
375
|
+
retryable: true,
|
|
376
|
+
retryReason: "managed_restart",
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
return {
|
|
380
|
+
ok: false,
|
|
381
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
382
|
+
message: `task ${cloud.assignment.taskId} run ${String(cloud.assignment.runEpoch)} is no longer current: Stop, teardown or a newer run took over its start`,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** The start's credentials closed for its run without an interruption:
|
|
387
|
+
* removed, refused or replaced in the cloud. */
|
|
388
|
+
function withdrawnStart(): TaskUpOutcome {
|
|
389
|
+
return { ok: false, code: HostErrorCode.Internal, message: CREDENTIALS_WITHDRAWN_MESSAGE };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** The last word before a start publishes its success: null while it may.
|
|
393
|
+
* Otherwise what it reports instead (`interrupted` when Stop, teardown, a
|
|
394
|
+
* newer run or shutdown took over): its interruption's result, or a
|
|
395
|
+
* withdrawn start's failure once its credentials are no longer open for
|
|
396
|
+
* its run. */
|
|
397
|
+
function cloudStartCannotPublish(
|
|
398
|
+
cloud: CloudCredentialTaskStart | null,
|
|
399
|
+
): { result: TaskUpOutcome; interrupted: boolean } | null {
|
|
400
|
+
if (!cloud?.start) return null;
|
|
401
|
+
const interrupted = interruptedStartResult(cloud);
|
|
402
|
+
if (interrupted) return { result: interrupted, interrupted: true };
|
|
403
|
+
try {
|
|
404
|
+
cloud.start.assertCurrent();
|
|
405
|
+
return null;
|
|
406
|
+
} catch {
|
|
407
|
+
return { result: withdrawnStart(), interrupted: false };
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** The long slot's operation, the start's hold let go as it ends: a hold
|
|
412
|
+
* never outlives the slot it was taken in, so a Stop or teardown queued
|
|
413
|
+
* behind it never waits on it. */
|
|
414
|
+
function releasingHold(
|
|
415
|
+
cloud: CloudCredentialTaskStart | null,
|
|
416
|
+
operation: () => Promise<TaskUpOutcome>,
|
|
417
|
+
): () => Promise<TaskUpOutcome> {
|
|
418
|
+
return async () => {
|
|
419
|
+
try {
|
|
420
|
+
return await operation();
|
|
421
|
+
} finally {
|
|
422
|
+
cloud?.hold?.release();
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
async function taskUpOn(
|
|
428
|
+
ctx: Parameters<TaskUpCommand>[0],
|
|
429
|
+
input: Parameters<TaskUpCommand>[1],
|
|
430
|
+
cloud: CloudCredentialTaskStart | null,
|
|
431
|
+
legacy: LegacyStartPermit | null,
|
|
432
|
+
): ReturnType<TaskUpCommand> {
|
|
433
|
+
const orchestrator = getOrchestrator();
|
|
434
|
+
let runEpochAdmission: ReturnType<typeof admitTaskRunEpoch>;
|
|
435
|
+
try {
|
|
436
|
+
if (input.task.runEpoch === undefined) {
|
|
437
|
+
// ADR-154: a start on the runner's own credentials only while the
|
|
438
|
+
// runner is still on them.
|
|
439
|
+
if (legacy?.valid() === false) return switchingToCloudCredentials();
|
|
440
|
+
// Preserve the v0 path exactly during the rolling window. It has no
|
|
441
|
+
// assignment floor to serialize and must keep failing cheap host
|
|
442
|
+
// preflights before entering the runtime lifecycle queue.
|
|
443
|
+
runEpochAdmission = admitTaskRunEpoch(input.task.id, undefined);
|
|
444
|
+
} else {
|
|
445
|
+
const admitted = await orchestrator.runTaskLifecycle(
|
|
446
|
+
input.task.id,
|
|
447
|
+
async () => {
|
|
448
|
+
// ADR-154: a start on the runner's own credentials is admitted
|
|
449
|
+
// only while the runner is still on them.
|
|
450
|
+
if (legacy?.valid() === false) {
|
|
451
|
+
return { ok: false as const, result: switchingToCloudCredentials() };
|
|
452
|
+
}
|
|
453
|
+
const admission = admitTaskRunEpoch(
|
|
454
|
+
input.task.id,
|
|
455
|
+
input.task.runEpoch,
|
|
456
|
+
);
|
|
457
|
+
if (admission.kind === "stale") {
|
|
458
|
+
return {
|
|
459
|
+
ok: false as const,
|
|
460
|
+
result: {
|
|
461
|
+
ok: false as const,
|
|
462
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
463
|
+
message:
|
|
464
|
+
`task ${input.task.id} run ${String(input.task.runEpoch)} is stale; ` +
|
|
465
|
+
`this station has already admitted run ${String(admission.currentRunEpoch)}`,
|
|
466
|
+
},
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
// ADR-154: the run's credential startup begins in this short slot;
|
|
470
|
+
// its predecessor's cleanup and its grant are awaited outside it.
|
|
471
|
+
if (cloud !== null) {
|
|
472
|
+
try {
|
|
473
|
+
cloud.start = cloud.runtime.beginStart(cloud.assignment);
|
|
474
|
+
} catch (error) {
|
|
475
|
+
return {
|
|
476
|
+
ok: false as const,
|
|
477
|
+
result: {
|
|
478
|
+
ok: false as const,
|
|
479
|
+
code: HostErrorCode.HostUnavailable,
|
|
480
|
+
message: `task ${input.task.id} could not begin on its cloud credentials: ${
|
|
481
|
+
error instanceof Error ? error.message : String(error)
|
|
482
|
+
}`,
|
|
483
|
+
retryable: true,
|
|
484
|
+
...(error instanceof StartInterruptedError ? { retryReason: "managed_restart" as const } : {}),
|
|
485
|
+
},
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
return { ok: true as const, admission };
|
|
490
|
+
},
|
|
491
|
+
);
|
|
492
|
+
if (!admitted.ok) return admitted.result;
|
|
493
|
+
runEpochAdmission = admitted.admission;
|
|
494
|
+
}
|
|
495
|
+
} catch (error) {
|
|
496
|
+
return {
|
|
497
|
+
ok: false,
|
|
498
|
+
code: HostErrorCode.Internal,
|
|
499
|
+
message:
|
|
500
|
+
error instanceof Error
|
|
501
|
+
? error.message
|
|
502
|
+
: "the task run epoch is invalid",
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
// Admission alone never proves a runtime belongs to this assignment. A
|
|
506
|
+
// failed prior start can leave an open floor beside the older runtime.
|
|
507
|
+
const mayReuseRuntime = canReuseTaskRuntimeForRun(
|
|
508
|
+
input.task.id,
|
|
509
|
+
input.task.runEpoch,
|
|
510
|
+
);
|
|
511
|
+
const admissionFailure = managedMaintenanceUnavailable();
|
|
512
|
+
if (admissionFailure) return admissionFailure;
|
|
513
|
+
const runtimePreparationFailure = await prepareContainerRuntimeForTask();
|
|
514
|
+
if (runtimePreparationFailure) return runtimePreparationFailure;
|
|
515
|
+
const runtimeFailure = runtimeUnavailable();
|
|
516
|
+
if (runtimeFailure) return runtimeFailure;
|
|
517
|
+
// The boot reconcile briefly replaces CLI shims in the shared volume.
|
|
518
|
+
// Do not create local lifecycle/channel state until its one-way latch has
|
|
519
|
+
// settled (success or best-effort failure), then revalidate both mutable
|
|
520
|
+
// admission authorities after the await.
|
|
521
|
+
await agentClisReady;
|
|
522
|
+
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
523
|
+
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
524
|
+
const readyRuntimeFailure = runtimeUnavailable();
|
|
525
|
+
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
526
|
+
try {
|
|
527
|
+
await waitForMcpGatewayReady();
|
|
528
|
+
} catch (error) {
|
|
529
|
+
console.warn(
|
|
530
|
+
`[task-gateway] refusing task ${input.task.id} before provisioning: ${
|
|
531
|
+
error instanceof Error ? error.message : String(error)
|
|
532
|
+
}`,
|
|
533
|
+
);
|
|
534
|
+
return {
|
|
535
|
+
ok: false,
|
|
536
|
+
code: HostErrorCode.HostUnavailable,
|
|
537
|
+
message:
|
|
538
|
+
"The task environment gateway is not ready on this host. Retry the task; if it persists, check the host service log.",
|
|
539
|
+
retryable: true,
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
// A task machine's own guards first: its runner's data directory root's
|
|
543
|
+
// alone, and the metadata service closed to the task's user. Without
|
|
544
|
+
// them the task does not start: it could read the sidecar's host token.
|
|
545
|
+
if (sidecarMode()) {
|
|
546
|
+
const guard = guardSidecarMachine({ dataDir: env.dataDir });
|
|
547
|
+
if (!guard.ok) {
|
|
548
|
+
console.warn(`[sidecar] refusing task ${input.task.id}: ${guard.reason}`);
|
|
549
|
+
return {
|
|
550
|
+
ok: false,
|
|
551
|
+
code: HostErrorCode.Internal,
|
|
552
|
+
message: `This task machine is not safe to run the task on: ${guard.reason}. Close the task and start it again on a new machine.`,
|
|
553
|
+
retryable: false,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
// A bindings-capable task sidecar must activate and commit its exact
|
|
558
|
+
// replica before any task state, /env generation or process can exist.
|
|
559
|
+
// Otherwise the source's intentional pre-activation legacy fallback
|
|
560
|
+
// could expose stored plaintext during the first reconciliation.
|
|
561
|
+
if (sidecarMode()) {
|
|
562
|
+
try {
|
|
563
|
+
// ADR-154: on cloud credentials the start's own grant is what readies
|
|
564
|
+
// it (awaited below); the v0 lease that readies the machine's stores
|
|
565
|
+
// never runs there.
|
|
566
|
+
if (cloud === null) await untilLeavingLegacy(legacy, awaitSidecarReady());
|
|
567
|
+
// ADR-154: and its env is the composer's, never the machine's replica.
|
|
568
|
+
if (cloud === null && projectEnvBindingCapability() !== null) {
|
|
569
|
+
await untilLeavingLegacy(legacy, awaitProjectEnvReplicaReady());
|
|
570
|
+
}
|
|
571
|
+
} catch (error) {
|
|
572
|
+
// ADR-154: the runner began leaving its own credentials, and with them
|
|
573
|
+
// the v0 lease this readiness comes from: the cloud starts the task
|
|
574
|
+
// again on cloud credentials.
|
|
575
|
+
if (legacy?.signal.aborted) return switchingToCloudCredentials();
|
|
576
|
+
console.warn(
|
|
577
|
+
`[task-env] refusing task ${input.task.id} before provisioning: ${
|
|
578
|
+
error instanceof Error ? error.message : String(error)
|
|
579
|
+
}`,
|
|
580
|
+
);
|
|
581
|
+
return {
|
|
582
|
+
ok: false,
|
|
583
|
+
code: HostErrorCode.HostUnavailable,
|
|
584
|
+
message:
|
|
585
|
+
"This task machine is still synchronizing its environment. Retry shortly.",
|
|
586
|
+
retryable: true,
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
// ADR-118 follow-up: each apple-container task is its own VM, and each
|
|
591
|
+
// VM pins host vnodes for its life. Refuse to start ANOTHER one when the
|
|
592
|
+
// table is nearly full — a clear, retryable refusal now beats ENFILE
|
|
593
|
+
// storms (and the suspected panic path) for the whole machine later.
|
|
594
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
595
|
+
const vnodeProblem = await vnodeAdmissionProblem();
|
|
596
|
+
if (vnodeProblem) {
|
|
597
|
+
return {
|
|
598
|
+
ok: false,
|
|
599
|
+
code: HostErrorCode.HostUnavailable,
|
|
600
|
+
message: vnodeProblem,
|
|
601
|
+
retryable: true,
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
// ADR-154: the grant installed and its gate open, its env snapshot exact:
|
|
606
|
+
// awaited outside every lifecycle slot, and bounded.
|
|
607
|
+
if (cloud?.start) {
|
|
608
|
+
try {
|
|
609
|
+
await cloud.runtime.waitForLaunchReady(cloud.start);
|
|
610
|
+
} catch (error) {
|
|
611
|
+
return (
|
|
612
|
+
interruptedStartResult(cloud) ?? {
|
|
613
|
+
ok: false,
|
|
614
|
+
code: HostErrorCode.HostUnavailable,
|
|
615
|
+
message: `The task's cloud credentials did not become usable (${
|
|
616
|
+
error instanceof Error ? error.message : String(error)
|
|
617
|
+
}). Retry the task.`,
|
|
618
|
+
retryable: true,
|
|
619
|
+
}
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
// Every local Docker task consumes one daemon address-pool subnet. A task
|
|
624
|
+
// that failed after Compose created its network can retain an empty subnet
|
|
625
|
+
// while it waits for Resume. Reclaim only networks whose trusted host row,
|
|
626
|
+
// exact Docker identity and two empty-container inventories all agree. The
|
|
627
|
+
// sweep runs before this task's lifecycle slot and never waits on another
|
|
628
|
+
// task's: a candidate whose slot is busy is skipped, so concurrent starts
|
|
629
|
+
// neither deadlock nor queue behind one another's provisioning.
|
|
630
|
+
if (
|
|
631
|
+
!sidecarMode() &&
|
|
632
|
+
!machineTasksEnabled() &&
|
|
633
|
+
pinnedContainerRuntimeProvider() === "docker"
|
|
634
|
+
) {
|
|
635
|
+
try {
|
|
636
|
+
const networkRecovery = await reclaimUnusedUaiComposeNetworks(
|
|
637
|
+
orchestrator,
|
|
638
|
+
);
|
|
639
|
+
if (networkRecovery.removed > 0) {
|
|
640
|
+
console.warn(
|
|
641
|
+
`[host-agent] reclaimed ${String(networkRecovery.removed)} empty UAI task ` +
|
|
642
|
+
`network(s) before starting ${input.task.id}`,
|
|
643
|
+
);
|
|
644
|
+
} else if (networkRecovery.inventory === "invalid") {
|
|
645
|
+
console.warn(
|
|
646
|
+
"[host-agent] skipped task-network recovery because Docker returned an invalid network inventory",
|
|
647
|
+
);
|
|
648
|
+
}
|
|
649
|
+
if (networkRecovery.busy > 0) {
|
|
650
|
+
console.log(
|
|
651
|
+
`[host-agent] task-network recovery left ${String(networkRecovery.busy)} ` +
|
|
652
|
+
`network(s) alone while their task was busy`,
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
} catch (error) {
|
|
656
|
+
// Best-effort capacity recovery must not block a task when Docker still
|
|
657
|
+
// has room. `docker compose up` remains the authoritative allocation.
|
|
658
|
+
console.warn(
|
|
659
|
+
`[host-agent] task-network recovery was unavailable: ${
|
|
660
|
+
error instanceof Error ? error.message : String(error)
|
|
661
|
+
}`,
|
|
662
|
+
);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
return orchestrator.runTaskLifecycle(input.task.id, releasingHold(cloud, async () => {
|
|
666
|
+
// A direct duplicate teardown may still be draining operations that
|
|
667
|
+
// target this task's stable compose/container names. Never recreate
|
|
668
|
+
// those names until every close caller has crossed the shared fence.
|
|
669
|
+
await orchestrator.waitForChannelClose(input.task.id);
|
|
670
|
+
// Admission is a short lifecycle transaction before the long host
|
|
671
|
+
// preflights above. A newer dispatch may have advanced the durable
|
|
672
|
+
// floor while this command was waiting; once this command owns the
|
|
673
|
+
// lifecycle slot, reject it before it can observe or mutate a newer
|
|
674
|
+
// assignment's runtime.
|
|
675
|
+
const assignmentState = readTaskRunEpochState(input.task.id);
|
|
676
|
+
const assignmentCurrent =
|
|
677
|
+
input.task.runEpoch === undefined
|
|
678
|
+
? assignmentState === null
|
|
679
|
+
: assignmentState !== null &&
|
|
680
|
+
assignmentState.assignmentOpen &&
|
|
681
|
+
assignmentState.highestRunEpoch === input.task.runEpoch;
|
|
682
|
+
if (!assignmentCurrent) {
|
|
683
|
+
const currentRunEpoch = assignmentState?.highestRunEpoch ?? 0;
|
|
684
|
+
return {
|
|
685
|
+
ok: false,
|
|
686
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
687
|
+
message:
|
|
688
|
+
`task ${input.task.id} run ${String(input.task.runEpoch)} is no longer current; ` +
|
|
689
|
+
`this station has admitted run ${String(currentRunEpoch)}`,
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
const inSlotAdmissionFailure = managedMaintenanceUnavailable();
|
|
693
|
+
if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
|
|
694
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
695
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
696
|
+
// ADR-154: nothing of a start on the runner's own credentials happens
|
|
697
|
+
// once the runner has begun leaving them.
|
|
698
|
+
if (legacy?.valid() === false) return switchingToCloudCredentials();
|
|
699
|
+
// ADR-154: the start holds its installed grant from here through the
|
|
700
|
+
// clone, provisioning and the runtime-ready writes; without it nothing
|
|
701
|
+
// starts.
|
|
702
|
+
if (cloud?.start) {
|
|
703
|
+
try {
|
|
704
|
+
cloud.hold = cloud.runtime.tryAcquireTaskUpHold(cloud.start);
|
|
705
|
+
} catch (error) {
|
|
706
|
+
return {
|
|
707
|
+
ok: false,
|
|
708
|
+
code: HostErrorCode.HostUnavailable,
|
|
709
|
+
message: `The task's lease names credentials that could not be read (${
|
|
710
|
+
error instanceof Error ? error.message : String(error)
|
|
711
|
+
}). Retry the task.`,
|
|
712
|
+
retryable: true,
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
if (cloud.hold === null) return interruptedStartResult(cloud) ?? withdrawnStart();
|
|
716
|
+
}
|
|
717
|
+
// A command retry that arrives after the original start completed must
|
|
718
|
+
// not rerender/compose-up/uai-init underneath live agent sessions. Treat
|
|
719
|
+
// that lost-response retry as the successful operation it already was:
|
|
720
|
+
// every cloud caller records a non-ok taskUp as an errored task.
|
|
721
|
+
const existingTask = getHostTask(input.task.id);
|
|
722
|
+
const incomingEnvSpec = taskProjectEnvSpec(input);
|
|
723
|
+
let existingGatewayReady = true;
|
|
724
|
+
if (existingTask && hostTaskEnvMode(existingTask) === "loader") {
|
|
725
|
+
// A lost-response retry is also the repair path after a host process
|
|
726
|
+
// restart: reattach the live resolver before any task process can be
|
|
727
|
+
// treated as reusable. Missing/corrupt durable identity forces normal
|
|
728
|
+
// recreation, which mints a generation and restarts its forwarder.
|
|
729
|
+
recordTaskProjectEnvSpec(input.task.id, input);
|
|
730
|
+
existingGatewayReady =
|
|
731
|
+
recoverTaskGatewayTask(input.task.id, {
|
|
732
|
+
// The generation serves the run its runtime was established for.
|
|
733
|
+
resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, existingTask.runEpoch > 0 ? existingTask.runEpoch : null),
|
|
734
|
+
runEpoch: existingTask.runEpoch > 0 ? existingTask.runEpoch : null,
|
|
735
|
+
}) !== null;
|
|
736
|
+
}
|
|
737
|
+
let runtimeQuarantinedForRecreate = false;
|
|
738
|
+
let channelClosedForRecreate = false;
|
|
739
|
+
if (
|
|
740
|
+
existingTask?.statusMirror === "running" &&
|
|
741
|
+
existingTask.environmentProvider === "apple-container"
|
|
742
|
+
) {
|
|
743
|
+
// ADR-106/round-4: the lost-response fast path must speak the
|
|
744
|
+
// provider's language — the Docker branch below ENOENTs on a
|
|
745
|
+
// Docker-free Mac and suspended the generation instead of returning
|
|
746
|
+
// the persisted success.
|
|
747
|
+
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
748
|
+
return {
|
|
749
|
+
ok: false,
|
|
750
|
+
code: HostErrorCode.Internal,
|
|
751
|
+
message:
|
|
752
|
+
`task ${input.task.id} is running but its persisted runtime ` +
|
|
753
|
+
"metadata is incomplete",
|
|
754
|
+
};
|
|
755
|
+
}
|
|
756
|
+
// Round 5: every await below can outlive the runtime generation
|
|
757
|
+
// (bridge revalidation, forced recheck). A superseded pass must
|
|
758
|
+
// observe only — never reopen consumers or mutate the container —
|
|
759
|
+
// the same discipline apple recovery applies.
|
|
760
|
+
// Round 6: the fence requires the OPERATIONAL generation, not merely
|
|
761
|
+
// the epoch — plain suspension (connection revalidation) keeps the
|
|
762
|
+
// epoch and ready status while the daemon is unverified.
|
|
763
|
+
const fastEpoch = containerRuntimeReadinessEpoch();
|
|
764
|
+
const fastStale = (): boolean =>
|
|
765
|
+
!containerRuntimeGenerationOperational(fastEpoch);
|
|
766
|
+
const staleRetry = {
|
|
767
|
+
ok: false as const,
|
|
768
|
+
code: HostErrorCode.HostUnavailable,
|
|
769
|
+
message:
|
|
770
|
+
`the container runtime generation changed while verifying task ${input.task.id}; retry`,
|
|
771
|
+
retryable: true,
|
|
772
|
+
};
|
|
773
|
+
const environment = await reconstructPersistedTaskEnvironment(
|
|
774
|
+
existingTask,
|
|
775
|
+
);
|
|
776
|
+
if (fastStale()) return staleRetry;
|
|
777
|
+
if (environment === null) {
|
|
778
|
+
// The persisted locator no longer reconstructs (e.g. an engine
|
|
779
|
+
// identity change) — recreate through the normal path.
|
|
780
|
+
suspendTaskGatewayTask(input.task.id);
|
|
781
|
+
await orchestrator.closeChannel(input.task.id);
|
|
782
|
+
// Session drain can outlive the generation too: a superseded pass
|
|
783
|
+
// must not reach recordTaskStarting/provisioning below.
|
|
784
|
+
if (fastStale()) return staleRetry;
|
|
785
|
+
channelClosedForRecreate = true;
|
|
786
|
+
} else {
|
|
787
|
+
const appleStatus = await environment.status();
|
|
788
|
+
if (fastStale()) return staleRetry;
|
|
789
|
+
if (appleStatus.state === "unknown") {
|
|
790
|
+
return {
|
|
791
|
+
ok: false,
|
|
792
|
+
code: HostErrorCode.HostUnavailable,
|
|
793
|
+
message:
|
|
794
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
795
|
+
appleStatus.detail,
|
|
796
|
+
retryable: true,
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
if (appleStatus.state === "running") {
|
|
800
|
+
const verdict = await proveAppleTaskRuntimeContract({
|
|
801
|
+
taskId: input.task.id,
|
|
802
|
+
containerName: appleTaskContainerName(input.task.id),
|
|
803
|
+
// The row's worktreePath is task-up's $task_dir — the same
|
|
804
|
+
// directory the apple locator records as hostWorktreePath.
|
|
805
|
+
hostWorktreePath: existingTask.worktreePath,
|
|
806
|
+
});
|
|
807
|
+
if (fastStale()) return staleRetry;
|
|
808
|
+
if (verdict.kind === "unreachable") {
|
|
809
|
+
return {
|
|
810
|
+
ok: false,
|
|
811
|
+
code: HostErrorCode.HostUnavailable,
|
|
812
|
+
message:
|
|
813
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
814
|
+
verdict.detail,
|
|
815
|
+
retryable: true,
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
let projectsHealthy = true;
|
|
819
|
+
for (const project of verdict.kind === "holds"
|
|
820
|
+
? input.projects
|
|
821
|
+
: []) {
|
|
822
|
+
const gitHealth = await environment.exec({
|
|
823
|
+
projectEnvProfile: "maintenance",
|
|
824
|
+
argv: [
|
|
825
|
+
"/usr/bin/env",
|
|
826
|
+
"-i",
|
|
827
|
+
"HOME=/home/node",
|
|
828
|
+
"PATH=/usr/bin:/bin",
|
|
829
|
+
"/usr/bin/git",
|
|
830
|
+
"rev-parse",
|
|
831
|
+
"--is-inside-work-tree",
|
|
832
|
+
],
|
|
833
|
+
user: "node",
|
|
834
|
+
cwd: `/workspace/${project.slug}`,
|
|
835
|
+
env: {},
|
|
836
|
+
timeoutMs: 10_000,
|
|
837
|
+
maxOutputBytes: 64 * 1024,
|
|
838
|
+
});
|
|
839
|
+
if (fastStale()) return staleRetry;
|
|
840
|
+
if (gitHealth.exitCode === null || gitHealth.signal !== null) {
|
|
841
|
+
return {
|
|
842
|
+
ok: false,
|
|
843
|
+
code: HostErrorCode.HostUnavailable,
|
|
844
|
+
message:
|
|
845
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
846
|
+
`project ${project.id}: the runtime did not answer`,
|
|
847
|
+
retryable: true,
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
if (
|
|
851
|
+
gitHealth.exitCode !== 0 ||
|
|
852
|
+
Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
|
|
853
|
+
"true"
|
|
854
|
+
) {
|
|
855
|
+
projectsHealthy = false;
|
|
856
|
+
console.warn(
|
|
857
|
+
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
|
|
858
|
+
);
|
|
859
|
+
break;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
if (
|
|
863
|
+
verdict.kind === "holds" &&
|
|
864
|
+
projectsHealthy &&
|
|
865
|
+
existingGatewayReady &&
|
|
866
|
+
mayReuseRuntime
|
|
867
|
+
) {
|
|
868
|
+
if (fastStale()) return staleRetry;
|
|
869
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
870
|
+
if (cannotPublish) return cannotPublish.result;
|
|
871
|
+
orchestrator.allowChannel(input.task.id);
|
|
872
|
+
return {
|
|
873
|
+
ok: true,
|
|
874
|
+
value: {
|
|
875
|
+
composeProject: existingTask.composeProject,
|
|
876
|
+
worktreePath: existingTask.worktreePath,
|
|
877
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
878
|
+
previewPorts: parsePreviewPortRuntimes(
|
|
879
|
+
existingTask.previewPorts,
|
|
880
|
+
),
|
|
881
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
882
|
+
},
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
// Violated contract or broken project repo: drain, stop, and
|
|
886
|
+
// prove before task-up recreates the same stable name.
|
|
887
|
+
if (fastStale()) return staleRetry;
|
|
888
|
+
suspendTaskGatewayTask(input.task.id);
|
|
889
|
+
await orchestrator.closeChannel(input.task.id);
|
|
890
|
+
channelClosedForRecreate = true;
|
|
891
|
+
if (fastStale()) return staleRetry;
|
|
892
|
+
await environment.stop();
|
|
893
|
+
const after = await environment.status();
|
|
894
|
+
if (fastStale()) return staleRetry;
|
|
895
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
896
|
+
return {
|
|
897
|
+
ok: false,
|
|
898
|
+
code: HostErrorCode.HostUnavailable,
|
|
899
|
+
message:
|
|
900
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
901
|
+
"could not be proven stopped; retry after the runtime is healthy",
|
|
902
|
+
retryable: true,
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
// Round 5: the container is PHYSICALLY stopped now, whatever the
|
|
906
|
+
// reason (violated contract or a broken project repo). A failed
|
|
907
|
+
// recreation must roll the mirror back to stopped — restoring
|
|
908
|
+
// "running" would advertise a container proven not to run.
|
|
909
|
+
runtimeQuarantinedForRecreate = true;
|
|
910
|
+
} else {
|
|
911
|
+
// Provider proved the container absent/stopped: normal self-heal
|
|
912
|
+
// recreate — and the rollback state must say stopped for the
|
|
913
|
+
// same reason as above.
|
|
914
|
+
if (fastStale()) return staleRetry;
|
|
915
|
+
suspendTaskGatewayTask(input.task.id);
|
|
916
|
+
await orchestrator.closeChannel(input.task.id);
|
|
917
|
+
// Fence AFTER the drain as well: provisioning must never start
|
|
918
|
+
// under a generation this pass no longer owns.
|
|
919
|
+
if (fastStale()) return staleRetry;
|
|
920
|
+
channelClosedForRecreate = true;
|
|
921
|
+
runtimeQuarantinedForRecreate = true;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
} else if (
|
|
925
|
+
existingTask?.statusMirror === "running" &&
|
|
926
|
+
isEnvironmentTransportProvider(existingTask.environmentProvider)
|
|
927
|
+
) {
|
|
928
|
+
// ADR-121: a machine-backed task revalidates through its own
|
|
929
|
+
// provider — the compose fast path below speaks docker and would
|
|
930
|
+
// misread a machine row. Running machine → persisted success;
|
|
931
|
+
// anything else falls through to provisionTaskEnvironment, whose
|
|
932
|
+
// machine provider owns recovery semantics.
|
|
933
|
+
try {
|
|
934
|
+
const environment = await reconstructPersistedTaskEnvironment(
|
|
935
|
+
existingTask,
|
|
936
|
+
);
|
|
937
|
+
if (environment) {
|
|
938
|
+
const machineStatus = await environment.status();
|
|
939
|
+
if (
|
|
940
|
+
machineStatus.state === "running" &&
|
|
941
|
+
existingGatewayReady &&
|
|
942
|
+
mayReuseRuntime
|
|
943
|
+
) {
|
|
944
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
945
|
+
if (cannotPublish) return cannotPublish.result;
|
|
946
|
+
orchestrator.allowChannel(input.task.id);
|
|
947
|
+
return {
|
|
948
|
+
ok: true,
|
|
949
|
+
value: {
|
|
950
|
+
composeProject: existingTask.composeProject ?? "",
|
|
951
|
+
worktreePath: existingTask.worktreePath ?? "",
|
|
952
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
953
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
954
|
+
},
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
} catch (error) {
|
|
959
|
+
console.warn(
|
|
960
|
+
`[machine] task ${input.task.id}: running-row revalidation failed, continuing to provision: ${error instanceof Error ? error.message : String(error)}`,
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
} else if (existingTask?.statusMirror === "running") {
|
|
964
|
+
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
965
|
+
return {
|
|
966
|
+
ok: false,
|
|
967
|
+
code: HostErrorCode.Internal,
|
|
968
|
+
message:
|
|
969
|
+
`task ${input.task.id} is running but its persisted runtime ` +
|
|
970
|
+
"metadata is incomplete",
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
// The mirror may be stale after a manual `docker compose down`.
|
|
974
|
+
// Confirm the app itself is live before treating this as a lost-response
|
|
975
|
+
// retry; otherwise fall through to the normal taskUp self-heal.
|
|
976
|
+
const appRuntime = await dockerCli(
|
|
977
|
+
[
|
|
978
|
+
"ps",
|
|
979
|
+
"-a",
|
|
980
|
+
"--filter",
|
|
981
|
+
`label=com.docker.compose.project=${existingTask.composeProject}`,
|
|
982
|
+
"--filter",
|
|
983
|
+
"label=com.docker.compose.service=app",
|
|
984
|
+
"--format",
|
|
985
|
+
"{{.State}}",
|
|
986
|
+
],
|
|
987
|
+
{ timeoutMs: 10_000 },
|
|
988
|
+
);
|
|
989
|
+
if (appRuntime.status !== 0) {
|
|
990
|
+
return dockerUnavailableResult(
|
|
991
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
992
|
+
(appRuntime.stderr.trim().slice(0, 200) ||
|
|
993
|
+
"docker did not answer"),
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
if (
|
|
997
|
+
appRuntime.stdout
|
|
998
|
+
.split(/\r?\n/)
|
|
999
|
+
.some((state) => state.trim() === "running")
|
|
1000
|
+
) {
|
|
1001
|
+
const appContainer = `${existingTask.composeProject}-app-1`;
|
|
1002
|
+
const runtimeContract = await inspectTaskRuntimeContainer(
|
|
1003
|
+
input.task.id,
|
|
1004
|
+
existingTask.composeProject,
|
|
1005
|
+
);
|
|
1006
|
+
if (runtimeContract.kind === "unreachable") {
|
|
1007
|
+
return dockerUnavailableResult(
|
|
1008
|
+
`could not inspect the exact runtime mount for task ${input.task.id}`,
|
|
1009
|
+
);
|
|
1010
|
+
}
|
|
1011
|
+
if (runtimeContract.kind === "failed") {
|
|
1012
|
+
return {
|
|
1013
|
+
ok: false,
|
|
1014
|
+
code: HostErrorCode.HostUnavailable,
|
|
1015
|
+
message:
|
|
1016
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
1017
|
+
runtimeContract.detail,
|
|
1018
|
+
retryable: true,
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
const reusable =
|
|
1022
|
+
runtimeContract.running &&
|
|
1023
|
+
!runtimeContract.restarting &&
|
|
1024
|
+
!runtimeContract.paused &&
|
|
1025
|
+
runtimeContract.sharedMount === "read-only" &&
|
|
1026
|
+
runtimeContract.taskEnvironment === "current" &&
|
|
1027
|
+
existingGatewayReady &&
|
|
1028
|
+
mayReuseRuntime;
|
|
1029
|
+
if (!reusable) {
|
|
1030
|
+
// Drain/tombstone before touching the exact container. task-up's
|
|
1031
|
+
// pre-materialization cleanup then removes this stopped immutable
|
|
1032
|
+
// generation, so Compose must create the current RO/private-prefix
|
|
1033
|
+
// contract rather than `docker start`ing it again.
|
|
1034
|
+
suspendTaskGatewayTask(input.task.id);
|
|
1035
|
+
await orchestrator.closeChannel(input.task.id);
|
|
1036
|
+
channelClosedForRecreate = true;
|
|
1037
|
+
const quarantined = await quarantineTaskRuntimeContainer({
|
|
1038
|
+
taskId: input.task.id,
|
|
1039
|
+
composeProject: existingTask.composeProject,
|
|
1040
|
+
expectedContainerId: runtimeContract.containerId,
|
|
1041
|
+
});
|
|
1042
|
+
if (quarantined === "unreachable") {
|
|
1043
|
+
return dockerUnavailableResult(
|
|
1044
|
+
`Docker became unavailable while quarantining task ${input.task.id}`,
|
|
1045
|
+
);
|
|
1046
|
+
}
|
|
1047
|
+
if (quarantined === "failed") {
|
|
1048
|
+
return {
|
|
1049
|
+
ok: false,
|
|
1050
|
+
code: HostErrorCode.HostUnavailable,
|
|
1051
|
+
message:
|
|
1052
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
1053
|
+
"could not be proven stopped; retry after Docker is healthy",
|
|
1054
|
+
retryable: true,
|
|
1055
|
+
};
|
|
1056
|
+
}
|
|
1057
|
+
runtimeQuarantinedForRecreate = true;
|
|
1058
|
+
}
|
|
1059
|
+
// Docker being up is not enough to prove a lost-response retry is
|
|
1060
|
+
// healthy. The worktree's `.git` file contains an absolute pointer
|
|
1061
|
+
// into the task-private repository; after a host workspace-root
|
|
1062
|
+
// migration the old container can keep running while every agent
|
|
1063
|
+
// sees `fatal: not a git repository`. Probe from INSIDE the live
|
|
1064
|
+
// container, which is the filesystem view the agents actually use.
|
|
1065
|
+
// Scratchpads have no projects, so this remains an O(1) fast path.
|
|
1066
|
+
let projectsHealthy = true;
|
|
1067
|
+
for (const project of reusable ? input.projects : []) {
|
|
1068
|
+
const gitHealth = await dockerCli(
|
|
1069
|
+
[
|
|
1070
|
+
"exec",
|
|
1071
|
+
"-u",
|
|
1072
|
+
"node",
|
|
1073
|
+
// docker exec inherits the Compose/project environment. Clear
|
|
1074
|
+
// loader and Git controls before the first executable, then
|
|
1075
|
+
// give Git a minimal environment of its own. Keep this aligned
|
|
1076
|
+
// with task-up.sh's managed container operations.
|
|
1077
|
+
"-e",
|
|
1078
|
+
"HOME=/home/node",
|
|
1079
|
+
"-e",
|
|
1080
|
+
"PATH=/usr/bin:/bin",
|
|
1081
|
+
"-e",
|
|
1082
|
+
"LD_PRELOAD=",
|
|
1083
|
+
"-e",
|
|
1084
|
+
"LD_LIBRARY_PATH=",
|
|
1085
|
+
"-e",
|
|
1086
|
+
"DYLD_INSERT_LIBRARIES=",
|
|
1087
|
+
"-e",
|
|
1088
|
+
"DYLD_LIBRARY_PATH=",
|
|
1089
|
+
"-e",
|
|
1090
|
+
"BASH_ENV=",
|
|
1091
|
+
"-e",
|
|
1092
|
+
"ENV=",
|
|
1093
|
+
"-e",
|
|
1094
|
+
"GIT_CONFIG=",
|
|
1095
|
+
"-e",
|
|
1096
|
+
"GIT_CONFIG_GLOBAL=",
|
|
1097
|
+
"-e",
|
|
1098
|
+
"GIT_CONFIG_SYSTEM=",
|
|
1099
|
+
"-e",
|
|
1100
|
+
"GIT_CONFIG_NOSYSTEM=",
|
|
1101
|
+
"-e",
|
|
1102
|
+
"GIT_CONFIG_COUNT=0",
|
|
1103
|
+
"-e",
|
|
1104
|
+
"GIT_EXEC_PATH=",
|
|
1105
|
+
"-e",
|
|
1106
|
+
"GIT_SSH=",
|
|
1107
|
+
"-e",
|
|
1108
|
+
"GIT_SSH_COMMAND=",
|
|
1109
|
+
"-e",
|
|
1110
|
+
"GIT_ASKPASS=",
|
|
1111
|
+
"-e",
|
|
1112
|
+
"SSH_ASKPASS=",
|
|
1113
|
+
"-w",
|
|
1114
|
+
`/workspace/${project.slug}`,
|
|
1115
|
+
appContainer,
|
|
1116
|
+
"/usr/bin/env",
|
|
1117
|
+
"-i",
|
|
1118
|
+
"HOME=/home/node",
|
|
1119
|
+
"PATH=/usr/bin:/bin",
|
|
1120
|
+
"/usr/bin/git",
|
|
1121
|
+
"rev-parse",
|
|
1122
|
+
"--is-inside-work-tree",
|
|
1123
|
+
],
|
|
1124
|
+
{ timeoutMs: 10_000 },
|
|
1125
|
+
);
|
|
1126
|
+
// A timeout/spawn error is not proof that the repository is
|
|
1127
|
+
// broken. Preserve the running task and let the caller retry once
|
|
1128
|
+
// Docker can answer, just as the runtime-state probe above does.
|
|
1129
|
+
if (gitHealth.status === null) {
|
|
1130
|
+
return dockerUnavailableResult(
|
|
1131
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
1132
|
+
`project ${project.id}: ` +
|
|
1133
|
+
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
1134
|
+
"docker did not answer"),
|
|
1135
|
+
);
|
|
1136
|
+
}
|
|
1137
|
+
if (
|
|
1138
|
+
gitHealth.status !== 0 ||
|
|
1139
|
+
gitHealth.stdout.trim() !== "true"
|
|
1140
|
+
) {
|
|
1141
|
+
if (gitHealth.status !== 0) {
|
|
1142
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
1143
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
1144
|
+
`project ${project.id}: ` +
|
|
1145
|
+
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
1146
|
+
`git exited ${String(gitHealth.status)}`),
|
|
1147
|
+
);
|
|
1148
|
+
if (unavailable) return unavailable;
|
|
1149
|
+
}
|
|
1150
|
+
projectsHealthy = false;
|
|
1151
|
+
console.warn(
|
|
1152
|
+
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
|
|
1153
|
+
);
|
|
1154
|
+
break;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
if (reusable && projectsHealthy) {
|
|
1158
|
+
// waitForChannelClose crossed the teardown fence above. A completed
|
|
1159
|
+
// close-only teardown may have left this running task tombstoned,
|
|
1160
|
+
// so the idempotent-success transition must reopen delivery just
|
|
1161
|
+
// like a fresh successful taskUp does.
|
|
1162
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
1163
|
+
if (cannotPublish) return cannotPublish.result;
|
|
1164
|
+
orchestrator.allowChannel(input.task.id);
|
|
1165
|
+
return {
|
|
1166
|
+
ok: true,
|
|
1167
|
+
value: {
|
|
1168
|
+
composeProject: existingTask.composeProject,
|
|
1169
|
+
worktreePath: existingTask.worktreePath,
|
|
1170
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
1171
|
+
previewPorts: parsePreviewPortRuntimes(
|
|
1172
|
+
existingTask.previewPorts,
|
|
1173
|
+
),
|
|
1174
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
1175
|
+
},
|
|
1176
|
+
};
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
// Docker positively proved the mirrored app is absent/stopped, or the
|
|
1180
|
+
// live container proved one of its project repositories is broken.
|
|
1181
|
+
// Drain and invalidate any stale in-memory channel before taskUp
|
|
1182
|
+
// recreates the same stable compose/container names and repairs Git.
|
|
1183
|
+
if (!channelClosedForRecreate) {
|
|
1184
|
+
suspendTaskGatewayTask(input.task.id);
|
|
1185
|
+
await orchestrator.closeChannel(input.task.id);
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
const priorRuntimeState = existingTask
|
|
1189
|
+
? runtimeQuarantinedForRecreate
|
|
1190
|
+
? {
|
|
1191
|
+
statusMirror: "stopped",
|
|
1192
|
+
codeServerPort: null,
|
|
1193
|
+
previewPorts: "[]",
|
|
1194
|
+
lockedAt: null,
|
|
1195
|
+
startedAt: existingTask.startedAt,
|
|
1196
|
+
endedAt: existingTask.endedAt,
|
|
1197
|
+
environmentProvider: existingTask.environmentProvider,
|
|
1198
|
+
environmentLocator: existingTask.environmentLocator,
|
|
1199
|
+
envMode: existingTask.envMode,
|
|
1200
|
+
projectEnvSpec: existingTask.projectEnvSpec,
|
|
1201
|
+
sharedFilesMode: existingTask.sharedFilesMode,
|
|
1202
|
+
runEpoch: existingTask.runEpoch,
|
|
1203
|
+
}
|
|
1204
|
+
: {
|
|
1205
|
+
statusMirror: existingTask.statusMirror,
|
|
1206
|
+
lockedAt: existingTask.lockedAt,
|
|
1207
|
+
startedAt: existingTask.startedAt,
|
|
1208
|
+
endedAt: existingTask.endedAt,
|
|
1209
|
+
environmentProvider: existingTask.environmentProvider,
|
|
1210
|
+
environmentLocator: existingTask.environmentLocator,
|
|
1211
|
+
envMode: existingTask.envMode,
|
|
1212
|
+
projectEnvSpec: existingTask.projectEnvSpec,
|
|
1213
|
+
sharedFilesMode: existingTask.sharedFilesMode,
|
|
1214
|
+
runEpoch: existingTask.runEpoch,
|
|
1215
|
+
}
|
|
1216
|
+
: {
|
|
1217
|
+
statusMirror: input.task.status,
|
|
1218
|
+
lockedAt: null,
|
|
1219
|
+
startedAt: null,
|
|
1220
|
+
endedAt: null,
|
|
1221
|
+
};
|
|
1222
|
+
// ADR-153 machines: a task with no row has never run on this machine,
|
|
1223
|
+
// so everything it runs comes through its hosts view. Recorded before
|
|
1224
|
+
// the row, and so before the task's first process; a task with a row
|
|
1225
|
+
// (a start driven again, a start after a stop) stands as recorded.
|
|
1226
|
+
const taskWasKnownBeforeAssignment =
|
|
1227
|
+
runEpochAdmission.kind === "accepted"
|
|
1228
|
+
? runEpochAdmission.hadTask
|
|
1229
|
+
: existingTask !== null;
|
|
1230
|
+
if (sidecarMode() && !taskWasKnownBeforeAssignment) {
|
|
1231
|
+
admitFreshTaskView(input.task.id);
|
|
1232
|
+
}
|
|
1233
|
+
if (input.task.runEpoch === undefined) {
|
|
1234
|
+
recordTaskStarting(input.task.id);
|
|
1235
|
+
} else {
|
|
1236
|
+
recordTaskStarting(input.task.id, input.task.runEpoch);
|
|
1237
|
+
}
|
|
1238
|
+
// Files-v1 recovery gets its mode only from a launch already admitted on
|
|
1239
|
+
// cloud credentials, never from the task-writable marker. A legacy start
|
|
1240
|
+
// must leave the column null so cutover can distinguish and retire it.
|
|
1241
|
+
const sharedFilesMode = cloud
|
|
1242
|
+
? recordTaskSharedFilesMode(input.task.id, input.task)
|
|
1243
|
+
: null;
|
|
1244
|
+
// Cross-process managed maintenance and task admission use a two-sided
|
|
1245
|
+
// handshake: update/uninstall writes its durable marker before checking
|
|
1246
|
+
// this DB row; task-up writes `starting` before checking admission again.
|
|
1247
|
+
// Whichever arrives second observes the first, so a runtime cannot be
|
|
1248
|
+
// swapped or removed while an unrecorded task enters Docker side effects.
|
|
1249
|
+
const admittedMaintenanceFailure = managedMaintenanceUnavailable();
|
|
1250
|
+
if (admittedMaintenanceFailure) {
|
|
1251
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1252
|
+
else deleteHostTask(input.task.id);
|
|
1253
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
1254
|
+
orchestrator.allowChannel(input.task.id);
|
|
1255
|
+
}
|
|
1256
|
+
return admittedMaintenanceFailure;
|
|
1257
|
+
}
|
|
1258
|
+
recordTaskOwner(
|
|
1259
|
+
input.task.id,
|
|
1260
|
+
input.task.ownerUserId,
|
|
1261
|
+
input.ownerEmail ?? null,
|
|
1262
|
+
input.ownerName ?? null,
|
|
1263
|
+
input.projects.map((p) => p.slug),
|
|
1264
|
+
// ADR-131 D3: persisted so recovery re-provisions the same scope.
|
|
1265
|
+
input.task.engineAccountScope ?? null,
|
|
1266
|
+
// Lead project first (position 0 wins env collisions, as in task-up).
|
|
1267
|
+
[...input.projects]
|
|
1268
|
+
.sort((a, b) => a.position - b.position)
|
|
1269
|
+
.map((p) => p.id),
|
|
1270
|
+
);
|
|
1271
|
+
recordTaskProjectEnvSpec(input.task.id, input);
|
|
1272
|
+
// ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
|
|
1273
|
+
// `uai` CLI's per-agent tokens can be minted now + after a host restart.
|
|
1274
|
+
if (input.task.cliSecret) {
|
|
1275
|
+
storeTaskCliSecret(input.task.id, input.task.cliSecret);
|
|
1276
|
+
}
|
|
1277
|
+
recordHostEvent(input.task.id, "task.created");
|
|
1278
|
+
let environmentLocator: TaskEnvironmentLocator | undefined;
|
|
1279
|
+
// The locator of everything this start may create, recorded before any
|
|
1280
|
+
// of it exists: once set, the row must keep it.
|
|
1281
|
+
let runtimePrepared = false;
|
|
1282
|
+
const recordPrepared = async (locator: TaskEnvironmentLocator): Promise<void> => {
|
|
1283
|
+
recordTaskEnvironmentPrepared(input.task.id, locator, "loader");
|
|
1284
|
+
runtimePrepared = true;
|
|
1285
|
+
};
|
|
1286
|
+
let registeredTaskGatewayGeneration: string | null = null;
|
|
1287
|
+
let result: TaskUpOutcome = await wrapAgent(ctx, "taskUp", async () => {
|
|
1288
|
+
// Self-heal the standard base image before task-up needs it. The
|
|
1289
|
+
// boot-time build is best-effort + one-shot: a host that started before
|
|
1290
|
+
// Docker was ready (common on Docker Desktop) never built
|
|
1291
|
+
// moondeck-standard:dev, so every task-up died on a doomed registry pull
|
|
1292
|
+
// (live 2026-07-22). This is idempotent — a fast inspect when present,
|
|
1293
|
+
// a real build only when missing/stale (Docker is definitely up now, a
|
|
1294
|
+
// task just arrived). On a build FAILURE, surface WHY to the cloud (not
|
|
1295
|
+
// just the host log — the operator can't read a remote user's machine).
|
|
1296
|
+
// A pass that succeeded within the hour is reused (2026-09-09: the
|
|
1297
|
+
// re-check cost 16 s of every task start); the hourly schedule in
|
|
1298
|
+
// main.ts and the boot pass keep it fresh.
|
|
1299
|
+
const img = await ensureStandardImage({ maxAgeMs: 60 * 60_000 });
|
|
1300
|
+
if (!img.ok) {
|
|
1301
|
+
throw new AgentError(
|
|
1302
|
+
"STANDARD_IMAGE_UNAVAILABLE",
|
|
1303
|
+
img.error
|
|
1304
|
+
? `could not prepare the host base image moondeck-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
|
|
1305
|
+
: "could not prepare the host base image moondeck-standard:dev — see the host log.",
|
|
1306
|
+
{},
|
|
1307
|
+
);
|
|
1308
|
+
}
|
|
1309
|
+
let gitCredential: TaskGithubGitCredential | null;
|
|
1310
|
+
try {
|
|
1311
|
+
// ADR-154: a start on cloud credentials clones with its lease's,
|
|
1312
|
+
// never the runner's own store.
|
|
1313
|
+
gitCredential = cloud === null && input.projects.some((project) =>
|
|
1314
|
+
isGithubRepo(project.repoUrl),
|
|
1315
|
+
)
|
|
1316
|
+
? await prepareTaskGithubGitCredential(input.task.ownerUserId)
|
|
1317
|
+
: null;
|
|
1318
|
+
} catch {
|
|
1319
|
+
throw new AgentError(
|
|
1320
|
+
"GITHUB_AUTH_UNAVAILABLE",
|
|
1321
|
+
"GitHub is connected on this host, but Moondeck could not prepare its credential for Git. Retry the task; if it persists, reconnect GitHub on this host.",
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
if (sidecarMode() && cloud === null) {
|
|
1325
|
+
// ADR-133: project env lives under /uai/projects/<id>, readable by
|
|
1326
|
+
// the lease only for parameters tagged with this org, and only the
|
|
1327
|
+
// task's own projects are worth listing — their ids are known here,
|
|
1328
|
+
// not at the first grant (docs/credential-store.md).
|
|
1329
|
+
const { reconcileProjectEnvFromStore } = await import("../lib/user-credentials-store");
|
|
1330
|
+
const projectIds = input.projects.map((project) => project.id);
|
|
1331
|
+
const env = await reconcileProjectEnvFromStore({ projectIds }).catch((error: unknown) => {
|
|
1332
|
+
console.warn(
|
|
1333
|
+
`[project-env] task ${input.task.id}: credential store read failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1334
|
+
);
|
|
1335
|
+
return null;
|
|
1336
|
+
});
|
|
1337
|
+
if (env && !env.skipped) {
|
|
1338
|
+
console.log(
|
|
1339
|
+
`[project-env] task ${input.task.id}: credential store reconciled for ${projectIds.length} project(s): ${env.imported} imported, ${env.removed} removed, ${env.errors} errors`,
|
|
1340
|
+
);
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
// Mint the authenticated task generation before its environment
|
|
1344
|
+
// starts. The declarations are pinned; values remain live DB reads.
|
|
1345
|
+
const gatewayRegistration = registerTaskGatewayTask(input.task.id, {
|
|
1346
|
+
resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null),
|
|
1347
|
+
runEpoch: input.task.runEpoch ?? null,
|
|
1348
|
+
});
|
|
1349
|
+
registeredTaskGatewayGeneration = gatewayRegistration.generation;
|
|
1350
|
+
const gatewaySecret = taskGatewayForwarderSecretInput(
|
|
1351
|
+
input.task.id,
|
|
1352
|
+
gatewayRegistration.generation,
|
|
1353
|
+
);
|
|
1354
|
+
if (gatewaySecret === null) {
|
|
1355
|
+
removeTaskGatewayTask(
|
|
1356
|
+
input.task.id,
|
|
1357
|
+
gatewayRegistration.generation,
|
|
1358
|
+
);
|
|
1359
|
+
registeredTaskGatewayGeneration = null;
|
|
1360
|
+
throw new Error("could not export the task gateway generation secret");
|
|
1361
|
+
}
|
|
1362
|
+
// Launch screen: the first stage boundary. A machine-backed task
|
|
1363
|
+
// spends the next minute launching an EC2 instance; a container
|
|
1364
|
+
// task goes straight to preparing its workspace.
|
|
1365
|
+
orchestrator.emitTaskStage(
|
|
1366
|
+
input.task.id,
|
|
1367
|
+
machineTasksEnabled() ? "machine" : "workspace",
|
|
1368
|
+
);
|
|
1369
|
+
try {
|
|
1370
|
+
const taskGateway = {
|
|
1371
|
+
generation: gatewayRegistration.generation,
|
|
1372
|
+
secret: gatewaySecret,
|
|
1373
|
+
};
|
|
1374
|
+
const provisioned = gitCredential
|
|
1375
|
+
? await gitCredential.run(() =>
|
|
1376
|
+
provisionTaskEnvironment(
|
|
1377
|
+
input,
|
|
1378
|
+
{
|
|
1379
|
+
githubCredentialSocket: gitCredential.socketPath,
|
|
1380
|
+
taskGateway,
|
|
1381
|
+
},
|
|
1382
|
+
recordPrepared,
|
|
1383
|
+
),
|
|
1384
|
+
)
|
|
1385
|
+
: await provisionTaskEnvironment(
|
|
1386
|
+
input,
|
|
1387
|
+
{
|
|
1388
|
+
taskGateway,
|
|
1389
|
+
...(cloud?.start && cloud.hold
|
|
1390
|
+
? {
|
|
1391
|
+
leased: leasedTaskUp(
|
|
1392
|
+
cloud.runtime,
|
|
1393
|
+
cloud.start,
|
|
1394
|
+
cloud.hold,
|
|
1395
|
+
sharedFilesMode ?? (() => {
|
|
1396
|
+
throw new Error("a cloud-credential start has no trusted Files mode");
|
|
1397
|
+
})(),
|
|
1398
|
+
),
|
|
1399
|
+
}
|
|
1400
|
+
: {}),
|
|
1401
|
+
},
|
|
1402
|
+
recordPrepared,
|
|
1403
|
+
);
|
|
1404
|
+
environmentLocator = provisioned.handle.descriptor.locator;
|
|
1405
|
+
// ADR-153 §4: the task's own schema observations, read from the
|
|
1406
|
+
// bases task-up recorded before its container started (a machine
|
|
1407
|
+
// captures its own). Stored once; a resume reuses them.
|
|
1408
|
+
const envSchemas = isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)
|
|
1409
|
+
? undefined
|
|
1410
|
+
: captureTaskEnvSchemas(
|
|
1411
|
+
input.task.id,
|
|
1412
|
+
incomingEnvSpec.projects.map((project) => project.projectId),
|
|
1413
|
+
);
|
|
1414
|
+
// ADR-153: before any consumer is let in, bring the task to now:
|
|
1415
|
+
// a CA successor minted while it started, and the domains its
|
|
1416
|
+
// policy has now rather than those task-up was handed. A failure
|
|
1417
|
+
// leaves those keys withheld, and the trust watcher retries.
|
|
1418
|
+
const secretsProxy = activeSecretsProxy();
|
|
1419
|
+
if (secretsProxy !== null) {
|
|
1420
|
+
await reconcileTaskProxyState(provisioned.handle, secretsProxy.runtime).catch((error: unknown) => {
|
|
1421
|
+
console.warn(
|
|
1422
|
+
`[secrets-proxy] task ${input.task.id}: could not bring the task's trust and pins up to date: ${
|
|
1423
|
+
error instanceof Error ? error.message : String(error)
|
|
1424
|
+
}`,
|
|
1425
|
+
);
|
|
1426
|
+
secretsProxy.runtime.requestTrustDelivery();
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
return envSchemas === undefined || envSchemas.length === 0
|
|
1430
|
+
? { ...provisioned.result, envMode: "loader" as const }
|
|
1431
|
+
: { ...provisioned.result, envMode: "loader" as const, envSchemas };
|
|
1432
|
+
} finally {
|
|
1433
|
+
gatewaySecret.fill(0);
|
|
1434
|
+
await gitCredential?.close().catch((error: unknown) => {
|
|
1435
|
+
// Cleanup must not replace the actual task-up result/error. The
|
|
1436
|
+
// credential is short-lived and close() also removes its private
|
|
1437
|
+
// directory in a finally block; retain diagnostics for operators.
|
|
1438
|
+
console.warn(
|
|
1439
|
+
`[github] task ${input.task.id}: credential cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1440
|
+
);
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
});
|
|
1444
|
+
// ADR-154: the last word before the start publishes its outcome. A
|
|
1445
|
+
// success needs the start still the task's current one, its credentials
|
|
1446
|
+
// open for its run; an interrupted start reports its interruption, never
|
|
1447
|
+
// what it failed of. Nothing it made is destroyed here: a teardown would
|
|
1448
|
+
// delete a resumed task's workspace, and whoever took over acts on the
|
|
1449
|
+
// runtime itself.
|
|
1450
|
+
const interrupted = interruptedStartResult(cloud);
|
|
1451
|
+
const replaced = result.ok
|
|
1452
|
+
? cloudStartCannotPublish(cloud)
|
|
1453
|
+
: interrupted
|
|
1454
|
+
? { result: interrupted, interrupted: true }
|
|
1455
|
+
: null;
|
|
1456
|
+
if (replaced) result = replaced.result;
|
|
1457
|
+
if (!result.ok && registeredTaskGatewayGeneration !== null) {
|
|
1458
|
+
forgetTaskEnvSnapshot(input.task.id);
|
|
1459
|
+
try {
|
|
1460
|
+
removeTaskGatewayTask(
|
|
1461
|
+
input.task.id,
|
|
1462
|
+
registeredTaskGatewayGeneration,
|
|
1463
|
+
);
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
// The in-process authority is revoked before durable deletion is
|
|
1466
|
+
// attempted. Preserve the task-up error while retaining a cleanup
|
|
1467
|
+
// diagnostic; a leftover file loads inactive after restart.
|
|
1468
|
+
console.warn(
|
|
1469
|
+
`[task-gateway] task ${input.task.id}: failed to remove the rejected generation: ${
|
|
1470
|
+
error instanceof Error ? error.message : String(error)
|
|
1471
|
+
}`,
|
|
1472
|
+
);
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
if (result.ok) {
|
|
1476
|
+
orchestrator.allowChannel(input.task.id);
|
|
1477
|
+
if (environmentLocator === undefined) {
|
|
1478
|
+
throw new Error("task environment provision returned no locator");
|
|
1479
|
+
}
|
|
1480
|
+
recordTaskUpResult(
|
|
1481
|
+
input.task.id,
|
|
1482
|
+
result.value,
|
|
1483
|
+
environmentLocator,
|
|
1484
|
+
"loader",
|
|
1485
|
+
);
|
|
1486
|
+
if (input.task.runEpoch !== undefined) {
|
|
1487
|
+
recordTaskRuntimeEstablished(
|
|
1488
|
+
input.task.id,
|
|
1489
|
+
input.task.runEpoch,
|
|
1490
|
+
);
|
|
1491
|
+
}
|
|
1492
|
+
recordHostEvent(input.task.id, "task.started");
|
|
1493
|
+
// ADR-153: with the proxy off, every start says so, since the task's
|
|
1494
|
+
// proxied keys are withheld rather than delivered.
|
|
1495
|
+
if (activeSecretsProxy() === null) {
|
|
1496
|
+
orchestrator.emitSystemNote(
|
|
1497
|
+
input.task.id,
|
|
1498
|
+
secretsProxyEnabled()
|
|
1499
|
+
? "The secrets proxy could not start on this host, so this task's proxied env vars are withheld. See the host log."
|
|
1500
|
+
: "The secrets proxy is off on this host (UAI_SECRETS_PROXY=off), so this task's proxied env vars are withheld.",
|
|
1501
|
+
);
|
|
1502
|
+
}
|
|
1503
|
+
// ADR-153 §4: one note naming the keys this task does not get, and
|
|
1504
|
+
// why, so nobody hunts for a value Uai withholds on purpose.
|
|
1505
|
+
try {
|
|
1506
|
+
const withheld = taskEnvWithheldNoteNow(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null);
|
|
1507
|
+
if (withheld !== null) orchestrator.emitSystemNote(input.task.id, withheld);
|
|
1508
|
+
} catch (error) {
|
|
1509
|
+
console.warn(
|
|
1510
|
+
`[project-env] task ${input.task.id}: withheld-keys note failed: ${
|
|
1511
|
+
error instanceof Error ? error.message : String(error)
|
|
1512
|
+
}`,
|
|
1513
|
+
);
|
|
1514
|
+
}
|
|
1515
|
+
// Degraded start (uai-init failed twice): the task is up, but say so
|
|
1516
|
+
// in the feed — a silent half-start reads as a broken product.
|
|
1517
|
+
if (result.value.initWarning) {
|
|
1518
|
+
orchestrator.emitSystemNote(
|
|
1519
|
+
input.task.id,
|
|
1520
|
+
result.value.initWarning,
|
|
1521
|
+
);
|
|
1522
|
+
}
|
|
1523
|
+
// ADR-122 phase 1: a project that declares @required env keys with
|
|
1524
|
+
// no value on this host gets ONE visible note now — not a blank 500
|
|
1525
|
+
// hours later (live 2026-08-25). Best-effort by design: a schema
|
|
1526
|
+
// read hiccup must never fail a completed task-up.
|
|
1527
|
+
void noteMissingRequiredEnvKeys(input).catch((err) =>
|
|
1528
|
+
console.warn(
|
|
1529
|
+
`[env-schema] task ${input.task.id}: validation note failed: ${
|
|
1530
|
+
err instanceof Error ? err.message : String(err)
|
|
1531
|
+
}`,
|
|
1532
|
+
),
|
|
1533
|
+
);
|
|
1534
|
+
// GitHub auth for the container is best-effort (ADR-027) and runs in the
|
|
1535
|
+
// background — it must never block or fail task-up. Awaiting it here would
|
|
1536
|
+
// couple the command result to a network token-exchange: a slow/hung
|
|
1537
|
+
// exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
|
|
1538
|
+
// and mark a running task as errored. The reconciler injects + schedules
|
|
1539
|
+
// (or emits a system note on failure) on its own; the agents come up
|
|
1540
|
+
// regardless and the token lands well before the first `gh` call.
|
|
1541
|
+
// ADR-121: machine-backed tasks take a direct environment injection —
|
|
1542
|
+
// the compose reconciler would probe for an app container, find none,
|
|
1543
|
+
// and silently skip. Reconnect re-injection for machines is a
|
|
1544
|
+
// follow-up alongside the machine recovery pass.
|
|
1545
|
+
if (cloud !== null) {
|
|
1546
|
+
// ADR-154: the runtime-ready writes put the lease's GitHub sign-in
|
|
1547
|
+
// and SSH key in; the runner's own store has nothing to add.
|
|
1548
|
+
} else if (isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)) {
|
|
1549
|
+
void machineTaskGithubAuth(
|
|
1550
|
+
input.task.id,
|
|
1551
|
+
input.task.ownerUserId,
|
|
1552
|
+
).catch((err) =>
|
|
1553
|
+
console.warn(
|
|
1554
|
+
`[github] machine task ${input.task.id}: auth injection failed: ${
|
|
1555
|
+
err instanceof Error ? err.message : String(err)
|
|
1556
|
+
}`,
|
|
1557
|
+
),
|
|
1558
|
+
);
|
|
1559
|
+
} else {
|
|
1560
|
+
void reconcileTaskGitAuth(
|
|
1561
|
+
input.task.id,
|
|
1562
|
+
input.task.ownerUserId,
|
|
1563
|
+
).catch((err) =>
|
|
1564
|
+
console.warn(
|
|
1565
|
+
`[github] task ${input.task.id}: post-start reconciliation failed: ${
|
|
1566
|
+
err instanceof Error ? err.message : String(err)
|
|
1567
|
+
}`,
|
|
1568
|
+
),
|
|
1569
|
+
);
|
|
1570
|
+
}
|
|
1571
|
+
} else if (replaced?.interrupted) {
|
|
1572
|
+
// ADR-154: Stop, teardown, a newer run or shutdown took over, and the
|
|
1573
|
+
// channel stays closed for it. A start that recorded its runtime's
|
|
1574
|
+
// locator (before any of that runtime existed) keeps it exactly, so
|
|
1575
|
+
// whoever took over stops, destroys or starts again that runtime and
|
|
1576
|
+
// nothing that may exist is forgotten; its credentials are cleaned up
|
|
1577
|
+
// while it still runs. A start that recorded none made nothing, and
|
|
1578
|
+
// leaves no trace.
|
|
1579
|
+
if (runtimePrepared) recordTaskError(input.task.id);
|
|
1580
|
+
else if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1581
|
+
else deleteHostTask(input.task.id);
|
|
1582
|
+
} else if (result.code === HostErrorCode.HostUnavailable) {
|
|
1583
|
+
// A daemon outage is retryable infrastructure state, not proof that
|
|
1584
|
+
// the task itself failed. Undo the optimistic local `starting` mirror
|
|
1585
|
+
// and normally reopen delivery; a runtime already proven stopped for
|
|
1586
|
+
// L5 recreation stays stopped/tombstoned for the cloud's retry.
|
|
1587
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1588
|
+
else deleteHostTask(input.task.id);
|
|
1589
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
1590
|
+
orchestrator.allowChannel(input.task.id);
|
|
1591
|
+
}
|
|
1592
|
+
} else {
|
|
1593
|
+
recordTaskError(input.task.id);
|
|
1594
|
+
}
|
|
1595
|
+
return result;
|
|
1596
|
+
}));
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
export const hostCommands: HostCommands = {
|
|
1600
|
+
// The hostId argument is cloud-side routing (host-scoped commands carry
|
|
1601
|
+
// their target in args, like filesOp); this process IS the target.
|
|
1602
|
+
async hostUpdate(ctx, _hostId) {
|
|
1603
|
+
logCommand(ctx, "hostUpdate");
|
|
1604
|
+
const pinned = sidecarRunnerIsPinned();
|
|
1605
|
+
if (pinned) return pinned;
|
|
1606
|
+
try {
|
|
1607
|
+
const { updateManagedRuntime } = await import("../lib/managed-runtime");
|
|
1608
|
+
const outcome = await updateManagedRuntime();
|
|
1609
|
+
if (outcome.status === "not-managed") {
|
|
1610
|
+
return {
|
|
1611
|
+
ok: false,
|
|
1612
|
+
code: HostErrorCode.Internal,
|
|
1613
|
+
message:
|
|
1614
|
+
"this host is not a managed install — update it where it runs",
|
|
1615
|
+
retryable: false,
|
|
1616
|
+
};
|
|
1617
|
+
}
|
|
1618
|
+
if (outcome.status === "deferred") {
|
|
1619
|
+
// A rollback hold is an operator verdict that the newer release is
|
|
1620
|
+
// unsafe on this host; a remote click must not override it.
|
|
1621
|
+
return {
|
|
1622
|
+
ok: true,
|
|
1623
|
+
value: { version: outcome.version, staged: false, deferred: true },
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
if (outcome.status === "current") {
|
|
1627
|
+
return { ok: true, value: { version: outcome.version, staged: false } };
|
|
1628
|
+
}
|
|
1629
|
+
managedHostRestartHook?.();
|
|
1630
|
+
return { ok: true, value: { version: outcome.version, staged: true } };
|
|
1631
|
+
} catch (error) {
|
|
1632
|
+
return {
|
|
1633
|
+
ok: false,
|
|
1634
|
+
code: HostErrorCode.Internal,
|
|
1635
|
+
message: `update failed: ${
|
|
1636
|
+
error instanceof Error ? error.message : "unknown error"
|
|
1637
|
+
}`,
|
|
1638
|
+
retryable: true,
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
},
|
|
1642
|
+
async hostRestart(ctx, _hostId) {
|
|
1643
|
+
logCommand(ctx, "hostRestart");
|
|
1644
|
+
const pinned = sidecarRunnerIsPinned();
|
|
1645
|
+
if (pinned) return pinned;
|
|
1646
|
+
if (!managedHostRestartHook) {
|
|
1647
|
+
return {
|
|
1648
|
+
ok: false,
|
|
1649
|
+
code: HostErrorCode.Internal,
|
|
1650
|
+
message: "restart is unavailable in this process",
|
|
1651
|
+
retryable: false,
|
|
1652
|
+
};
|
|
1653
|
+
}
|
|
1654
|
+
// The result frame flushes before exit: shutdown drains in-flight
|
|
1655
|
+
// commands (this one included) before the process terminates.
|
|
1656
|
+
managedHostRestartHook();
|
|
1657
|
+
return { ok: true, value: undefined };
|
|
1658
|
+
},
|
|
1659
|
+
async taskUp(ctx, input) {
|
|
1660
|
+
logCommand(ctx, "taskUp", input.task.id);
|
|
1661
|
+
// ADR-154: on cloud credentials every start goes through the composer,
|
|
1662
|
+
// and none starts while the runner's activation is pending.
|
|
1663
|
+
const mode = credentialsMode();
|
|
1664
|
+
if (mode === "pending") return switchingToCloudCredentials();
|
|
1665
|
+
// On the runner's own credentials, a counted permit from here to the
|
|
1666
|
+
// outcome: leaving them waits for it.
|
|
1667
|
+
let legacy: LegacyStartPermit | null = null;
|
|
1668
|
+
if (mode === "legacy") {
|
|
1669
|
+
legacy = admitLegacyStart();
|
|
1670
|
+
if (legacy === null) return switchingToCloudCredentials();
|
|
1671
|
+
}
|
|
1672
|
+
let cloud: CloudCredentialTaskStart | null = null;
|
|
1673
|
+
if (mode === "active") {
|
|
1674
|
+
const runtime = cloudCredentialRuntime();
|
|
1675
|
+
if (runtime === null || input.task.runEpoch === undefined) {
|
|
1676
|
+
return {
|
|
1677
|
+
ok: false,
|
|
1678
|
+
code: HostErrorCode.Internal,
|
|
1679
|
+
message: "This host runs tasks only on cloud credentials, and could not start this one on them.",
|
|
1680
|
+
retryable: false,
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
cloud = { runtime, assignment: { taskId: input.task.id, runEpoch: input.task.runEpoch }, start: null, hold: null };
|
|
1684
|
+
// A newer run's start ends an older one's before it queues behind it.
|
|
1685
|
+
runtime.interruptStarts(input.task.id, { below: input.task.runEpoch });
|
|
1686
|
+
}
|
|
1687
|
+
let outcome: TaskUpOutcome | undefined;
|
|
1688
|
+
try {
|
|
1689
|
+
outcome = await taskUpOn(ctx, input, cloud, legacy);
|
|
1690
|
+
return outcome;
|
|
1691
|
+
} finally {
|
|
1692
|
+
legacy?.release();
|
|
1693
|
+
// Its hold already let go with the long slot: the cleanup an abandon
|
|
1694
|
+
// runs never waits on it.
|
|
1695
|
+
if (cloud?.start) {
|
|
1696
|
+
if (outcome?.ok === true) cloud.runtime.endStart(cloud.start);
|
|
1697
|
+
else await cloud.runtime.abandonStart(cloud.start, cloud.hold);
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
},
|
|
1701
|
+
|
|
1702
|
+
async taskDown(ctx, input) {
|
|
1703
|
+
logCommand(ctx, "taskDown", input.taskId);
|
|
1704
|
+
const orphanGc = input.cleanup?.kind === "orphan-gc";
|
|
1705
|
+
// ADR-154: teardown ends its run's start (orphan GC: every start of the
|
|
1706
|
+
// task) before it queues behind it.
|
|
1707
|
+
const interrupting = activeCloudCredentials();
|
|
1708
|
+
if (interrupting) {
|
|
1709
|
+
if (orphanGc) interrupting.interruptStarts(input.taskId);
|
|
1710
|
+
else if (input.task.runEpoch !== undefined) interrupting.interruptStarts(input.taskId, { exact: input.task.runEpoch });
|
|
1711
|
+
}
|
|
1712
|
+
// A GC command is already authoritative evidence that this cloud task is
|
|
1713
|
+
// gone. Enter the lifecycle slot and tombstone its channel even when the
|
|
1714
|
+
// runtime is temporarily unavailable; the durable cloud intent will retry
|
|
1715
|
+
// destruction later. Ordinary teardown retains its fast preflight.
|
|
1716
|
+
if (!orphanGc) {
|
|
1717
|
+
// Teardown-specific admission: `checking` with a pinned backend ADMITS
|
|
1718
|
+
// teardown. Gating kills on `operational` created the 2026-08-16 ring —
|
|
1719
|
+
// a task holding the shared volume blocked the very maintenance whose
|
|
1720
|
+
// completion the kill was waiting on. Removing load is how a degraded
|
|
1721
|
+
// host heals.
|
|
1722
|
+
const runtimeFailure = teardownRuntimeUnavailable();
|
|
1723
|
+
if (runtimeFailure) return runtimeFailure;
|
|
1724
|
+
}
|
|
1725
|
+
const orchestrator = getOrchestrator();
|
|
1726
|
+
return orchestrator.runTaskLifecycle(input.taskId, async () => {
|
|
1727
|
+
// Preserve ordinary teardown's runtime-race behavior: if the daemon was
|
|
1728
|
+
// lost while waiting for the lifecycle slot, leave its channel open.
|
|
1729
|
+
if (!orphanGc) {
|
|
1730
|
+
const inSlotRuntimeFailure = teardownRuntimeUnavailable();
|
|
1731
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
1732
|
+
}
|
|
1733
|
+
let environment: TaskEnvironmentHandle<TaskDownResult> | null;
|
|
1734
|
+
if (!orphanGc) {
|
|
1735
|
+
try {
|
|
1736
|
+
environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
1737
|
+
} catch (error) {
|
|
1738
|
+
return failFromUnknown(error);
|
|
1739
|
+
}
|
|
1740
|
+
} else {
|
|
1741
|
+
environment = null;
|
|
1742
|
+
}
|
|
1743
|
+
// Close the exact v1 assignment before any terminal side effect. A
|
|
1744
|
+
// delayed teardown for an older run must never tombstone or destroy the
|
|
1745
|
+
// newer run. Orphan GC is authoritative for the task id and therefore
|
|
1746
|
+
// closes whichever assignment still survives locally.
|
|
1747
|
+
let v1AssignmentClosed = false;
|
|
1748
|
+
let closedRunEpoch: number | null = null;
|
|
1749
|
+
const assignmentState = readTaskRunEpochState(input.taskId);
|
|
1750
|
+
if (assignmentState !== null || input.task.runEpoch !== undefined) {
|
|
1751
|
+
const requestedRunEpoch = orphanGc
|
|
1752
|
+
? assignmentState?.highestRunEpoch
|
|
1753
|
+
: input.task.runEpoch;
|
|
1754
|
+
if (assignmentState === null || requestedRunEpoch === undefined) {
|
|
1755
|
+
return {
|
|
1756
|
+
ok: false,
|
|
1757
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
1758
|
+
message:
|
|
1759
|
+
`task ${input.taskId} teardown does not name its current run`,
|
|
1760
|
+
};
|
|
1761
|
+
}
|
|
1762
|
+
const closed = closeTaskRunAssignment(
|
|
1763
|
+
input.taskId,
|
|
1764
|
+
requestedRunEpoch,
|
|
1765
|
+
);
|
|
1766
|
+
if (closed.kind === "missing" || closed.kind === "mismatch") {
|
|
1767
|
+
return {
|
|
1768
|
+
ok: false,
|
|
1769
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
1770
|
+
message:
|
|
1771
|
+
`task ${input.taskId} run ${String(requestedRunEpoch)} is no longer current`,
|
|
1772
|
+
};
|
|
1773
|
+
}
|
|
1774
|
+
v1AssignmentClosed = true;
|
|
1775
|
+
closedRunEpoch = requestedRunEpoch;
|
|
1776
|
+
}
|
|
1777
|
+
// Tombstone + close before container destruction. A concurrent cloud
|
|
1778
|
+
// ensure cannot reopen the channel while teardown is awaiting Docker.
|
|
1779
|
+
const suspendedTaskGateway = suspendTaskGatewayTask(input.taskId);
|
|
1780
|
+
// ADR-153: the suspended generation is no longer served; end the
|
|
1781
|
+
// proxy connections it already had, too.
|
|
1782
|
+
activeSecretsProxy()?.runtime.stopTask(input.taskId);
|
|
1783
|
+
await orchestrator.closeChannel(input.taskId);
|
|
1784
|
+
if (orphanGc) {
|
|
1785
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
1786
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
1787
|
+
try {
|
|
1788
|
+
environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
1789
|
+
} catch (error) {
|
|
1790
|
+
return failFromUnknown(error);
|
|
1791
|
+
}
|
|
1792
|
+
// Channel startup and runtime recovery launch SSH reconciliation in
|
|
1793
|
+
// the background. Fence every future launch and join the exact older
|
|
1794
|
+
// ensure before container destruction; otherwise a paused health probe
|
|
1795
|
+
// could materialize the private key after localStatePruned was sent.
|
|
1796
|
+
await retireTaskSshIdentityEnsures(input.taskId);
|
|
1797
|
+
}
|
|
1798
|
+
// Ad-hoc preview proxies share this task's compose network and stable
|
|
1799
|
+
// names. Remove them inside the same lifecycle slot before compose down.
|
|
1800
|
+
try {
|
|
1801
|
+
await stopPreviewSidecars(input.taskId);
|
|
1802
|
+
} catch (err) {
|
|
1803
|
+
console.warn(
|
|
1804
|
+
`[host-agent] task ${input.taskId}: preview cleanup failed: ${
|
|
1805
|
+
err instanceof Error ? err.message : String(err)
|
|
1806
|
+
}`,
|
|
1807
|
+
);
|
|
1808
|
+
}
|
|
1809
|
+
// ADR-154 §6: the closed run's credentials cleaned up, every path
|
|
1810
|
+
// proven gone (or the runtime destroyed), before the runtime goes; by
|
|
1811
|
+
// the composer as it is now, in the slot.
|
|
1812
|
+
const credentials = activeCloudCredentials();
|
|
1813
|
+
if (credentials && closedRunEpoch !== null) {
|
|
1814
|
+
try {
|
|
1815
|
+
await credentials.closeAssignment({ taskId: input.taskId, runEpoch: closedRunEpoch }, "teardown");
|
|
1816
|
+
} catch (error) {
|
|
1817
|
+
return failFromUnknown(error);
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
const result = await wrapAgent(ctx, "taskDown", async () => {
|
|
1821
|
+
if (environment === null) return agent.taskDown(input);
|
|
1822
|
+
// taskDown is TERMINAL — its only cloud callers are task close and
|
|
1823
|
+
// failed-occurrence/orphan cleanup; pause rides the separate stopTask
|
|
1824
|
+
// command. Machines therefore TERMINATE here: a closed task can never
|
|
1825
|
+
// be resumed, so a stopped-but-kept machine is a per-month disk leak
|
|
1826
|
+
// with no consumer (owner, 2026-08-29 — 16 parked disks after one
|
|
1827
|
+
// evening of testing). The task branch lives on the remote; the
|
|
1828
|
+
// machine's disk is not a preservation surface once the task ends.
|
|
1829
|
+
if (isEnvironmentTransportProvider(environment.descriptor.locator.provider)) {
|
|
1830
|
+
stopMachineGatewayTunnel(input.taskId);
|
|
1831
|
+
}
|
|
1832
|
+
return taskDownResultForInput(input, await environment.teardown());
|
|
1833
|
+
});
|
|
1834
|
+
if (result.ok) {
|
|
1835
|
+
if (orphanGc) {
|
|
1836
|
+
try {
|
|
1837
|
+
// task-down.sh is destructive but its exit alone is not the proof:
|
|
1838
|
+
// independently re-query every exact Docker label and the one
|
|
1839
|
+
// derived task directory before deleting the remaining authority.
|
|
1840
|
+
await proveOrphanTaskResourcesAbsent(input.taskId);
|
|
1841
|
+
clearTaskGatewayAclStrict(input.taskId);
|
|
1842
|
+
removeTaskCliSecretStrict(input.taskId);
|
|
1843
|
+
removeTaskIdentityStrict(input.taskId);
|
|
1844
|
+
removeTaskGatewayTask(input.taskId);
|
|
1845
|
+
// No longer served: the proxy forgets its placeholders, and no
|
|
1846
|
+
// write still queued for the task may land or record.
|
|
1847
|
+
activeSecretsProxy()?.runtime.dropTask(input.taskId);
|
|
1848
|
+
invalidateTaskWrites(input.taskId);
|
|
1849
|
+
forgetTaskProxyReadiness(input.taskId);
|
|
1850
|
+
forgetTaskEnvSnapshot(input.taskId);
|
|
1851
|
+
forgetTaskEnvSchemaObservations(input.taskId);
|
|
1852
|
+
clearRefresh(input.taskId);
|
|
1853
|
+
// Last, atomically. Do not record taskDown/task.ended first: both
|
|
1854
|
+
// would recreate the very inventory evidence being pruned.
|
|
1855
|
+
purgeHostTaskState(input.taskId);
|
|
1856
|
+
return ok({ ...result.value, localStatePruned: true as const });
|
|
1857
|
+
} catch (error) {
|
|
1858
|
+
console.warn(
|
|
1859
|
+
`[host-agent] task ${input.taskId}: orphan cleanup proof failed: ${
|
|
1860
|
+
error instanceof Error ? error.message : String(error)
|
|
1861
|
+
}`,
|
|
1862
|
+
);
|
|
1863
|
+
return failFromUnknown(error);
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
// The live generation was suspended before channel drain. Delete its
|
|
1867
|
+
// durable state only after destruction succeeds; on failure the task
|
|
1868
|
+
// is reopened below with the same on-disk MCP definitions.
|
|
1869
|
+
try {
|
|
1870
|
+
removeTaskGatewayTask(input.taskId);
|
|
1871
|
+
} catch (error) {
|
|
1872
|
+
// Destruction succeeded and the live authority was already revoked,
|
|
1873
|
+
// but keep the lifecycle retryable until the durable identity is
|
|
1874
|
+
// gone. Never reopen a channel onto the destroyed runtime.
|
|
1875
|
+
recordTaskError(input.taskId);
|
|
1876
|
+
return failFromUnknown(error);
|
|
1877
|
+
}
|
|
1878
|
+
// No longer served: the proxy forgets its placeholders, and no write
|
|
1879
|
+
// still queued for the task may land or record.
|
|
1880
|
+
activeSecretsProxy()?.runtime.dropTask(input.taskId);
|
|
1881
|
+
invalidateTaskWrites(input.taskId);
|
|
1882
|
+
forgetTaskProxyReadiness(input.taskId);
|
|
1883
|
+
forgetTaskEnvSnapshot(input.taskId);
|
|
1884
|
+
forgetTaskEnvSchemaObservations(input.taskId);
|
|
1885
|
+
clearTaskGatewayAcl(input.taskId);
|
|
1886
|
+
recordTaskDown(input.taskId, result.value.status);
|
|
1887
|
+
recordHostEvent(input.taskId, "task.ended");
|
|
1888
|
+
clearRefresh(input.taskId);
|
|
1889
|
+
} else {
|
|
1890
|
+
// Destruction failed; let a subsequent ensure restore the still-running
|
|
1891
|
+
// task instead of leaving it permanently tombstoned.
|
|
1892
|
+
if (!orphanGc && !v1AssignmentClosed) {
|
|
1893
|
+
if (suspendedTaskGateway) {
|
|
1894
|
+
reactivateTaskGatewayTask(suspendedTaskGateway);
|
|
1895
|
+
}
|
|
1896
|
+
orchestrator.allowChannel(input.taskId);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
return result;
|
|
1900
|
+
});
|
|
1901
|
+
},
|
|
1902
|
+
|
|
1903
|
+
async taskStop(ctx, taskId, input) {
|
|
1904
|
+
logCommand(ctx, "taskStop", taskId, input.reason);
|
|
1905
|
+
const orchestrator = getOrchestrator();
|
|
1906
|
+
// ADR-154: a manual Stop ends its run's start before it queues behind
|
|
1907
|
+
// it; an idle Stop never does, since a start vetoes it in the slot.
|
|
1908
|
+
const interrupting = activeCloudCredentials();
|
|
1909
|
+
if (interrupting && input.reason !== "idle" && input.runEpoch !== undefined) {
|
|
1910
|
+
interrupting.interruptStarts(taskId, { exact: input.runEpoch });
|
|
1911
|
+
}
|
|
1912
|
+
const result = await orchestrator.stopTask(taskId, {
|
|
1913
|
+
// ADR-154: the run Stop closes, cleaned up by the composer as it is
|
|
1914
|
+
// when Stop gets there, in the slot.
|
|
1915
|
+
closeCredentials: (runEpoch: number) =>
|
|
1916
|
+
activeCloudCredentials()?.closeAssignment({ taskId, runEpoch }, "stop") ?? Promise.resolve(),
|
|
1917
|
+
// Resolve legacy versus v1 only after Stop owns the task lifecycle slot.
|
|
1918
|
+
// Otherwise task-up can admit v1 between an outside read and the stop.
|
|
1919
|
+
admitAssignment: () => {
|
|
1920
|
+
if (input.runEpoch === undefined) {
|
|
1921
|
+
return readTaskRunEpochState(taskId) === null
|
|
1922
|
+
? { kind: "legacy" }
|
|
1923
|
+
: { kind: "stale" };
|
|
1924
|
+
}
|
|
1925
|
+
const closed = closeTaskRunAssignmentForStop(taskId, input.runEpoch);
|
|
1926
|
+
return closed.kind === "closed" || closed.kind === "already_closed"
|
|
1927
|
+
? { kind: "closed", runEpoch: input.runEpoch }
|
|
1928
|
+
: { kind: "stale" };
|
|
1929
|
+
},
|
|
1930
|
+
// ADR-125: idle stops are the cloud's judgment from what it can see;
|
|
1931
|
+
// only the host can see a turn that is mid-flight RIGHT NOW. Evaluate
|
|
1932
|
+
// immediately before assignment closure. Manual pause has no veto.
|
|
1933
|
+
shouldVeto:
|
|
1934
|
+
input.reason === "idle"
|
|
1935
|
+
? () => {
|
|
1936
|
+
const activeAt = orchestrator.lastChannelActivity(taskId);
|
|
1937
|
+
return (
|
|
1938
|
+
activeAt !== null && Date.now() - activeAt < IDLE_STOP_VETO_MS
|
|
1939
|
+
);
|
|
1940
|
+
}
|
|
1941
|
+
: undefined,
|
|
1942
|
+
});
|
|
1943
|
+
if (result.ok && result.vetoed) {
|
|
1944
|
+
return { ok: true, value: { vetoed: "active" as const } };
|
|
1945
|
+
}
|
|
1946
|
+
if (!result.ok) {
|
|
1947
|
+
return {
|
|
1948
|
+
ok: false,
|
|
1949
|
+
code: result.assignmentStale
|
|
1950
|
+
? HostErrorCode.TaskAlreadyRunning
|
|
1951
|
+
: HostErrorCode.Internal,
|
|
1952
|
+
message: result.error ?? "stop failed",
|
|
1953
|
+
...(result.assignmentStale ? {} : { retryable: true }),
|
|
1954
|
+
};
|
|
1955
|
+
}
|
|
1956
|
+
return { ok: true, value: {} };
|
|
1957
|
+
},
|
|
1958
|
+
async taskStatus(ctx, taskId) {
|
|
1959
|
+
logCommand(ctx, "taskStatus", taskId);
|
|
1960
|
+
const runtimeFailure = runtimeUnavailable();
|
|
1961
|
+
if (runtimeFailure) return runtimeFailure;
|
|
1962
|
+
const result = await wrapAgent(ctx, "taskStatus", async () => {
|
|
1963
|
+
const environment = await reconstructHostTaskEnvironment(taskId);
|
|
1964
|
+
if (environment === null) {
|
|
1965
|
+
return {
|
|
1966
|
+
composeRunning: false,
|
|
1967
|
+
containers: [],
|
|
1968
|
+
worktreePresent: false,
|
|
1969
|
+
};
|
|
1970
|
+
}
|
|
1971
|
+
return taskEnvironmentStatusResult(await environment.status());
|
|
1972
|
+
});
|
|
1973
|
+
// ADR-051: attach the published preview ports from the host DB so the
|
|
1974
|
+
// cloud can backfill its mirror on reconcile (tasks launched before the
|
|
1975
|
+
// cloud started recording them at task-up).
|
|
1976
|
+
if (result.ok) {
|
|
1977
|
+
const ports = parsePreviewPortRuntimes(
|
|
1978
|
+
getHostTask(taskId)?.previewPorts,
|
|
1979
|
+
);
|
|
1980
|
+
if (ports.length > 0) {
|
|
1981
|
+
return { ...result, value: { ...result.value, previewPorts: ports } };
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
return result;
|
|
1985
|
+
},
|
|
1986
|
+
|
|
1987
|
+
async channelEnsure(ctx, input) {
|
|
1988
|
+
logCommand(ctx, "channelEnsure", input.taskId);
|
|
1989
|
+
if (credentialsMode() === "pending") {
|
|
1990
|
+
return switchingToCloudCredentials();
|
|
1991
|
+
}
|
|
1992
|
+
const admissionFailure = managedMaintenanceUnavailable();
|
|
1993
|
+
if (admissionFailure) return admissionFailure;
|
|
1994
|
+
const runtimeFailure = runtimeUnavailable();
|
|
1995
|
+
if (runtimeFailure) return runtimeFailure;
|
|
1996
|
+
await agentClisReady;
|
|
1997
|
+
if (credentialsMode() === "pending") {
|
|
1998
|
+
return switchingToCloudCredentials();
|
|
1999
|
+
}
|
|
2000
|
+
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
2001
|
+
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
2002
|
+
const readyRuntimeFailure = runtimeUnavailable();
|
|
2003
|
+
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
2004
|
+
try {
|
|
2005
|
+
getOrchestrator().registerChannelSpec(normalizeChannelSpec(input));
|
|
2006
|
+
const ready = await getOrchestrator().ensureStarted(input.taskId);
|
|
2007
|
+
// Open mode historically treats ensure as best-effort: a stopped task or
|
|
2008
|
+
// transient spawn failure still acknowledges the idempotent command.
|
|
2009
|
+
// Secretary mode is different because its CLI is the only delegation
|
|
2010
|
+
// path, so starting without it would create a human-facing agent that
|
|
2011
|
+
// cannot hand work to the crew.
|
|
2012
|
+
if (!ready && input.mode === "secretary") {
|
|
2013
|
+
return {
|
|
2014
|
+
ok: false,
|
|
2015
|
+
code: HostErrorCode.TaskNotRunning,
|
|
2016
|
+
message:
|
|
2017
|
+
"Secretary channel could not start; verify the task CLI and token were materialized",
|
|
2018
|
+
retryable: false,
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
return ok(undefined);
|
|
2022
|
+
} catch (err) {
|
|
2023
|
+
return failFromUnknown(err);
|
|
2024
|
+
}
|
|
2025
|
+
},
|
|
2026
|
+
|
|
2027
|
+
async channelTeardown(ctx, taskId) {
|
|
2028
|
+
logCommand(ctx, "channelTeardown", taskId);
|
|
2029
|
+
const orchestrator = getOrchestrator();
|
|
2030
|
+
return orchestrator.runTaskLifecycle(taskId, async () => {
|
|
2031
|
+
try {
|
|
2032
|
+
await orchestrator.closeChannel(taskId);
|
|
2033
|
+
// ADR-057: the task's gateway routes die with the channel.
|
|
2034
|
+
clearTaskGatewayAcl(taskId);
|
|
2035
|
+
return ok(undefined);
|
|
2036
|
+
} catch (err) {
|
|
2037
|
+
return failFromUnknown(err);
|
|
2038
|
+
}
|
|
2039
|
+
});
|
|
2040
|
+
},
|
|
2041
|
+
|
|
2042
|
+
async channelDeliver(ctx, taskId, agentId, text, origin, assignment) {
|
|
2043
|
+
logCommand(ctx, "channelDeliver", taskId, agentId);
|
|
2044
|
+
if (credentialsMode() === "pending") {
|
|
2045
|
+
return switchingToCloudCredentials();
|
|
2046
|
+
}
|
|
2047
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2048
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2049
|
+
try {
|
|
2050
|
+
// An inbound turn is activity: the idle veto must cover the gap between
|
|
2051
|
+
// this delivery landing and the agent's first output event.
|
|
2052
|
+
getOrchestrator().markChannelActivity(taskId);
|
|
2053
|
+
const result = await getOrchestrator().deliver(taskId, agentId, text, origin, assignment);
|
|
2054
|
+
if (!result.ok) {
|
|
2055
|
+
return {
|
|
2056
|
+
ok: false,
|
|
2057
|
+
code: mapDeliverError(result.error),
|
|
2058
|
+
message: result.error,
|
|
2059
|
+
};
|
|
2060
|
+
}
|
|
2061
|
+
return ok(undefined);
|
|
2062
|
+
} catch (err) {
|
|
2063
|
+
return failFromUnknown(err);
|
|
2064
|
+
}
|
|
2065
|
+
},
|
|
2066
|
+
|
|
2067
|
+
async previewEnsure(ctx, taskId, name, containerPort) {
|
|
2068
|
+
logCommand(ctx, "previewEnsure", taskId, name);
|
|
2069
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2070
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2071
|
+
const orchestrator = getOrchestrator();
|
|
2072
|
+
return orchestrator.runTaskLifecycle(taskId, async () => {
|
|
2073
|
+
try {
|
|
2074
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
2075
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
2076
|
+
const task = getHostTask(taskId);
|
|
2077
|
+
if (!task) {
|
|
2078
|
+
return {
|
|
2079
|
+
ok: false as const,
|
|
2080
|
+
code: HostErrorCode.TaskNotFound,
|
|
2081
|
+
message: `no such task: ${taskId}`,
|
|
2082
|
+
};
|
|
2083
|
+
}
|
|
2084
|
+
if (task.statusMirror !== "running") {
|
|
2085
|
+
return ok({ hostPort: null });
|
|
2086
|
+
}
|
|
2087
|
+
if (!task.composeProject) return ok({ hostPort: null });
|
|
2088
|
+
const declared = parsePreviewPortRuntimes(task.previewPorts).find(
|
|
2089
|
+
(port) => port.name === name,
|
|
2090
|
+
);
|
|
2091
|
+
// A stored host port is never routing truth after an unnoticed daemon
|
|
2092
|
+
// restart. Prove the exact app container and current mapping before
|
|
2093
|
+
// either returning a declared port or creating an ad-hoc sidecar.
|
|
2094
|
+
const appProof = await inspectTunnelContainer(
|
|
2095
|
+
`${task.composeProject}-app-1`,
|
|
2096
|
+
containerPort,
|
|
2097
|
+
);
|
|
2098
|
+
if (appProof.kind === "daemon-unavailable") {
|
|
2099
|
+
return dockerUnavailableResult(
|
|
2100
|
+
`could not verify preview runtime for task ${taskId}: ${appProof.detail}`,
|
|
2101
|
+
);
|
|
2102
|
+
}
|
|
2103
|
+
// The proof awaited Docker. A concurrent runtime recheck can invalidate
|
|
2104
|
+
// its generation before this continuation resumes, so fence the result
|
|
2105
|
+
// again before returning any route to the cloud.
|
|
2106
|
+
const postInspectRuntimeFailure = runtimeUnavailable();
|
|
2107
|
+
if (postInspectRuntimeFailure) return postInspectRuntimeFailure;
|
|
2108
|
+
if (appProof.kind !== "running") return ok({ hostPort: null });
|
|
2109
|
+
// Task-up published port (preview enabled at launch) wins, but use the
|
|
2110
|
+
// mapping Docker just proved rather than the persisted hostPort.
|
|
2111
|
+
if (declared) {
|
|
2112
|
+
return ok({ hostPort: appProof.published?.port ?? null });
|
|
2113
|
+
}
|
|
2114
|
+
// Else start (or find) the node-proxy sidecar NOW — the same one the
|
|
2115
|
+
// tunnel lazy-starts on first access — and hand back its host port.
|
|
2116
|
+
const hostPort = await ensurePreviewSidecar({
|
|
2117
|
+
taskId,
|
|
2118
|
+
composeProject: task.composeProject,
|
|
2119
|
+
name,
|
|
2120
|
+
containerPort,
|
|
2121
|
+
});
|
|
2122
|
+
const postSidecarRuntimeFailure = runtimeUnavailable();
|
|
2123
|
+
if (postSidecarRuntimeFailure) return postSidecarRuntimeFailure;
|
|
2124
|
+
if (hostPort === null) {
|
|
2125
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
2126
|
+
`could not prepare preview runtime for task ${taskId}`,
|
|
2127
|
+
);
|
|
2128
|
+
if (unavailable) return unavailable;
|
|
2129
|
+
}
|
|
2130
|
+
return ok({ hostPort });
|
|
2131
|
+
} catch (err) {
|
|
2132
|
+
return failFromUnknown(err);
|
|
2133
|
+
}
|
|
2134
|
+
});
|
|
2135
|
+
},
|
|
2136
|
+
|
|
2137
|
+
// ADR-116: remove a labeled engine account. Delegates to the same internals
|
|
2138
|
+
// the local UI uses; `default:*` refusal and idempotency live there. The
|
|
2139
|
+
// dispatcher re-advertises capabilities after an ok result.
|
|
2140
|
+
async engineAccountRemove(ctx, input) {
|
|
2141
|
+
logCommand(ctx, "engineAccountRemove", input.id);
|
|
2142
|
+
try {
|
|
2143
|
+
const result = removeEngineAccount(input.id, input.scope);
|
|
2144
|
+
if (!result.ok) {
|
|
2145
|
+
return ok({ removed: false, message: result.message });
|
|
2146
|
+
}
|
|
2147
|
+
return ok({ removed: true });
|
|
2148
|
+
} catch (err) {
|
|
2149
|
+
return failFromUnknown(err);
|
|
2150
|
+
}
|
|
2151
|
+
},
|
|
2152
|
+
|
|
2153
|
+
async engineAccountRename(ctx, input) {
|
|
2154
|
+
logCommand(ctx, "engineAccountRename", input.id);
|
|
2155
|
+
try {
|
|
2156
|
+
const result = renameEngineAccount(input.id, input.label, input.scope);
|
|
2157
|
+
return ok(
|
|
2158
|
+
result.ok
|
|
2159
|
+
? { renamed: true }
|
|
2160
|
+
: { renamed: false, message: result.message },
|
|
2161
|
+
);
|
|
2162
|
+
} catch (err) {
|
|
2163
|
+
return failFromUnknown(err);
|
|
2164
|
+
}
|
|
2165
|
+
},
|
|
2166
|
+
|
|
2167
|
+
async engineAccountReorder(ctx, input) {
|
|
2168
|
+
logCommand(ctx, "engineAccountReorder", input.kind);
|
|
2169
|
+
try {
|
|
2170
|
+
const result = reorderEngineAccounts(input.kind, input.ids, input.scope);
|
|
2171
|
+
return ok(
|
|
2172
|
+
result.ok
|
|
2173
|
+
? { reordered: true }
|
|
2174
|
+
: { reordered: false, message: result.message },
|
|
2175
|
+
);
|
|
2176
|
+
} catch (err) {
|
|
2177
|
+
return failFromUnknown(err);
|
|
2178
|
+
}
|
|
2179
|
+
},
|
|
2180
|
+
|
|
2181
|
+
async engineDisconnect(ctx, input) {
|
|
2182
|
+
logCommand(ctx, "engineDisconnect", input.engine);
|
|
2183
|
+
try {
|
|
2184
|
+
disconnectEngine(input.engine);
|
|
2185
|
+
return ok({ disconnected: true });
|
|
2186
|
+
} catch (err) {
|
|
2187
|
+
return failFromUnknown(err);
|
|
2188
|
+
}
|
|
2189
|
+
},
|
|
2190
|
+
|
|
2191
|
+
async channelResolvePermission(ctx, taskId, agentId, requestId, decision) {
|
|
2192
|
+
logCommand(ctx, "channelResolvePermission", taskId, agentId, requestId);
|
|
2193
|
+
try {
|
|
2194
|
+
const okResolved = await getOrchestrator().resolvePermission(
|
|
2195
|
+
taskId,
|
|
2196
|
+
agentId,
|
|
2197
|
+
requestId,
|
|
2198
|
+
toLegacyDecision(decision),
|
|
2199
|
+
);
|
|
2200
|
+
if (!okResolved) {
|
|
2201
|
+
return {
|
|
2202
|
+
ok: false,
|
|
2203
|
+
code: HostErrorCode.NoSuchPermission,
|
|
2204
|
+
message: "no live session for that permission request",
|
|
2205
|
+
};
|
|
2206
|
+
}
|
|
2207
|
+
return ok(undefined);
|
|
2208
|
+
} catch (err) {
|
|
2209
|
+
return failFromUnknown(err);
|
|
2210
|
+
}
|
|
2211
|
+
},
|
|
2212
|
+
|
|
2213
|
+
async cloneRepo(ctx, input) {
|
|
2214
|
+
logCommand(ctx, "cloneRepo", input.projectId);
|
|
2215
|
+
try {
|
|
2216
|
+
return ok(cloneRepo(input));
|
|
2217
|
+
} catch (err) {
|
|
2218
|
+
return failFromUnknown(err);
|
|
2219
|
+
}
|
|
2220
|
+
},
|
|
2221
|
+
|
|
2222
|
+
async taskDiff(ctx, input) {
|
|
2223
|
+
logCommand(ctx, "taskDiff", input.taskId);
|
|
2224
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2225
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2226
|
+
try {
|
|
2227
|
+
const task = getHostTask(input.taskId);
|
|
2228
|
+
// #cb3j0z: Git never runs on the host against a task's repositories;
|
|
2229
|
+
// what a task configured there would run as the runner's user. A
|
|
2230
|
+
// running task must reconstruct its exact persisted provider identity
|
|
2231
|
+
// (malformed/foreign metadata is not permission to run Git against a
|
|
2232
|
+
// different filesystem view) and diffs there; a stopped one diffs in
|
|
2233
|
+
// a throwaway sandbox of its retained worktree.
|
|
2234
|
+
if (task?.statusMirror !== "running") {
|
|
2235
|
+
return ok(
|
|
2236
|
+
await stoppedTaskDiff(input, task ? { environmentProvider: task.environmentProvider ?? null } : null),
|
|
2237
|
+
);
|
|
2238
|
+
}
|
|
2239
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
2240
|
+
if (environment === null) {
|
|
2241
|
+
throw new Error(
|
|
2242
|
+
`running task ${input.taskId} has no persisted task environment`,
|
|
2243
|
+
);
|
|
2244
|
+
}
|
|
2245
|
+
return ok(await buildTaskDiff(input, environment));
|
|
2246
|
+
} catch (err) {
|
|
2247
|
+
return failFromUnknown(err);
|
|
2248
|
+
}
|
|
2249
|
+
},
|
|
2250
|
+
|
|
2251
|
+
// ADR-161 §2.3: add a repository to a running task. Each command runs in
|
|
2252
|
+
// the task's live-project queue; the publication steps take its lifecycle
|
|
2253
|
+
// slot (lib/task-project-ops.ts).
|
|
2254
|
+
async taskProjectPrepare(ctx, input) {
|
|
2255
|
+
logCommand(ctx, "taskProjectPrepare", input.taskId, input.operationId);
|
|
2256
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2257
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2258
|
+
try {
|
|
2259
|
+
return ok(await taskProjectOps().prepare(input));
|
|
2260
|
+
} catch (err) {
|
|
2261
|
+
return failFromUnknown(err);
|
|
2262
|
+
}
|
|
2263
|
+
},
|
|
2264
|
+
|
|
2265
|
+
async taskProjectActivate(ctx, input) {
|
|
2266
|
+
logCommand(ctx, "taskProjectActivate", input.taskId, input.operationId);
|
|
2267
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2268
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2269
|
+
try {
|
|
2270
|
+
return ok(await taskProjectOps().activate(input));
|
|
2271
|
+
} catch (err) {
|
|
2272
|
+
return failFromUnknown(err);
|
|
2273
|
+
}
|
|
2274
|
+
},
|
|
2275
|
+
|
|
2276
|
+
async taskProjectCompensate(ctx, input) {
|
|
2277
|
+
logCommand(ctx, "taskProjectCompensate", input.taskId, input.operationId);
|
|
2278
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2279
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2280
|
+
try {
|
|
2281
|
+
return ok(await taskProjectOps().compensate(input));
|
|
2282
|
+
} catch (err) {
|
|
2283
|
+
return failFromUnknown(err);
|
|
2284
|
+
}
|
|
2285
|
+
},
|
|
2286
|
+
|
|
2287
|
+
async taskProjectStatus(ctx, input) {
|
|
2288
|
+
logCommand(ctx, "taskProjectStatus", input.taskId, input.operationId);
|
|
2289
|
+
const runtimeFailure = runtimeUnavailable();
|
|
2290
|
+
if (runtimeFailure) return runtimeFailure;
|
|
2291
|
+
try {
|
|
2292
|
+
return ok(await taskProjectOps().status(input));
|
|
2293
|
+
} catch (err) {
|
|
2294
|
+
return failFromUnknown(err);
|
|
2295
|
+
}
|
|
2296
|
+
},
|
|
2297
|
+
|
|
2298
|
+
async attachmentWrite(ctx, input) {
|
|
2299
|
+
logCommand(ctx, "attachmentWrite", input.taskId, input.filename);
|
|
2300
|
+
try {
|
|
2301
|
+
// ADR-121 strangler: attachments go through the environment's file
|
|
2302
|
+
// ops, so this call site no longer knows where the workspace lives —
|
|
2303
|
+
// host-side providers implement them as the exact direct fs access
|
|
2304
|
+
// this used to be; machine providers reach over their transport. A
|
|
2305
|
+
// row without a reconstructible environment (legacy) keeps the
|
|
2306
|
+
// host-FS path unchanged.
|
|
2307
|
+
const environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
2308
|
+
if (environment) {
|
|
2309
|
+
await environment.writeWorkspaceFile(
|
|
2310
|
+
attachmentEnvironmentPath(environment, input.filename),
|
|
2311
|
+
Buffer.from(input.dataBase64, "base64"),
|
|
2312
|
+
);
|
|
2313
|
+
return ok(undefined);
|
|
2314
|
+
}
|
|
2315
|
+
await writeAttachment(
|
|
2316
|
+
input.taskId,
|
|
2317
|
+
input.filename,
|
|
2318
|
+
Buffer.from(input.dataBase64, "base64"),
|
|
2319
|
+
);
|
|
2320
|
+
return ok(undefined);
|
|
2321
|
+
} catch (err) {
|
|
2322
|
+
return failFromUnknown(err);
|
|
2323
|
+
}
|
|
2324
|
+
},
|
|
2325
|
+
|
|
2326
|
+
async workspaceFileRead(ctx, input) {
|
|
2327
|
+
try {
|
|
2328
|
+
const path = normalizedWorkspacePath(assertShareablePath(input.path));
|
|
2329
|
+
const maxBytes = Math.min(8 * 1024 * 1024, Math.max(0, input.maxBytes));
|
|
2330
|
+
const environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
2331
|
+
const bytes = environment ? await environment.readWorkspaceFile(`${environment.descriptor.workspacePath}/${path.slice("/workspace/".length)}`, { maxBytes: maxBytes + 1 }) :
|
|
2332
|
+
await readBoundedFile(taskWorkspaceDir(input.taskId), path.slice("/workspace/".length).split("/"), { maxBytes: maxBytes + 1 });
|
|
2333
|
+
if (!bytes) return { ok: false, code: HostErrorCode.TaskNotFound, message: "no such workspace file" };
|
|
2334
|
+
if (bytes.byteLength > maxBytes) return { ok: false, code: HostErrorCode.PayloadTooLarge, message: "Ask the agent to uai share this file (live preview is limited to 8 MiB)." };
|
|
2335
|
+
return ok({ dataBase64: Buffer.from(bytes).toString("base64"), size: bytes.byteLength });
|
|
2336
|
+
} catch { return { ok: false, code: HostErrorCode.TaskNotFound, message: "no such workspace file" }; }
|
|
2337
|
+
},
|
|
2338
|
+
|
|
2339
|
+
async attachmentRead(ctx, input) {
|
|
2340
|
+
logCommand(ctx, "attachmentRead", input.taskId, input.filename);
|
|
2341
|
+
try {
|
|
2342
|
+
const environment = await reconstructHostTaskEnvironment(input.taskId);
|
|
2343
|
+
const bytes = environment
|
|
2344
|
+
? await environment.readWorkspaceFile(
|
|
2345
|
+
attachmentEnvironmentPath(environment, input.filename),
|
|
2346
|
+
)
|
|
2347
|
+
: await readAttachment(input.taskId, input.filename);
|
|
2348
|
+
if (!bytes) {
|
|
2349
|
+
return {
|
|
2350
|
+
ok: false,
|
|
2351
|
+
code: HostErrorCode.TaskNotFound,
|
|
2352
|
+
message: `no such attachment: ${input.filename}`,
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
return ok({ dataBase64: Buffer.from(bytes).toString("base64") });
|
|
2356
|
+
} catch (err) {
|
|
2357
|
+
return failFromUnknown(err);
|
|
2358
|
+
}
|
|
2359
|
+
},
|
|
2360
|
+
|
|
2361
|
+
async filesOp(ctx, input) {
|
|
2362
|
+
logCommand(ctx, "filesOp", input.op, `${input.scope}:${input.path}`);
|
|
2363
|
+
try {
|
|
2364
|
+
return ok(await handleFilesOp(input));
|
|
2365
|
+
} catch (err) {
|
|
2366
|
+
return failFromUnknown(err);
|
|
2367
|
+
}
|
|
2368
|
+
},
|
|
2369
|
+
|
|
2370
|
+
async sidecarAssign(ctx, input) {
|
|
2371
|
+
logCommand(ctx, "sidecarAssign", input.taskId);
|
|
2372
|
+
try {
|
|
2373
|
+
const { applySidecarAssign } = await import("../lib/sidecar-assign");
|
|
2374
|
+
applySidecarAssign(input.taskId);
|
|
2375
|
+
return ok(undefined);
|
|
2376
|
+
} catch (err) {
|
|
2377
|
+
return failFromUnknown(err);
|
|
2378
|
+
}
|
|
2379
|
+
},
|
|
2380
|
+
|
|
2381
|
+
async channelInterrupt(ctx, taskId, agentId) {
|
|
2382
|
+
logCommand(ctx, "channelInterrupt", taskId, agentId);
|
|
2383
|
+
try {
|
|
2384
|
+
const result = await getOrchestrator().interrupt(taskId, agentId);
|
|
2385
|
+
if (!result.ok) {
|
|
2386
|
+
return {
|
|
2387
|
+
ok: false,
|
|
2388
|
+
code: HostErrorCode.TaskNotRunning,
|
|
2389
|
+
message: result.error,
|
|
2390
|
+
};
|
|
2391
|
+
}
|
|
2392
|
+
return ok(undefined);
|
|
2393
|
+
} catch (err) {
|
|
2394
|
+
return failFromUnknown(err);
|
|
2395
|
+
}
|
|
2396
|
+
},
|
|
2397
|
+
|
|
2398
|
+
async appendTranscript(_ctx, taskId, author, text, targets) {
|
|
2399
|
+
// Per-message + high-frequency, so no logCommand (avoid log spam).
|
|
2400
|
+
try {
|
|
2401
|
+
// ADR-121: a machine-backed workspace lives in the machine's world —
|
|
2402
|
+
// the append must travel over the environment transport. Container
|
|
2403
|
+
// tasks keep the direct host-FS write (their workspace is a bind
|
|
2404
|
+
// mount and the sync path is cheaper than a docker exec per message).
|
|
2405
|
+
const task = getHostTask(taskId);
|
|
2406
|
+
if (isEnvironmentTransportProvider(task?.environmentProvider)) {
|
|
2407
|
+
const environment = await reconstructHostTaskEnvironment(taskId);
|
|
2408
|
+
if (environment) {
|
|
2409
|
+
await appendTranscriptViaEnvironment(
|
|
2410
|
+
taskId,
|
|
2411
|
+
environment,
|
|
2412
|
+
author,
|
|
2413
|
+
text,
|
|
2414
|
+
targets,
|
|
2415
|
+
);
|
|
2416
|
+
return ok(undefined);
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
writeTranscript(taskId, author, text, targets);
|
|
2420
|
+
return ok(undefined);
|
|
2421
|
+
} catch (err) {
|
|
2422
|
+
return failFromUnknown(err);
|
|
2423
|
+
}
|
|
2424
|
+
},
|
|
2425
|
+
};
|
|
2426
|
+
|
|
2427
|
+
function normalizeChannelSpec(input: ChannelEnsureInput): ChannelEnsureInput {
|
|
2428
|
+
return { ...input, workspacePath: "/workspace" };
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
/**
|
|
2432
|
+
* ADR-121: gh auth for a machine-backed task, injected over the environment
|
|
2433
|
+
* transport (token via exec stdin — same `gh auth login --with-token` +
|
|
2434
|
+
* `setup-git` gesture as containers, no compose probing). Best-effort like
|
|
2435
|
+
* the compose reconciler: absence of a GitHub connection is a quiet no-op.
|
|
2436
|
+
*/
|
|
2437
|
+
/** ADR-154: only while the runner is on its own credentials. Leaving them
|
|
2438
|
+
* waits for one under way, and the write is fenced on them: none starts
|
|
2439
|
+
* once leaving has begun. */
|
|
2440
|
+
async function machineTaskGithubAuth(
|
|
2441
|
+
taskId: string,
|
|
2442
|
+
userId: string,
|
|
2443
|
+
): Promise<void> {
|
|
2444
|
+
await whileOnOwnCredentials(async () => {
|
|
2445
|
+
const environment = await reconstructHostTaskEnvironment(taskId);
|
|
2446
|
+
if (!environment) return;
|
|
2447
|
+
const token = await requestAccessToken(userId);
|
|
2448
|
+
if (!token || !onOwnCredentials()) return;
|
|
2449
|
+
// Machine work is not container work: no docker/apple runtime admission.
|
|
2450
|
+
await injectIntoContainer(
|
|
2451
|
+
taskId,
|
|
2452
|
+
token.accessToken,
|
|
2453
|
+
undefined,
|
|
2454
|
+
() => {},
|
|
2455
|
+
environment,
|
|
2456
|
+
);
|
|
2457
|
+
});
|
|
2458
|
+
}
|
|
2459
|
+
|
|
2460
|
+
async function reconstructHostTaskEnvironment(
|
|
2461
|
+
taskId: string,
|
|
2462
|
+
): Promise<TaskEnvironmentHandle<TaskDownResult> | null> {
|
|
2463
|
+
const task = getHostTask(taskId);
|
|
2464
|
+
return task === null ? null : reconstructPersistedTaskEnvironment(task);
|
|
2465
|
+
}
|
|
2466
|
+
|
|
2467
|
+
function taskEnvironmentStatusResult(status: TaskEnvironmentStatus) {
|
|
2468
|
+
if (status.state === "unknown") {
|
|
2469
|
+
throw new Error(`task environment status is unknown: ${status.detail}`);
|
|
2470
|
+
}
|
|
2471
|
+
return {
|
|
2472
|
+
composeRunning: status.state === "running",
|
|
2473
|
+
containers: [...status.instances],
|
|
2474
|
+
worktreePresent: status.workspacePresent,
|
|
2475
|
+
};
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
function taskDownResultForInput(
|
|
2479
|
+
input: TaskDownInput,
|
|
2480
|
+
providerResult: TaskDownResult,
|
|
2481
|
+
): TaskDownResult {
|
|
2482
|
+
const status = input.task.status;
|
|
2483
|
+
return {
|
|
2484
|
+
...providerResult,
|
|
2485
|
+
status: status === "shipped" || status === "error" ? status : "killed",
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
/** Attachment location inside the environment — same shape every provider
|
|
2490
|
+
* serves (`<workspace>/.moondeck/attachments/<name>`), basename-hardened exactly
|
|
2491
|
+
* like the legacy host-side safePath. */
|
|
2492
|
+
function attachmentEnvironmentPath(
|
|
2493
|
+
environment: { descriptor: { workspacePath: string } },
|
|
2494
|
+
filename: string,
|
|
2495
|
+
): string {
|
|
2496
|
+
const name = filename.split("/").pop() ?? "";
|
|
2497
|
+
if (name === "" || name === "." || name === ".." || name.includes("\0")) {
|
|
2498
|
+
throw new Error("invalid attachment filename");
|
|
2499
|
+
}
|
|
2500
|
+
return `${environment.descriptor.workspacePath}/.moondeck/attachments/${name}`;
|
|
2501
|
+
}
|
|
2502
|
+
|
|
2503
|
+
/** ADR-122 phase 1: read each project's committed .env.schema from the
|
|
2504
|
+
* fresh workspace, compare @required keys against the union of values this
|
|
2505
|
+
* host actually injected (env is merged across projects, so any project's
|
|
2506
|
+
* value covers the key), and post one note naming what's missing. */
|
|
2507
|
+
async function noteMissingRequiredEnvKeys(
|
|
2508
|
+
input: TaskLaunchInput,
|
|
2509
|
+
): Promise<void> {
|
|
2510
|
+
const provided = new Set<string>();
|
|
2511
|
+
for (const project of input.projects) {
|
|
2512
|
+
for (const key of Object.keys(getDecryptedForProject(project.id))) {
|
|
2513
|
+
provided.add(key);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
const perProject: Array<{ slug: string; missing: string[] }> = [];
|
|
2517
|
+
for (const project of input.projects) {
|
|
2518
|
+
let content: string;
|
|
2519
|
+
try {
|
|
2520
|
+
content = readFileSync(
|
|
2521
|
+
resolve(taskProjectWorktree(input.task.id, project.slug), ".env.schema"),
|
|
2522
|
+
"utf8",
|
|
2523
|
+
);
|
|
2524
|
+
} catch {
|
|
2525
|
+
continue; // No schema, no opinion — today's behavior, unchanged.
|
|
2526
|
+
}
|
|
2527
|
+
perProject.push({
|
|
2528
|
+
slug: project.slug,
|
|
2529
|
+
missing: missingRequiredKeys(parseEnvSchema(content), provided),
|
|
2530
|
+
});
|
|
2531
|
+
}
|
|
2532
|
+
const note = envSchemaNote(perProject);
|
|
2533
|
+
if (note) getOrchestrator().emitSystemNote(input.task.id, note);
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
function logCommand(
|
|
2537
|
+
ctx: CommandContext,
|
|
2538
|
+
command: keyof HostCommands,
|
|
2539
|
+
...parts: string[]
|
|
2540
|
+
): void {
|
|
2541
|
+
console.log(`[host-agent] ${ctx.commandId} ${command} ${parts.join(" ")}`);
|
|
2542
|
+
}
|
|
2543
|
+
|
|
2544
|
+
async function wrapAgent<T>(
|
|
2545
|
+
_ctx: CommandContext,
|
|
2546
|
+
_command: keyof HostCommands,
|
|
2547
|
+
fn: () => Promise<T>,
|
|
2548
|
+
): Promise<HostCommandResult<T>> {
|
|
2549
|
+
try {
|
|
2550
|
+
return ok(await fn());
|
|
2551
|
+
} catch (err) {
|
|
2552
|
+
if (AgentError.is(err) && isRuntimeSensitiveAgentError(err.code)) {
|
|
2553
|
+
if (err.code === "DOCKER_PS_FAILED") {
|
|
2554
|
+
return dockerUnavailableResult(err.message);
|
|
2555
|
+
}
|
|
2556
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(err.message);
|
|
2557
|
+
if (unavailable) return unavailable;
|
|
2558
|
+
}
|
|
2559
|
+
return failFromUnknown(err);
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
|
|
2563
|
+
function ok<T>(value: T): HostCommandResult<T> {
|
|
2564
|
+
return { ok: true, value };
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
/** Docker-dependent commands must not reinterpret an unavailable or
|
|
2568
|
+
* half-recovered backend as an absent task/container. */
|
|
2569
|
+
function runtimeUnavailable(): HostCommandResult<never> | null {
|
|
2570
|
+
const message = containerRuntimeProblem();
|
|
2571
|
+
if (!message) return null;
|
|
2572
|
+
return {
|
|
2573
|
+
ok: false,
|
|
2574
|
+
code: HostErrorCode.HostUnavailable,
|
|
2575
|
+
message,
|
|
2576
|
+
retryable: true,
|
|
2577
|
+
};
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
function teardownRuntimeUnavailable(): HostCommandResult<never> | null {
|
|
2581
|
+
const message = containerRuntimeTeardownProblem();
|
|
2582
|
+
if (!message) return null;
|
|
2583
|
+
return {
|
|
2584
|
+
ok: false,
|
|
2585
|
+
code: HostErrorCode.HostUnavailable,
|
|
2586
|
+
message,
|
|
2587
|
+
retryable: true,
|
|
2588
|
+
};
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
/** ADR-125: an idle stop is vetoed when the channel showed any activity
|
|
2592
|
+
* this recently — a delivery landing or any session event. */
|
|
2593
|
+
const IDLE_STOP_VETO_MS = 120_000;
|
|
2594
|
+
|
|
2595
|
+
const RUNTIME_TASK_ADMISSION_WAIT_MS = 3_000;
|
|
2596
|
+
|
|
2597
|
+
/**
|
|
2598
|
+
* Runtime activation is intentionally detached from one cloud command's
|
|
2599
|
+
* 15-minute deadline. A first attempt joins for a few seconds, then returns a
|
|
2600
|
+
* narrowly-tagged retry while the single detection/activation promise keeps
|
|
2601
|
+
* running. Every retry rechecks the exact generation and durable provider
|
|
2602
|
+
* claim before task-owned side effects begin.
|
|
2603
|
+
*/
|
|
2604
|
+
async function prepareContainerRuntimeForTask(): Promise<
|
|
2605
|
+
HostCommandResult<never> | null
|
|
2606
|
+
> {
|
|
2607
|
+
// ADR-133 sidecar mode: the task runs on this machine; there is no
|
|
2608
|
+
// container runtime to prepare (live 2026-09-10, hostless smoke 3: this
|
|
2609
|
+
// step answered with the no-runtime message before the gated check ran).
|
|
2610
|
+
if (sidecarMode()) return null;
|
|
2611
|
+
let prepared:
|
|
2612
|
+
| Awaited<ReturnType<typeof ensureContainerRuntimeForTask>>
|
|
2613
|
+
| null;
|
|
2614
|
+
try {
|
|
2615
|
+
prepared = await settleWithin(
|
|
2616
|
+
ensureContainerRuntimeForTask(),
|
|
2617
|
+
RUNTIME_TASK_ADMISSION_WAIT_MS,
|
|
2618
|
+
);
|
|
2619
|
+
} catch (error) {
|
|
2620
|
+
return runtimeSetupRetry(
|
|
2621
|
+
`container runtime setup failed: ${boundedFailureMessage(error)}`,
|
|
2622
|
+
);
|
|
2623
|
+
}
|
|
2624
|
+
if (prepared === null) {
|
|
2625
|
+
return runtimeSetupRetry(
|
|
2626
|
+
"The bundled container runtime is still being provisioned on this host.",
|
|
2627
|
+
);
|
|
2628
|
+
}
|
|
2629
|
+
if (prepared.status !== "ready") {
|
|
2630
|
+
return {
|
|
2631
|
+
ok: false,
|
|
2632
|
+
code: HostErrorCode.HostUnavailable,
|
|
2633
|
+
message:
|
|
2634
|
+
prepared.status === "no-runtime"
|
|
2635
|
+
? prepared.message
|
|
2636
|
+
: "The container runtime is still being detected on this host.",
|
|
2637
|
+
retryable: true,
|
|
2638
|
+
};
|
|
2639
|
+
}
|
|
2640
|
+
const epoch = containerRuntimeReadinessEpoch();
|
|
2641
|
+
const operational = await waitForContainerRuntimeOperational(
|
|
2642
|
+
epoch,
|
|
2643
|
+
RUNTIME_TASK_ADMISSION_WAIT_MS,
|
|
2644
|
+
);
|
|
2645
|
+
if (!operational) {
|
|
2646
|
+
const failure = runtimeUnavailable();
|
|
2647
|
+
if (failure && !failure.ok && !failure.message.includes("still preparing")) {
|
|
2648
|
+
return failure;
|
|
2649
|
+
}
|
|
2650
|
+
return runtimeSetupRetry(
|
|
2651
|
+
"The container runtime is completing image maintenance and task recovery.",
|
|
2652
|
+
);
|
|
2653
|
+
}
|
|
2654
|
+
try {
|
|
2655
|
+
if (await claimContainerRuntimeForTask(epoch)) return null;
|
|
2656
|
+
} catch (error) {
|
|
2657
|
+
return runtimeSetupRetry(
|
|
2658
|
+
`The container runtime could not persist its task-state claim: ${boundedFailureMessage(error)}`,
|
|
2659
|
+
);
|
|
2660
|
+
}
|
|
2661
|
+
return runtimeSetupRetry(
|
|
2662
|
+
"The container runtime changed while task admission was being prepared.",
|
|
2663
|
+
);
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
function runtimeSetupRetry(message: string): HostCommandResult<never> {
|
|
2667
|
+
return {
|
|
2668
|
+
ok: false,
|
|
2669
|
+
code: HostErrorCode.HostUnavailable,
|
|
2670
|
+
message,
|
|
2671
|
+
retryable: true,
|
|
2672
|
+
retryReason: "runtime_setup",
|
|
2673
|
+
};
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
function settleWithin<T>(promise: Promise<T>, timeoutMs: number): Promise<T | null> {
|
|
2677
|
+
return new Promise<T | null>((resolve, reject) => {
|
|
2678
|
+
let settled = false;
|
|
2679
|
+
const timer = setTimeout(() => {
|
|
2680
|
+
if (settled) return;
|
|
2681
|
+
settled = true;
|
|
2682
|
+
resolve(null);
|
|
2683
|
+
}, timeoutMs);
|
|
2684
|
+
timer.unref?.();
|
|
2685
|
+
void promise.then(
|
|
2686
|
+
(value) => {
|
|
2687
|
+
if (settled) return;
|
|
2688
|
+
settled = true;
|
|
2689
|
+
clearTimeout(timer);
|
|
2690
|
+
resolve(value);
|
|
2691
|
+
},
|
|
2692
|
+
(error: unknown) => {
|
|
2693
|
+
if (settled) return;
|
|
2694
|
+
settled = true;
|
|
2695
|
+
clearTimeout(timer);
|
|
2696
|
+
reject(error);
|
|
2697
|
+
},
|
|
2698
|
+
);
|
|
2699
|
+
});
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
function boundedFailureMessage(error: unknown): string {
|
|
2703
|
+
return (error instanceof Error ? error.message : String(error)).slice(0, 512);
|
|
2704
|
+
}
|
|
2705
|
+
|
|
2706
|
+
/** Managed installation, activation, and uninstall drain instead of admitting work. */
|
|
2707
|
+
let lastAdmissionRefusalLogAt = 0;
|
|
2708
|
+
let lastAdmissionReclaimAt = 0;
|
|
2709
|
+
let admissionReclaimInFlight: Promise<"cleared" | "kept" | "none"> | null =
|
|
2710
|
+
null;
|
|
2711
|
+
function managedMaintenanceUnavailable(): HostCommandResult<never> | null {
|
|
2712
|
+
const blocker = managedRuntimeAdmissionBlocker();
|
|
2713
|
+
if (blocker === null) return null;
|
|
2714
|
+
const now = Date.now();
|
|
2715
|
+
// Mid-run self-heal (the abandoned-uninstall marker recurred WHILE the
|
|
2716
|
+
// service was up, 2026-08-19 — boot-time reclaim alone cannot cover a
|
|
2717
|
+
// writer on a schedule). Reclaim only ever touches a provably-untorn
|
|
2718
|
+
// "checking" marker, under the operation lock; rate-limited so the
|
|
2719
|
+
// cloud's retry hammer cannot churn the lock. The command that observed
|
|
2720
|
+
// the marker still refuses — the NEXT retry finds admission open.
|
|
2721
|
+
if (now - lastAdmissionReclaimAt > 30_000 && !admissionReclaimInFlight) {
|
|
2722
|
+
lastAdmissionReclaimAt = now;
|
|
2723
|
+
admissionReclaimInFlight = reclaimAbandonedMaintenanceCheck();
|
|
2724
|
+
void admissionReclaimInFlight
|
|
2725
|
+
.then((verdict) => {
|
|
2726
|
+
if (verdict === "cleared") {
|
|
2727
|
+
console.warn(
|
|
2728
|
+
"[host-agent] cleared an abandoned maintenance marker mid-run (a managed uninstall/update died before any teardown); task admission reopened",
|
|
2729
|
+
);
|
|
2730
|
+
}
|
|
2731
|
+
})
|
|
2732
|
+
.catch((error) => {
|
|
2733
|
+
console.warn(
|
|
2734
|
+
`[host-agent] mid-run maintenance-marker reclaim failed: ${
|
|
2735
|
+
error instanceof Error ? error.message : String(error)
|
|
2736
|
+
}`,
|
|
2737
|
+
);
|
|
2738
|
+
})
|
|
2739
|
+
.finally(() => {
|
|
2740
|
+
admissionReclaimInFlight = null;
|
|
2741
|
+
});
|
|
2742
|
+
}
|
|
2743
|
+
// A refusal must NAME its gate somewhere an operator can find it. Three
|
|
2744
|
+
// stacked silent refusals cost hours live (2026-08-18); rate-limited so a
|
|
2745
|
+
// retrying cloud cannot flood the log.
|
|
2746
|
+
if (now - lastAdmissionRefusalLogAt > 60_000) {
|
|
2747
|
+
lastAdmissionRefusalLogAt = now;
|
|
2748
|
+
console.warn(
|
|
2749
|
+
`[host-agent] refusing task admission: managed-maintenance marker present at ${blocker}`,
|
|
2750
|
+
);
|
|
2751
|
+
}
|
|
2752
|
+
return {
|
|
2753
|
+
ok: false,
|
|
2754
|
+
code: HostErrorCode.HostUnavailable,
|
|
2755
|
+
message: "managed host maintenance is waiting for this runtime to restart",
|
|
2756
|
+
retryable: true,
|
|
2757
|
+
retryReason: "managed_restart",
|
|
2758
|
+
};
|
|
2759
|
+
}
|
|
2760
|
+
|
|
2761
|
+
function dockerUnavailableResult<T = never>(
|
|
2762
|
+
message: string,
|
|
2763
|
+
): HostCommandResult<T> {
|
|
2764
|
+
// Invalidate synchronously so a second command in this event-loop turn
|
|
2765
|
+
// cannot touch the daemon under a stale ready verdict. Detection/recovery is
|
|
2766
|
+
// asynchronous and deliberately does not delay the retryable response.
|
|
2767
|
+
suspendContainerRuntime(true);
|
|
2768
|
+
void initializeContainerRuntime();
|
|
2769
|
+
return {
|
|
2770
|
+
ok: false,
|
|
2771
|
+
code: HostErrorCode.HostUnavailable,
|
|
2772
|
+
message,
|
|
2773
|
+
retryable: true,
|
|
2774
|
+
};
|
|
2775
|
+
}
|
|
2776
|
+
|
|
2777
|
+
async function runtimeFailureIfDaemonUnavailable<T = never>(
|
|
2778
|
+
message: string,
|
|
2779
|
+
): Promise<HostCommandResult<T> | null> {
|
|
2780
|
+
return (await reprobeContainerRuntimeMachineIdentity()) !== null
|
|
2781
|
+
? null
|
|
2782
|
+
: dockerUnavailableResult(message);
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
function isRuntimeSensitiveAgentError(code: string): boolean {
|
|
2786
|
+
return (
|
|
2787
|
+
code === "STANDARD_IMAGE_UNAVAILABLE" ||
|
|
2788
|
+
code === "STANDARD_IMAGE_MISSING" ||
|
|
2789
|
+
code === "COMPOSE_UP_FAILED" ||
|
|
2790
|
+
code === "COMPOSE_DOWN_FAILED" ||
|
|
2791
|
+
code === "CONTAINER_INIT_FAILED" ||
|
|
2792
|
+
code === "DOCKER_PS_FAILED"
|
|
2793
|
+
);
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2796
|
+
function failFromUnknown<T = never>(err: unknown): HostCommandResult<T> {
|
|
2797
|
+
if (err instanceof ContainerRuntimeUnavailableError) {
|
|
2798
|
+
return {
|
|
2799
|
+
ok: false,
|
|
2800
|
+
code: HostErrorCode.HostUnavailable,
|
|
2801
|
+
message: err.message,
|
|
2802
|
+
retryable: true,
|
|
2803
|
+
};
|
|
2804
|
+
}
|
|
2805
|
+
if (AgentError.is(err)) {
|
|
2806
|
+
return {
|
|
2807
|
+
ok: false,
|
|
2808
|
+
code: mapAgentError(err.code),
|
|
2809
|
+
message: err.message,
|
|
2810
|
+
retryable: isRetryableAgentError(err.code),
|
|
2811
|
+
};
|
|
2812
|
+
}
|
|
2813
|
+
return {
|
|
2814
|
+
ok: false,
|
|
2815
|
+
code: HostErrorCode.Internal,
|
|
2816
|
+
message: err instanceof Error ? err.message : String(err),
|
|
2817
|
+
};
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
function mapAgentError(code: string): HostErrorCode {
|
|
2821
|
+
switch (code) {
|
|
2822
|
+
case "TASK_NOT_FOUND":
|
|
2823
|
+
return HostErrorCode.TaskNotFound;
|
|
2824
|
+
case "PROJECT_NOT_FOUND":
|
|
2825
|
+
return HostErrorCode.ProjectNotFound;
|
|
2826
|
+
case "TEMPLATE_NOT_FOUND":
|
|
2827
|
+
return HostErrorCode.TemplateNotFound;
|
|
2828
|
+
case "COMPOSE_UP_FAILED":
|
|
2829
|
+
return HostErrorCode.ComposeUpFailed;
|
|
2830
|
+
case "COMPOSE_DOWN_FAILED":
|
|
2831
|
+
return HostErrorCode.ComposeDownFailed;
|
|
2832
|
+
case "WORKTREE_FAILED":
|
|
2833
|
+
return HostErrorCode.WorktreeFailed;
|
|
2834
|
+
case "CLONE_FAILED":
|
|
2835
|
+
return HostErrorCode.CloneFailed;
|
|
2836
|
+
case "GITHUB_SSH_ACCESS_DENIED":
|
|
2837
|
+
return HostErrorCode.GitHubSshAccessDenied;
|
|
2838
|
+
case "GITHUB_TOKEN_ACCESS_DENIED":
|
|
2839
|
+
return HostErrorCode.GitHubTokenAccessDenied;
|
|
2840
|
+
case "GITHUB_AUTH_UNAVAILABLE":
|
|
2841
|
+
return HostErrorCode.GitHubAuthUnavailable;
|
|
2842
|
+
case "GITHUB_CONNECTION_REQUIRED":
|
|
2843
|
+
return HostErrorCode.GitHubConnectionRequired;
|
|
2844
|
+
case "FETCH_FAILED":
|
|
2845
|
+
return HostErrorCode.FetchFailed;
|
|
2846
|
+
case "RENDER_FAILED":
|
|
2847
|
+
return HostErrorCode.RenderFailed;
|
|
2848
|
+
case "CONTAINER_INIT_FAILED":
|
|
2849
|
+
return HostErrorCode.ContainerInitFailed;
|
|
2850
|
+
case "DB_FAILED":
|
|
2851
|
+
return HostErrorCode.DbFailed;
|
|
2852
|
+
case "DOCKER_PS_FAILED":
|
|
2853
|
+
return HostErrorCode.HostUnavailable;
|
|
2854
|
+
default:
|
|
2855
|
+
return HostErrorCode.Internal;
|
|
2856
|
+
}
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
/** Host-side predicate only; API-boundary normalization already canonicalizes
|
|
2860
|
+
* normal GitHub projects. Keep scratchpads/GitLab/local remotes independent of
|
|
2861
|
+
* an unrelated broken GitHub connection on the same host. */
|
|
2862
|
+
function isGithubRepo(input: string): boolean {
|
|
2863
|
+
const value = input.trim();
|
|
2864
|
+
if (/^(?:[^@/]+@)?(?:www\.)?github\.com:/i.test(value)) return true;
|
|
2865
|
+
if (/^[\w.-]+\/[\w.-]+(?:\.git)?\/?$/i.test(value)) return true;
|
|
2866
|
+
try {
|
|
2867
|
+
const url = new URL(value);
|
|
2868
|
+
return /^(?:www\.)?github\.com$/i.test(url.hostname);
|
|
2869
|
+
} catch {
|
|
2870
|
+
return false;
|
|
2871
|
+
}
|
|
2872
|
+
}
|
|
2873
|
+
|
|
2874
|
+
function mapDeliverError(message: string): HostErrorCode {
|
|
2875
|
+
if (message.includes("not found")) return HostErrorCode.TaskNotFound;
|
|
2876
|
+
if (message.includes("not running")) return HostErrorCode.TaskNotRunning;
|
|
2877
|
+
if (message.includes("no such agent")) return HostErrorCode.NoSuchAgent;
|
|
2878
|
+
return HostErrorCode.Internal;
|
|
2879
|
+
}
|
|
2880
|
+
|
|
2881
|
+
function isRetryableAgentError(code: string): boolean {
|
|
2882
|
+
return (
|
|
2883
|
+
code === "FETCH_FAILED" ||
|
|
2884
|
+
code === "CONTAINER_INIT_FAILED" ||
|
|
2885
|
+
code === "GITHUB_AUTH_UNAVAILABLE" ||
|
|
2886
|
+
code === "DOCKER_PS_FAILED"
|
|
2887
|
+
);
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
function toLegacyDecision(decision: PermissionDecision): "accept" | "decline" {
|
|
2891
|
+
return decision.kind;
|
|
2892
|
+
}
|
|
2893
|
+
|
|
2894
|
+
export type {
|
|
2895
|
+
ChannelEnsureInput,
|
|
2896
|
+
TaskDiffInput,
|
|
2897
|
+
TaskDownInput,
|
|
2898
|
+
TaskLaunchInput,
|
|
2899
|
+
};
|