@runuai/host 0.9.148 → 0.9.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0024_task_env_mode.sql +9 -0
- package/db/migrations/0025_project_env_authority.sql +147 -0
- package/db/migrations/0026_project_env_replica.sql +150 -0
- package/db/migrations/meta/_journal.json +21 -0
- package/db/schema.ts +312 -2
- package/images/standard/Dockerfile +44 -6
- package/images/standard/container/uai-init +85 -2
- package/images/standard/container/uai-task-forwarder.mjs +303 -0
- package/lib/agent-cli.ts +18 -1
- package/lib/agent.ts +99 -34
- package/lib/agents/codex.ts +9 -1
- package/lib/agents/cursor.ts +15 -1
- package/lib/agents/grok.ts +8 -0
- package/lib/agents/kimi.ts +1 -0
- package/lib/agents/machine-durable.ts +2 -0
- package/lib/agents/opencode.ts +1 -0
- package/lib/agents/proc.ts +6 -1
- package/lib/agents/transport.ts +9 -5
- package/lib/browser-testing.ts +6 -0
- package/lib/codex-auth.ts +2 -0
- package/lib/credential-store.ts +141 -6
- package/lib/engine-accounts.ts +1 -0
- package/lib/env-file.ts +7 -0
- package/lib/env-schema.ts +841 -51
- package/lib/git-diff.ts +1 -0
- package/lib/git-identity.ts +2 -0
- package/lib/github-tokens.ts +1 -0
- package/lib/host-env.ts +98 -18
- package/lib/lease.ts +11 -3
- package/lib/machine-gateway-tunnel.ts +195 -17
- package/lib/machine-project-env-loader.ts +152 -0
- package/lib/managed-runtime.ts +270 -51
- package/lib/mcp-gateway.ts +777 -26
- package/lib/orchestrator.ts +391 -69
- package/lib/project-env-authority.ts +1319 -0
- package/lib/project-env-feature.ts +74 -0
- package/lib/project-env-loader.ts +237 -0
- package/lib/project-env-replica.ts +2094 -0
- package/lib/project-env-schema-capture.ts +342 -0
- package/lib/project-env-staging.ts +148 -0
- package/lib/project-env-task-policy.ts +176 -0
- package/lib/runtime-archive.ts +79 -16
- package/lib/runtime-authority.ts +44 -0
- package/lib/runtime-state.ts +175 -10
- package/lib/secrets-proxy/address.ts +72 -0
- package/lib/secrets-proxy/audit.ts +30 -0
- package/lib/secrets-proxy/ca-store.ts +200 -0
- package/lib/secrets-proxy/ca.ts +470 -0
- package/lib/secrets-proxy/container-steps.ts +138 -0
- package/lib/secrets-proxy/der.ts +273 -0
- package/lib/secrets-proxy/engine.ts +1633 -0
- package/lib/secrets-proxy/host.ts +93 -0
- package/lib/secrets-proxy/hostname.ts +60 -0
- package/lib/secrets-proxy/placeholders.ts +196 -0
- package/lib/secrets-proxy/policy.ts +141 -0
- package/lib/secrets-proxy/readiness.ts +148 -0
- package/lib/secrets-proxy/runtime.ts +582 -0
- package/lib/secrets-proxy/scrub.ts +559 -0
- package/lib/secrets-proxy/sidecar-view.ts +139 -0
- package/lib/secrets-proxy/task-scripts.ts +262 -0
- package/lib/secrets-proxy/task-trust.ts +574 -0
- package/lib/secrets-proxy/trust-watch.ts +80 -0
- package/lib/shared-files-s3.ts +3 -0
- package/lib/sidecar-guard.ts +58 -0
- package/lib/sidecar-resume.ts +51 -3
- package/lib/standard-image.ts +50 -12
- package/lib/task-env-forwarder.ts +228 -0
- package/lib/task-env-source.ts +193 -0
- package/lib/task-environment/apple-container.ts +45 -11
- package/lib/task-environment/docker.ts +51 -12
- package/lib/task-environment/index.ts +6 -2
- package/lib/task-environment/machine-task-up.ts +258 -25
- package/lib/task-environment/machine.ts +80 -22
- package/lib/task-environment/registry.ts +3 -2
- package/lib/task-environment/self-identity.ts +20 -0
- package/lib/task-environment/self.ts +92 -16
- package/lib/task-environment/types.ts +63 -0
- package/lib/task-gateway-port.ts +22 -0
- package/lib/task-gateway-protocol.ts +190 -0
- package/lib/task-gateway-recovery.ts +44 -0
- package/lib/task-gateway-state.ts +497 -0
- package/lib/transcript.ts +1 -0
- package/package.json +13 -1
- package/scripts/agent/env-hook.sh +626 -0
- package/scripts/agent/task-up.sh +476 -27
- package/scripts/agent/uai-env-exec.sh +26 -0
- package/src/cli.ts +26 -3
- package/src/index.ts +273 -19
- package/src/main.ts +939 -48
- package/src/project-env-latest.ts +130 -0
- package/src/project-env-policy.ts +281 -0
- package/src/project-env-wording.ts +47 -0
- package/src/protocol.ts +2219 -15
package/lib/orchestrator.ts
CHANGED
|
@@ -50,7 +50,12 @@ import { ACTIVE_STATUSES, isActive, isOnDashboard } from "./task-status";
|
|
|
50
50
|
import { isSafeHostTaskId } from "./task-identity";
|
|
51
51
|
import { projectEnvForTask } from "./host-env";
|
|
52
52
|
import { shadowRoute } from "./routing/shadow";
|
|
53
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
getHostTask,
|
|
55
|
+
hostTaskEnvMode,
|
|
56
|
+
hostTaskProjectIds,
|
|
57
|
+
upsertHostTask,
|
|
58
|
+
} from "./runtime-state";
|
|
54
59
|
import { clearRefresh, reconcileTaskGitAuth } from "./github-tokens";
|
|
55
60
|
import { ensureTaskSshIdentity, setupTaskGitIdentity } from "./git-identity";
|
|
56
61
|
import { dockerCli } from "./docker-exec";
|
|
@@ -83,7 +88,20 @@ import {
|
|
|
83
88
|
appleTaskStoreVolumeName,
|
|
84
89
|
parseAppleTaskEnvironmentLocator,
|
|
85
90
|
} from "./task-environment/apple-container";
|
|
86
|
-
import {
|
|
91
|
+
import {
|
|
92
|
+
DOCKER_TASK_ENVIRONMENT_PROVIDER,
|
|
93
|
+
parseDockerTaskEnvironmentLocator,
|
|
94
|
+
} from "./task-environment/docker";
|
|
95
|
+
import { APPLE_TASK_ENVIRONMENT_PROVIDER } from "./task-environment/apple-container";
|
|
96
|
+
import { activeSecretsProxy } from "./secrets-proxy/host";
|
|
97
|
+
import { usableProxiedKeys } from "./secrets-proxy/policy";
|
|
98
|
+
import { forgetTaskProxyReadiness } from "./secrets-proxy/readiness";
|
|
99
|
+
import {
|
|
100
|
+
invalidateTaskWrites,
|
|
101
|
+
applyTaskRootSteps,
|
|
102
|
+
taskWritesQuiescing,
|
|
103
|
+
type RecoveryProxyFailure,
|
|
104
|
+
} from "./secrets-proxy/task-trust";
|
|
87
105
|
import type {
|
|
88
106
|
TaskEnvironmentDescriptor,
|
|
89
107
|
TaskEnvironmentHandle,
|
|
@@ -97,9 +115,15 @@ import {
|
|
|
97
115
|
type SkillExec,
|
|
98
116
|
writeAgentSkills,
|
|
99
117
|
} from "./skills";
|
|
100
|
-
import {
|
|
118
|
+
import {
|
|
119
|
+
ensureMachineGatewayTunnel,
|
|
120
|
+
ensureMachineGatewayTunnelReady,
|
|
121
|
+
MACHINE_GATEWAY_TUNNEL_PORT,
|
|
122
|
+
} from "./machine-gateway-tunnel";
|
|
123
|
+
import { ensureContainerTaskEnvForwarder } from "./task-env-forwarder";
|
|
101
124
|
import {
|
|
102
125
|
CONTAINER_CLI_PATH,
|
|
126
|
+
agentCliArgv,
|
|
103
127
|
agentCliEnv,
|
|
104
128
|
apiUrlFromCloudUrl,
|
|
105
129
|
loadTaskCliSecret,
|
|
@@ -126,6 +150,12 @@ import {
|
|
|
126
150
|
type ResolvedEngineAccount,
|
|
127
151
|
} from "./engine-accounts";
|
|
128
152
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
153
|
+
import {
|
|
154
|
+
reactivateTaskGatewayTask,
|
|
155
|
+
removeTaskGatewayTask,
|
|
156
|
+
suspendTaskGatewayTask,
|
|
157
|
+
type TaskGatewayRegistration,
|
|
158
|
+
} from "./task-gateway-state";
|
|
129
159
|
import {
|
|
130
160
|
PREVIEW_SIDECAR_TASK_LABEL,
|
|
131
161
|
stopPreviewSidecars,
|
|
@@ -150,6 +180,7 @@ import {
|
|
|
150
180
|
SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
|
|
151
181
|
} from "../src/protocol";
|
|
152
182
|
import { isEnvironmentTransportProvider } from "./task-environment/types";
|
|
183
|
+
import { forgetTaskEnvSnapshot } from "./task-env-source";
|
|
153
184
|
|
|
154
185
|
export type HostEventSubscriber = (event: HostEvent) => void;
|
|
155
186
|
|
|
@@ -708,6 +739,7 @@ export class Orchestrator {
|
|
|
708
739
|
channel.sharedFiles,
|
|
709
740
|
spec.mode,
|
|
710
741
|
spec.secretaryAgentId,
|
|
742
|
+
taskProxiedKeys(channel.taskId),
|
|
711
743
|
),
|
|
712
744
|
);
|
|
713
745
|
}
|
|
@@ -937,6 +969,7 @@ export class Orchestrator {
|
|
|
937
969
|
spec.sharedFiles ?? "ro",
|
|
938
970
|
spec.mode,
|
|
939
971
|
spec.secretaryAgentId,
|
|
972
|
+
taskProxiedKeys(taskId),
|
|
940
973
|
),
|
|
941
974
|
);
|
|
942
975
|
// ADR-046: materialise this agent's skills to its per-agent SKILL.md in
|
|
@@ -2112,6 +2145,7 @@ export class Orchestrator {
|
|
|
2112
2145
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2113
2146
|
channel,
|
|
2114
2147
|
agent,
|
|
2148
|
+
task,
|
|
2115
2149
|
),
|
|
2116
2150
|
agentEnv: this.accountAgentEnv(
|
|
2117
2151
|
channel,
|
|
@@ -2124,7 +2158,7 @@ export class Orchestrator {
|
|
|
2124
2158
|
cliSecret,
|
|
2125
2159
|
this.isPromptLevelSecretary(channel, agent.id),
|
|
2126
2160
|
),
|
|
2127
|
-
|
|
2161
|
+
task,
|
|
2128
2162
|
),
|
|
2129
2163
|
},
|
|
2130
2164
|
);
|
|
@@ -2153,6 +2187,7 @@ export class Orchestrator {
|
|
|
2153
2187
|
private attachCompatibilityKeyFor(
|
|
2154
2188
|
channel: Channel,
|
|
2155
2189
|
agent: RosterAgent,
|
|
2190
|
+
task: { envMode?: string | null },
|
|
2156
2191
|
): string | undefined {
|
|
2157
2192
|
// Key the durable runner to the policy actually passed to the adapter.
|
|
2158
2193
|
// The restricted profile is dormant today, but giving it the normal-tools
|
|
@@ -2168,9 +2203,11 @@ export class Orchestrator {
|
|
|
2168
2203
|
// Preserve legacy null/policy-only attachment for agents that choose no
|
|
2169
2204
|
// launch flags. Explicit model/effort gets a stable key so a host restart
|
|
2170
2205
|
// cannot attach a runner created under different settings.
|
|
2171
|
-
|
|
2206
|
+
const envMode = hostTaskEnvMode(task);
|
|
2207
|
+
if (envMode === "baked" && !model && !effort) return policy;
|
|
2172
2208
|
return JSON.stringify([
|
|
2173
|
-
"roster-launch-v1",
|
|
2209
|
+
envMode === "baked" ? "roster-launch-v1" : "roster-launch-v2",
|
|
2210
|
+
...(envMode === "loader" ? [envMode] : []),
|
|
2174
2211
|
agent.kind,
|
|
2175
2212
|
model,
|
|
2176
2213
|
effort,
|
|
@@ -2209,10 +2246,13 @@ export class Orchestrator {
|
|
|
2209
2246
|
channel: Channel,
|
|
2210
2247
|
agent: RosterAgent,
|
|
2211
2248
|
base: Record<string, string>,
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2249
|
+
task: {
|
|
2250
|
+
environmentProvider: string | null;
|
|
2251
|
+
projectIds?: string | null;
|
|
2252
|
+
envMode?: string | null;
|
|
2253
|
+
},
|
|
2215
2254
|
): Record<string, string> {
|
|
2255
|
+
const floor = this.machineProjectEnv(task);
|
|
2216
2256
|
if (!this.hasDesiredLaunchSettings(channel, agent.id, agent)) {
|
|
2217
2257
|
channel.accountByAgent.delete(agent.id);
|
|
2218
2258
|
return { ...floor, ...base };
|
|
@@ -2230,18 +2270,22 @@ export class Orchestrator {
|
|
|
2230
2270
|
}
|
|
2231
2271
|
|
|
2232
2272
|
/**
|
|
2233
|
-
*
|
|
2234
|
-
*
|
|
2235
|
-
*
|
|
2236
|
-
*
|
|
2237
|
-
* tasks — their env already carries it and must not be doubled into argv.
|
|
2273
|
+
* Rollout compatibility for a machine/self task created before ADR-153.
|
|
2274
|
+
* Those runtimes had no container env: project values rode every agent
|
|
2275
|
+
* spawn. Loader tasks must never take this path; they fetch through the
|
|
2276
|
+
* gateway, and adding values here would freeze them and bypass declarations.
|
|
2238
2277
|
*/
|
|
2239
2278
|
private machineProjectEnv(task: {
|
|
2240
2279
|
environmentProvider: string | null;
|
|
2241
2280
|
projectIds?: string | null;
|
|
2281
|
+
envMode?: string | null;
|
|
2242
2282
|
}): Record<string, string> {
|
|
2243
|
-
|
|
2244
|
-
|
|
2283
|
+
if (
|
|
2284
|
+
hostTaskEnvMode(task) !== "baked" ||
|
|
2285
|
+
!isEnvironmentTransportProvider(task.environmentProvider)
|
|
2286
|
+
) {
|
|
2287
|
+
return {};
|
|
2288
|
+
}
|
|
2245
2289
|
return projectEnvForTask(hostTaskProjectIds(task));
|
|
2246
2290
|
}
|
|
2247
2291
|
|
|
@@ -2386,6 +2430,7 @@ export class Orchestrator {
|
|
|
2386
2430
|
try {
|
|
2387
2431
|
const res = await environment.exec({
|
|
2388
2432
|
argv: ["/bin/sh", "-c", codexHomeSeedCommand(home, perAgent)],
|
|
2433
|
+
projectEnvProfile: "maintenance",
|
|
2389
2434
|
user: "root",
|
|
2390
2435
|
env: RUNTIME_AUTHORITY_ENV,
|
|
2391
2436
|
timeoutMs: 30_000,
|
|
@@ -2577,6 +2622,7 @@ export class Orchestrator {
|
|
|
2577
2622
|
const argv = cmd as [string, ...string[]];
|
|
2578
2623
|
const result = await environment.exec({
|
|
2579
2624
|
argv,
|
|
2625
|
+
projectEnvProfile: "maintenance",
|
|
2580
2626
|
cwd,
|
|
2581
2627
|
user: "node",
|
|
2582
2628
|
env: RUNTIME_AUTHORITY_ENV,
|
|
@@ -2740,13 +2786,14 @@ export class Orchestrator {
|
|
|
2740
2786
|
{
|
|
2741
2787
|
taskId: channel.taskId,
|
|
2742
2788
|
agent,
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2789
|
+
systemPreamble: channel.preambles.get(agent.id) ?? "",
|
|
2790
|
+
executionProfile: this.executionProfileFor(channel, agent.id),
|
|
2791
|
+
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
2792
|
+
channel,
|
|
2793
|
+
agent,
|
|
2794
|
+
task,
|
|
2795
|
+
),
|
|
2796
|
+
agentEnv: this.accountAgentEnv(
|
|
2750
2797
|
channel,
|
|
2751
2798
|
agent,
|
|
2752
2799
|
agentCliEnv(
|
|
@@ -2757,7 +2804,7 @@ export class Orchestrator {
|
|
|
2757
2804
|
cliSecret,
|
|
2758
2805
|
this.isPromptLevelSecretary(channel, agent.id),
|
|
2759
2806
|
),
|
|
2760
|
-
|
|
2807
|
+
task,
|
|
2761
2808
|
),
|
|
2762
2809
|
},
|
|
2763
2810
|
);
|
|
@@ -3204,7 +3251,8 @@ export class Orchestrator {
|
|
|
3204
3251
|
exec: async (argv) => {
|
|
3205
3252
|
if (!handle || !canExec) return null;
|
|
3206
3253
|
const r = await handle.exec({
|
|
3207
|
-
argv,
|
|
3254
|
+
argv: agentCliArgv(handle, argv),
|
|
3255
|
+
projectEnvProfile: "task",
|
|
3208
3256
|
cwd: scratchDir,
|
|
3209
3257
|
user: "node",
|
|
3210
3258
|
env,
|
|
@@ -3860,10 +3908,15 @@ export class Orchestrator {
|
|
|
3860
3908
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
3861
3909
|
channel,
|
|
3862
3910
|
agent,
|
|
3911
|
+
task,
|
|
3863
3912
|
),
|
|
3864
3913
|
agentEnv: boundAccount
|
|
3865
|
-
? {
|
|
3866
|
-
|
|
3914
|
+
? {
|
|
3915
|
+
...this.machineProjectEnv(task),
|
|
3916
|
+
...boundAccount.execEnv,
|
|
3917
|
+
...base,
|
|
3918
|
+
}
|
|
3919
|
+
: this.accountAgentEnv(channel, agent, base, task),
|
|
3867
3920
|
});
|
|
3868
3921
|
if (!session) {
|
|
3869
3922
|
if (channel.rosterSessionChanges.has(agentId)) {
|
|
@@ -4062,8 +4115,13 @@ export class Orchestrator {
|
|
|
4062
4115
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
4063
4116
|
channel,
|
|
4064
4117
|
agent,
|
|
4118
|
+
task,
|
|
4065
4119
|
),
|
|
4066
|
-
agentEnv: {
|
|
4120
|
+
agentEnv: {
|
|
4121
|
+
...this.machineProjectEnv(task),
|
|
4122
|
+
...next.execEnv,
|
|
4123
|
+
...base,
|
|
4124
|
+
},
|
|
4067
4125
|
},
|
|
4068
4126
|
recovery,
|
|
4069
4127
|
);
|
|
@@ -4267,8 +4325,13 @@ export class Orchestrator {
|
|
|
4267
4325
|
attachCompatibilityKey: this.attachCompatibilityKeyFor(
|
|
4268
4326
|
channel,
|
|
4269
4327
|
agent,
|
|
4328
|
+
task,
|
|
4270
4329
|
),
|
|
4271
|
-
agentEnv: {
|
|
4330
|
+
agentEnv: {
|
|
4331
|
+
...this.machineProjectEnv(task),
|
|
4332
|
+
...account.execEnv,
|
|
4333
|
+
...base,
|
|
4334
|
+
},
|
|
4272
4335
|
},
|
|
4273
4336
|
recovery,
|
|
4274
4337
|
);
|
|
@@ -4559,18 +4622,38 @@ export class Orchestrator {
|
|
|
4559
4622
|
* tombstone prevents any agent process from respawning during maintenance. */
|
|
4560
4623
|
async quarantineChannelForRecovery(taskId: string): Promise<void> {
|
|
4561
4624
|
const currentSpec = this.channelSpecs.get(taskId);
|
|
4625
|
+
const suspended = suspendTaskGatewayTask(taskId);
|
|
4626
|
+
if (suspended) this.suspendedTaskGateways.set(taskId, suspended);
|
|
4562
4627
|
await this.closeChannel(taskId);
|
|
4563
4628
|
if (currentSpec && !this.channelSpecs.has(taskId)) {
|
|
4564
4629
|
this.channelSpecs.set(taskId, currentSpec);
|
|
4565
4630
|
}
|
|
4566
4631
|
}
|
|
4567
4632
|
|
|
4633
|
+
/** Reopen only the authenticated task gateway for sanctioned recovery
|
|
4634
|
+
* work. The channel tombstone remains in place until allowChannel(), so
|
|
4635
|
+
* agents cannot respawn while uai-init's dependency worker refreshes env. */
|
|
4636
|
+
reactivateTaskGatewayForRecovery(taskId: string): void {
|
|
4637
|
+
const suspended = this.suspendedTaskGateways.get(taskId);
|
|
4638
|
+
if (!suspended) return;
|
|
4639
|
+
if (!reactivateTaskGatewayTask(suspended)) {
|
|
4640
|
+
throw new Error(
|
|
4641
|
+
`task ${taskId} gateway generation could not be reactivated for recovery`,
|
|
4642
|
+
);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
|
|
4568
4646
|
/** A successful task-up is the only transition that reopens a tombstoned
|
|
4569
4647
|
* task id after stop/teardown. */
|
|
4570
4648
|
allowChannel(taskId: string): void {
|
|
4571
4649
|
// A caller that forgot the task-up fence cannot reopen the id while old
|
|
4572
4650
|
// external operations are still draining.
|
|
4573
4651
|
if (this.channelClosures.has(taskId)) return;
|
|
4652
|
+
const suspended = this.suspendedTaskGateways.get(taskId);
|
|
4653
|
+
if (suspended) {
|
|
4654
|
+
reactivateTaskGatewayTask(suspended);
|
|
4655
|
+
this.suspendedTaskGateways.delete(taskId);
|
|
4656
|
+
}
|
|
4574
4657
|
this.blockedTasks.delete(taskId);
|
|
4575
4658
|
}
|
|
4576
4659
|
|
|
@@ -4587,6 +4670,10 @@ export class Orchestrator {
|
|
|
4587
4670
|
* (ADR-125) reads this — the cloud decides idleness from what it can see,
|
|
4588
4671
|
* but only the host can see a turn that is mid-flight RIGHT NOW. */
|
|
4589
4672
|
readonly #channelActivity = new Map<string, number>();
|
|
4673
|
+
private readonly suspendedTaskGateways = new Map<
|
|
4674
|
+
string,
|
|
4675
|
+
TaskGatewayRegistration
|
|
4676
|
+
>();
|
|
4590
4677
|
|
|
4591
4678
|
markChannelActivity(taskId: string): void {
|
|
4592
4679
|
this.#channelActivity.set(taskId, Date.now());
|
|
@@ -4612,7 +4699,11 @@ export class Orchestrator {
|
|
|
4612
4699
|
if (!task) return { ok: false, error: "unknown task" };
|
|
4613
4700
|
const project = task.composeProject;
|
|
4614
4701
|
if (!project) return { ok: false, error: "task has no running stack" };
|
|
4615
|
-
//
|
|
4702
|
+
// Revoke env/proxy access before a potentially long session drain, then
|
|
4703
|
+
// close sessions so nothing races to respawn them mid-stop.
|
|
4704
|
+
const suspendedGateway = suspendTaskGatewayTask(taskId);
|
|
4705
|
+
// ADR-153: no longer served; end its open proxy connections too.
|
|
4706
|
+
activeSecretsProxy()?.runtime.stopTask(taskId);
|
|
4616
4707
|
await this.closeChannel(taskId);
|
|
4617
4708
|
try {
|
|
4618
4709
|
await stopPreviewSidecars(taskId);
|
|
@@ -4633,11 +4724,13 @@ export class Orchestrator {
|
|
|
4633
4724
|
try {
|
|
4634
4725
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
4635
4726
|
if (!environment) {
|
|
4727
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
4636
4728
|
this.allowChannel(taskId);
|
|
4637
4729
|
return { ok: false, error: "task has no persisted environment" };
|
|
4638
4730
|
}
|
|
4639
4731
|
await environment.stop();
|
|
4640
4732
|
} catch (err) {
|
|
4733
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
4641
4734
|
this.allowChannel(taskId);
|
|
4642
4735
|
return {
|
|
4643
4736
|
ok: false,
|
|
@@ -4645,6 +4738,24 @@ export class Orchestrator {
|
|
|
4645
4738
|
};
|
|
4646
4739
|
}
|
|
4647
4740
|
upsertHostTask(taskId, { statusMirror: "stopped" });
|
|
4741
|
+
// The next start installs the CA again before anything reads the env;
|
|
4742
|
+
// no write still queued for the stopped container may land or record.
|
|
4743
|
+
invalidateTaskWrites(taskId);
|
|
4744
|
+
forgetTaskProxyReadiness(taskId);
|
|
4745
|
+
try {
|
|
4746
|
+
removeTaskGatewayTask(taskId);
|
|
4747
|
+
} catch (error) {
|
|
4748
|
+
// The runtime is stopped and the in-memory generation is already
|
|
4749
|
+
// inactive. A leftover durable file also loads inactive; explicit
|
|
4750
|
+
// Resume rotates it through task-up.
|
|
4751
|
+
console.warn(
|
|
4752
|
+
`[task-gateway] ${taskId}: paused with stale inactive gateway state: ${
|
|
4753
|
+
error instanceof Error ? error.message : String(error)
|
|
4754
|
+
}`,
|
|
4755
|
+
);
|
|
4756
|
+
}
|
|
4757
|
+
// Nothing serves the task's env now: no value of it stays in memory.
|
|
4758
|
+
forgetTaskEnvSnapshot(taskId);
|
|
4648
4759
|
clearTaskGatewayAcl(taskId);
|
|
4649
4760
|
return { ok: true };
|
|
4650
4761
|
}
|
|
@@ -4802,6 +4913,7 @@ async function repairClaudeConfigInContainer(
|
|
|
4802
4913
|
if (environment) {
|
|
4803
4914
|
const write = await environment.exec({
|
|
4804
4915
|
argv: ["/bin/sh", "-c", repairScript],
|
|
4916
|
+
projectEnvProfile: "maintenance",
|
|
4805
4917
|
env: RUNTIME_AUTHORITY_ENV,
|
|
4806
4918
|
stdin: content,
|
|
4807
4919
|
timeoutMs: 30_000,
|
|
@@ -4898,6 +5010,28 @@ export function assembleFirstTurnPrompt(
|
|
|
4898
5010
|
* to every turn; only `initialPrompt` is delivered as the first user
|
|
4899
5011
|
* turn (see `assembleFirstTurnPrompt`).
|
|
4900
5012
|
*/
|
|
5013
|
+
/** The task's usable proxied keys right now (ADR-153 §5): its agents are
|
|
5014
|
+
* told which env vars hold placeholders, and for which domains; a withheld
|
|
5015
|
+
* key is absent from the task, so it is not mentioned. */
|
|
5016
|
+
function taskProxiedKeys(taskId: string): { key: string; domains: string[] }[] {
|
|
5017
|
+
// The same gate as the placeholder and the injection.
|
|
5018
|
+
return usableProxiedKeys(taskId).map((key) => ({ key: key.key, domains: [...key.domains] }));
|
|
5019
|
+
}
|
|
5020
|
+
|
|
5021
|
+
/** ADR-153 §5's line, with the exact key-to-domain mapping. */
|
|
5022
|
+
export function proxiedEnvLine(keys: readonly { readonly key: string; readonly domains: readonly string[] }[]): string {
|
|
5023
|
+
const listed = [...keys]
|
|
5024
|
+
.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
|
|
5025
|
+
.map((entry) => `\`${entry.key}\` (${entry.domains.join(", ")})`)
|
|
5026
|
+
.join(", ");
|
|
5027
|
+
const domainCount = new Set(keys.flatMap((entry) => entry.domains)).size;
|
|
5028
|
+
return (
|
|
5029
|
+
`These env vars hold placeholders, not the real secrets: ${listed}. Uai's proxy puts the real value in on ` +
|
|
5030
|
+
`HTTPS requests to ${domainCount === 1 ? "that domain" : "those domains"}, so use them as they are — don't ` +
|
|
5031
|
+
`replace, re-fetch or "fix" them. They won't work anywhere else.`
|
|
5032
|
+
);
|
|
5033
|
+
}
|
|
5034
|
+
|
|
4901
5035
|
export function buildSystemPreamble(
|
|
4902
5036
|
roster: Roster,
|
|
4903
5037
|
agent: RosterAgent,
|
|
@@ -4910,6 +5044,7 @@ export function buildSystemPreamble(
|
|
|
4910
5044
|
sharedFiles?: string,
|
|
4911
5045
|
mode?: "open" | "secretary",
|
|
4912
5046
|
secretaryAgentId?: string,
|
|
5047
|
+
proxiedKeys?: readonly { readonly key: string; readonly domains: readonly string[] }[],
|
|
4913
5048
|
): string {
|
|
4914
5049
|
const channelList = roster
|
|
4915
5050
|
.map((a) =>
|
|
@@ -5355,6 +5490,10 @@ export function buildSystemPreamble(
|
|
|
5355
5490
|
"",
|
|
5356
5491
|
]
|
|
5357
5492
|
: []),
|
|
5493
|
+
// ADR-153 §5: proxied keys reach the task as placeholders.
|
|
5494
|
+
...(proxiedKeys !== undefined && proxiedKeys.length > 0
|
|
5495
|
+
? ["## Proxied env vars", "", proxiedEnvLine(proxiedKeys), ""]
|
|
5496
|
+
: []),
|
|
5358
5497
|
// ADR-048: tell every agent with persona-granted permissions about its
|
|
5359
5498
|
// `uai` CLI. Secretary mode adds dispatch but does not invent any other
|
|
5360
5499
|
// permission: an empty persona permission set remains empty.
|
|
@@ -5550,7 +5689,15 @@ function beginRuntimeRecovery(
|
|
|
5550
5689
|
const predecessor = recoveryInFlight;
|
|
5551
5690
|
const sweep = async (): Promise<boolean> => {
|
|
5552
5691
|
try {
|
|
5553
|
-
return await recoverRunningTasks({
|
|
5692
|
+
return await recoverRunningTasks({
|
|
5693
|
+
lifecycle,
|
|
5694
|
+
// Boot recovery deliberately keeps its per-task barriers instead of
|
|
5695
|
+
// entering lifecycle slots, but still needs the live orchestrator to
|
|
5696
|
+
// suspend/reactivate gateway generations around uai-init.
|
|
5697
|
+
consumerLifecycle:
|
|
5698
|
+
lifecycle ?? globalForOrchestrator.__uaiOrchestrator,
|
|
5699
|
+
isCurrent,
|
|
5700
|
+
});
|
|
5554
5701
|
} catch (error) {
|
|
5555
5702
|
console.error(
|
|
5556
5703
|
"[orchestrator] recovery: unhandled failure",
|
|
@@ -6971,6 +7118,9 @@ export async function recoverRunningTasks(opts?: {
|
|
|
6971
7118
|
/** Post-boot recovery must queue behind live taskUp/taskDown operations.
|
|
6972
7119
|
* Boot recovery omits this and uses its earlier per-task barriers instead. */
|
|
6973
7120
|
lifecycle?: Orchestrator;
|
|
7121
|
+
/** Consumer quarantine/admission hooks without changing boot recovery's
|
|
7122
|
+
* barrier-based sequencing. Production boot supplies the live singleton. */
|
|
7123
|
+
consumerLifecycle?: Orchestrator;
|
|
6974
7124
|
/** Test seam; production waits for the shared image/CLI-volume reconcile. */
|
|
6975
7125
|
maintenanceReady?: Promise<void>;
|
|
6976
7126
|
/** A daemon-generation fence. False cancels every remaining side effect. */
|
|
@@ -6990,6 +7140,7 @@ export async function recoverRunningTasks(opts?: {
|
|
|
6990
7140
|
const result = await recoveryPass(
|
|
6991
7141
|
pendingTaskIds,
|
|
6992
7142
|
opts?.lifecycle,
|
|
7143
|
+
opts?.consumerLifecycle,
|
|
6993
7144
|
opts?.maintenanceReady ?? agentClisReady,
|
|
6994
7145
|
opts?.isCurrent ?? (() => true),
|
|
6995
7146
|
);
|
|
@@ -7034,6 +7185,7 @@ interface RecoveryPassResult {
|
|
|
7034
7185
|
async function recoveryPass(
|
|
7035
7186
|
onlyTaskIds?: ReadonlySet<string>,
|
|
7036
7187
|
lifecycle?: Orchestrator,
|
|
7188
|
+
consumerLifecycle?: Orchestrator,
|
|
7037
7189
|
maintenanceReady: Promise<void> = agentClisReady,
|
|
7038
7190
|
isCurrent: () => boolean = () => true,
|
|
7039
7191
|
): Promise<RecoveryPassResult> {
|
|
@@ -7091,6 +7243,7 @@ async function recoveryPass(
|
|
|
7091
7243
|
task,
|
|
7092
7244
|
maintenanceReady,
|
|
7093
7245
|
isCurrent,
|
|
7246
|
+
consumerLifecycle,
|
|
7094
7247
|
);
|
|
7095
7248
|
if (recovered) {
|
|
7096
7249
|
if (!lifecycle) finishTaskRecovery(task.taskId);
|
|
@@ -7285,15 +7438,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7285
7438
|
/** Keep the MCP gateway reverse tunnel alive for a machine channel. The
|
|
7286
7439
|
* handle exposes tunnel argv only on the machine provider; anything else is
|
|
7287
7440
|
* a quiet no-op. */
|
|
7288
|
-
function ensureMachineGatewayTunnelFor(
|
|
7441
|
+
async function ensureMachineGatewayTunnelFor(
|
|
7289
7442
|
taskId: string,
|
|
7290
7443
|
handle: TaskEnvironmentHandle<unknown> | null,
|
|
7291
|
-
|
|
7444
|
+
waitUntilReady = false,
|
|
7445
|
+
): Promise<void> {
|
|
7292
7446
|
const argv = handle?.gatewayTunnelSshArgs;
|
|
7293
7447
|
if (!argv) return;
|
|
7294
|
-
|
|
7295
|
-
|
|
7296
|
-
|
|
7448
|
+
const source = { gatewayTunnelSshArgs: argv.bind(handle) };
|
|
7449
|
+
const loaderPort =
|
|
7450
|
+
handle.descriptor.envMode === "loader"
|
|
7451
|
+
? MACHINE_GATEWAY_TUNNEL_PORT
|
|
7452
|
+
: undefined;
|
|
7453
|
+
if (waitUntilReady && loaderPort !== undefined) {
|
|
7454
|
+
await ensureMachineGatewayTunnelReady(taskId, source, loaderPort);
|
|
7455
|
+
return;
|
|
7456
|
+
}
|
|
7457
|
+
ensureMachineGatewayTunnel(taskId, source, loaderPort);
|
|
7297
7458
|
}
|
|
7298
7459
|
|
|
7299
7460
|
async function materializeAgentCli(
|
|
@@ -7328,25 +7489,105 @@ async function recoverPersistedTaskEnvironment(
|
|
|
7328
7489
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
7329
7490
|
if (environment === null) return true;
|
|
7330
7491
|
if (!isCurrent()) return true;
|
|
7331
|
-
|
|
7332
|
-
|
|
7333
|
-
|
|
7334
|
-
|
|
7335
|
-
|
|
7336
|
-
|
|
7337
|
-
|
|
7338
|
-
|
|
7339
|
-
|
|
7340
|
-
|
|
7341
|
-
|
|
7342
|
-
|
|
7343
|
-
|
|
7344
|
-
|
|
7345
|
-
|
|
7492
|
+
// ADR-153: a recovery may restart the task's container or VM, which takes
|
|
7493
|
+
// the proxy's CA, chain and names with it, and its forwarder. Before the
|
|
7494
|
+
// provider touches it, nothing recorded counts any more and no write queued
|
|
7495
|
+
// or under way may land: a reconcile in flight ends with its exec under
|
|
7496
|
+
// way, and none pins again until this recovery's own slot has the
|
|
7497
|
+
// forwarder running.
|
|
7498
|
+
invalidateTaskWrites(task.taskId);
|
|
7499
|
+
forgetTaskProxyReadiness(task.taskId);
|
|
7500
|
+
// A running machine's editor can submit a command before any agent channel
|
|
7501
|
+
// is reconstructed. Re-arm the authenticated gateway tunnel at recovery,
|
|
7502
|
+
// not only at the first session spawn, so that command can refresh env.
|
|
7503
|
+
void ensureMachineGatewayTunnelFor(task.taskId, environment);
|
|
7504
|
+
let consumersPrepared = false;
|
|
7505
|
+
let tunnelReadinessFailed = false;
|
|
7506
|
+
const prepareConsumers = async (): Promise<void> => {
|
|
7507
|
+
if (!isCurrent()) return;
|
|
7508
|
+
try {
|
|
7509
|
+
await ensureMachineGatewayTunnelFor(task.taskId, environment, true);
|
|
7510
|
+
} catch (error) {
|
|
7511
|
+
tunnelReadinessFailed = true;
|
|
7512
|
+
console.warn(
|
|
7513
|
+
`[machine-tunnel] ${task.taskId}: recovery deferred until the gateway tunnel is ready (${error instanceof Error ? error.message : String(error)})`,
|
|
7514
|
+
);
|
|
7515
|
+
throw error;
|
|
7516
|
+
}
|
|
7517
|
+
// ADR-153 "Order", in one slot of the task's write queue: the CA (or,
|
|
7518
|
+
// with the proxy off, the pins out), then the forwarder, then the pins,
|
|
7519
|
+
// so no name is pinned with nothing listening behind it. The forwarder
|
|
7520
|
+
// carries the generation's secret into the task as root, and the
|
|
7521
|
+
// reactivation opens the gateway to it: neither once this recovery is
|
|
7522
|
+
// stale or the runner is shutting down. Every exec is fenced, so
|
|
7523
|
+
// shutdown waits for one under way rather than exiting under it, and
|
|
7524
|
+
// refuses (throws) any that would start after.
|
|
7525
|
+
if (!isCurrent()) return;
|
|
7526
|
+
const secretsProxy = activeSecretsProxy();
|
|
7527
|
+
let proxyFailure: RecoveryProxyFailure | null;
|
|
7528
|
+
try {
|
|
7529
|
+
proxyFailure = await applyTaskRootSteps(
|
|
7530
|
+
environment,
|
|
7531
|
+
environment.descriptor.envMode === "loader" ? { runtime: secretsProxy?.runtime ?? null } : null,
|
|
7532
|
+
(fenced) => ensureContainerTaskEnvForwarder(fenced),
|
|
7533
|
+
isCurrent,
|
|
7534
|
+
);
|
|
7535
|
+
} catch (error) {
|
|
7536
|
+
// Made stale meanwhile, its exec refused: recover() handles a stale
|
|
7537
|
+
// recovery itself.
|
|
7538
|
+
if (!isCurrent()) return;
|
|
7539
|
+
throw error;
|
|
7540
|
+
}
|
|
7541
|
+
if (proxyFailure !== null) {
|
|
7542
|
+
console.warn(
|
|
7543
|
+
`[secrets-proxy] ${task.taskId}: recovery could not reconcile the proxy's pins and trust: ${
|
|
7544
|
+
proxyFailure.error instanceof Error ? proxyFailure.error.message : String(proxyFailure.error)
|
|
7545
|
+
}`,
|
|
7546
|
+
);
|
|
7547
|
+
// A failure leaves the task up without the proxy: the trust watcher
|
|
7548
|
+
// retries, and its proxied keys stay withheld until the CA and the
|
|
7549
|
+
// pins are in.
|
|
7550
|
+
secretsProxy?.runtime.requestTrustDelivery();
|
|
7551
|
+
}
|
|
7552
|
+
if (!isCurrent()) return;
|
|
7553
|
+
// Shutdown began during the launch: not prepared, so the consumers stay
|
|
7554
|
+
// unadmitted, as for any failed preparation.
|
|
7555
|
+
if (taskWritesQuiescing()) throw new Error("the runner is shutting down");
|
|
7556
|
+
lifecycle?.reactivateTaskGatewayForRecovery(task.taskId);
|
|
7557
|
+
consumersPrepared = true;
|
|
7558
|
+
};
|
|
7559
|
+
try {
|
|
7560
|
+
const result = await environment.recover({
|
|
7561
|
+
maintenanceReady,
|
|
7562
|
+
isCurrent,
|
|
7563
|
+
...(lifecycle
|
|
7564
|
+
? {
|
|
7565
|
+
quarantineConsumers: () =>
|
|
7566
|
+
lifecycle.quarantineChannelForRecovery(task.taskId),
|
|
7567
|
+
prepareConsumers,
|
|
7568
|
+
allowConsumers: () => lifecycle.allowChannel(task.taskId),
|
|
7569
|
+
ensureConsumersStarted: async () => {
|
|
7570
|
+
await lifecycle.ensureStarted(task.taskId);
|
|
7571
|
+
},
|
|
7572
|
+
}
|
|
7573
|
+
: {}),
|
|
7574
|
+
});
|
|
7575
|
+
if (result.outcome === "recovered" && !consumersPrepared && isCurrent()) {
|
|
7576
|
+
const current = getHostTask(task.taskId);
|
|
7577
|
+
if (current?.statusMirror && isActive(current.statusMirror)) {
|
|
7578
|
+
await prepareConsumers();
|
|
7579
|
+
}
|
|
7580
|
+
}
|
|
7581
|
+
return result.outcome === "recovered";
|
|
7582
|
+
} catch (error) {
|
|
7583
|
+
if (tunnelReadinessFailed) return false;
|
|
7584
|
+
throw error;
|
|
7585
|
+
}
|
|
7346
7586
|
}
|
|
7347
7587
|
|
|
7348
7588
|
interface RecoveryTaskConsumers {
|
|
7349
7589
|
quarantineChannelForRecovery(taskId: string): Promise<void>;
|
|
7590
|
+
prepareChannelForRecovery(taskId: string): Promise<void>;
|
|
7350
7591
|
allowChannel(taskId: string): void;
|
|
7351
7592
|
ensureStarted(taskId: string): Promise<boolean | void>;
|
|
7352
7593
|
}
|
|
@@ -7809,6 +8050,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7809
8050
|
'[ "$(/usr/bin/stat -c %U /opt/uai/store/pnpm)" = "node" ] || { echo "store dir is not node-owned" >&2; exit 1; }; ' +
|
|
7810
8051
|
"fi",
|
|
7811
8052
|
],
|
|
8053
|
+
projectEnvProfile: "maintenance",
|
|
7812
8054
|
user: "root",
|
|
7813
8055
|
env: {},
|
|
7814
8056
|
timeoutMs: 30_000,
|
|
@@ -7835,23 +8077,47 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7835
8077
|
);
|
|
7836
8078
|
return settleFromStatus(afterOverlay);
|
|
7837
8079
|
}
|
|
8080
|
+
// The dependency worker inside uai-init performs its one loader sync. Its
|
|
8081
|
+
// root forwarder and authenticated generation must be live first, while
|
|
8082
|
+
// public agent admission remains quarantined until initialization ends.
|
|
8083
|
+
let consumersPreparedForInit = false;
|
|
8084
|
+
const reclosePreparedConsumers = async (): Promise<void> => {
|
|
8085
|
+
if (!consumersPreparedForInit) return;
|
|
8086
|
+
consumersPreparedForInit = false;
|
|
8087
|
+
await context.quarantineConsumers?.();
|
|
8088
|
+
};
|
|
8089
|
+
if (context.prepareConsumers) {
|
|
8090
|
+
await context.prepareConsumers();
|
|
8091
|
+
consumersPreparedForInit = true;
|
|
8092
|
+
}
|
|
8093
|
+
if (!context.isCurrent()) {
|
|
8094
|
+
await reclosePreparedConsumers();
|
|
8095
|
+
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
8096
|
+
}
|
|
7838
8097
|
// Dependencies + code-server. The in-container deadline (TERM then KILL)
|
|
7839
8098
|
// means a host-side expiry can never leave the initializer mutating the
|
|
7840
8099
|
// workspace while agents use it; exit 78 is a runtime-authority violation
|
|
7841
8100
|
// and quarantines exactly like at task-up.
|
|
7842
|
-
const init = await environment
|
|
7843
|
-
|
|
7844
|
-
|
|
7845
|
-
|
|
7846
|
-
|
|
7847
|
-
|
|
7848
|
-
|
|
7849
|
-
|
|
7850
|
-
|
|
7851
|
-
|
|
7852
|
-
|
|
7853
|
-
|
|
8101
|
+
const init = await environment
|
|
8102
|
+
.exec({
|
|
8103
|
+
argv: [
|
|
8104
|
+
"/usr/bin/timeout",
|
|
8105
|
+
"--signal=TERM",
|
|
8106
|
+
"--kill-after=15s",
|
|
8107
|
+
"570s",
|
|
8108
|
+
"/usr/local/bin/uai-init",
|
|
8109
|
+
],
|
|
8110
|
+
projectEnvProfile: "maintenance",
|
|
8111
|
+
env: {},
|
|
8112
|
+
timeoutMs: 10 * 60_000,
|
|
8113
|
+
maxOutputBytes: 1024 * 1024,
|
|
8114
|
+
})
|
|
8115
|
+
.catch(async (error: unknown) => {
|
|
8116
|
+
await reclosePreparedConsumers();
|
|
8117
|
+
throw error;
|
|
8118
|
+
});
|
|
7854
8119
|
if (!context.isCurrent()) {
|
|
8120
|
+
await reclosePreparedConsumers();
|
|
7855
8121
|
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
7856
8122
|
}
|
|
7857
8123
|
if (
|
|
@@ -7860,6 +8126,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7860
8126
|
init.stdoutTruncated ||
|
|
7861
8127
|
init.stderrTruncated
|
|
7862
8128
|
) {
|
|
8129
|
+
await reclosePreparedConsumers();
|
|
7863
8130
|
// The host-side client died (deadline, overflow, forced settlement)
|
|
7864
8131
|
// without a real exit code, so the in-container initializer may STILL be
|
|
7865
8132
|
// mutating dependencies for several minutes. Never expose the task on an
|
|
@@ -7882,6 +8149,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7882
8149
|
return settleFromStatus(after);
|
|
7883
8150
|
}
|
|
7884
8151
|
if (init.exitCode === 78) {
|
|
8152
|
+
await reclosePreparedConsumers();
|
|
7885
8153
|
await environment.stop();
|
|
7886
8154
|
const after = await environment.status();
|
|
7887
8155
|
if (after.state === "running" || after.state === "unknown") {
|
|
@@ -7900,6 +8168,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7900
8168
|
return settleFromStatus(after);
|
|
7901
8169
|
}
|
|
7902
8170
|
if (init.exitCode === 127) {
|
|
8171
|
+
await reclosePreparedConsumers();
|
|
7903
8172
|
// Command-not-found from a read-only bind mount the guest cannot touch:
|
|
7904
8173
|
// the initializer itself is unrunnable — live 2026-08-26, containers
|
|
7905
8174
|
// whose versioned asset tree a host update had pruned looped here as
|
|
@@ -7930,6 +8199,7 @@ export async function recoverAppleTaskEnvironment(
|
|
|
7930
8199
|
);
|
|
7931
8200
|
}
|
|
7932
8201
|
if (!context.isCurrent()) {
|
|
8202
|
+
await reclosePreparedConsumers();
|
|
7933
8203
|
return { outcome: "deferred", detail: "runtime generation superseded" };
|
|
7934
8204
|
}
|
|
7935
8205
|
db_setRuntime(descriptor.taskId, { codeServerPort: null });
|
|
@@ -7983,6 +8253,7 @@ async function recoverDockerTaskEnvironment(
|
|
|
7983
8253
|
|
|
7984
8254
|
const lifecycleCallbacks = [
|
|
7985
8255
|
context.quarantineConsumers,
|
|
8256
|
+
context.prepareConsumers,
|
|
7986
8257
|
context.allowConsumers,
|
|
7987
8258
|
context.ensureConsumersStarted,
|
|
7988
8259
|
];
|
|
@@ -7998,6 +8269,12 @@ async function recoverDockerTaskEnvironment(
|
|
|
7998
8269
|
}
|
|
7999
8270
|
await context.quarantineConsumers!();
|
|
8000
8271
|
},
|
|
8272
|
+
prepareChannelForRecovery: async (taskId) => {
|
|
8273
|
+
if (taskId !== descriptor.taskId) {
|
|
8274
|
+
throw new Error("task environment recovery consumer identity mismatch");
|
|
8275
|
+
}
|
|
8276
|
+
await context.prepareConsumers!();
|
|
8277
|
+
},
|
|
8001
8278
|
allowChannel: (taskId) => {
|
|
8002
8279
|
if (taskId !== descriptor.taskId) {
|
|
8003
8280
|
throw new Error("task environment recovery consumer identity mismatch");
|
|
@@ -8205,6 +8482,13 @@ async function recoverOneDockerTask(
|
|
|
8205
8482
|
);
|
|
8206
8483
|
if (!isCurrent()) return true;
|
|
8207
8484
|
if (lifecycle) {
|
|
8485
|
+
await lifecycle.prepareChannelForRecovery(task.taskId);
|
|
8486
|
+
if (!isCurrent()) {
|
|
8487
|
+
// Preparation may have reactivated this task's gateway generation.
|
|
8488
|
+
// A superseded pass must close that authority again before returning.
|
|
8489
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
8490
|
+
return true;
|
|
8491
|
+
}
|
|
8208
8492
|
lifecycle.allowChannel(task.taskId);
|
|
8209
8493
|
await lifecycle.ensureStarted(task.taskId);
|
|
8210
8494
|
if (!isCurrent()) {
|
|
@@ -8340,6 +8624,23 @@ async function recoverOneDockerTask(
|
|
|
8340
8624
|
}
|
|
8341
8625
|
if (!isCurrent()) return true;
|
|
8342
8626
|
}
|
|
8627
|
+
// uai-init's dependency worker loads the task env once. Recreate the
|
|
8628
|
+
// forwarder and reactivate only its authenticated gateway before that
|
|
8629
|
+
// worker runs; the channel itself stays tombstoned until recovery finishes.
|
|
8630
|
+
let consumersPreparedForInit = false;
|
|
8631
|
+
const reclosePreparedConsumers = async (): Promise<void> => {
|
|
8632
|
+
if (!consumersPreparedForInit || !lifecycle) return;
|
|
8633
|
+
consumersPreparedForInit = false;
|
|
8634
|
+
await lifecycle.quarantineChannelForRecovery(task.taskId);
|
|
8635
|
+
};
|
|
8636
|
+
if (lifecycle) {
|
|
8637
|
+
await lifecycle.prepareChannelForRecovery(task.taskId);
|
|
8638
|
+
consumersPreparedForInit = true;
|
|
8639
|
+
if (!isCurrent()) {
|
|
8640
|
+
await reclosePreparedConsumers();
|
|
8641
|
+
return true;
|
|
8642
|
+
}
|
|
8643
|
+
}
|
|
8343
8644
|
// uai-init reinstalls workspace deps (pnpm/npm install) — minutes on a big
|
|
8344
8645
|
// repo. dockerCli's 30s default would SIGKILL it mid-install.
|
|
8345
8646
|
const initResult = await dockerCli(
|
|
@@ -8431,9 +8732,16 @@ async function recoverOneDockerTask(
|
|
|
8431
8732
|
RECOVERY_INIT_PATH,
|
|
8432
8733
|
],
|
|
8433
8734
|
{ timeoutMs: 10 * 60_000 },
|
|
8434
|
-
)
|
|
8435
|
-
|
|
8735
|
+
).catch(async (error: unknown) => {
|
|
8736
|
+
await reclosePreparedConsumers();
|
|
8737
|
+
throw error;
|
|
8738
|
+
});
|
|
8739
|
+
if (!isCurrent()) {
|
|
8740
|
+
await reclosePreparedConsumers();
|
|
8741
|
+
return true;
|
|
8742
|
+
}
|
|
8436
8743
|
if (initResult.status === 78) {
|
|
8744
|
+
await reclosePreparedConsumers();
|
|
8437
8745
|
const quarantined = await quarantineRunningContainer(containerName);
|
|
8438
8746
|
if (!isCurrent()) return true;
|
|
8439
8747
|
if (quarantined === "unreachable") return false;
|
|
@@ -8456,9 +8764,20 @@ async function recoverOneDockerTask(
|
|
|
8456
8764
|
`[orchestrator] recovery: ${task.taskId} uai-init failed; container is up but Editor may be down`,
|
|
8457
8765
|
);
|
|
8458
8766
|
}
|
|
8459
|
-
const portResult = await dockerPort(containerName, 8080)
|
|
8460
|
-
|
|
8461
|
-
|
|
8767
|
+
const portResult = await dockerPort(containerName, 8080).catch(
|
|
8768
|
+
async (error: unknown) => {
|
|
8769
|
+
await reclosePreparedConsumers();
|
|
8770
|
+
throw error;
|
|
8771
|
+
},
|
|
8772
|
+
);
|
|
8773
|
+
if (!isCurrent()) {
|
|
8774
|
+
await reclosePreparedConsumers();
|
|
8775
|
+
return true;
|
|
8776
|
+
}
|
|
8777
|
+
if (!portResult.reachable) {
|
|
8778
|
+
await reclosePreparedConsumers();
|
|
8779
|
+
return false;
|
|
8780
|
+
}
|
|
8462
8781
|
const port = portResult.port;
|
|
8463
8782
|
db_setRuntime(task.taskId, {
|
|
8464
8783
|
codeServerPort: port,
|
|
@@ -8475,7 +8794,10 @@ async function recoverOneDockerTask(
|
|
|
8475
8794
|
console.log(
|
|
8476
8795
|
`[orchestrator] recovery: ${task.taskId} resumed (port ${port ?? "?"})`,
|
|
8477
8796
|
);
|
|
8478
|
-
if (!isCurrent())
|
|
8797
|
+
if (!isCurrent()) {
|
|
8798
|
+
await reclosePreparedConsumers();
|
|
8799
|
+
return true;
|
|
8800
|
+
}
|
|
8479
8801
|
if (lifecycle) {
|
|
8480
8802
|
lifecycle.allowChannel(task.taskId);
|
|
8481
8803
|
await lifecycle.ensureStarted(task.taskId);
|