@runuai/host 0.9.150 → 0.9.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1880 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1837 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +650 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
package/src/index.ts
CHANGED
|
@@ -1,9 +1,28 @@
|
|
|
1
1
|
import {
|
|
2
2
|
agent,
|
|
3
3
|
AgentError,
|
|
4
|
+
CREDENTIALS_WITHDRAWN_MESSAGE,
|
|
4
5
|
toolStderrDetail,
|
|
6
|
+
type LeasedTaskUp,
|
|
5
7
|
type TaskDownResult,
|
|
6
8
|
} from "../lib/agent";
|
|
9
|
+
import {
|
|
10
|
+
activeCloudCredentials,
|
|
11
|
+
admitLegacyStart,
|
|
12
|
+
cloudCredentialRuntime,
|
|
13
|
+
credentialsMode,
|
|
14
|
+
onOwnCredentials,
|
|
15
|
+
whileOnOwnCredentials,
|
|
16
|
+
type LegacyStartPermit,
|
|
17
|
+
} from "../lib/cloud-credential-host";
|
|
18
|
+
import { CredentialLeaseUnavailableError, type TaskAssignment } from "../lib/cloud-credential-leases";
|
|
19
|
+
import {
|
|
20
|
+
StartInterruptedError,
|
|
21
|
+
type CloudCredentialRuntime,
|
|
22
|
+
type TaskCredentialStart,
|
|
23
|
+
type TaskUpHold,
|
|
24
|
+
} from "../lib/cloud-credential-runtime";
|
|
25
|
+
import { writeTaskGithub, writeTaskSshKey } from "../lib/credential-consumers/github";
|
|
7
26
|
import { agentClisReady, ensureStandardImage } from "../lib/standard-image";
|
|
8
27
|
import { cloneRepo } from "../lib/repo-clone";
|
|
9
28
|
import { handleFilesOp } from "../lib/shared-files";
|
|
@@ -43,16 +62,23 @@ import {
|
|
|
43
62
|
appendTranscriptViaEnvironment,
|
|
44
63
|
} from "../lib/transcript";
|
|
45
64
|
import { buildTaskDiff } from "../lib/task-diff";
|
|
65
|
+
import { stoppedTaskDiff } from "../lib/task-diff-sandbox";
|
|
46
66
|
import {
|
|
67
|
+
admitTaskRunEpoch,
|
|
68
|
+
canReuseTaskRuntimeForRun,
|
|
69
|
+
closeTaskRunAssignment,
|
|
70
|
+
closeTaskRunAssignmentForStop,
|
|
47
71
|
deleteHostTask,
|
|
48
72
|
getHostTask,
|
|
49
73
|
hostTaskEnvMode,
|
|
74
|
+
readTaskRunEpochState,
|
|
50
75
|
recordHostEvent,
|
|
51
76
|
recordTaskDown,
|
|
52
77
|
recordTaskEnvironmentPrepared,
|
|
53
78
|
recordTaskError,
|
|
54
79
|
recordTaskOwner,
|
|
55
80
|
recordTaskProjectEnvSpec,
|
|
81
|
+
recordTaskRuntimeEstablished,
|
|
56
82
|
recordTaskStarting,
|
|
57
83
|
recordTaskUpResult,
|
|
58
84
|
taskProjectEnvSpec,
|
|
@@ -78,7 +104,10 @@ import { activeSecretsProxy } from "../lib/secrets-proxy/host";
|
|
|
78
104
|
import { secretsProxyEnabled } from "../lib/secrets-proxy/runtime";
|
|
79
105
|
import { forgetTaskProxyReadiness } from "../lib/secrets-proxy/readiness";
|
|
80
106
|
import { admitFreshTaskView } from "../lib/secrets-proxy/sidecar-view";
|
|
81
|
-
import {
|
|
107
|
+
import {
|
|
108
|
+
invalidateTaskWrites,
|
|
109
|
+
reconcileTaskProxyState,
|
|
110
|
+
} from "../lib/secrets-proxy/task-trust";
|
|
82
111
|
import { parsePreviewPortRuntimes } from "../lib/preview-ports";
|
|
83
112
|
import { dockerCli } from "../lib/docker-exec";
|
|
84
113
|
import {
|
|
@@ -187,7 +216,7 @@ export {
|
|
|
187
216
|
export { cloneRepo } from "../lib/repo-clone";
|
|
188
217
|
export type { CloneRepoInput, CloneRepoResult } from "../lib/repo-clone";
|
|
189
218
|
|
|
190
|
-
export { parseGitDiff
|
|
219
|
+
export { parseGitDiff } from "../lib/git-diff";
|
|
191
220
|
export type { DiffFile, DiffHunk, DiffLine, DiffStatus } from "../lib/git-diff";
|
|
192
221
|
|
|
193
222
|
export {
|
|
@@ -242,88 +271,292 @@ function sidecarRunnerIsPinned(): HostCommandResult<never> | null {
|
|
|
242
271
|
};
|
|
243
272
|
}
|
|
244
273
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
+
type TaskUpCommand = HostCommands["taskUp"];
|
|
275
|
+
type TaskUpOutcome = Awaited<ReturnType<TaskUpCommand>>;
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* ADR-154: a start on cloud credentials, as task-up carries it. It begins in
|
|
279
|
+
* the short admission slot (the run's startup; every older start of the task
|
|
280
|
+
* interrupted), is awaited outside every slot, holds its installed grant
|
|
281
|
+
* through the long slot (the clone, provisioning and the runtime-ready
|
|
282
|
+
* writes), publishes its success only while it is still the task's current
|
|
283
|
+
* start with its credentials open, and is settled after: ended, or abandoned
|
|
284
|
+
* with its hold let go first.
|
|
285
|
+
*/
|
|
286
|
+
interface CloudCredentialTaskStart {
|
|
287
|
+
readonly runtime: CloudCredentialRuntime;
|
|
288
|
+
readonly assignment: TaskAssignment;
|
|
289
|
+
start: TaskCredentialStart | null;
|
|
290
|
+
hold: TaskUpHold | null;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** The start's lease for task-up: its whole grant's hold, and what the
|
|
294
|
+
* runner writes into the runtime once it exists (the owner's GitHub sign-in
|
|
295
|
+
* and SSH key, recorded for cleanup) before anything in it runs. */
|
|
296
|
+
function leasedTaskUp(start: TaskCredentialStart, hold: TaskUpHold): LeasedTaskUp {
|
|
297
|
+
return {
|
|
298
|
+
// The whole grant's signal: its gate closing, a cleanup reaching it or
|
|
299
|
+
// the start being interrupted all end task-up.
|
|
300
|
+
clone: { ...hold.git, signal: hold.signal },
|
|
301
|
+
runtimeReady: async (environment, signal) => {
|
|
302
|
+
for (const write of [writeTaskGithub, writeTaskSshKey]) {
|
|
303
|
+
if (signal.aborted) throw new CredentialLeaseUnavailableError("the task's credentials were withdrawn");
|
|
304
|
+
await write(environment, start.assignment);
|
|
274
305
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
306
|
+
},
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** `wait`, given up (rejecting) as soon as the runner begins leaving its own
|
|
311
|
+
* credentials, when what it waits for may never come. */
|
|
312
|
+
function untilLeavingLegacy<T>(legacy: LegacyStartPermit | null, wait: Promise<T>): Promise<T> {
|
|
313
|
+
if (legacy === null) return wait;
|
|
314
|
+
const signal = legacy.signal;
|
|
315
|
+
return new Promise<T>((resolve, reject) => {
|
|
316
|
+
const onAbort = (): void => reject(signal.reason);
|
|
317
|
+
if (signal.aborted) onAbort();
|
|
318
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
319
|
+
wait.then(
|
|
320
|
+
(value) => {
|
|
321
|
+
signal.removeEventListener("abort", onAbort);
|
|
322
|
+
resolve(value);
|
|
323
|
+
},
|
|
324
|
+
(error: unknown) => {
|
|
325
|
+
signal.removeEventListener("abort", onAbort);
|
|
326
|
+
reject(error);
|
|
327
|
+
},
|
|
328
|
+
);
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** A start the runner does not take while it switches its tasks to cloud
|
|
333
|
+
* credentials: the cloud starts it again shortly. */
|
|
334
|
+
function switchingToCloudCredentials(): TaskUpOutcome {
|
|
335
|
+
return {
|
|
336
|
+
ok: false,
|
|
337
|
+
code: HostErrorCode.HostUnavailable,
|
|
338
|
+
message: "This host is switching its tasks to cloud credentials; the task starts once it is done.",
|
|
339
|
+
retryable: true,
|
|
340
|
+
retryReason: "managed_restart",
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** What a start reports when Stop, teardown, a newer run or shutdown
|
|
345
|
+
* interrupted it, in place of its own outcome; null when none did. The
|
|
346
|
+
* interruption that won decides: shutdown asks the cloud to start the
|
|
347
|
+
* task again once the runner is back, anything else leaves a run that is
|
|
348
|
+
* no longer current. */
|
|
349
|
+
function interruptedStartResult(cloud: CloudCredentialTaskStart | null): TaskUpOutcome | null {
|
|
350
|
+
if (!cloud?.start?.signal.aborted) return null;
|
|
351
|
+
const reason: unknown = cloud.start.signal.reason;
|
|
352
|
+
if (reason instanceof StartInterruptedError && reason.by === "shutdown") {
|
|
353
|
+
return {
|
|
354
|
+
ok: false,
|
|
355
|
+
code: HostErrorCode.HostUnavailable,
|
|
356
|
+
message: "This host is restarting; the task starts once it is back.",
|
|
357
|
+
retryable: true,
|
|
358
|
+
retryReason: "managed_restart",
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
return {
|
|
362
|
+
ok: false,
|
|
363
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
364
|
+
message: `task ${cloud.assignment.taskId} run ${String(cloud.assignment.runEpoch)} is no longer current: Stop, teardown or a newer run took over its start`,
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** The start's credentials closed for its run without an interruption:
|
|
369
|
+
* removed, refused or replaced in the cloud. */
|
|
370
|
+
function withdrawnStart(): TaskUpOutcome {
|
|
371
|
+
return { ok: false, code: HostErrorCode.Internal, message: CREDENTIALS_WITHDRAWN_MESSAGE };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** The last word before a start publishes its success: null while it may.
|
|
375
|
+
* Otherwise what it reports instead (`interrupted` when Stop, teardown, a
|
|
376
|
+
* newer run or shutdown took over): its interruption's result, or a
|
|
377
|
+
* withdrawn start's failure once its credentials are no longer open for
|
|
378
|
+
* its run. */
|
|
379
|
+
function cloudStartCannotPublish(
|
|
380
|
+
cloud: CloudCredentialTaskStart | null,
|
|
381
|
+
): { result: TaskUpOutcome; interrupted: boolean } | null {
|
|
382
|
+
if (!cloud?.start) return null;
|
|
383
|
+
const interrupted = interruptedStartResult(cloud);
|
|
384
|
+
if (interrupted) return { result: interrupted, interrupted: true };
|
|
385
|
+
try {
|
|
386
|
+
cloud.start.assertCurrent();
|
|
387
|
+
return null;
|
|
388
|
+
} catch {
|
|
389
|
+
return { result: withdrawnStart(), interrupted: false };
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** The long slot's operation, the start's hold let go as it ends: a hold
|
|
394
|
+
* never outlives the slot it was taken in, so a Stop or teardown queued
|
|
395
|
+
* behind it never waits on it. */
|
|
396
|
+
function releasingHold(
|
|
397
|
+
cloud: CloudCredentialTaskStart | null,
|
|
398
|
+
operation: () => Promise<TaskUpOutcome>,
|
|
399
|
+
): () => Promise<TaskUpOutcome> {
|
|
400
|
+
return async () => {
|
|
401
|
+
try {
|
|
402
|
+
return await operation();
|
|
403
|
+
} finally {
|
|
404
|
+
cloud?.hold?.release();
|
|
286
405
|
}
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
async function taskUpOn(
|
|
410
|
+
ctx: Parameters<TaskUpCommand>[0],
|
|
411
|
+
input: Parameters<TaskUpCommand>[1],
|
|
412
|
+
cloud: CloudCredentialTaskStart | null,
|
|
413
|
+
legacy: LegacyStartPermit | null,
|
|
414
|
+
): ReturnType<TaskUpCommand> {
|
|
415
|
+
const orchestrator = getOrchestrator();
|
|
416
|
+
let runEpochAdmission: ReturnType<typeof admitTaskRunEpoch>;
|
|
417
|
+
try {
|
|
418
|
+
if (input.task.runEpoch === undefined) {
|
|
419
|
+
// ADR-154: a start on the runner's own credentials only while the
|
|
420
|
+
// runner is still on them.
|
|
421
|
+
if (legacy?.valid() === false) return switchingToCloudCredentials();
|
|
422
|
+
// Preserve the v0 path exactly during the rolling window. It has no
|
|
423
|
+
// assignment floor to serialize and must keep failing cheap host
|
|
424
|
+
// preflights before entering the runtime lifecycle queue.
|
|
425
|
+
runEpochAdmission = admitTaskRunEpoch(input.task.id, undefined);
|
|
426
|
+
} else {
|
|
427
|
+
const admitted = await orchestrator.runTaskLifecycle(
|
|
428
|
+
input.task.id,
|
|
429
|
+
async () => {
|
|
430
|
+
// ADR-154: a start on the runner's own credentials is admitted
|
|
431
|
+
// only while the runner is still on them.
|
|
432
|
+
if (legacy?.valid() === false) {
|
|
433
|
+
return { ok: false as const, result: switchingToCloudCredentials() };
|
|
434
|
+
}
|
|
435
|
+
const admission = admitTaskRunEpoch(
|
|
436
|
+
input.task.id,
|
|
437
|
+
input.task.runEpoch,
|
|
438
|
+
);
|
|
439
|
+
if (admission.kind === "stale") {
|
|
440
|
+
return {
|
|
441
|
+
ok: false as const,
|
|
442
|
+
result: {
|
|
443
|
+
ok: false as const,
|
|
444
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
445
|
+
message:
|
|
446
|
+
`task ${input.task.id} run ${String(input.task.runEpoch)} is stale; ` +
|
|
447
|
+
`this runner has already admitted run ${String(admission.currentRunEpoch)}`,
|
|
448
|
+
},
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
// ADR-154: the run's credential startup begins in this short slot;
|
|
452
|
+
// its predecessor's cleanup and its grant are awaited outside it.
|
|
453
|
+
if (cloud !== null) {
|
|
454
|
+
try {
|
|
455
|
+
cloud.start = cloud.runtime.beginStart(cloud.assignment);
|
|
456
|
+
} catch (error) {
|
|
457
|
+
return {
|
|
458
|
+
ok: false as const,
|
|
459
|
+
result: {
|
|
460
|
+
ok: false as const,
|
|
461
|
+
code: HostErrorCode.HostUnavailable,
|
|
462
|
+
message: `task ${input.task.id} could not begin on its cloud credentials: ${
|
|
463
|
+
error instanceof Error ? error.message : String(error)
|
|
464
|
+
}`,
|
|
465
|
+
retryable: true,
|
|
466
|
+
...(error instanceof StartInterruptedError ? { retryReason: "managed_restart" as const } : {}),
|
|
467
|
+
},
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
return { ok: true as const, admission };
|
|
472
|
+
},
|
|
473
|
+
);
|
|
474
|
+
if (!admitted.ok) return admitted.result;
|
|
475
|
+
runEpochAdmission = admitted.admission;
|
|
476
|
+
}
|
|
477
|
+
} catch (error) {
|
|
478
|
+
return {
|
|
479
|
+
ok: false,
|
|
480
|
+
code: HostErrorCode.Internal,
|
|
481
|
+
message:
|
|
482
|
+
error instanceof Error
|
|
483
|
+
? error.message
|
|
484
|
+
: "the task run epoch is invalid",
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
// Admission alone never proves a runtime belongs to this assignment. A
|
|
488
|
+
// failed prior start can leave an open floor beside the older runtime.
|
|
489
|
+
const mayReuseRuntime = canReuseTaskRuntimeForRun(
|
|
490
|
+
input.task.id,
|
|
491
|
+
input.task.runEpoch,
|
|
492
|
+
);
|
|
493
|
+
const admissionFailure = managedMaintenanceUnavailable();
|
|
494
|
+
if (admissionFailure) return admissionFailure;
|
|
495
|
+
const runtimePreparationFailure = await prepareContainerRuntimeForTask();
|
|
496
|
+
if (runtimePreparationFailure) return runtimePreparationFailure;
|
|
497
|
+
const runtimeFailure = runtimeUnavailable();
|
|
498
|
+
if (runtimeFailure) return runtimeFailure;
|
|
499
|
+
// The boot reconcile briefly replaces CLI shims in the shared volume.
|
|
500
|
+
// Do not create local lifecycle/channel state until its one-way latch has
|
|
501
|
+
// settled (success or best-effort failure), then revalidate both mutable
|
|
502
|
+
// admission authorities after the await.
|
|
503
|
+
await agentClisReady;
|
|
504
|
+
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
505
|
+
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
506
|
+
const readyRuntimeFailure = runtimeUnavailable();
|
|
507
|
+
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
508
|
+
try {
|
|
509
|
+
await waitForMcpGatewayReady();
|
|
510
|
+
} catch (error) {
|
|
511
|
+
console.warn(
|
|
512
|
+
`[task-gateway] refusing task ${input.task.id} before provisioning: ${
|
|
513
|
+
error instanceof Error ? error.message : String(error)
|
|
514
|
+
}`,
|
|
515
|
+
);
|
|
516
|
+
return {
|
|
517
|
+
ok: false,
|
|
518
|
+
code: HostErrorCode.HostUnavailable,
|
|
519
|
+
message:
|
|
520
|
+
"The task environment gateway is not ready on this host. Retry the task; if it persists, check the host service log.",
|
|
521
|
+
retryable: true,
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
// A task machine's own guards first: its runner's data directory root's
|
|
525
|
+
// alone, and the metadata service closed to the task's user. Without
|
|
526
|
+
// them the task does not start: it could read the sidecar's host token.
|
|
527
|
+
if (sidecarMode()) {
|
|
528
|
+
const guard = guardSidecarMachine({ dataDir: env.dataDir });
|
|
529
|
+
if (!guard.ok) {
|
|
530
|
+
console.warn(`[sidecar] refusing task ${input.task.id}: ${guard.reason}`);
|
|
293
531
|
return {
|
|
294
532
|
ok: false,
|
|
295
533
|
code: HostErrorCode.Internal,
|
|
296
|
-
message:
|
|
534
|
+
message: `This task machine is not safe to run the task on: ${guard.reason}. Close the task and start it again on a new machine.`,
|
|
297
535
|
retryable: false,
|
|
298
536
|
};
|
|
299
537
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
logCommand(ctx, "taskUp", input.task.id);
|
|
307
|
-
const admissionFailure = managedMaintenanceUnavailable();
|
|
308
|
-
if (admissionFailure) return admissionFailure;
|
|
309
|
-
const runtimePreparationFailure = await prepareContainerRuntimeForTask();
|
|
310
|
-
if (runtimePreparationFailure) return runtimePreparationFailure;
|
|
311
|
-
const runtimeFailure = runtimeUnavailable();
|
|
312
|
-
if (runtimeFailure) return runtimeFailure;
|
|
313
|
-
// The boot reconcile briefly replaces CLI shims in the shared volume.
|
|
314
|
-
// Do not create local lifecycle/channel state until its one-way latch has
|
|
315
|
-
// settled (success or best-effort failure), then revalidate both mutable
|
|
316
|
-
// admission authorities after the await.
|
|
317
|
-
await agentClisReady;
|
|
318
|
-
const readyAdmissionFailure = managedMaintenanceUnavailable();
|
|
319
|
-
if (readyAdmissionFailure) return readyAdmissionFailure;
|
|
320
|
-
const readyRuntimeFailure = runtimeUnavailable();
|
|
321
|
-
if (readyRuntimeFailure) return readyRuntimeFailure;
|
|
538
|
+
}
|
|
539
|
+
// A bindings-capable task sidecar must activate and commit its exact
|
|
540
|
+
// replica before any task state, /env generation or process can exist.
|
|
541
|
+
// Otherwise the source's intentional pre-activation legacy fallback
|
|
542
|
+
// could expose stored plaintext during the first reconciliation.
|
|
543
|
+
if (sidecarMode()) {
|
|
322
544
|
try {
|
|
323
|
-
|
|
545
|
+
// ADR-154: on cloud credentials the start's own grant is what readies
|
|
546
|
+
// it (awaited below); the v0 lease that readies the machine's stores
|
|
547
|
+
// never runs there.
|
|
548
|
+
if (cloud === null) await untilLeavingLegacy(legacy, awaitSidecarReady());
|
|
549
|
+
// ADR-154: and its env is the composer's, never the machine's replica.
|
|
550
|
+
if (cloud === null && projectEnvBindingCapability() !== null) {
|
|
551
|
+
await untilLeavingLegacy(legacy, awaitProjectEnvReplicaReady());
|
|
552
|
+
}
|
|
324
553
|
} catch (error) {
|
|
554
|
+
// ADR-154: the runner began leaving its own credentials, and with them
|
|
555
|
+
// the v0 lease this readiness comes from: the cloud starts the task
|
|
556
|
+
// again on cloud credentials.
|
|
557
|
+
if (legacy?.signal.aborted) return switchingToCloudCredentials();
|
|
325
558
|
console.warn(
|
|
326
|
-
`[task-
|
|
559
|
+
`[task-env] refusing task ${input.task.id} before provisioning: ${
|
|
327
560
|
error instanceof Error ? error.message : String(error)
|
|
328
561
|
}`,
|
|
329
562
|
);
|
|
@@ -331,881 +564,1072 @@ export const hostCommands: HostCommands = {
|
|
|
331
564
|
ok: false,
|
|
332
565
|
code: HostErrorCode.HostUnavailable,
|
|
333
566
|
message:
|
|
334
|
-
"
|
|
567
|
+
"This task machine is still synchronizing its environment. Retry shortly.",
|
|
335
568
|
retryable: true,
|
|
336
569
|
};
|
|
337
570
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
571
|
+
}
|
|
572
|
+
// ADR-118 follow-up: each apple-container task is its own VM, and each
|
|
573
|
+
// VM pins host vnodes for its life. Refuse to start ANOTHER one when the
|
|
574
|
+
// table is nearly full — a clear, retryable refusal now beats ENFILE
|
|
575
|
+
// storms (and the suspected panic path) for the whole machine later.
|
|
576
|
+
if (pinnedContainerRuntimeProvider() === "apple-container") {
|
|
577
|
+
const vnodeProblem = await vnodeAdmissionProblem();
|
|
578
|
+
if (vnodeProblem) {
|
|
579
|
+
return {
|
|
580
|
+
ok: false,
|
|
581
|
+
code: HostErrorCode.HostUnavailable,
|
|
582
|
+
message: vnodeProblem,
|
|
583
|
+
retryable: true,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
// ADR-154: the grant installed and its gate open, its env snapshot exact:
|
|
588
|
+
// awaited outside every lifecycle slot, and bounded.
|
|
589
|
+
if (cloud?.start) {
|
|
590
|
+
try {
|
|
591
|
+
await cloud.runtime.waitForLaunchReady(cloud.start);
|
|
592
|
+
} catch (error) {
|
|
593
|
+
return (
|
|
594
|
+
interruptedStartResult(cloud) ?? {
|
|
346
595
|
ok: false,
|
|
347
|
-
code: HostErrorCode.
|
|
348
|
-
message: `
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
596
|
+
code: HostErrorCode.HostUnavailable,
|
|
597
|
+
message: `The task's cloud credentials did not become usable (${
|
|
598
|
+
error instanceof Error ? error.message : String(error)
|
|
599
|
+
}). Retry the task.`,
|
|
600
|
+
retryable: true,
|
|
601
|
+
}
|
|
602
|
+
);
|
|
352
603
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
|
|
604
|
+
}
|
|
605
|
+
return orchestrator.runTaskLifecycle(input.task.id, releasingHold(cloud, async () => {
|
|
606
|
+
// A direct duplicate teardown may still be draining operations that
|
|
607
|
+
// target this task's stable compose/container names. Never recreate
|
|
608
|
+
// those names until every close caller has crossed the shared fence.
|
|
609
|
+
await orchestrator.waitForChannelClose(input.task.id);
|
|
610
|
+
// Admission is a short lifecycle transaction before the long host
|
|
611
|
+
// preflights above. A newer dispatch may have advanced the durable
|
|
612
|
+
// floor while this command was waiting; once this command owns the
|
|
613
|
+
// lifecycle slot, reject it before it can observe or mutate a newer
|
|
614
|
+
// assignment's runtime.
|
|
615
|
+
const assignmentState = readTaskRunEpochState(input.task.id);
|
|
616
|
+
const assignmentCurrent =
|
|
617
|
+
input.task.runEpoch === undefined
|
|
618
|
+
? assignmentState === null
|
|
619
|
+
: assignmentState !== null &&
|
|
620
|
+
assignmentState.assignmentOpen &&
|
|
621
|
+
assignmentState.highestRunEpoch === input.task.runEpoch;
|
|
622
|
+
if (!assignmentCurrent) {
|
|
623
|
+
const currentRunEpoch = assignmentState?.highestRunEpoch ?? 0;
|
|
624
|
+
return {
|
|
625
|
+
ok: false,
|
|
626
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
627
|
+
message:
|
|
628
|
+
`task ${input.task.id} run ${String(input.task.runEpoch)} is no longer current; ` +
|
|
629
|
+
`this runner has admitted run ${String(currentRunEpoch)}`,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
const inSlotAdmissionFailure = managedMaintenanceUnavailable();
|
|
633
|
+
if (inSlotAdmissionFailure) return inSlotAdmissionFailure;
|
|
634
|
+
const inSlotRuntimeFailure = runtimeUnavailable();
|
|
635
|
+
if (inSlotRuntimeFailure) return inSlotRuntimeFailure;
|
|
636
|
+
// ADR-154: nothing of a start on the runner's own credentials happens
|
|
637
|
+
// once the runner has begun leaving them.
|
|
638
|
+
if (legacy?.valid() === false) return switchingToCloudCredentials();
|
|
639
|
+
// ADR-154: the start holds its installed grant from here through the
|
|
640
|
+
// clone, provisioning and the runtime-ready writes; without it nothing
|
|
641
|
+
// starts.
|
|
642
|
+
if (cloud?.start) {
|
|
358
643
|
try {
|
|
359
|
-
|
|
360
|
-
if (projectEnvBindingCapability() !== null) {
|
|
361
|
-
await awaitProjectEnvReplicaReady();
|
|
362
|
-
}
|
|
644
|
+
cloud.hold = cloud.runtime.tryAcquireTaskUpHold(cloud.start);
|
|
363
645
|
} catch (error) {
|
|
364
|
-
console.warn(
|
|
365
|
-
`[task-env] refusing task ${input.task.id} before provisioning: ${
|
|
366
|
-
error instanceof Error ? error.message : String(error)
|
|
367
|
-
}`,
|
|
368
|
-
);
|
|
369
646
|
return {
|
|
370
647
|
ok: false,
|
|
371
648
|
code: HostErrorCode.HostUnavailable,
|
|
372
|
-
message:
|
|
373
|
-
|
|
649
|
+
message: `The task's lease names credentials that could not be read (${
|
|
650
|
+
error instanceof Error ? error.message : String(error)
|
|
651
|
+
}). Retry the task.`,
|
|
374
652
|
retryable: true,
|
|
375
653
|
};
|
|
376
654
|
}
|
|
655
|
+
if (cloud.hold === null) return interruptedStartResult(cloud) ?? withdrawnStart();
|
|
656
|
+
}
|
|
657
|
+
// A command retry that arrives after the original start completed must
|
|
658
|
+
// not rerender/compose-up/uai-init underneath live agent sessions. Treat
|
|
659
|
+
// that lost-response retry as the successful operation it already was:
|
|
660
|
+
// every cloud caller records a non-ok taskUp as an errored task.
|
|
661
|
+
const existingTask = getHostTask(input.task.id);
|
|
662
|
+
const incomingEnvSpec = taskProjectEnvSpec(input);
|
|
663
|
+
let existingGatewayReady = true;
|
|
664
|
+
if (existingTask && hostTaskEnvMode(existingTask) === "loader") {
|
|
665
|
+
// A lost-response retry is also the repair path after a host process
|
|
666
|
+
// restart: reattach the live resolver before any task process can be
|
|
667
|
+
// treated as reusable. Missing/corrupt durable identity forces normal
|
|
668
|
+
// recreation, which mints a generation and restarts its forwarder.
|
|
669
|
+
recordTaskProjectEnvSpec(input.task.id, input);
|
|
670
|
+
existingGatewayReady =
|
|
671
|
+
recoverTaskGatewayTask(input.task.id, {
|
|
672
|
+
// The generation serves the run its runtime was established for.
|
|
673
|
+
resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, existingTask.runEpoch > 0 ? existingTask.runEpoch : null),
|
|
674
|
+
runEpoch: existingTask.runEpoch > 0 ? existingTask.runEpoch : null,
|
|
675
|
+
}) !== null;
|
|
377
676
|
}
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
677
|
+
let runtimeQuarantinedForRecreate = false;
|
|
678
|
+
let channelClosedForRecreate = false;
|
|
679
|
+
if (
|
|
680
|
+
existingTask?.statusMirror === "running" &&
|
|
681
|
+
existingTask.environmentProvider === "apple-container"
|
|
682
|
+
) {
|
|
683
|
+
// ADR-106/round-4: the lost-response fast path must speak the
|
|
684
|
+
// provider's language — the Docker branch below ENOENTs on a
|
|
685
|
+
// Docker-free Mac and suspended the generation instead of returning
|
|
686
|
+
// the persisted success.
|
|
687
|
+
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
385
688
|
return {
|
|
386
689
|
ok: false,
|
|
387
|
-
code: HostErrorCode.
|
|
388
|
-
message:
|
|
389
|
-
|
|
690
|
+
code: HostErrorCode.Internal,
|
|
691
|
+
message:
|
|
692
|
+
`task ${input.task.id} is running but its persisted runtime ` +
|
|
693
|
+
"metadata is incomplete",
|
|
390
694
|
};
|
|
391
695
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
//
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
//
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
) {
|
|
427
|
-
// ADR-106/round-4: the lost-response fast path must speak the
|
|
428
|
-
// provider's language — the Docker branch below ENOENTs on a
|
|
429
|
-
// Docker-free Mac and suspended the generation instead of returning
|
|
430
|
-
// the persisted success.
|
|
431
|
-
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
696
|
+
// Round 5: every await below can outlive the runtime generation
|
|
697
|
+
// (bridge revalidation, forced recheck). A superseded pass must
|
|
698
|
+
// observe only — never reopen consumers or mutate the container —
|
|
699
|
+
// the same discipline apple recovery applies.
|
|
700
|
+
// Round 6: the fence requires the OPERATIONAL generation, not merely
|
|
701
|
+
// the epoch — plain suspension (connection revalidation) keeps the
|
|
702
|
+
// epoch and ready status while the daemon is unverified.
|
|
703
|
+
const fastEpoch = containerRuntimeReadinessEpoch();
|
|
704
|
+
const fastStale = (): boolean =>
|
|
705
|
+
!containerRuntimeGenerationOperational(fastEpoch);
|
|
706
|
+
const staleRetry = {
|
|
707
|
+
ok: false as const,
|
|
708
|
+
code: HostErrorCode.HostUnavailable,
|
|
709
|
+
message:
|
|
710
|
+
`the container runtime generation changed while verifying task ${input.task.id}; retry`,
|
|
711
|
+
retryable: true,
|
|
712
|
+
};
|
|
713
|
+
const environment = await reconstructPersistedTaskEnvironment(
|
|
714
|
+
existingTask,
|
|
715
|
+
);
|
|
716
|
+
if (fastStale()) return staleRetry;
|
|
717
|
+
if (environment === null) {
|
|
718
|
+
// The persisted locator no longer reconstructs (e.g. an engine
|
|
719
|
+
// identity change) — recreate through the normal path.
|
|
720
|
+
suspendTaskGatewayTask(input.task.id);
|
|
721
|
+
await orchestrator.closeChannel(input.task.id);
|
|
722
|
+
// Session drain can outlive the generation too: a superseded pass
|
|
723
|
+
// must not reach recordTaskStarting/provisioning below.
|
|
724
|
+
if (fastStale()) return staleRetry;
|
|
725
|
+
channelClosedForRecreate = true;
|
|
726
|
+
} else {
|
|
727
|
+
const appleStatus = await environment.status();
|
|
728
|
+
if (fastStale()) return staleRetry;
|
|
729
|
+
if (appleStatus.state === "unknown") {
|
|
432
730
|
return {
|
|
433
731
|
ok: false,
|
|
434
|
-
code: HostErrorCode.
|
|
732
|
+
code: HostErrorCode.HostUnavailable,
|
|
435
733
|
message:
|
|
436
|
-
`task ${input.task.id}
|
|
437
|
-
|
|
734
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
735
|
+
appleStatus.detail,
|
|
736
|
+
retryable: true,
|
|
438
737
|
};
|
|
439
738
|
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
const fastStale = (): boolean =>
|
|
449
|
-
!containerRuntimeGenerationOperational(fastEpoch);
|
|
450
|
-
const staleRetry = {
|
|
451
|
-
ok: false as const,
|
|
452
|
-
code: HostErrorCode.HostUnavailable,
|
|
453
|
-
message:
|
|
454
|
-
`the container runtime generation changed while verifying task ${input.task.id}; retry`,
|
|
455
|
-
retryable: true,
|
|
456
|
-
};
|
|
457
|
-
const environment = await reconstructPersistedTaskEnvironment(
|
|
458
|
-
existingTask,
|
|
459
|
-
);
|
|
460
|
-
if (fastStale()) return staleRetry;
|
|
461
|
-
if (environment === null) {
|
|
462
|
-
// The persisted locator no longer reconstructs (e.g. an engine
|
|
463
|
-
// identity change) — recreate through the normal path.
|
|
464
|
-
suspendTaskGatewayTask(input.task.id);
|
|
465
|
-
await orchestrator.closeChannel(input.task.id);
|
|
466
|
-
// Session drain can outlive the generation too: a superseded pass
|
|
467
|
-
// must not reach recordTaskStarting/provisioning below.
|
|
468
|
-
if (fastStale()) return staleRetry;
|
|
469
|
-
channelClosedForRecreate = true;
|
|
470
|
-
} else {
|
|
471
|
-
const appleStatus = await environment.status();
|
|
739
|
+
if (appleStatus.state === "running") {
|
|
740
|
+
const verdict = await proveAppleTaskRuntimeContract({
|
|
741
|
+
taskId: input.task.id,
|
|
742
|
+
containerName: appleTaskContainerName(input.task.id),
|
|
743
|
+
// The row's worktreePath is task-up's $task_dir — the same
|
|
744
|
+
// directory the apple locator records as hostWorktreePath.
|
|
745
|
+
hostWorktreePath: existingTask.worktreePath,
|
|
746
|
+
});
|
|
472
747
|
if (fastStale()) return staleRetry;
|
|
473
|
-
if (
|
|
748
|
+
if (verdict.kind === "unreachable") {
|
|
474
749
|
return {
|
|
475
750
|
ok: false,
|
|
476
751
|
code: HostErrorCode.HostUnavailable,
|
|
477
752
|
message:
|
|
478
|
-
`could not
|
|
479
|
-
|
|
753
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
754
|
+
verdict.detail,
|
|
480
755
|
retryable: true,
|
|
481
756
|
};
|
|
482
757
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
758
|
+
let projectsHealthy = true;
|
|
759
|
+
for (const project of verdict.kind === "holds"
|
|
760
|
+
? input.projects
|
|
761
|
+
: []) {
|
|
762
|
+
const gitHealth = await environment.exec({
|
|
763
|
+
projectEnvProfile: "maintenance",
|
|
764
|
+
argv: [
|
|
765
|
+
"/usr/bin/env",
|
|
766
|
+
"-i",
|
|
767
|
+
"HOME=/home/node",
|
|
768
|
+
"PATH=/usr/bin:/bin",
|
|
769
|
+
"/usr/bin/git",
|
|
770
|
+
"rev-parse",
|
|
771
|
+
"--is-inside-work-tree",
|
|
772
|
+
],
|
|
773
|
+
user: "node",
|
|
774
|
+
cwd: `/workspace/${project.slug}`,
|
|
775
|
+
env: {},
|
|
776
|
+
timeoutMs: 10_000,
|
|
777
|
+
maxOutputBytes: 64 * 1024,
|
|
490
778
|
});
|
|
491
779
|
if (fastStale()) return staleRetry;
|
|
492
|
-
if (
|
|
780
|
+
if (gitHealth.exitCode === null || gitHealth.signal !== null) {
|
|
493
781
|
return {
|
|
494
782
|
ok: false,
|
|
495
783
|
code: HostErrorCode.HostUnavailable,
|
|
496
784
|
message:
|
|
497
|
-
`could not
|
|
498
|
-
|
|
785
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
786
|
+
`project ${project.id}: the runtime did not answer`,
|
|
499
787
|
retryable: true,
|
|
500
788
|
};
|
|
501
789
|
}
|
|
502
|
-
let projectsHealthy = true;
|
|
503
|
-
for (const project of verdict.kind === "holds"
|
|
504
|
-
? input.projects
|
|
505
|
-
: []) {
|
|
506
|
-
const gitHealth = await environment.exec({
|
|
507
|
-
projectEnvProfile: "maintenance",
|
|
508
|
-
argv: [
|
|
509
|
-
"/usr/bin/env",
|
|
510
|
-
"-i",
|
|
511
|
-
"HOME=/home/node",
|
|
512
|
-
"PATH=/usr/bin:/bin",
|
|
513
|
-
"/usr/bin/git",
|
|
514
|
-
"rev-parse",
|
|
515
|
-
"--is-inside-work-tree",
|
|
516
|
-
],
|
|
517
|
-
user: "node",
|
|
518
|
-
cwd: `/workspace/${project.slug}`,
|
|
519
|
-
env: {},
|
|
520
|
-
timeoutMs: 10_000,
|
|
521
|
-
maxOutputBytes: 64 * 1024,
|
|
522
|
-
});
|
|
523
|
-
if (fastStale()) return staleRetry;
|
|
524
|
-
if (gitHealth.exitCode === null || gitHealth.signal !== null) {
|
|
525
|
-
return {
|
|
526
|
-
ok: false,
|
|
527
|
-
code: HostErrorCode.HostUnavailable,
|
|
528
|
-
message:
|
|
529
|
-
`could not verify Git state for task ${input.task.id} ` +
|
|
530
|
-
`project ${project.id}: the runtime did not answer`,
|
|
531
|
-
retryable: true,
|
|
532
|
-
};
|
|
533
|
-
}
|
|
534
|
-
if (
|
|
535
|
-
gitHealth.exitCode !== 0 ||
|
|
536
|
-
Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
|
|
537
|
-
"true"
|
|
538
|
-
) {
|
|
539
|
-
projectsHealthy = false;
|
|
540
|
-
console.warn(
|
|
541
|
-
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
|
|
542
|
-
);
|
|
543
|
-
break;
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
790
|
if (
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
791
|
+
gitHealth.exitCode !== 0 ||
|
|
792
|
+
Buffer.from(gitHealth.stdout).toString("utf8").trim() !==
|
|
793
|
+
"true"
|
|
550
794
|
) {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
composeProject: existingTask.composeProject,
|
|
557
|
-
worktreePath: existingTask.worktreePath,
|
|
558
|
-
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
559
|
-
previewPorts: parsePreviewPortRuntimes(
|
|
560
|
-
existingTask.previewPorts,
|
|
561
|
-
),
|
|
562
|
-
envMode: hostTaskEnvMode(existingTask),
|
|
563
|
-
},
|
|
564
|
-
};
|
|
565
|
-
}
|
|
566
|
-
// Violated contract or broken project repo: drain, stop, and
|
|
567
|
-
// prove before task-up recreates the same stable name.
|
|
568
|
-
if (fastStale()) return staleRetry;
|
|
569
|
-
suspendTaskGatewayTask(input.task.id);
|
|
570
|
-
await orchestrator.closeChannel(input.task.id);
|
|
571
|
-
channelClosedForRecreate = true;
|
|
572
|
-
if (fastStale()) return staleRetry;
|
|
573
|
-
await environment.stop();
|
|
574
|
-
const after = await environment.status();
|
|
575
|
-
if (fastStale()) return staleRetry;
|
|
576
|
-
if (after.state === "running" || after.state === "unknown") {
|
|
577
|
-
return {
|
|
578
|
-
ok: false,
|
|
579
|
-
code: HostErrorCode.HostUnavailable,
|
|
580
|
-
message:
|
|
581
|
-
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
582
|
-
"could not be proven stopped; retry after the runtime is healthy",
|
|
583
|
-
retryable: true,
|
|
584
|
-
};
|
|
795
|
+
projectsHealthy = false;
|
|
796
|
+
console.warn(
|
|
797
|
+
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task`,
|
|
798
|
+
);
|
|
799
|
+
break;
|
|
585
800
|
}
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
// recreate — and the rollback state must say stopped for the
|
|
594
|
-
// same reason as above.
|
|
595
|
-
if (fastStale()) return staleRetry;
|
|
596
|
-
suspendTaskGatewayTask(input.task.id);
|
|
597
|
-
await orchestrator.closeChannel(input.task.id);
|
|
598
|
-
// Fence AFTER the drain as well: provisioning must never start
|
|
599
|
-
// under a generation this pass no longer owns.
|
|
801
|
+
}
|
|
802
|
+
if (
|
|
803
|
+
verdict.kind === "holds" &&
|
|
804
|
+
projectsHealthy &&
|
|
805
|
+
existingGatewayReady &&
|
|
806
|
+
mayReuseRuntime
|
|
807
|
+
) {
|
|
600
808
|
if (fastStale()) return staleRetry;
|
|
601
|
-
|
|
602
|
-
|
|
809
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
810
|
+
if (cannotPublish) return cannotPublish.result;
|
|
811
|
+
orchestrator.allowChannel(input.task.id);
|
|
812
|
+
return {
|
|
813
|
+
ok: true,
|
|
814
|
+
value: {
|
|
815
|
+
composeProject: existingTask.composeProject,
|
|
816
|
+
worktreePath: existingTask.worktreePath,
|
|
817
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
818
|
+
previewPorts: parsePreviewPortRuntimes(
|
|
819
|
+
existingTask.previewPorts,
|
|
820
|
+
),
|
|
821
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
822
|
+
},
|
|
823
|
+
};
|
|
603
824
|
}
|
|
825
|
+
// Violated contract or broken project repo: drain, stop, and
|
|
826
|
+
// prove before task-up recreates the same stable name.
|
|
827
|
+
if (fastStale()) return staleRetry;
|
|
828
|
+
suspendTaskGatewayTask(input.task.id);
|
|
829
|
+
await orchestrator.closeChannel(input.task.id);
|
|
830
|
+
channelClosedForRecreate = true;
|
|
831
|
+
if (fastStale()) return staleRetry;
|
|
832
|
+
await environment.stop();
|
|
833
|
+
const after = await environment.status();
|
|
834
|
+
if (fastStale()) return staleRetry;
|
|
835
|
+
if (after.state === "running" || after.state === "unknown") {
|
|
836
|
+
return {
|
|
837
|
+
ok: false,
|
|
838
|
+
code: HostErrorCode.HostUnavailable,
|
|
839
|
+
message:
|
|
840
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
841
|
+
"could not be proven stopped; retry after the runtime is healthy",
|
|
842
|
+
retryable: true,
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
// Round 5: the container is PHYSICALLY stopped now, whatever the
|
|
846
|
+
// reason (violated contract or a broken project repo). A failed
|
|
847
|
+
// recreation must roll the mirror back to stopped — restoring
|
|
848
|
+
// "running" would advertise a container proven not to run.
|
|
849
|
+
runtimeQuarantinedForRecreate = true;
|
|
850
|
+
} else {
|
|
851
|
+
// Provider proved the container absent/stopped: normal self-heal
|
|
852
|
+
// recreate — and the rollback state must say stopped for the
|
|
853
|
+
// same reason as above.
|
|
854
|
+
if (fastStale()) return staleRetry;
|
|
855
|
+
suspendTaskGatewayTask(input.task.id);
|
|
856
|
+
await orchestrator.closeChannel(input.task.id);
|
|
857
|
+
// Fence AFTER the drain as well: provisioning must never start
|
|
858
|
+
// under a generation this pass no longer owns.
|
|
859
|
+
if (fastStale()) return staleRetry;
|
|
860
|
+
channelClosedForRecreate = true;
|
|
861
|
+
runtimeQuarantinedForRecreate = true;
|
|
604
862
|
}
|
|
605
|
-
}
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
)
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
863
|
+
}
|
|
864
|
+
} else if (
|
|
865
|
+
existingTask?.statusMirror === "running" &&
|
|
866
|
+
isEnvironmentTransportProvider(existingTask.environmentProvider)
|
|
867
|
+
) {
|
|
868
|
+
// ADR-121: a machine-backed task revalidates through its own
|
|
869
|
+
// provider — the compose fast path below speaks docker and would
|
|
870
|
+
// misread a machine row. Running machine → persisted success;
|
|
871
|
+
// anything else falls through to provisionTaskEnvironment, whose
|
|
872
|
+
// machine provider owns recovery semantics.
|
|
873
|
+
try {
|
|
874
|
+
const environment = await reconstructPersistedTaskEnvironment(
|
|
875
|
+
existingTask,
|
|
876
|
+
);
|
|
877
|
+
if (environment) {
|
|
878
|
+
const machineStatus = await environment.status();
|
|
879
|
+
if (
|
|
880
|
+
machineStatus.state === "running" &&
|
|
881
|
+
existingGatewayReady &&
|
|
882
|
+
mayReuseRuntime
|
|
883
|
+
) {
|
|
884
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
885
|
+
if (cannotPublish) return cannotPublish.result;
|
|
886
|
+
orchestrator.allowChannel(input.task.id);
|
|
887
|
+
return {
|
|
888
|
+
ok: true,
|
|
889
|
+
value: {
|
|
890
|
+
composeProject: existingTask.composeProject ?? "",
|
|
891
|
+
worktreePath: existingTask.worktreePath ?? "",
|
|
892
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
893
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
894
|
+
},
|
|
895
|
+
};
|
|
632
896
|
}
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
|
|
897
|
+
}
|
|
898
|
+
} catch (error) {
|
|
899
|
+
console.warn(
|
|
900
|
+
`[machine] task ${input.task.id}: running-row revalidation failed, continuing to provision: ${error instanceof Error ? error.message : String(error)}`,
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
} else if (existingTask?.statusMirror === "running") {
|
|
904
|
+
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
905
|
+
return {
|
|
906
|
+
ok: false,
|
|
907
|
+
code: HostErrorCode.Internal,
|
|
908
|
+
message:
|
|
909
|
+
`task ${input.task.id} is running but its persisted runtime ` +
|
|
910
|
+
"metadata is incomplete",
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
// The mirror may be stale after a manual `docker compose down`.
|
|
914
|
+
// Confirm the app itself is live before treating this as a lost-response
|
|
915
|
+
// retry; otherwise fall through to the normal taskUp self-heal.
|
|
916
|
+
const appRuntime = await dockerCli(
|
|
917
|
+
[
|
|
918
|
+
"ps",
|
|
919
|
+
"-a",
|
|
920
|
+
"--filter",
|
|
921
|
+
`label=com.docker.compose.project=${existingTask.composeProject}`,
|
|
922
|
+
"--filter",
|
|
923
|
+
"label=com.docker.compose.service=app",
|
|
924
|
+
"--format",
|
|
925
|
+
"{{.State}}",
|
|
926
|
+
],
|
|
927
|
+
{ timeoutMs: 10_000 },
|
|
928
|
+
);
|
|
929
|
+
if (appRuntime.status !== 0) {
|
|
930
|
+
return dockerUnavailableResult(
|
|
931
|
+
`could not verify runtime state for task ${input.task.id}: ` +
|
|
932
|
+
(appRuntime.stderr.trim().slice(0, 200) ||
|
|
933
|
+
"docker did not answer"),
|
|
934
|
+
);
|
|
935
|
+
}
|
|
936
|
+
if (
|
|
937
|
+
appRuntime.stdout
|
|
938
|
+
.split(/\r?\n/)
|
|
939
|
+
.some((state) => state.trim() === "running")
|
|
940
|
+
) {
|
|
941
|
+
const appContainer = `${existingTask.composeProject}-app-1`;
|
|
942
|
+
const runtimeContract = await inspectTaskRuntimeContainer(
|
|
943
|
+
input.task.id,
|
|
944
|
+
existingTask.composeProject,
|
|
945
|
+
);
|
|
946
|
+
if (runtimeContract.kind === "unreachable") {
|
|
947
|
+
return dockerUnavailableResult(
|
|
948
|
+
`could not inspect the exact runtime mount for task ${input.task.id}`,
|
|
636
949
|
);
|
|
637
950
|
}
|
|
638
|
-
|
|
639
|
-
if (!existingTask.composeProject || !existingTask.worktreePath) {
|
|
951
|
+
if (runtimeContract.kind === "failed") {
|
|
640
952
|
return {
|
|
641
953
|
ok: false,
|
|
642
|
-
code: HostErrorCode.
|
|
954
|
+
code: HostErrorCode.HostUnavailable,
|
|
643
955
|
message:
|
|
644
|
-
`task ${input.task.id}
|
|
645
|
-
|
|
956
|
+
`could not prove the exact runtime contract for task ${input.task.id}: ` +
|
|
957
|
+
runtimeContract.detail,
|
|
958
|
+
retryable: true,
|
|
646
959
|
};
|
|
647
960
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
)
|
|
670
|
-
}
|
|
671
|
-
if (
|
|
672
|
-
appRuntime.stdout
|
|
673
|
-
.split(/\r?\n/)
|
|
674
|
-
.some((state) => state.trim() === "running")
|
|
675
|
-
) {
|
|
676
|
-
const appContainer = `${existingTask.composeProject}-app-1`;
|
|
677
|
-
const runtimeContract = await inspectTaskRuntimeContainer(
|
|
678
|
-
input.task.id,
|
|
679
|
-
existingTask.composeProject,
|
|
680
|
-
);
|
|
681
|
-
if (runtimeContract.kind === "unreachable") {
|
|
961
|
+
const reusable =
|
|
962
|
+
runtimeContract.running &&
|
|
963
|
+
!runtimeContract.restarting &&
|
|
964
|
+
!runtimeContract.paused &&
|
|
965
|
+
runtimeContract.sharedMount === "read-only" &&
|
|
966
|
+
runtimeContract.taskEnvironment === "current" &&
|
|
967
|
+
existingGatewayReady &&
|
|
968
|
+
mayReuseRuntime;
|
|
969
|
+
if (!reusable) {
|
|
970
|
+
// Drain/tombstone before touching the exact container. task-up's
|
|
971
|
+
// pre-materialization cleanup then removes this stopped immutable
|
|
972
|
+
// generation, so Compose must create the current RO/private-prefix
|
|
973
|
+
// contract rather than `docker start`ing it again.
|
|
974
|
+
suspendTaskGatewayTask(input.task.id);
|
|
975
|
+
await orchestrator.closeChannel(input.task.id);
|
|
976
|
+
channelClosedForRecreate = true;
|
|
977
|
+
const quarantined = await quarantineTaskRuntimeContainer({
|
|
978
|
+
taskId: input.task.id,
|
|
979
|
+
composeProject: existingTask.composeProject,
|
|
980
|
+
expectedContainerId: runtimeContract.containerId,
|
|
981
|
+
});
|
|
982
|
+
if (quarantined === "unreachable") {
|
|
682
983
|
return dockerUnavailableResult(
|
|
683
|
-
`
|
|
984
|
+
`Docker became unavailable while quarantining task ${input.task.id}`,
|
|
684
985
|
);
|
|
685
986
|
}
|
|
686
|
-
if (
|
|
987
|
+
if (quarantined === "failed") {
|
|
687
988
|
return {
|
|
688
989
|
ok: false,
|
|
689
990
|
code: HostErrorCode.HostUnavailable,
|
|
690
991
|
message:
|
|
691
|
-
`
|
|
692
|
-
|
|
992
|
+
`task ${input.task.id} has a non-current runtime contract and ` +
|
|
993
|
+
"could not be proven stopped; retry after Docker is healthy",
|
|
693
994
|
retryable: true,
|
|
694
995
|
};
|
|
695
996
|
}
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
"GIT_CONFIG_NOSYSTEM=",
|
|
775
|
-
"-e",
|
|
776
|
-
"GIT_CONFIG_COUNT=0",
|
|
777
|
-
"-e",
|
|
778
|
-
"GIT_EXEC_PATH=",
|
|
779
|
-
"-e",
|
|
780
|
-
"GIT_SSH=",
|
|
781
|
-
"-e",
|
|
782
|
-
"GIT_SSH_COMMAND=",
|
|
783
|
-
"-e",
|
|
784
|
-
"GIT_ASKPASS=",
|
|
785
|
-
"-e",
|
|
786
|
-
"SSH_ASKPASS=",
|
|
787
|
-
"-w",
|
|
788
|
-
`/workspace/${project.slug}`,
|
|
789
|
-
appContainer,
|
|
790
|
-
"/usr/bin/env",
|
|
791
|
-
"-i",
|
|
792
|
-
"HOME=/home/node",
|
|
793
|
-
"PATH=/usr/bin:/bin",
|
|
794
|
-
"/usr/bin/git",
|
|
795
|
-
"rev-parse",
|
|
796
|
-
"--is-inside-work-tree",
|
|
797
|
-
],
|
|
798
|
-
{ timeoutMs: 10_000 },
|
|
997
|
+
runtimeQuarantinedForRecreate = true;
|
|
998
|
+
}
|
|
999
|
+
// Docker being up is not enough to prove a lost-response retry is
|
|
1000
|
+
// healthy. The worktree's `.git` file contains an absolute pointer
|
|
1001
|
+
// into the task-private repository; after a host workspace-root
|
|
1002
|
+
// migration the old container can keep running while every agent
|
|
1003
|
+
// sees `fatal: not a git repository`. Probe from INSIDE the live
|
|
1004
|
+
// container, which is the filesystem view the agents actually use.
|
|
1005
|
+
// Scratchpads have no projects, so this remains an O(1) fast path.
|
|
1006
|
+
let projectsHealthy = true;
|
|
1007
|
+
for (const project of reusable ? input.projects : []) {
|
|
1008
|
+
const gitHealth = await dockerCli(
|
|
1009
|
+
[
|
|
1010
|
+
"exec",
|
|
1011
|
+
"-u",
|
|
1012
|
+
"node",
|
|
1013
|
+
// docker exec inherits the Compose/project environment. Clear
|
|
1014
|
+
// loader and Git controls before the first executable, then
|
|
1015
|
+
// give Git a minimal environment of its own. Keep this aligned
|
|
1016
|
+
// with task-up.sh's managed container operations.
|
|
1017
|
+
"-e",
|
|
1018
|
+
"HOME=/home/node",
|
|
1019
|
+
"-e",
|
|
1020
|
+
"PATH=/usr/bin:/bin",
|
|
1021
|
+
"-e",
|
|
1022
|
+
"LD_PRELOAD=",
|
|
1023
|
+
"-e",
|
|
1024
|
+
"LD_LIBRARY_PATH=",
|
|
1025
|
+
"-e",
|
|
1026
|
+
"DYLD_INSERT_LIBRARIES=",
|
|
1027
|
+
"-e",
|
|
1028
|
+
"DYLD_LIBRARY_PATH=",
|
|
1029
|
+
"-e",
|
|
1030
|
+
"BASH_ENV=",
|
|
1031
|
+
"-e",
|
|
1032
|
+
"ENV=",
|
|
1033
|
+
"-e",
|
|
1034
|
+
"GIT_CONFIG=",
|
|
1035
|
+
"-e",
|
|
1036
|
+
"GIT_CONFIG_GLOBAL=",
|
|
1037
|
+
"-e",
|
|
1038
|
+
"GIT_CONFIG_SYSTEM=",
|
|
1039
|
+
"-e",
|
|
1040
|
+
"GIT_CONFIG_NOSYSTEM=",
|
|
1041
|
+
"-e",
|
|
1042
|
+
"GIT_CONFIG_COUNT=0",
|
|
1043
|
+
"-e",
|
|
1044
|
+
"GIT_EXEC_PATH=",
|
|
1045
|
+
"-e",
|
|
1046
|
+
"GIT_SSH=",
|
|
1047
|
+
"-e",
|
|
1048
|
+
"GIT_SSH_COMMAND=",
|
|
1049
|
+
"-e",
|
|
1050
|
+
"GIT_ASKPASS=",
|
|
1051
|
+
"-e",
|
|
1052
|
+
"SSH_ASKPASS=",
|
|
1053
|
+
"-w",
|
|
1054
|
+
`/workspace/${project.slug}`,
|
|
1055
|
+
appContainer,
|
|
1056
|
+
"/usr/bin/env",
|
|
1057
|
+
"-i",
|
|
1058
|
+
"HOME=/home/node",
|
|
1059
|
+
"PATH=/usr/bin:/bin",
|
|
1060
|
+
"/usr/bin/git",
|
|
1061
|
+
"rev-parse",
|
|
1062
|
+
"--is-inside-work-tree",
|
|
1063
|
+
],
|
|
1064
|
+
{ timeoutMs: 10_000 },
|
|
1065
|
+
);
|
|
1066
|
+
// A timeout/spawn error is not proof that the repository is
|
|
1067
|
+
// broken. Preserve the running task and let the caller retry once
|
|
1068
|
+
// Docker can answer, just as the runtime-state probe above does.
|
|
1069
|
+
if (gitHealth.status === null) {
|
|
1070
|
+
return dockerUnavailableResult(
|
|
1071
|
+
`could not verify Git state for task ${input.task.id} ` +
|
|
1072
|
+
`project ${project.id}: ` +
|
|
1073
|
+
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
1074
|
+
"docker did not answer"),
|
|
799
1075
|
);
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
1076
|
+
}
|
|
1077
|
+
if (
|
|
1078
|
+
gitHealth.status !== 0 ||
|
|
1079
|
+
gitHealth.stdout.trim() !== "true"
|
|
1080
|
+
) {
|
|
1081
|
+
if (gitHealth.status !== 0) {
|
|
1082
|
+
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
805
1083
|
`could not verify Git state for task ${input.task.id} ` +
|
|
806
1084
|
`project ${project.id}: ` +
|
|
807
1085
|
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
808
|
-
|
|
1086
|
+
`git exited ${String(gitHealth.status)}`),
|
|
809
1087
|
);
|
|
1088
|
+
if (unavailable) return unavailable;
|
|
810
1089
|
}
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
gitHealth.
|
|
814
|
-
)
|
|
815
|
-
|
|
816
|
-
const unavailable = await runtimeFailureIfDaemonUnavailable(
|
|
817
|
-
`could not verify Git state for task ${input.task.id} ` +
|
|
818
|
-
`project ${project.id}: ` +
|
|
819
|
-
(gitHealth.stderr.trim().slice(0, 200) ||
|
|
820
|
-
`git exited ${String(gitHealth.status)}`),
|
|
821
|
-
);
|
|
822
|
-
if (unavailable) return unavailable;
|
|
823
|
-
}
|
|
824
|
-
projectsHealthy = false;
|
|
825
|
-
console.warn(
|
|
826
|
-
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
|
|
827
|
-
);
|
|
828
|
-
break;
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
if (reusable && projectsHealthy) {
|
|
832
|
-
// waitForChannelClose crossed the teardown fence above. A completed
|
|
833
|
-
// close-only teardown may have left this running task tombstoned,
|
|
834
|
-
// so the idempotent-success transition must reopen delivery just
|
|
835
|
-
// like a fresh successful taskUp does.
|
|
836
|
-
orchestrator.allowChannel(input.task.id);
|
|
837
|
-
return {
|
|
838
|
-
ok: true,
|
|
839
|
-
value: {
|
|
840
|
-
composeProject: existingTask.composeProject,
|
|
841
|
-
worktreePath: existingTask.worktreePath,
|
|
842
|
-
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
843
|
-
previewPorts: parsePreviewPortRuntimes(
|
|
844
|
-
existingTask.previewPorts,
|
|
845
|
-
),
|
|
846
|
-
envMode: hostTaskEnvMode(existingTask),
|
|
847
|
-
},
|
|
848
|
-
};
|
|
1090
|
+
projectsHealthy = false;
|
|
1091
|
+
console.warn(
|
|
1092
|
+
`[host-agent] task ${input.task.id}: project ${project.id} Git health check failed; rebuilding the running task (${gitHealth.stderr.trim().slice(0, 200) || `git exited ${String(gitHealth.status)}`})`,
|
|
1093
|
+
);
|
|
1094
|
+
break;
|
|
849
1095
|
}
|
|
850
1096
|
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
1097
|
+
if (reusable && projectsHealthy) {
|
|
1098
|
+
// waitForChannelClose crossed the teardown fence above. A completed
|
|
1099
|
+
// close-only teardown may have left this running task tombstoned,
|
|
1100
|
+
// so the idempotent-success transition must reopen delivery just
|
|
1101
|
+
// like a fresh successful taskUp does.
|
|
1102
|
+
const cannotPublish = cloudStartCannotPublish(cloud);
|
|
1103
|
+
if (cannotPublish) return cannotPublish.result;
|
|
1104
|
+
orchestrator.allowChannel(input.task.id);
|
|
1105
|
+
return {
|
|
1106
|
+
ok: true,
|
|
1107
|
+
value: {
|
|
1108
|
+
composeProject: existingTask.composeProject,
|
|
1109
|
+
worktreePath: existingTask.worktreePath,
|
|
1110
|
+
codeServerPort: existingTask.codeServerPort ?? undefined,
|
|
1111
|
+
previewPorts: parsePreviewPortRuntimes(
|
|
1112
|
+
existingTask.previewPorts,
|
|
1113
|
+
),
|
|
1114
|
+
envMode: hostTaskEnvMode(existingTask),
|
|
1115
|
+
},
|
|
1116
|
+
};
|
|
858
1117
|
}
|
|
859
1118
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
statusMirror: existingTask.statusMirror,
|
|
876
|
-
lockedAt: existingTask.lockedAt,
|
|
877
|
-
startedAt: existingTask.startedAt,
|
|
878
|
-
endedAt: existingTask.endedAt,
|
|
879
|
-
environmentProvider: existingTask.environmentProvider,
|
|
880
|
-
environmentLocator: existingTask.environmentLocator,
|
|
881
|
-
envMode: existingTask.envMode,
|
|
882
|
-
projectEnvSpec: existingTask.projectEnvSpec,
|
|
883
|
-
}
|
|
884
|
-
: {
|
|
885
|
-
statusMirror: input.task.status,
|
|
1119
|
+
// Docker positively proved the mirrored app is absent/stopped, or the
|
|
1120
|
+
// live container proved one of its project repositories is broken.
|
|
1121
|
+
// Drain and invalidate any stale in-memory channel before taskUp
|
|
1122
|
+
// recreates the same stable compose/container names and repairs Git.
|
|
1123
|
+
if (!channelClosedForRecreate) {
|
|
1124
|
+
suspendTaskGatewayTask(input.task.id);
|
|
1125
|
+
await orchestrator.closeChannel(input.task.id);
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
const priorRuntimeState = existingTask
|
|
1129
|
+
? runtimeQuarantinedForRecreate
|
|
1130
|
+
? {
|
|
1131
|
+
statusMirror: "stopped",
|
|
1132
|
+
codeServerPort: null,
|
|
1133
|
+
previewPorts: "[]",
|
|
886
1134
|
lockedAt: null,
|
|
887
|
-
startedAt:
|
|
888
|
-
endedAt:
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
1135
|
+
startedAt: existingTask.startedAt,
|
|
1136
|
+
endedAt: existingTask.endedAt,
|
|
1137
|
+
environmentProvider: existingTask.environmentProvider,
|
|
1138
|
+
environmentLocator: existingTask.environmentLocator,
|
|
1139
|
+
envMode: existingTask.envMode,
|
|
1140
|
+
projectEnvSpec: existingTask.projectEnvSpec,
|
|
1141
|
+
runEpoch: existingTask.runEpoch,
|
|
1142
|
+
}
|
|
1143
|
+
: {
|
|
1144
|
+
statusMirror: existingTask.statusMirror,
|
|
1145
|
+
lockedAt: existingTask.lockedAt,
|
|
1146
|
+
startedAt: existingTask.startedAt,
|
|
1147
|
+
endedAt: existingTask.endedAt,
|
|
1148
|
+
environmentProvider: existingTask.environmentProvider,
|
|
1149
|
+
environmentLocator: existingTask.environmentLocator,
|
|
1150
|
+
envMode: existingTask.envMode,
|
|
1151
|
+
projectEnvSpec: existingTask.projectEnvSpec,
|
|
1152
|
+
runEpoch: existingTask.runEpoch,
|
|
1153
|
+
}
|
|
1154
|
+
: {
|
|
1155
|
+
statusMirror: input.task.status,
|
|
1156
|
+
lockedAt: null,
|
|
1157
|
+
startedAt: null,
|
|
1158
|
+
endedAt: null,
|
|
1159
|
+
};
|
|
1160
|
+
// ADR-153 machines: a task with no row has never run on this machine,
|
|
1161
|
+
// so everything it runs comes through its hosts view. Recorded before
|
|
1162
|
+
// the row, and so before the task's first process; a task with a row
|
|
1163
|
+
// (a start driven again, a start after a stop) stands as recorded.
|
|
1164
|
+
const taskWasKnownBeforeAssignment =
|
|
1165
|
+
runEpochAdmission.kind === "accepted"
|
|
1166
|
+
? runEpochAdmission.hadTask
|
|
1167
|
+
: existingTask !== null;
|
|
1168
|
+
if (sidecarMode() && !taskWasKnownBeforeAssignment) {
|
|
1169
|
+
admitFreshTaskView(input.task.id);
|
|
1170
|
+
}
|
|
1171
|
+
if (input.task.runEpoch === undefined) {
|
|
895
1172
|
recordTaskStarting(input.task.id);
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
1173
|
+
} else {
|
|
1174
|
+
recordTaskStarting(input.task.id, input.task.runEpoch);
|
|
1175
|
+
}
|
|
1176
|
+
// Cross-process managed maintenance and task admission use a two-sided
|
|
1177
|
+
// handshake: update/uninstall writes its durable marker before checking
|
|
1178
|
+
// this DB row; task-up writes `starting` before checking admission again.
|
|
1179
|
+
// Whichever arrives second observes the first, so a runtime cannot be
|
|
1180
|
+
// swapped or removed while an unrecorded task enters Docker side effects.
|
|
1181
|
+
const admittedMaintenanceFailure = managedMaintenanceUnavailable();
|
|
1182
|
+
if (admittedMaintenanceFailure) {
|
|
1183
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1184
|
+
else deleteHostTask(input.task.id);
|
|
1185
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
1186
|
+
orchestrator.allowChannel(input.task.id);
|
|
909
1187
|
}
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1188
|
+
return admittedMaintenanceFailure;
|
|
1189
|
+
}
|
|
1190
|
+
recordTaskOwner(
|
|
1191
|
+
input.task.id,
|
|
1192
|
+
input.task.ownerUserId,
|
|
1193
|
+
input.ownerEmail ?? null,
|
|
1194
|
+
input.ownerName ?? null,
|
|
1195
|
+
input.projects.map((p) => p.slug),
|
|
1196
|
+
// ADR-131 D3: persisted so recovery re-provisions the same scope.
|
|
1197
|
+
input.task.engineAccountScope ?? null,
|
|
1198
|
+
// Lead project first (position 0 wins env collisions, as in task-up).
|
|
1199
|
+
[...input.projects]
|
|
1200
|
+
.sort((a, b) => a.position - b.position)
|
|
1201
|
+
.map((p) => p.id),
|
|
1202
|
+
);
|
|
1203
|
+
recordTaskProjectEnvSpec(input.task.id, input);
|
|
1204
|
+
// ADR-048: persist the cloud-issued per-task cli secret (host-private) so the
|
|
1205
|
+
// `uai` CLI's per-agent tokens can be minted now + after a host restart.
|
|
1206
|
+
if (input.task.cliSecret) {
|
|
1207
|
+
storeTaskCliSecret(input.task.id, input.task.cliSecret);
|
|
1208
|
+
}
|
|
1209
|
+
recordHostEvent(input.task.id, "task.created");
|
|
1210
|
+
let environmentLocator: TaskEnvironmentLocator | undefined;
|
|
1211
|
+
// The locator of everything this start may create, recorded before any
|
|
1212
|
+
// of it exists: once set, the row must keep it.
|
|
1213
|
+
let runtimePrepared = false;
|
|
1214
|
+
const recordPrepared = async (locator: TaskEnvironmentLocator): Promise<void> => {
|
|
1215
|
+
recordTaskEnvironmentPrepared(input.task.id, locator, "loader");
|
|
1216
|
+
runtimePrepared = true;
|
|
1217
|
+
};
|
|
1218
|
+
let registeredTaskGatewayGeneration: string | null = null;
|
|
1219
|
+
let result: TaskUpOutcome = await wrapAgent(ctx, "taskUp", async () => {
|
|
1220
|
+
// Self-heal the standard base image before task-up needs it. The
|
|
1221
|
+
// boot-time build is best-effort + one-shot: a host that started before
|
|
1222
|
+
// Docker was ready (common on Docker Desktop) never built
|
|
1223
|
+
// uai-standard:dev, so every task-up died on a doomed registry pull
|
|
1224
|
+
// (live 2026-07-22). This is idempotent — a fast inspect when present,
|
|
1225
|
+
// a real build only when missing/stale (Docker is definitely up now, a
|
|
1226
|
+
// task just arrived). On a build FAILURE, surface WHY to the cloud (not
|
|
1227
|
+
// just the host log — the operator can't read a remote user's machine).
|
|
1228
|
+
// A pass that succeeded within the hour is reused (2026-09-09: the
|
|
1229
|
+
// re-check cost 16 s of every task start); the hourly schedule in
|
|
1230
|
+
// main.ts and the boot pass keep it fresh.
|
|
1231
|
+
const img = await ensureStandardImage({ maxAgeMs: 60 * 60_000 });
|
|
1232
|
+
if (!img.ok) {
|
|
1233
|
+
throw new AgentError(
|
|
1234
|
+
"STANDARD_IMAGE_UNAVAILABLE",
|
|
1235
|
+
img.error
|
|
1236
|
+
? `could not prepare the host base image uai-standard:dev — the build failed. ${toolStderrDetail(img.error)}`
|
|
1237
|
+
: "could not prepare the host base image uai-standard:dev — see the host log.",
|
|
1238
|
+
{},
|
|
1239
|
+
);
|
|
928
1240
|
}
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
let gitCredential: TaskGithubGitCredential | null;
|
|
955
|
-
try {
|
|
956
|
-
gitCredential = input.projects.some((project) =>
|
|
957
|
-
isGithubRepo(project.repoUrl),
|
|
958
|
-
)
|
|
959
|
-
? await prepareTaskGithubGitCredential(input.task.ownerUserId)
|
|
960
|
-
: null;
|
|
961
|
-
} catch {
|
|
962
|
-
throw new AgentError(
|
|
963
|
-
"GITHUB_AUTH_UNAVAILABLE",
|
|
964
|
-
"GitHub is connected on this host, but Uai could not prepare its credential for Git. Retry the task; if it persists, reconnect GitHub on this host.",
|
|
1241
|
+
let gitCredential: TaskGithubGitCredential | null;
|
|
1242
|
+
try {
|
|
1243
|
+
// ADR-154: a start on cloud credentials clones with its lease's,
|
|
1244
|
+
// never the runner's own store.
|
|
1245
|
+
gitCredential = cloud === null && input.projects.some((project) =>
|
|
1246
|
+
isGithubRepo(project.repoUrl),
|
|
1247
|
+
)
|
|
1248
|
+
? await prepareTaskGithubGitCredential(input.task.ownerUserId)
|
|
1249
|
+
: null;
|
|
1250
|
+
} catch {
|
|
1251
|
+
throw new AgentError(
|
|
1252
|
+
"GITHUB_AUTH_UNAVAILABLE",
|
|
1253
|
+
"GitHub is connected on this host, but Uai could not prepare its credential for Git. Retry the task; if it persists, reconnect GitHub on this host.",
|
|
1254
|
+
);
|
|
1255
|
+
}
|
|
1256
|
+
if (sidecarMode() && cloud === null) {
|
|
1257
|
+
// ADR-133: project env lives under /uai/projects/<id>, readable by
|
|
1258
|
+
// the lease only for parameters tagged with this org, and only the
|
|
1259
|
+
// task's own projects are worth listing — their ids are known here,
|
|
1260
|
+
// not at the first grant (docs/credential-store.md).
|
|
1261
|
+
const { reconcileProjectEnvFromStore } = await import("../lib/user-credentials-store");
|
|
1262
|
+
const projectIds = input.projects.map((project) => project.id);
|
|
1263
|
+
const env = await reconcileProjectEnvFromStore({ projectIds }).catch((error: unknown) => {
|
|
1264
|
+
console.warn(
|
|
1265
|
+
`[project-env] task ${input.task.id}: credential store read failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
965
1266
|
);
|
|
966
|
-
|
|
967
|
-
if (sidecarMode()) {
|
|
968
|
-
// ADR-133: project env lives under /uai/projects/<id>, readable by
|
|
969
|
-
// the lease only for parameters tagged with this org, and only the
|
|
970
|
-
// task's own projects are worth listing — their ids are known here,
|
|
971
|
-
// not at the first grant (docs/credential-store.md).
|
|
972
|
-
const { reconcileProjectEnvFromStore } = await import("../lib/user-credentials-store");
|
|
973
|
-
const projectIds = input.projects.map((project) => project.id);
|
|
974
|
-
const env = await reconcileProjectEnvFromStore({ projectIds }).catch((error: unknown) => {
|
|
975
|
-
console.warn(
|
|
976
|
-
`[project-env] task ${input.task.id}: credential store read failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
977
|
-
);
|
|
978
|
-
return null;
|
|
979
|
-
});
|
|
980
|
-
if (env && !env.skipped) {
|
|
981
|
-
console.log(
|
|
982
|
-
`[project-env] task ${input.task.id}: credential store reconciled for ${projectIds.length} project(s): ${env.imported} imported, ${env.removed} removed, ${env.errors} errors`,
|
|
983
|
-
);
|
|
984
|
-
}
|
|
985
|
-
}
|
|
986
|
-
// Mint the authenticated task generation before its environment
|
|
987
|
-
// starts. The declarations are pinned; values remain live DB reads.
|
|
988
|
-
const gatewayRegistration = registerTaskGatewayTask(input.task.id, {
|
|
989
|
-
resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec),
|
|
1267
|
+
return null;
|
|
990
1268
|
});
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
gatewayRegistration.generation,
|
|
995
|
-
);
|
|
996
|
-
if (gatewaySecret === null) {
|
|
997
|
-
removeTaskGatewayTask(
|
|
998
|
-
input.task.id,
|
|
999
|
-
gatewayRegistration.generation,
|
|
1269
|
+
if (env && !env.skipped) {
|
|
1270
|
+
console.log(
|
|
1271
|
+
`[project-env] task ${input.task.id}: credential store reconciled for ${projectIds.length} project(s): ${env.imported} imported, ${env.removed} removed, ${env.errors} errors`,
|
|
1000
1272
|
);
|
|
1001
|
-
registeredTaskGatewayGeneration = null;
|
|
1002
|
-
throw new Error("could not export the task gateway generation secret");
|
|
1003
1273
|
}
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1274
|
+
}
|
|
1275
|
+
// Mint the authenticated task generation before its environment
|
|
1276
|
+
// starts. The declarations are pinned; values remain live DB reads.
|
|
1277
|
+
const gatewayRegistration = registerTaskGatewayTask(input.task.id, {
|
|
1278
|
+
resolveEnv: taskEnvResolver(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null),
|
|
1279
|
+
runEpoch: input.task.runEpoch ?? null,
|
|
1280
|
+
});
|
|
1281
|
+
registeredTaskGatewayGeneration = gatewayRegistration.generation;
|
|
1282
|
+
const gatewaySecret = taskGatewayForwarderSecretInput(
|
|
1283
|
+
input.task.id,
|
|
1284
|
+
gatewayRegistration.generation,
|
|
1285
|
+
);
|
|
1286
|
+
if (gatewaySecret === null) {
|
|
1287
|
+
removeTaskGatewayTask(
|
|
1008
1288
|
input.task.id,
|
|
1009
|
-
|
|
1289
|
+
gatewayRegistration.generation,
|
|
1010
1290
|
);
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
);
|
|
1030
|
-
},
|
|
1031
|
-
),
|
|
1032
|
-
)
|
|
1033
|
-
: await provisionTaskEnvironment(
|
|
1291
|
+
registeredTaskGatewayGeneration = null;
|
|
1292
|
+
throw new Error("could not export the task gateway generation secret");
|
|
1293
|
+
}
|
|
1294
|
+
// Launch screen: the first stage boundary. A machine-backed task
|
|
1295
|
+
// spends the next minute launching an EC2 instance; a container
|
|
1296
|
+
// task goes straight to preparing its workspace.
|
|
1297
|
+
orchestrator.emitTaskStage(
|
|
1298
|
+
input.task.id,
|
|
1299
|
+
machineTasksEnabled() ? "machine" : "workspace",
|
|
1300
|
+
);
|
|
1301
|
+
try {
|
|
1302
|
+
const taskGateway = {
|
|
1303
|
+
generation: gatewayRegistration.generation,
|
|
1304
|
+
secret: gatewaySecret,
|
|
1305
|
+
};
|
|
1306
|
+
const provisioned = gitCredential
|
|
1307
|
+
? await gitCredential.run(() =>
|
|
1308
|
+
provisionTaskEnvironment(
|
|
1034
1309
|
input,
|
|
1035
|
-
{
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
input.task.id,
|
|
1039
|
-
locator,
|
|
1040
|
-
"loader",
|
|
1041
|
-
);
|
|
1310
|
+
{
|
|
1311
|
+
githubCredentialSocket: gitCredential.socketPath,
|
|
1312
|
+
taskGateway,
|
|
1042
1313
|
},
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
: { ...provisioned.result, envMode: "loader" as const, envSchemas };
|
|
1072
|
-
} finally {
|
|
1073
|
-
gatewaySecret.fill(0);
|
|
1074
|
-
await gitCredential?.close().catch((error: unknown) => {
|
|
1075
|
-
// Cleanup must not replace the actual task-up result/error. The
|
|
1076
|
-
// credential is short-lived and close() also removes its private
|
|
1077
|
-
// directory in a finally block; retain diagnostics for operators.
|
|
1314
|
+
recordPrepared,
|
|
1315
|
+
),
|
|
1316
|
+
)
|
|
1317
|
+
: await provisionTaskEnvironment(
|
|
1318
|
+
input,
|
|
1319
|
+
{
|
|
1320
|
+
taskGateway,
|
|
1321
|
+
...(cloud?.start && cloud.hold ? { leased: leasedTaskUp(cloud.start, cloud.hold) } : {}),
|
|
1322
|
+
},
|
|
1323
|
+
recordPrepared,
|
|
1324
|
+
);
|
|
1325
|
+
environmentLocator = provisioned.handle.descriptor.locator;
|
|
1326
|
+
// ADR-153 §4: the task's own schema observations, read from the
|
|
1327
|
+
// bases task-up recorded before its container started (a machine
|
|
1328
|
+
// captures its own). Stored once; a resume reuses them.
|
|
1329
|
+
const envSchemas = isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)
|
|
1330
|
+
? undefined
|
|
1331
|
+
: captureTaskEnvSchemas(
|
|
1332
|
+
input.task.id,
|
|
1333
|
+
incomingEnvSpec.projects.map((project) => project.projectId),
|
|
1334
|
+
);
|
|
1335
|
+
// ADR-153: before any consumer is let in, bring the task to now:
|
|
1336
|
+
// a CA successor minted while it started, and the domains its
|
|
1337
|
+
// policy has now rather than those task-up was handed. A failure
|
|
1338
|
+
// leaves those keys withheld, and the trust watcher retries.
|
|
1339
|
+
const secretsProxy = activeSecretsProxy();
|
|
1340
|
+
if (secretsProxy !== null) {
|
|
1341
|
+
await reconcileTaskProxyState(provisioned.handle, secretsProxy.runtime).catch((error: unknown) => {
|
|
1078
1342
|
console.warn(
|
|
1079
|
-
`[
|
|
1343
|
+
`[secrets-proxy] task ${input.task.id}: could not bring the task's trust and pins up to date: ${
|
|
1344
|
+
error instanceof Error ? error.message : String(error)
|
|
1345
|
+
}`,
|
|
1080
1346
|
);
|
|
1347
|
+
secretsProxy.runtime.requestTrustDelivery();
|
|
1081
1348
|
});
|
|
1082
1349
|
}
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
)
|
|
1091
|
-
|
|
1092
|
-
// The in-process authority is revoked before durable deletion is
|
|
1093
|
-
// attempted. Preserve the task-up error while retaining a cleanup
|
|
1094
|
-
// diagnostic; a leftover file loads inactive after restart.
|
|
1350
|
+
return envSchemas === undefined || envSchemas.length === 0
|
|
1351
|
+
? { ...provisioned.result, envMode: "loader" as const }
|
|
1352
|
+
: { ...provisioned.result, envMode: "loader" as const, envSchemas };
|
|
1353
|
+
} finally {
|
|
1354
|
+
gatewaySecret.fill(0);
|
|
1355
|
+
await gitCredential?.close().catch((error: unknown) => {
|
|
1356
|
+
// Cleanup must not replace the actual task-up result/error. The
|
|
1357
|
+
// credential is short-lived and close() also removes its private
|
|
1358
|
+
// directory in a finally block; retain diagnostics for operators.
|
|
1095
1359
|
console.warn(
|
|
1096
|
-
`[
|
|
1097
|
-
error instanceof Error ? error.message : String(error)
|
|
1098
|
-
}`,
|
|
1360
|
+
`[github] task ${input.task.id}: credential cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1099
1361
|
);
|
|
1100
|
-
}
|
|
1362
|
+
});
|
|
1101
1363
|
}
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1364
|
+
});
|
|
1365
|
+
// ADR-154: the last word before the start publishes its outcome. A
|
|
1366
|
+
// success needs the start still the task's current one, its credentials
|
|
1367
|
+
// open for its run; an interrupted start reports its interruption, never
|
|
1368
|
+
// what it failed of. Nothing it made is destroyed here: a teardown would
|
|
1369
|
+
// delete a resumed task's workspace, and whoever took over acts on the
|
|
1370
|
+
// runtime itself.
|
|
1371
|
+
const interrupted = interruptedStartResult(cloud);
|
|
1372
|
+
const replaced = result.ok
|
|
1373
|
+
? cloudStartCannotPublish(cloud)
|
|
1374
|
+
: interrupted
|
|
1375
|
+
? { result: interrupted, interrupted: true }
|
|
1376
|
+
: null;
|
|
1377
|
+
if (replaced) result = replaced.result;
|
|
1378
|
+
if (!result.ok && registeredTaskGatewayGeneration !== null) {
|
|
1379
|
+
forgetTaskEnvSnapshot(input.task.id);
|
|
1380
|
+
try {
|
|
1381
|
+
removeTaskGatewayTask(
|
|
1108
1382
|
input.task.id,
|
|
1109
|
-
|
|
1110
|
-
environmentLocator,
|
|
1111
|
-
"loader",
|
|
1383
|
+
registeredTaskGatewayGeneration,
|
|
1112
1384
|
);
|
|
1113
|
-
|
|
1114
|
-
//
|
|
1115
|
-
//
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1385
|
+
} catch (error) {
|
|
1386
|
+
// The in-process authority is revoked before durable deletion is
|
|
1387
|
+
// attempted. Preserve the task-up error while retaining a cleanup
|
|
1388
|
+
// diagnostic; a leftover file loads inactive after restart.
|
|
1389
|
+
console.warn(
|
|
1390
|
+
`[task-gateway] task ${input.task.id}: failed to remove the rejected generation: ${
|
|
1391
|
+
error instanceof Error ? error.message : String(error)
|
|
1392
|
+
}`,
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
if (result.ok) {
|
|
1397
|
+
orchestrator.allowChannel(input.task.id);
|
|
1398
|
+
if (environmentLocator === undefined) {
|
|
1399
|
+
throw new Error("task environment provision returned no locator");
|
|
1400
|
+
}
|
|
1401
|
+
recordTaskUpResult(
|
|
1402
|
+
input.task.id,
|
|
1403
|
+
result.value,
|
|
1404
|
+
environmentLocator,
|
|
1405
|
+
"loader",
|
|
1406
|
+
);
|
|
1407
|
+
if (input.task.runEpoch !== undefined) {
|
|
1408
|
+
recordTaskRuntimeEstablished(
|
|
1409
|
+
input.task.id,
|
|
1410
|
+
input.task.runEpoch,
|
|
1411
|
+
);
|
|
1412
|
+
}
|
|
1413
|
+
recordHostEvent(input.task.id, "task.started");
|
|
1414
|
+
// ADR-153: with the proxy off, every start says so, since the task's
|
|
1415
|
+
// proxied keys are withheld rather than delivered.
|
|
1416
|
+
if (activeSecretsProxy() === null) {
|
|
1417
|
+
orchestrator.emitSystemNote(
|
|
1418
|
+
input.task.id,
|
|
1419
|
+
secretsProxyEnabled()
|
|
1420
|
+
? "The secrets proxy could not start on this host, so this task's proxied env vars are withheld. See the host log."
|
|
1421
|
+
: "The secrets proxy is off on this host (UAI_SECRETS_PROXY=off), so this task's proxied env vars are withheld.",
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
// ADR-153 §4: one note naming the keys this task does not get, and
|
|
1425
|
+
// why, so nobody hunts for a value Uai withholds on purpose.
|
|
1426
|
+
try {
|
|
1427
|
+
const withheld = taskEnvWithheldNoteNow(input.task.id, incomingEnvSpec, input.task.runEpoch ?? null);
|
|
1428
|
+
if (withheld !== null) orchestrator.emitSystemNote(input.task.id, withheld);
|
|
1429
|
+
} catch (error) {
|
|
1430
|
+
console.warn(
|
|
1431
|
+
`[project-env] task ${input.task.id}: withheld-keys note failed: ${
|
|
1432
|
+
error instanceof Error ? error.message : String(error)
|
|
1433
|
+
}`,
|
|
1434
|
+
);
|
|
1435
|
+
}
|
|
1436
|
+
// Degraded start (uai-init failed twice): the task is up, but say so
|
|
1437
|
+
// in the feed — a silent half-start reads as a broken product.
|
|
1438
|
+
if (result.value.initWarning) {
|
|
1439
|
+
orchestrator.emitSystemNote(
|
|
1440
|
+
input.task.id,
|
|
1441
|
+
result.value.initWarning,
|
|
1442
|
+
);
|
|
1443
|
+
}
|
|
1444
|
+
// ADR-122 phase 1: a project that declares @required env keys with
|
|
1445
|
+
// no value on this host gets ONE visible note now — not a blank 500
|
|
1446
|
+
// hours later (live 2026-08-25). Best-effort by design: a schema
|
|
1447
|
+
// read hiccup must never fail a completed task-up.
|
|
1448
|
+
void noteMissingRequiredEnvKeys(input).catch((err) =>
|
|
1449
|
+
console.warn(
|
|
1450
|
+
`[env-schema] task ${input.task.id}: validation note failed: ${
|
|
1451
|
+
err instanceof Error ? err.message : String(err)
|
|
1452
|
+
}`,
|
|
1453
|
+
),
|
|
1454
|
+
);
|
|
1455
|
+
// GitHub auth for the container is best-effort (ADR-027) and runs in the
|
|
1456
|
+
// background — it must never block or fail task-up. Awaiting it here would
|
|
1457
|
+
// couple the command result to a network token-exchange: a slow/hung
|
|
1458
|
+
// exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
|
|
1459
|
+
// and mark a running task as errored. The reconciler injects + schedules
|
|
1460
|
+
// (or emits a system note on failure) on its own; the agents come up
|
|
1461
|
+
// regardless and the token lands well before the first `gh` call.
|
|
1462
|
+
// ADR-121: machine-backed tasks take a direct environment injection —
|
|
1463
|
+
// the compose reconciler would probe for an app container, find none,
|
|
1464
|
+
// and silently skip. Reconnect re-injection for machines is a
|
|
1465
|
+
// follow-up alongside the machine recovery pass.
|
|
1466
|
+
if (cloud !== null) {
|
|
1467
|
+
// ADR-154: the runtime-ready writes put the lease's GitHub sign-in
|
|
1468
|
+
// and SSH key in; the runner's own store has nothing to add.
|
|
1469
|
+
} else if (isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)) {
|
|
1470
|
+
void machineTaskGithubAuth(
|
|
1471
|
+
input.task.id,
|
|
1472
|
+
input.task.ownerUserId,
|
|
1473
|
+
).catch((err) =>
|
|
1130
1474
|
console.warn(
|
|
1131
|
-
`[
|
|
1132
|
-
|
|
1475
|
+
`[github] machine task ${input.task.id}: auth injection failed: ${
|
|
1476
|
+
err instanceof Error ? err.message : String(err)
|
|
1133
1477
|
}`,
|
|
1134
|
-
)
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
result.value.initWarning,
|
|
1142
|
-
);
|
|
1143
|
-
}
|
|
1144
|
-
// ADR-122 phase 1: a project that declares @required env keys with
|
|
1145
|
-
// no value on this host gets ONE visible note now — not a blank 500
|
|
1146
|
-
// hours later (live 2026-08-25). Best-effort by design: a schema
|
|
1147
|
-
// read hiccup must never fail a completed task-up.
|
|
1148
|
-
void noteMissingRequiredEnvKeys(input).catch((err) =>
|
|
1478
|
+
),
|
|
1479
|
+
);
|
|
1480
|
+
} else {
|
|
1481
|
+
void reconcileTaskGitAuth(
|
|
1482
|
+
input.task.id,
|
|
1483
|
+
input.task.ownerUserId,
|
|
1484
|
+
).catch((err) =>
|
|
1149
1485
|
console.warn(
|
|
1150
|
-
`[
|
|
1486
|
+
`[github] task ${input.task.id}: post-start reconciliation failed: ${
|
|
1151
1487
|
err instanceof Error ? err.message : String(err)
|
|
1152
1488
|
}`,
|
|
1153
1489
|
),
|
|
1154
1490
|
);
|
|
1155
|
-
// GitHub auth for the container is best-effort (ADR-027) and runs in the
|
|
1156
|
-
// background — it must never block or fail task-up. Awaiting it here would
|
|
1157
|
-
// couple the command result to a network token-exchange: a slow/hung
|
|
1158
|
-
// exchange (e.g. cloud mid-deploy) would trip the cloud's command timeout
|
|
1159
|
-
// and mark a running task as errored. The reconciler injects + schedules
|
|
1160
|
-
// (or emits a system note on failure) on its own; the agents come up
|
|
1161
|
-
// regardless and the token lands well before the first `gh` call.
|
|
1162
|
-
// ADR-121: machine-backed tasks take a direct environment injection —
|
|
1163
|
-
// the compose reconciler would probe for an app container, find none,
|
|
1164
|
-
// and silently skip. Reconnect re-injection for machines is a
|
|
1165
|
-
// follow-up alongside the machine recovery pass.
|
|
1166
|
-
if (isEnvironmentTransportProvider(getHostTask(input.task.id)?.environmentProvider)) {
|
|
1167
|
-
void machineTaskGithubAuth(
|
|
1168
|
-
input.task.id,
|
|
1169
|
-
input.task.ownerUserId,
|
|
1170
|
-
).catch((err) =>
|
|
1171
|
-
console.warn(
|
|
1172
|
-
`[github] machine task ${input.task.id}: auth injection failed: ${
|
|
1173
|
-
err instanceof Error ? err.message : String(err)
|
|
1174
|
-
}`,
|
|
1175
|
-
),
|
|
1176
|
-
);
|
|
1177
|
-
} else {
|
|
1178
|
-
void reconcileTaskGitAuth(
|
|
1179
|
-
input.task.id,
|
|
1180
|
-
input.task.ownerUserId,
|
|
1181
|
-
).catch((err) =>
|
|
1182
|
-
console.warn(
|
|
1183
|
-
`[github] task ${input.task.id}: post-start reconciliation failed: ${
|
|
1184
|
-
err instanceof Error ? err.message : String(err)
|
|
1185
|
-
}`,
|
|
1186
|
-
),
|
|
1187
|
-
);
|
|
1188
|
-
}
|
|
1189
|
-
} else if (result.code === HostErrorCode.HostUnavailable) {
|
|
1190
|
-
// A daemon outage is retryable infrastructure state, not proof that
|
|
1191
|
-
// the task itself failed. Undo the optimistic local `starting` mirror
|
|
1192
|
-
// and normally reopen delivery; a runtime already proven stopped for
|
|
1193
|
-
// L5 recreation stays stopped/tombstoned for the cloud's retry.
|
|
1194
|
-
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1195
|
-
else deleteHostTask(input.task.id);
|
|
1196
|
-
if (!runtimeQuarantinedForRecreate) {
|
|
1197
|
-
orchestrator.allowChannel(input.task.id);
|
|
1198
|
-
}
|
|
1199
|
-
} else {
|
|
1200
|
-
recordTaskError(input.task.id);
|
|
1201
1491
|
}
|
|
1202
|
-
|
|
1203
|
-
|
|
1492
|
+
} else if (replaced?.interrupted) {
|
|
1493
|
+
// ADR-154: Stop, teardown, a newer run or shutdown took over, and the
|
|
1494
|
+
// channel stays closed for it. A start that recorded its runtime's
|
|
1495
|
+
// locator (before any of that runtime existed) keeps it exactly, so
|
|
1496
|
+
// whoever took over stops, destroys or starts again that runtime and
|
|
1497
|
+
// nothing that may exist is forgotten; its credentials are cleaned up
|
|
1498
|
+
// while it still runs. A start that recorded none made nothing, and
|
|
1499
|
+
// leaves no trace.
|
|
1500
|
+
if (runtimePrepared) recordTaskError(input.task.id);
|
|
1501
|
+
else if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1502
|
+
else deleteHostTask(input.task.id);
|
|
1503
|
+
} else if (result.code === HostErrorCode.HostUnavailable) {
|
|
1504
|
+
// A daemon outage is retryable infrastructure state, not proof that
|
|
1505
|
+
// the task itself failed. Undo the optimistic local `starting` mirror
|
|
1506
|
+
// and normally reopen delivery; a runtime already proven stopped for
|
|
1507
|
+
// L5 recreation stays stopped/tombstoned for the cloud's retry.
|
|
1508
|
+
if (existingTask) upsertHostTask(input.task.id, priorRuntimeState);
|
|
1509
|
+
else deleteHostTask(input.task.id);
|
|
1510
|
+
if (!runtimeQuarantinedForRecreate) {
|
|
1511
|
+
orchestrator.allowChannel(input.task.id);
|
|
1512
|
+
}
|
|
1513
|
+
} else {
|
|
1514
|
+
recordTaskError(input.task.id);
|
|
1515
|
+
}
|
|
1516
|
+
return result;
|
|
1517
|
+
}));
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
export const hostCommands: HostCommands = {
|
|
1521
|
+
// The hostId argument is cloud-side routing (host-scoped commands carry
|
|
1522
|
+
// their target in args, like filesOp); this process IS the target.
|
|
1523
|
+
async hostUpdate(ctx, _hostId) {
|
|
1524
|
+
logCommand(ctx, "hostUpdate");
|
|
1525
|
+
const pinned = sidecarRunnerIsPinned();
|
|
1526
|
+
if (pinned) return pinned;
|
|
1527
|
+
try {
|
|
1528
|
+
const { updateManagedRuntime } = await import("../lib/managed-runtime");
|
|
1529
|
+
const outcome = await updateManagedRuntime();
|
|
1530
|
+
if (outcome.status === "not-managed") {
|
|
1531
|
+
return {
|
|
1532
|
+
ok: false,
|
|
1533
|
+
code: HostErrorCode.Internal,
|
|
1534
|
+
message:
|
|
1535
|
+
"this host is not a managed install — update it where it runs",
|
|
1536
|
+
retryable: false,
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
if (outcome.status === "deferred") {
|
|
1540
|
+
// A rollback hold is an operator verdict that the newer release is
|
|
1541
|
+
// unsafe on this host; a remote click must not override it.
|
|
1542
|
+
return {
|
|
1543
|
+
ok: true,
|
|
1544
|
+
value: { version: outcome.version, staged: false, deferred: true },
|
|
1545
|
+
};
|
|
1546
|
+
}
|
|
1547
|
+
if (outcome.status === "current") {
|
|
1548
|
+
return { ok: true, value: { version: outcome.version, staged: false } };
|
|
1549
|
+
}
|
|
1550
|
+
managedHostRestartHook?.();
|
|
1551
|
+
return { ok: true, value: { version: outcome.version, staged: true } };
|
|
1552
|
+
} catch (error) {
|
|
1553
|
+
return {
|
|
1554
|
+
ok: false,
|
|
1555
|
+
code: HostErrorCode.Internal,
|
|
1556
|
+
message: `update failed: ${
|
|
1557
|
+
error instanceof Error ? error.message : "unknown error"
|
|
1558
|
+
}`,
|
|
1559
|
+
retryable: true,
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
},
|
|
1563
|
+
async hostRestart(ctx, _hostId) {
|
|
1564
|
+
logCommand(ctx, "hostRestart");
|
|
1565
|
+
const pinned = sidecarRunnerIsPinned();
|
|
1566
|
+
if (pinned) return pinned;
|
|
1567
|
+
if (!managedHostRestartHook) {
|
|
1568
|
+
return {
|
|
1569
|
+
ok: false,
|
|
1570
|
+
code: HostErrorCode.Internal,
|
|
1571
|
+
message: "restart is unavailable in this process",
|
|
1572
|
+
retryable: false,
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
// The result frame flushes before exit: shutdown drains in-flight
|
|
1576
|
+
// commands (this one included) before the process terminates.
|
|
1577
|
+
managedHostRestartHook();
|
|
1578
|
+
return { ok: true, value: undefined };
|
|
1579
|
+
},
|
|
1580
|
+
async taskUp(ctx, input) {
|
|
1581
|
+
logCommand(ctx, "taskUp", input.task.id);
|
|
1582
|
+
// ADR-154: on cloud credentials every start goes through the composer,
|
|
1583
|
+
// and none starts while the runner's activation is pending.
|
|
1584
|
+
const mode = credentialsMode();
|
|
1585
|
+
if (mode === "pending") return switchingToCloudCredentials();
|
|
1586
|
+
// On the runner's own credentials, a counted permit from here to the
|
|
1587
|
+
// outcome: leaving them waits for it.
|
|
1588
|
+
let legacy: LegacyStartPermit | null = null;
|
|
1589
|
+
if (mode === "legacy") {
|
|
1590
|
+
legacy = admitLegacyStart();
|
|
1591
|
+
if (legacy === null) return switchingToCloudCredentials();
|
|
1592
|
+
}
|
|
1593
|
+
let cloud: CloudCredentialTaskStart | null = null;
|
|
1594
|
+
if (mode === "active") {
|
|
1595
|
+
const runtime = cloudCredentialRuntime();
|
|
1596
|
+
if (runtime === null || input.task.runEpoch === undefined) {
|
|
1597
|
+
return {
|
|
1598
|
+
ok: false,
|
|
1599
|
+
code: HostErrorCode.Internal,
|
|
1600
|
+
message: "This host runs tasks only on cloud credentials, and could not start this one on them.",
|
|
1601
|
+
retryable: false,
|
|
1602
|
+
};
|
|
1603
|
+
}
|
|
1604
|
+
cloud = { runtime, assignment: { taskId: input.task.id, runEpoch: input.task.runEpoch }, start: null, hold: null };
|
|
1605
|
+
// A newer run's start ends an older one's before it queues behind it.
|
|
1606
|
+
runtime.interruptStarts(input.task.id, { below: input.task.runEpoch });
|
|
1607
|
+
}
|
|
1608
|
+
let outcome: TaskUpOutcome | undefined;
|
|
1609
|
+
try {
|
|
1610
|
+
outcome = await taskUpOn(ctx, input, cloud, legacy);
|
|
1611
|
+
return outcome;
|
|
1612
|
+
} finally {
|
|
1613
|
+
legacy?.release();
|
|
1614
|
+
// Its hold already let go with the long slot: the cleanup an abandon
|
|
1615
|
+
// runs never waits on it.
|
|
1616
|
+
if (cloud?.start) {
|
|
1617
|
+
if (outcome?.ok === true) cloud.runtime.endStart(cloud.start);
|
|
1618
|
+
else await cloud.runtime.abandonStart(cloud.start, cloud.hold);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1204
1621
|
},
|
|
1205
1622
|
|
|
1206
1623
|
async taskDown(ctx, input) {
|
|
1207
1624
|
logCommand(ctx, "taskDown", input.taskId);
|
|
1208
1625
|
const orphanGc = input.cleanup?.kind === "orphan-gc";
|
|
1626
|
+
// ADR-154: teardown ends its run's start (orphan GC: every start of the
|
|
1627
|
+
// task) before it queues behind it.
|
|
1628
|
+
const interrupting = activeCloudCredentials();
|
|
1629
|
+
if (interrupting) {
|
|
1630
|
+
if (orphanGc) interrupting.interruptStarts(input.taskId);
|
|
1631
|
+
else if (input.task.runEpoch !== undefined) interrupting.interruptStarts(input.taskId, { exact: input.task.runEpoch });
|
|
1632
|
+
}
|
|
1209
1633
|
// A GC command is already authoritative evidence that this cloud task is
|
|
1210
1634
|
// gone. Enter the lifecycle slot and tombstone its channel even when the
|
|
1211
1635
|
// runtime is temporarily unavailable; the durable cloud intent will retry
|
|
@@ -1237,6 +1661,40 @@ export const hostCommands: HostCommands = {
|
|
|
1237
1661
|
} else {
|
|
1238
1662
|
environment = null;
|
|
1239
1663
|
}
|
|
1664
|
+
// Close the exact v1 assignment before any terminal side effect. A
|
|
1665
|
+
// delayed teardown for an older run must never tombstone or destroy the
|
|
1666
|
+
// newer run. Orphan GC is authoritative for the task id and therefore
|
|
1667
|
+
// closes whichever assignment still survives locally.
|
|
1668
|
+
let v1AssignmentClosed = false;
|
|
1669
|
+
let closedRunEpoch: number | null = null;
|
|
1670
|
+
const assignmentState = readTaskRunEpochState(input.taskId);
|
|
1671
|
+
if (assignmentState !== null || input.task.runEpoch !== undefined) {
|
|
1672
|
+
const requestedRunEpoch = orphanGc
|
|
1673
|
+
? assignmentState?.highestRunEpoch
|
|
1674
|
+
: input.task.runEpoch;
|
|
1675
|
+
if (assignmentState === null || requestedRunEpoch === undefined) {
|
|
1676
|
+
return {
|
|
1677
|
+
ok: false,
|
|
1678
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
1679
|
+
message:
|
|
1680
|
+
`task ${input.taskId} teardown does not name its current run`,
|
|
1681
|
+
};
|
|
1682
|
+
}
|
|
1683
|
+
const closed = closeTaskRunAssignment(
|
|
1684
|
+
input.taskId,
|
|
1685
|
+
requestedRunEpoch,
|
|
1686
|
+
);
|
|
1687
|
+
if (closed.kind === "missing" || closed.kind === "mismatch") {
|
|
1688
|
+
return {
|
|
1689
|
+
ok: false,
|
|
1690
|
+
code: HostErrorCode.TaskAlreadyRunning,
|
|
1691
|
+
message:
|
|
1692
|
+
`task ${input.taskId} run ${String(requestedRunEpoch)} is no longer current`,
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
v1AssignmentClosed = true;
|
|
1696
|
+
closedRunEpoch = requestedRunEpoch;
|
|
1697
|
+
}
|
|
1240
1698
|
// Tombstone + close before container destruction. A concurrent cloud
|
|
1241
1699
|
// ensure cannot reopen the channel while teardown is awaiting Docker.
|
|
1242
1700
|
const suspendedTaskGateway = suspendTaskGatewayTask(input.taskId);
|
|
@@ -1269,6 +1727,17 @@ export const hostCommands: HostCommands = {
|
|
|
1269
1727
|
}`,
|
|
1270
1728
|
);
|
|
1271
1729
|
}
|
|
1730
|
+
// ADR-154 §6: the closed run's credentials cleaned up, every path
|
|
1731
|
+
// proven gone (or the runtime destroyed), before the runtime goes; by
|
|
1732
|
+
// the composer as it is now, in the slot.
|
|
1733
|
+
const credentials = activeCloudCredentials();
|
|
1734
|
+
if (credentials && closedRunEpoch !== null) {
|
|
1735
|
+
try {
|
|
1736
|
+
await credentials.closeAssignment({ taskId: input.taskId, runEpoch: closedRunEpoch }, "teardown");
|
|
1737
|
+
} catch (error) {
|
|
1738
|
+
return failFromUnknown(error);
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1272
1741
|
const result = await wrapAgent(ctx, "taskDown", async () => {
|
|
1273
1742
|
if (environment === null) return agent.taskDown(input);
|
|
1274
1743
|
// taskDown is TERMINAL — its only cloud callers are task close and
|
|
@@ -1341,7 +1810,7 @@ export const hostCommands: HostCommands = {
|
|
|
1341
1810
|
} else {
|
|
1342
1811
|
// Destruction failed; let a subsequent ensure restore the still-running
|
|
1343
1812
|
// task instead of leaving it permanently tombstoned.
|
|
1344
|
-
if (!orphanGc) {
|
|
1813
|
+
if (!orphanGc && !v1AssignmentClosed) {
|
|
1345
1814
|
if (suspendedTaskGateway) {
|
|
1346
1815
|
reactivateTaskGatewayTask(suspendedTaskGateway);
|
|
1347
1816
|
}
|
|
@@ -1354,22 +1823,55 @@ export const hostCommands: HostCommands = {
|
|
|
1354
1823
|
|
|
1355
1824
|
async taskStop(ctx, taskId, input) {
|
|
1356
1825
|
logCommand(ctx, "taskStop", taskId, input.reason);
|
|
1357
|
-
|
|
1358
|
-
//
|
|
1359
|
-
//
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1826
|
+
const orchestrator = getOrchestrator();
|
|
1827
|
+
// ADR-154: a manual Stop ends its run's start before it queues behind
|
|
1828
|
+
// it; an idle Stop never does, since a start vetoes it in the slot.
|
|
1829
|
+
const interrupting = activeCloudCredentials();
|
|
1830
|
+
if (interrupting && input.reason !== "idle" && input.runEpoch !== undefined) {
|
|
1831
|
+
interrupting.interruptStarts(taskId, { exact: input.runEpoch });
|
|
1832
|
+
}
|
|
1833
|
+
const result = await orchestrator.stopTask(taskId, {
|
|
1834
|
+
// ADR-154: the run Stop closes, cleaned up by the composer as it is
|
|
1835
|
+
// when Stop gets there, in the slot.
|
|
1836
|
+
closeCredentials: (runEpoch: number) =>
|
|
1837
|
+
activeCloudCredentials()?.closeAssignment({ taskId, runEpoch }, "stop") ?? Promise.resolve(),
|
|
1838
|
+
// Resolve legacy versus v1 only after Stop owns the task lifecycle slot.
|
|
1839
|
+
// Otherwise task-up can admit v1 between an outside read and the stop.
|
|
1840
|
+
admitAssignment: () => {
|
|
1841
|
+
if (input.runEpoch === undefined) {
|
|
1842
|
+
return readTaskRunEpochState(taskId) === null
|
|
1843
|
+
? { kind: "legacy" }
|
|
1844
|
+
: { kind: "stale" };
|
|
1845
|
+
}
|
|
1846
|
+
const closed = closeTaskRunAssignmentForStop(taskId, input.runEpoch);
|
|
1847
|
+
return closed.kind === "closed" || closed.kind === "already_closed"
|
|
1848
|
+
? { kind: "closed", runEpoch: input.runEpoch }
|
|
1849
|
+
: { kind: "stale" };
|
|
1850
|
+
},
|
|
1851
|
+
// ADR-125: idle stops are the cloud's judgment from what it can see;
|
|
1852
|
+
// only the host can see a turn that is mid-flight RIGHT NOW. Evaluate
|
|
1853
|
+
// immediately before assignment closure. Manual pause has no veto.
|
|
1854
|
+
shouldVeto:
|
|
1855
|
+
input.reason === "idle"
|
|
1856
|
+
? () => {
|
|
1857
|
+
const activeAt = orchestrator.lastChannelActivity(taskId);
|
|
1858
|
+
return (
|
|
1859
|
+
activeAt !== null && Date.now() - activeAt < IDLE_STOP_VETO_MS
|
|
1860
|
+
);
|
|
1861
|
+
}
|
|
1862
|
+
: undefined,
|
|
1863
|
+
});
|
|
1864
|
+
if (result.ok && result.vetoed) {
|
|
1865
|
+
return { ok: true, value: { vetoed: "active" as const } };
|
|
1365
1866
|
}
|
|
1366
|
-
const result = await getOrchestrator().stopTask(taskId);
|
|
1367
1867
|
if (!result.ok) {
|
|
1368
1868
|
return {
|
|
1369
1869
|
ok: false,
|
|
1370
|
-
code:
|
|
1870
|
+
code: result.assignmentStale
|
|
1871
|
+
? HostErrorCode.TaskAlreadyRunning
|
|
1872
|
+
: HostErrorCode.Internal,
|
|
1371
1873
|
message: result.error ?? "stop failed",
|
|
1372
|
-
retryable: true,
|
|
1874
|
+
...(result.assignmentStale ? {} : { retryable: true }),
|
|
1373
1875
|
};
|
|
1374
1876
|
}
|
|
1375
1877
|
return { ok: true, value: {} };
|
|
@@ -1635,15 +2137,19 @@ export const hostCommands: HostCommands = {
|
|
|
1635
2137
|
if (runtimeFailure) return runtimeFailure;
|
|
1636
2138
|
try {
|
|
1637
2139
|
const task = getHostTask(input.taskId);
|
|
1638
|
-
//
|
|
1639
|
-
//
|
|
1640
|
-
// its exact persisted provider identity
|
|
1641
|
-
// not permission to run Git against a
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
2140
|
+
// #cb3j0z: Git never runs on the host against a task's repositories;
|
|
2141
|
+
// what a task configured there would run as the runner's user. A
|
|
2142
|
+
// running task must reconstruct its exact persisted provider identity
|
|
2143
|
+
// (malformed/foreign metadata is not permission to run Git against a
|
|
2144
|
+
// different filesystem view) and diffs there; a stopped one diffs in
|
|
2145
|
+
// a throwaway sandbox of its retained worktree.
|
|
2146
|
+
if (task?.statusMirror !== "running") {
|
|
2147
|
+
return ok(
|
|
2148
|
+
await stoppedTaskDiff(input, task ? { environmentProvider: task.environmentProvider ?? null } : null),
|
|
2149
|
+
);
|
|
2150
|
+
}
|
|
2151
|
+
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
2152
|
+
if (environment === null) {
|
|
1647
2153
|
throw new Error(
|
|
1648
2154
|
`running task ${input.taskId} has no persisted task environment`,
|
|
1649
2155
|
);
|
|
@@ -1671,7 +2177,7 @@ export const hostCommands: HostCommands = {
|
|
|
1671
2177
|
);
|
|
1672
2178
|
return ok(undefined);
|
|
1673
2179
|
}
|
|
1674
|
-
writeAttachment(
|
|
2180
|
+
await writeAttachment(
|
|
1675
2181
|
input.taskId,
|
|
1676
2182
|
input.filename,
|
|
1677
2183
|
Buffer.from(input.dataBase64, "base64"),
|
|
@@ -1690,7 +2196,7 @@ export const hostCommands: HostCommands = {
|
|
|
1690
2196
|
? await environment.readWorkspaceFile(
|
|
1691
2197
|
attachmentEnvironmentPath(environment, input.filename),
|
|
1692
2198
|
)
|
|
1693
|
-
: readAttachment(input.taskId, input.filename);
|
|
2199
|
+
: await readAttachment(input.taskId, input.filename);
|
|
1694
2200
|
if (!bytes) {
|
|
1695
2201
|
return {
|
|
1696
2202
|
ok: false,
|
|
@@ -1780,22 +2286,27 @@ function normalizeChannelSpec(input: ChannelEnsureInput): ChannelEnsureInput {
|
|
|
1780
2286
|
* `setup-git` gesture as containers, no compose probing). Best-effort like
|
|
1781
2287
|
* the compose reconciler: absence of a GitHub connection is a quiet no-op.
|
|
1782
2288
|
*/
|
|
2289
|
+
/** ADR-154: only while the runner is on its own credentials. Leaving them
|
|
2290
|
+
* waits for one under way, and the write is fenced on them: none starts
|
|
2291
|
+
* once leaving has begun. */
|
|
1783
2292
|
async function machineTaskGithubAuth(
|
|
1784
2293
|
taskId: string,
|
|
1785
2294
|
userId: string,
|
|
1786
2295
|
): Promise<void> {
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
2296
|
+
await whileOnOwnCredentials(async () => {
|
|
2297
|
+
const environment = await reconstructHostTaskEnvironment(taskId);
|
|
2298
|
+
if (!environment) return;
|
|
2299
|
+
const token = await requestAccessToken(userId);
|
|
2300
|
+
if (!token || !onOwnCredentials()) return;
|
|
2301
|
+
// Machine work is not container work: no docker/apple runtime admission.
|
|
2302
|
+
await injectIntoContainer(
|
|
2303
|
+
taskId,
|
|
2304
|
+
token.accessToken,
|
|
2305
|
+
undefined,
|
|
2306
|
+
() => {},
|
|
2307
|
+
environment,
|
|
2308
|
+
);
|
|
2309
|
+
});
|
|
1799
2310
|
}
|
|
1800
2311
|
|
|
1801
2312
|
async function reconstructHostTaskEnvironment(
|