@runuai/host 0.9.150 → 0.9.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1876 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1830 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +623 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
package/lib/orchestrator.ts
CHANGED
|
@@ -51,6 +51,7 @@ import { isSafeHostTaskId } from "./task-identity";
|
|
|
51
51
|
import { projectEnvForTask } from "./host-env";
|
|
52
52
|
import { shadowRoute } from "./routing/shadow";
|
|
53
53
|
import {
|
|
54
|
+
closeCurrentTaskRunAssignmentForStoppedRuntime,
|
|
54
55
|
getHostTask,
|
|
55
56
|
hostTaskEnvMode,
|
|
56
57
|
hostTaskProjectIds,
|
|
@@ -79,6 +80,7 @@ import {
|
|
|
79
80
|
} from "./container-runtime";
|
|
80
81
|
import {
|
|
81
82
|
reconstructPersistedTaskEnvironment,
|
|
83
|
+
reconstructTaskEnvironment,
|
|
82
84
|
setAppleTaskEnvironmentRecoveryDriver,
|
|
83
85
|
setDockerTaskEnvironmentRecoveryDriver,
|
|
84
86
|
} from "./task-environment";
|
|
@@ -137,6 +139,14 @@ import {
|
|
|
137
139
|
type BrowserSetup,
|
|
138
140
|
} from "./browser-testing";
|
|
139
141
|
import { injectCodexIntoContainer } from "./codex-auth";
|
|
142
|
+
import { runtimeNamespaceOf, type RuntimeNamespace } from "./task-environment/workspace-files";
|
|
143
|
+
import {
|
|
144
|
+
holdOwnCredentials,
|
|
145
|
+
LeftOwnCredentialsError,
|
|
146
|
+
onOwnCredentials,
|
|
147
|
+
onOwnCredentialsEnvironment,
|
|
148
|
+
whileOnOwnCredentials,
|
|
149
|
+
} from "./cloud-credential-host";
|
|
140
150
|
import { isCredentialInvalidSignal, requestCredentialRefresh } from "./token-broker-local";
|
|
141
151
|
import {
|
|
142
152
|
codexHomeIsolatable,
|
|
@@ -147,8 +157,51 @@ import {
|
|
|
147
157
|
pickEngineAccount,
|
|
148
158
|
provisionEngineAccounts,
|
|
149
159
|
resolveEngineAccounts,
|
|
160
|
+
selectAccount,
|
|
150
161
|
type ResolvedEngineAccount,
|
|
151
162
|
} from "./engine-accounts";
|
|
163
|
+
import {
|
|
164
|
+
ENGINE_ACCOUNT_HOMES,
|
|
165
|
+
EngineCredentialWriteRefused,
|
|
166
|
+
engineSpawnEnv,
|
|
167
|
+
engineVersionHeldBack,
|
|
168
|
+
ensureEngineHome,
|
|
169
|
+
escalateEngineLaunchFence,
|
|
170
|
+
isLeasedEngineAccount,
|
|
171
|
+
leasedEnginesFor,
|
|
172
|
+
onLeasedEnginesChanged,
|
|
173
|
+
refreshLeasedEngineAccount,
|
|
174
|
+
setEngineAdmissionFloorRaiser,
|
|
175
|
+
syncEngineHomes,
|
|
176
|
+
waitForEngineGate,
|
|
177
|
+
type LeasedEngineAccount,
|
|
178
|
+
type TaskEngineLease,
|
|
179
|
+
} from "./credential-consumers/engines";
|
|
180
|
+
import {
|
|
181
|
+
admitEngineLaunch,
|
|
182
|
+
anchoredEnvironment,
|
|
183
|
+
dischargeEngineRevocations,
|
|
184
|
+
ENGINE_ACCOUNT_ENV,
|
|
185
|
+
ENGINE_RUN_ENV,
|
|
186
|
+
EngineAnchorLost,
|
|
187
|
+
EngineLaunchRefused,
|
|
188
|
+
newEngineLaunchSlotId,
|
|
189
|
+
raiseEngineAdmissionFloors,
|
|
190
|
+
retireEngineLaunchSlot,
|
|
191
|
+
sameRuntime,
|
|
192
|
+
stopAnchoredEngines,
|
|
193
|
+
type EngineAdmissionFloor,
|
|
194
|
+
type EngineLaunchIdentity,
|
|
195
|
+
} from "./credential-consumers/engine-anchor";
|
|
196
|
+
import { CredentialLeaseUnavailableError } from "./cloud-credential-leases";
|
|
197
|
+
import type { TaskCredentialCleanupRequest } from "./task-credential-cleanup";
|
|
198
|
+
import {
|
|
199
|
+
forgetLeasedEngineRuntimes,
|
|
200
|
+
listCredentialCleanupPaths,
|
|
201
|
+
listLeasedEngineRuntimes,
|
|
202
|
+
rememberLeasedEngineRuntime,
|
|
203
|
+
} from "./credential-cleanup-inventory";
|
|
204
|
+
import { ENGINE_CREDENTIAL_FILES, ENGINE_ENV_CREDENTIAL_KEYS } from "../src/cloud-credentials";
|
|
152
205
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
153
206
|
import {
|
|
154
207
|
reactivateTaskGatewayTask,
|
|
@@ -179,7 +232,11 @@ import {
|
|
|
179
232
|
MAX_SECRETARY_DISPATCH_ID_CHARS,
|
|
180
233
|
SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
|
|
181
234
|
} from "../src/protocol";
|
|
182
|
-
import {
|
|
235
|
+
import {
|
|
236
|
+
isEnvironmentTransportProvider,
|
|
237
|
+
parseTaskEnvironmentLocator,
|
|
238
|
+
serializeTaskEnvironmentLocator,
|
|
239
|
+
} from "./task-environment/types";
|
|
183
240
|
import { forgetTaskEnvSnapshot } from "./task-env-source";
|
|
184
241
|
|
|
185
242
|
export type HostEventSubscriber = (event: HostEvent) => void;
|
|
@@ -385,6 +442,41 @@ interface Channel {
|
|
|
385
442
|
/** ADR-076: per-agent account-rotation budget (reset on a successful turn) —
|
|
386
443
|
* bounds a rate-limit storm across accounts, mirroring the respawn budget. */
|
|
387
444
|
rotations: Map<string, number>;
|
|
445
|
+
/** ADR-154: the task's AI accounts come from its lease. Sticky for the
|
|
446
|
+
* channel: once on cloud credentials, never the runner's own accounts. */
|
|
447
|
+
leased: boolean;
|
|
448
|
+
/** ADR-154 §8: rewrites of the task's account homes after a change, one
|
|
449
|
+
* after another. */
|
|
450
|
+
leasedSync: Promise<void> | null;
|
|
451
|
+
/** ADR-154 §6: agents a cleanup stopped, started again on their next
|
|
452
|
+
* account once the task's gate opens. */
|
|
453
|
+
leasedRespawn: Set<string>;
|
|
454
|
+
/** ADR-154 §6: the exact lease material each agent's session was picked
|
|
455
|
+
* on, so a spawn starts only on what the task still holds and a cleanup
|
|
456
|
+
* of an older run never stops a newer run's agent. */
|
|
457
|
+
leasedIdentityByAgent: Map<string, LeasedIdentity>;
|
|
458
|
+
/** ADR-154 §5: agents down for want of a usable account; the gate's next
|
|
459
|
+
* opening starts them again. */
|
|
460
|
+
leasedBlocked: Set<string>;
|
|
461
|
+
/** ADR-154 §8: a failed rewrite of the account homes, retried. */
|
|
462
|
+
leasedSyncRetry: ReturnType<typeof setTimeout> | null;
|
|
463
|
+
/** ADR-154 §5, §6: agents wait to be started again (stopped by a cleanup,
|
|
464
|
+
* down for want of an account, or their spawn's material moved); kept
|
|
465
|
+
* until a start pass actually ran, so a failed pass does not lose it. */
|
|
466
|
+
leasedWakePending: boolean;
|
|
467
|
+
/** ADR-154 §6: engines being created on a leased account (null: on an
|
|
468
|
+
* agent's task token alone, the Managed Engine) right now, each until its
|
|
469
|
+
* create settles. Every cleanup attempt on the account (on every one, for
|
|
470
|
+
* a whole run's) waits for them before it proves the engines gone,
|
|
471
|
+
* whatever has become of the agent's binding meanwhile. */
|
|
472
|
+
leasedCreating: Set<{
|
|
473
|
+
agentId: string;
|
|
474
|
+
accountId: string | null;
|
|
475
|
+
runEpoch: number;
|
|
476
|
+
settled: Promise<void>;
|
|
477
|
+
}>;
|
|
478
|
+
/** ADR-154 §6: each agent's current spawn of a leased engine. */
|
|
479
|
+
leasedLaunches: Map<string, LeasedLaunchSlot>;
|
|
388
480
|
}
|
|
389
481
|
|
|
390
482
|
interface AcceptedPrompt {
|
|
@@ -405,6 +497,119 @@ interface RecoveryFence {
|
|
|
405
497
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
406
498
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
407
499
|
const MAX_ROTATIONS_PER_AGENT = 6;
|
|
500
|
+
/** ADR-154 §5: how long a spawn waits for the task's leased AI accounts
|
|
501
|
+
* before the agent fails with "credentials unavailable". */
|
|
502
|
+
const LEASED_ACCOUNT_WAIT_MS = 60_000;
|
|
503
|
+
/** ADR-154 §6: how long the cleanup's quiesce step waits for a spawn
|
|
504
|
+
* already creating an engine on a removed account. */
|
|
505
|
+
const LEASED_QUIESCE_EXIT_MS = 20_000;
|
|
506
|
+
/** ADR-141: the shadow router reads the tail of the task's chat (it keeps
|
|
507
|
+
* 1,200 characters) and its classifier's answer, never more. */
|
|
508
|
+
const SHADOW_CONTEXT_BYTES = 8 * 1024;
|
|
509
|
+
const SHADOW_OUTPUT_BYTES = 64 * 1024;
|
|
510
|
+
/** ADR-154 §6: the markers every process of a leased engine inherits: its
|
|
511
|
+
* run and its account. A launch carrying them is an engine launch, started
|
|
512
|
+
* only under the engine supervisor. */
|
|
513
|
+
const LEASED_RUN_ENV = ENGINE_RUN_ENV;
|
|
514
|
+
const LEASED_ACCOUNT_ENV = ENGINE_ACCOUNT_ENV;
|
|
515
|
+
/**
|
|
516
|
+
* ADR-154 §6: every variable an env-token account may set, blank. A leased
|
|
517
|
+
* spawn carries these under its account's own values: an explicit blank
|
|
518
|
+
* wins over an adapter's inheritance of the runner's variable (Claude's
|
|
519
|
+
* passEnv, Grok's XAI_API_KEY, Cursor's fallback) and over the container's
|
|
520
|
+
* env, so no engine ever runs on the runner's own credential beside, or
|
|
521
|
+
* instead of, the task's.
|
|
522
|
+
*/
|
|
523
|
+
const LEASED_CREDENTIAL_BLANKS: Readonly<Record<string, string>> = Object.freeze(
|
|
524
|
+
Object.fromEntries(
|
|
525
|
+
[...new Set(Object.values(ENGINE_ENV_CREDENTIAL_KEYS).flat())].map((name) => [name, ""]),
|
|
526
|
+
),
|
|
527
|
+
);
|
|
528
|
+
/** ADR-128: the Managed Engine. It needs no AI account (it runs on Uai's
|
|
529
|
+
* gateway), but its agent's task token is its bearer: on a leased task its
|
|
530
|
+
* launches are admitted as that token (ADR-154 §6). */
|
|
531
|
+
const MANAGED_ENGINE_KIND = "uai";
|
|
532
|
+
/** ADR-154 §10: kinds whose agents carry no credential at all; every other
|
|
533
|
+
* kind outside the leased lanes (Kimi) stays down on a leased task until
|
|
534
|
+
* its lane exists. */
|
|
535
|
+
const CREDENTIAL_FREE_KINDS: ReadonlySet<string> = new Set(["mock"]);
|
|
536
|
+
/** ADR-154 §8: how soon a failed rewrite of a task's account homes is tried
|
|
537
|
+
* again. */
|
|
538
|
+
const LEASED_SYNC_RETRY_MS = 30_000;
|
|
539
|
+
|
|
540
|
+
/** ADR-154 §6: the exact lease material a spawn starts on: an AI account's,
|
|
541
|
+
* or, for the Managed Engine, the task's open run alone (its agent's task
|
|
542
|
+
* token is its credential). */
|
|
543
|
+
type LeasedIdentity =
|
|
544
|
+
| { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number }
|
|
545
|
+
| { subject: "task-token"; runEpoch: number };
|
|
546
|
+
|
|
547
|
+
type SpawnArgs = Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">;
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* ADR-154 §6: one spawn of an agent's leased engine. Its launches are
|
|
551
|
+
* admitted only while it is the agent's current spawn and, once its create
|
|
552
|
+
* settled, only while the session it bound is the agent's live one: a spawn
|
|
553
|
+
* replaced (on the same material too), retired, removed, never bound, or
|
|
554
|
+
* whose session reported its end, launches nothing more, and once retired,
|
|
555
|
+
* nothing it was admitted for starts in its runtime (`retireLeasedLaunch`).
|
|
556
|
+
*/
|
|
557
|
+
interface LeasedLaunchSlot {
|
|
558
|
+
created: boolean;
|
|
559
|
+
session: AgentSession | null;
|
|
560
|
+
/** What its launches are admitted as, its slot's id included, and where. */
|
|
561
|
+
readonly identity: EngineLaunchIdentity & { slotId: string };
|
|
562
|
+
readonly environment: TaskEnvironmentHandle<unknown>;
|
|
563
|
+
/** It began writing an admission: its retirement reaches its runtime. */
|
|
564
|
+
admitted: boolean;
|
|
565
|
+
retired: boolean;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** ADR-154 §6: a retired slot not yet retired in its runtime. */
|
|
569
|
+
interface RetiredLaunchSlot {
|
|
570
|
+
readonly agentId: string;
|
|
571
|
+
readonly slot: LeasedLaunchSlot;
|
|
572
|
+
/** The attempt under way, if one is. */
|
|
573
|
+
attempt: Promise<void> | null;
|
|
574
|
+
/** A cleanup of its run that covers it proves what it fences: a failure
|
|
575
|
+
* after is not escalated. */
|
|
576
|
+
covered: boolean;
|
|
577
|
+
/** Its run's full cleanup was asked for once an attempt failed. */
|
|
578
|
+
escalated: boolean;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function newLeasedLaunchSlot(
|
|
582
|
+
taskId: string,
|
|
583
|
+
identity: LeasedIdentity,
|
|
584
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
585
|
+
): LeasedLaunchSlot {
|
|
586
|
+
return {
|
|
587
|
+
created: false,
|
|
588
|
+
session: null,
|
|
589
|
+
identity: { taskId, slotId: newEngineLaunchSlotId(), ...identity },
|
|
590
|
+
environment,
|
|
591
|
+
admitted: false,
|
|
592
|
+
retired: false,
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function sameLeasedIdentity(a: LeasedIdentity, b: LeasedIdentity): boolean {
|
|
597
|
+
if (a.subject === "account" && b.subject === "account") {
|
|
598
|
+
return (
|
|
599
|
+
a.runEpoch === b.runEpoch &&
|
|
600
|
+
a.credentialId === b.credentialId &&
|
|
601
|
+
a.revision === b.revision &&
|
|
602
|
+
a.version === b.version
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
return a.subject === b.subject && a.runEpoch === b.runEpoch;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/** The engine kinds whose agents run on an AI account from the lease. */
|
|
609
|
+
const LEASED_ENGINE_KINDS: ReadonlySet<string> = new Set([
|
|
610
|
+
...Object.keys(ENGINE_ENV_CREDENTIAL_KEYS),
|
|
611
|
+
...Object.keys(ENGINE_CREDENTIAL_FILES),
|
|
612
|
+
]);
|
|
408
613
|
/** A burned respawn budget resets after this quiet period (see reconcile). */
|
|
409
614
|
const RESPAWN_COOLDOWN_MS = 10 * 60_000;
|
|
410
615
|
/** ADR-053: gap between browser-install retries. Ensures fire every couple of
|
|
@@ -504,6 +709,20 @@ async function sshIdentityDepsFor(task: {
|
|
|
504
709
|
return { environment: () => environment };
|
|
505
710
|
}
|
|
506
711
|
|
|
712
|
+
export type TaskStopAssignmentAdmission =
|
|
713
|
+
| { kind: "legacy" }
|
|
714
|
+
| { kind: "closed"; runEpoch: number }
|
|
715
|
+
| { kind: "stale" };
|
|
716
|
+
|
|
717
|
+
export interface OrchestratorTaskStopResult {
|
|
718
|
+
ok: boolean;
|
|
719
|
+
error?: string;
|
|
720
|
+
assignmentStale?: boolean;
|
|
721
|
+
vetoed?: boolean;
|
|
722
|
+
/** Exact v1 run stopped under the lifecycle lock. Absent for legacy. */
|
|
723
|
+
stoppedRunEpoch?: number;
|
|
724
|
+
}
|
|
725
|
+
|
|
507
726
|
export class Orchestrator {
|
|
508
727
|
private readonly channels = new Map<string, Channel>();
|
|
509
728
|
private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
|
|
@@ -517,8 +736,15 @@ export class Orchestrator {
|
|
|
517
736
|
* later command cannot overtake an earlier one after channel close. */
|
|
518
737
|
private readonly taskLifecycleTails = new Map<string, Promise<void>>();
|
|
519
738
|
private readonly hostSubscribers = new Set<HostEventSubscriber>();
|
|
739
|
+
/** ADR-154 §6: launch slots retired whose retirement has not yet reached
|
|
740
|
+
* their runtime, per task, whatever channel they were in: an agent's next
|
|
741
|
+
* spawn launches there only once they have (`retireLeasedLaunch`). */
|
|
742
|
+
private readonly leasedRetired = new Map<string, Set<RetiredLaunchSlot>>();
|
|
520
743
|
|
|
521
|
-
constructor(private readonly factory: AgentSessionFactory) {
|
|
744
|
+
constructor(private readonly factory: AgentSessionFactory) {
|
|
745
|
+
onLeasedEnginesChanged((taskId) => this.leasedEnginesChanged(taskId));
|
|
746
|
+
setEngineAdmissionFloorRaiser((assignment, floors) => this.raiseLeasedAdmissionFloors(assignment, floors));
|
|
747
|
+
}
|
|
522
748
|
|
|
523
749
|
// -- subscriptions --------------------------------------------------------
|
|
524
750
|
|
|
@@ -645,7 +871,7 @@ export class Orchestrator {
|
|
|
645
871
|
for (const agent of added) {
|
|
646
872
|
// Per-agent link/document skills, like getOrCreateChannel does at
|
|
647
873
|
// channel birth. Best-effort.
|
|
648
|
-
writeAgentSkills(channel.taskId, agent);
|
|
874
|
+
writeAgentSkills(channel.taskId, agent, () => this.workspaceRuntime(channel.taskId));
|
|
649
875
|
}
|
|
650
876
|
for (const { agent, reason } of changed) {
|
|
651
877
|
const recoveryAbortPublished = this.supersedeRecovery(
|
|
@@ -665,6 +891,9 @@ export class Orchestrator {
|
|
|
665
891
|
);
|
|
666
892
|
}
|
|
667
893
|
channel.accountByAgent.delete(agent.id);
|
|
894
|
+
// ADR-154 §6: the retired spawn launches nothing more on its material.
|
|
895
|
+
channel.leasedIdentityByAgent.delete(agent.id);
|
|
896
|
+
void this.retireLeasedLaunch(channel, agent.id, channel.leasedLaunches.get(agent.id));
|
|
668
897
|
channel.rotations.delete(agent.id);
|
|
669
898
|
// Crash/recovery budgets belong to the retired launch generation. A
|
|
670
899
|
// newly selected engine/model/effort must get an immediate clean start,
|
|
@@ -814,6 +1043,9 @@ export class Orchestrator {
|
|
|
814
1043
|
channel.respawnLastAt.delete(agentId);
|
|
815
1044
|
channel.everBound.delete(agentId);
|
|
816
1045
|
channel.accountByAgent.delete(agentId);
|
|
1046
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
1047
|
+
// ADR-154 §6: its spawn admits nothing from here (see below for the end).
|
|
1048
|
+
void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
|
|
817
1049
|
channel.lastPrompt.delete(agentId);
|
|
818
1050
|
channel.rotations.delete(agentId);
|
|
819
1051
|
|
|
@@ -831,6 +1063,10 @@ export class Orchestrator {
|
|
|
831
1063
|
// factory await and closes the stale process instead of binding it.
|
|
832
1064
|
await inFlightCreate.catch(() => null);
|
|
833
1065
|
}
|
|
1066
|
+
// The removal ends only once its spawns are retired in their runtime, or
|
|
1067
|
+
// their run's cleanup is asked for (retireInRuntime): the current one, and
|
|
1068
|
+
// any whose retirement an earlier event, or this removal's drain, began.
|
|
1069
|
+
await Promise.all(this.leasedRetirementsUnderWay(channel.taskId, agentId));
|
|
834
1070
|
if (!this.isActiveChannel(channel)) return;
|
|
835
1071
|
// A newer ensure can re-add the same id while the old close/create drains.
|
|
836
1072
|
// That spec owns the status now; do not stop its bound generation or emit
|
|
@@ -972,9 +1208,6 @@ export class Orchestrator {
|
|
|
972
1208
|
taskProxiedKeys(taskId),
|
|
973
1209
|
),
|
|
974
1210
|
);
|
|
975
|
-
// ADR-046: materialise this agent's skills to its per-agent SKILL.md in
|
|
976
|
-
// the workspace (bind-mounted into the container). Best-effort.
|
|
977
|
-
writeAgentSkills(taskId, agent);
|
|
978
1211
|
// ADR-022: any agent with a non-empty initialPrompt opens a first
|
|
979
1212
|
// turn at container-ready. The rest stay silent until addressed.
|
|
980
1213
|
const firstTurn = assembleFirstTurnPrompt(
|
|
@@ -1048,8 +1281,24 @@ export class Orchestrator {
|
|
|
1048
1281
|
accountByAgent: new Map(),
|
|
1049
1282
|
lastPrompt: new Map(),
|
|
1050
1283
|
rotations: new Map(),
|
|
1284
|
+
leased: false,
|
|
1285
|
+
leasedSync: null,
|
|
1286
|
+
leasedRespawn: new Set(),
|
|
1287
|
+
leasedIdentityByAgent: new Map(),
|
|
1288
|
+
leasedBlocked: new Set(),
|
|
1289
|
+
leasedSyncRetry: null,
|
|
1290
|
+
leasedWakePending: false,
|
|
1291
|
+
leasedCreating: new Set(),
|
|
1292
|
+
leasedLaunches: new Map(),
|
|
1051
1293
|
};
|
|
1052
1294
|
this.channels.set(taskId, channel);
|
|
1295
|
+
// ADR-046: materialise each agent's skills to its per-agent SKILL.md in
|
|
1296
|
+
// the workspace (bind-mounted into the container), best-effort, once the
|
|
1297
|
+
// channel can give the task's runtime: where only it can make the
|
|
1298
|
+
// agent's directory (macOS, #cb3j0z), it is asked for then.
|
|
1299
|
+
for (const agent of roster) {
|
|
1300
|
+
writeAgentSkills(taskId, agent, () => this.workspaceRuntime(taskId));
|
|
1301
|
+
}
|
|
1053
1302
|
return channel;
|
|
1054
1303
|
}
|
|
1055
1304
|
|
|
@@ -1255,22 +1504,90 @@ export class Orchestrator {
|
|
|
1255
1504
|
* copies use this gate: a false→true toggle queues migration after the older
|
|
1256
1505
|
* copy, so that copy cannot land last and erase the managed browser block.
|
|
1257
1506
|
*/
|
|
1258
|
-
private codexHomes(hasCodex: boolean,
|
|
1507
|
+
private codexHomes(hasCodex: boolean, channel: Channel): string[] {
|
|
1259
1508
|
if (!hasCodex) return [];
|
|
1260
|
-
|
|
1261
|
-
.map((account) => account.configDir?.containerDir)
|
|
1262
|
-
.filter((home): home is string => Boolean(home));
|
|
1263
|
-
return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
|
|
1509
|
+
return [...new Set([DEFAULT_CODEX_HOME, ...this.accountHomes(channel, "codex")])];
|
|
1264
1510
|
}
|
|
1265
1511
|
|
|
1266
|
-
/** Every GROK_HOME a task
|
|
1267
|
-
*
|
|
1268
|
-
private grokHomes(hasGrok: boolean,
|
|
1512
|
+
/** Every GROK_HOME a task may run under: the default plus each sign-in
|
|
1513
|
+
* account's isolated home (API-key accounts use the default). */
|
|
1514
|
+
private grokHomes(hasGrok: boolean, channel: Channel): string[] {
|
|
1269
1515
|
if (!hasGrok) return [];
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1516
|
+
return [...new Set([DEFAULT_GROK_HOME, ...this.accountHomes(channel, "grok")])];
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
/** The homes `kind`'s config-directory accounts live in: the task's leased
|
|
1520
|
+
* ones, or the runner's own for a task not on cloud credentials. A leased
|
|
1521
|
+
* Codex account's home is listed with every Codex agent's own home on it,
|
|
1522
|
+
* so the task's managed config (browser, MCP) reaches the homes the
|
|
1523
|
+
* agents run in, now and whenever it changes. */
|
|
1524
|
+
private accountHomes(channel: Channel, kind: string): string[] {
|
|
1525
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1526
|
+
return resolveEngineAccounts(kind, {}, channel.engineAccountScope)
|
|
1527
|
+
.map((account) => account.configDir?.containerDir)
|
|
1528
|
+
.filter((home): home is string => Boolean(home));
|
|
1529
|
+
}
|
|
1530
|
+
const homes = (leasedEnginesFor(channel.taskId)?.accounts ?? [])
|
|
1531
|
+
.filter((account) => account.engineKind === kind && account.home !== null)
|
|
1532
|
+
.map((account) => account.home!);
|
|
1533
|
+
if (kind !== "codex") return homes;
|
|
1534
|
+
const agents = channel.roster.filter((agent) => agent.kind === "codex");
|
|
1535
|
+
return homes.flatMap((home) => [home, ...agents.map((agent) => codexPerAgentHome(home, agent.id))]);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
/** ADR-154: whether this channel's AI accounts come from the task's lease
|
|
1539
|
+
* (the engine consumer holds a record for the task, or the runner is no
|
|
1540
|
+
* longer on its own credentials at all: no default slot, so an agent
|
|
1541
|
+
* reattached before its grant waits for it). Sticky: a channel that was
|
|
1542
|
+
* on cloud credentials never falls back to the runner's own accounts,
|
|
1543
|
+
* not even after the record is gone. */
|
|
1544
|
+
private isLeasedChannel(channel: Channel): boolean {
|
|
1545
|
+
if (!channel.leased && (leasedEnginesFor(channel.taskId) !== undefined || !onOwnCredentials())) {
|
|
1546
|
+
channel.leased = true;
|
|
1547
|
+
}
|
|
1548
|
+
return channel.leased;
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/**
|
|
1552
|
+
* The accounts `kind` may run on, in try order: the task's leased ones
|
|
1553
|
+
* that are readable now (none while its gate is closed), or the runner's
|
|
1554
|
+
* own for a task not on cloud credentials. A leased account carries no env
|
|
1555
|
+
* and no directory here: the spawn boundary applies it (leasedSpawnArgs),
|
|
1556
|
+
* and no runner-side copy ever names it.
|
|
1557
|
+
*/
|
|
1558
|
+
private engineAccounts(channel: Channel, kind: string): ResolvedEngineAccount[] {
|
|
1559
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1560
|
+
return resolveEngineAccounts(kind, {}, channel.engineAccountScope);
|
|
1561
|
+
}
|
|
1562
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
1563
|
+
if (!lease?.open) return [];
|
|
1564
|
+
return lease.accounts
|
|
1565
|
+
.filter(
|
|
1566
|
+
(account): account is LeasedEngineAccount =>
|
|
1567
|
+
account.engineKind === kind && isLeasedEngineAccount(account),
|
|
1568
|
+
)
|
|
1569
|
+
.map((account) => ({
|
|
1570
|
+
id: account.credentialId,
|
|
1571
|
+
kind,
|
|
1572
|
+
label: account.label,
|
|
1573
|
+
isDefault: false,
|
|
1574
|
+
scope: account.scope,
|
|
1575
|
+
authKind: account.authKind,
|
|
1576
|
+
execEnv: {},
|
|
1577
|
+
}));
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/** ADR-076 selection over `engineAccounts`: the first healthy account in
|
|
1581
|
+
* try order, skipping `exclude`. */
|
|
1582
|
+
private pickAccount(
|
|
1583
|
+
channel: Channel,
|
|
1584
|
+
kind: string,
|
|
1585
|
+
exclude?: Set<string>,
|
|
1586
|
+
): ResolvedEngineAccount | null {
|
|
1587
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1588
|
+
return pickEngineAccount(kind, { scope: channel.engineAccountScope, exclude });
|
|
1589
|
+
}
|
|
1590
|
+
return selectAccount(this.engineAccounts(channel, kind), { exclude });
|
|
1274
1591
|
}
|
|
1275
1592
|
|
|
1276
1593
|
private noteProvisionMutation(
|
|
@@ -1299,9 +1616,12 @@ export class Orchestrator {
|
|
|
1299
1616
|
if (this.isActiveChannel(channel)) channel.configGeneration += 1;
|
|
1300
1617
|
}
|
|
1301
1618
|
|
|
1302
|
-
private provisionFingerprint(kinds: readonly string[],
|
|
1619
|
+
private provisionFingerprint(kinds: readonly string[], channel: Channel): string {
|
|
1620
|
+
// ADR-154: a leased task's accounts are never copied from the runner;
|
|
1621
|
+
// the spawn boundary writes each one's files through the engine consumer.
|
|
1622
|
+
if (this.isLeasedChannel(channel)) return "leased";
|
|
1303
1623
|
const accounts = kinds.flatMap((kind) =>
|
|
1304
|
-
resolveEngineAccounts(kind, {},
|
|
1624
|
+
resolveEngineAccounts(kind, {}, channel.engineAccountScope)
|
|
1305
1625
|
.filter((account) => !account.isDefault && account.configDir)
|
|
1306
1626
|
.map((account) => ({
|
|
1307
1627
|
kind,
|
|
@@ -1374,13 +1694,13 @@ export class Orchestrator {
|
|
|
1374
1694
|
const requestedHasCodex =
|
|
1375
1695
|
hasCodex || channel.roster.some((agent) => agent.kind === "codex");
|
|
1376
1696
|
const requestedHomes = requestedBrowser
|
|
1377
|
-
? this.codexHomes(requestedHasCodex, channel
|
|
1697
|
+
? this.codexHomes(requestedHasCodex, channel)
|
|
1378
1698
|
: [];
|
|
1379
1699
|
const requestedKinds = [
|
|
1380
1700
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1381
1701
|
];
|
|
1382
1702
|
const requestedProvisionFingerprint =
|
|
1383
|
-
this.provisionFingerprint(requestedKinds, channel
|
|
1703
|
+
this.provisionFingerprint(requestedKinds, channel);
|
|
1384
1704
|
const changedProvisionKinds = this.changedProvisionKinds(
|
|
1385
1705
|
channel.provisionAttemptedFingerprint,
|
|
1386
1706
|
requestedProvisionFingerprint,
|
|
@@ -1389,11 +1709,14 @@ export class Orchestrator {
|
|
|
1389
1709
|
const provisionKinds = forceProvision
|
|
1390
1710
|
? requestedKinds
|
|
1391
1711
|
: changedProvisionKinds;
|
|
1392
|
-
const
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
)
|
|
1396
|
-
|
|
1712
|
+
const leased = this.isLeasedChannel(channel);
|
|
1713
|
+
const mutableProvisionKinds = leased
|
|
1714
|
+
? []
|
|
1715
|
+
: provisionKinds.filter((kind) =>
|
|
1716
|
+
resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
|
|
1717
|
+
(account) => !account.isDefault && Boolean(account.configDir),
|
|
1718
|
+
),
|
|
1719
|
+
);
|
|
1397
1720
|
const mayProvisionConfig = mutableProvisionKinds.length > 0;
|
|
1398
1721
|
|
|
1399
1722
|
const inFlight = channel.browserSetup;
|
|
@@ -1410,13 +1733,13 @@ export class Orchestrator {
|
|
|
1410
1733
|
requestedHasCodex ||
|
|
1411
1734
|
channel.roster.some((agent) => agent.kind === "codex");
|
|
1412
1735
|
const latestHomes = latestBrowser
|
|
1413
|
-
? this.codexHomes(latestHasCodex, channel
|
|
1736
|
+
? this.codexHomes(latestHasCodex, channel)
|
|
1414
1737
|
: [];
|
|
1415
1738
|
const latestKinds = [
|
|
1416
1739
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1417
1740
|
];
|
|
1418
1741
|
const latestProvisionFingerprint =
|
|
1419
|
-
this.provisionFingerprint(latestKinds, channel
|
|
1742
|
+
this.provisionFingerprint(latestKinds, channel);
|
|
1420
1743
|
if (
|
|
1421
1744
|
(latestBrowser &&
|
|
1422
1745
|
coverage?.browserAttempted !== true &&
|
|
@@ -1452,7 +1775,12 @@ export class Orchestrator {
|
|
|
1452
1775
|
forceProvision ||
|
|
1453
1776
|
channel.provisionAttemptedFingerprint !==
|
|
1454
1777
|
requestedProvisionFingerprint;
|
|
1455
|
-
if (shouldProvision) {
|
|
1778
|
+
if (shouldProvision && leased) {
|
|
1779
|
+
// ADR-154: nothing is copied from the runner for a leased task.
|
|
1780
|
+
coverage.provisionAttempted = true;
|
|
1781
|
+
channel.provisionAttemptedFingerprint = requestedProvisionFingerprint;
|
|
1782
|
+
channel.provisionRetryFingerprint = undefined;
|
|
1783
|
+
} else if (shouldProvision) {
|
|
1456
1784
|
coverage.provisionAttempted = true;
|
|
1457
1785
|
if (mayProvisionConfig) {
|
|
1458
1786
|
// Fence a factory that started before the copy without declaring a
|
|
@@ -1462,21 +1790,27 @@ export class Orchestrator {
|
|
|
1462
1790
|
}
|
|
1463
1791
|
let copiedOrUnknown = false;
|
|
1464
1792
|
try {
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1793
|
+
// ADR-154: the runner's own accounts, copied only while it is on
|
|
1794
|
+
// its own credentials, counted (leaving waits for the copy);
|
|
1795
|
+
// once leaving has begun, nothing is copied.
|
|
1796
|
+
const provisioning = whileOnOwnCredentials(() =>
|
|
1797
|
+
channel.agentHandle
|
|
1798
|
+
? provisionEngineAccounts(
|
|
1799
|
+
channel.containerName,
|
|
1800
|
+
provisionKinds,
|
|
1801
|
+
{},
|
|
1802
|
+
channel.agentHandle,
|
|
1803
|
+
channel.engineAccountScope,
|
|
1804
|
+
)
|
|
1805
|
+
: provisionEngineAccounts(
|
|
1806
|
+
channel.containerName,
|
|
1807
|
+
provisionKinds,
|
|
1808
|
+
{},
|
|
1809
|
+
undefined,
|
|
1810
|
+
channel.engineAccountScope,
|
|
1811
|
+
),
|
|
1812
|
+
);
|
|
1813
|
+
const provisioned = (await provisioning) ?? { copied: 0, failed: 0, mayHaveMutated: false };
|
|
1480
1814
|
copiedOrUnknown =
|
|
1481
1815
|
provisioned.copied > 0 || provisioned.mayHaveMutated;
|
|
1482
1816
|
if (provisioned.failed > 0) {
|
|
@@ -1537,7 +1871,7 @@ export class Orchestrator {
|
|
|
1537
1871
|
requestedHomes,
|
|
1538
1872
|
channel.agentHandle,
|
|
1539
1873
|
requestedKinds,
|
|
1540
|
-
this.grokHomes(requestedKinds.includes("grok"), channel
|
|
1874
|
+
this.grokHomes(requestedKinds.includes("grok"), channel),
|
|
1541
1875
|
)
|
|
1542
1876
|
: await setupBrowserTesting(
|
|
1543
1877
|
channel.taskId,
|
|
@@ -1546,7 +1880,7 @@ export class Orchestrator {
|
|
|
1546
1880
|
requestedHomes,
|
|
1547
1881
|
undefined,
|
|
1548
1882
|
requestedKinds,
|
|
1549
|
-
this.grokHomes(requestedKinds.includes("grok"), channel
|
|
1883
|
+
this.grokHomes(requestedKinds.includes("grok"), channel),
|
|
1550
1884
|
);
|
|
1551
1885
|
} catch (err) {
|
|
1552
1886
|
console.warn(
|
|
@@ -1617,15 +1951,12 @@ export class Orchestrator {
|
|
|
1617
1951
|
requestedHasCodex ||
|
|
1618
1952
|
channel.roster.some((agent) => agent.kind === "codex");
|
|
1619
1953
|
const latestHomes = latestBrowser
|
|
1620
|
-
? this.codexHomes(latestHasCodex, channel
|
|
1954
|
+
? this.codexHomes(latestHasCodex, channel)
|
|
1621
1955
|
: [];
|
|
1622
1956
|
const latestKinds = [
|
|
1623
1957
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1624
1958
|
];
|
|
1625
|
-
const latestProvisionFingerprint = this.provisionFingerprint(
|
|
1626
|
-
latestKinds,
|
|
1627
|
-
channel.engineAccountScope,
|
|
1628
|
-
);
|
|
1959
|
+
const latestProvisionFingerprint = this.provisionFingerprint(latestKinds, channel);
|
|
1629
1960
|
if (
|
|
1630
1961
|
(latestBrowser &&
|
|
1631
1962
|
!coverage.browserAttempted &&
|
|
@@ -1714,12 +2045,9 @@ export class Orchestrator {
|
|
|
1714
2045
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1715
2046
|
].sort();
|
|
1716
2047
|
const codexHomes = engineKinds.includes("codex")
|
|
1717
|
-
? this.codexHomes(true, channel
|
|
2048
|
+
? this.codexHomes(true, channel)
|
|
1718
2049
|
: [];
|
|
1719
|
-
const grokHomes = this.grokHomes(
|
|
1720
|
-
engineKinds.includes("grok"),
|
|
1721
|
-
channel.engineAccountScope,
|
|
1722
|
-
);
|
|
2050
|
+
const grokHomes = this.grokHomes(engineKinds.includes("grok"), channel);
|
|
1723
2051
|
const fingerprint = JSON.stringify({
|
|
1724
2052
|
connections,
|
|
1725
2053
|
engineKinds,
|
|
@@ -2053,6 +2381,9 @@ export class Orchestrator {
|
|
|
2053
2381
|
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
2054
2382
|
return false;
|
|
2055
2383
|
}
|
|
2384
|
+
// ADR-154 §5: down for want of a usable account until the gate next
|
|
2385
|
+
// opens (which starts it again), not retried on every pass.
|
|
2386
|
+
if (channel.leasedBlocked.has(agent.id)) return false;
|
|
2056
2387
|
// Crash-loop budget: an agent whose session keeps dying stops being
|
|
2057
2388
|
// respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
|
|
2058
2389
|
// but a cold budget resets so the task can heal after the cause clears.
|
|
@@ -2162,7 +2493,12 @@ export class Orchestrator {
|
|
|
2162
2493
|
),
|
|
2163
2494
|
},
|
|
2164
2495
|
);
|
|
2165
|
-
if (!session)
|
|
2496
|
+
if (!session) {
|
|
2497
|
+
// Credentials unavailable is this agent's alone: its siblings
|
|
2498
|
+
// still start.
|
|
2499
|
+
if (channel.leasedBlocked.has(agent.id)) continue;
|
|
2500
|
+
return;
|
|
2501
|
+
}
|
|
2166
2502
|
this.startNextPendingPrompt(channel, agent.id);
|
|
2167
2503
|
}
|
|
2168
2504
|
} finally {
|
|
@@ -2257,6 +2593,9 @@ export class Orchestrator {
|
|
|
2257
2593
|
channel.accountByAgent.delete(agent.id);
|
|
2258
2594
|
return { ...floor, ...base };
|
|
2259
2595
|
}
|
|
2596
|
+
// ADR-154: a leased task's account is picked and applied at the spawn
|
|
2597
|
+
// boundary (leasedSpawnArgs), which can wait for the task's gate.
|
|
2598
|
+
if (this.isLeasedChannel(channel)) return { ...floor, ...base };
|
|
2260
2599
|
const account = pickEngineAccount(agent.kind, {
|
|
2261
2600
|
scope: channel.engineAccountScope,
|
|
2262
2601
|
});
|
|
@@ -2345,6 +2684,11 @@ export class Orchestrator {
|
|
|
2345
2684
|
) {
|
|
2346
2685
|
return;
|
|
2347
2686
|
}
|
|
2687
|
+
// ADR-154 §6: an engine that ended launches nothing more from now,
|
|
2688
|
+
// whatever its terminal handling still awaits (the runtime's proof).
|
|
2689
|
+
if (event.type === "error" || event.type === "exit") {
|
|
2690
|
+
this.endLeasedLaunches(channel, agentId, session);
|
|
2691
|
+
}
|
|
2348
2692
|
const operation = eventTail
|
|
2349
2693
|
.catch(() => {})
|
|
2350
2694
|
.then(async () => {
|
|
@@ -2411,6 +2755,835 @@ export class Orchestrator {
|
|
|
2411
2755
|
return operation;
|
|
2412
2756
|
}
|
|
2413
2757
|
|
|
2758
|
+
/**
|
|
2759
|
+
* ADR-154 §5, §6, §10: what a spawn on the task's leased AI account takes,
|
|
2760
|
+
* and the exact lease material it starts on. The agent runs on the account
|
|
2761
|
+
* it is bound to while that account is still served readable, else on the
|
|
2762
|
+
* first healthy one in try order (the owner's accounts, then the
|
|
2763
|
+
* workspace's). An env-token account's values ride the spawn, synchronously
|
|
2764
|
+
* when nothing has to be waited for. A config-directory account's home gets
|
|
2765
|
+
* the task's managed config first (browser, MCP), then its files, through
|
|
2766
|
+
* the engine consumer, in the home the agent runs in (Codex's own copy,
|
|
2767
|
+
* seeded from the account's home without its credential files). While the
|
|
2768
|
+
* task's gate is closed it waits, bounded; null when no account can be
|
|
2769
|
+
* used, and the agent stays down with "credentials unavailable", never on
|
|
2770
|
+
* the runner's own accounts, until the gate next opens.
|
|
2771
|
+
*/
|
|
2772
|
+
private leasedSpawnArgs(
|
|
2773
|
+
channel: Channel,
|
|
2774
|
+
agentId: string,
|
|
2775
|
+
args: SpawnArgs,
|
|
2776
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
2777
|
+
):
|
|
2778
|
+
| { args: SpawnArgs; identity: LeasedIdentity }
|
|
2779
|
+
| null
|
|
2780
|
+
| Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
|
|
2781
|
+
const kind = args.agent?.kind ?? "";
|
|
2782
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
2783
|
+
if (lease?.open) {
|
|
2784
|
+
const account = this.leasedAccountFor(channel, agentId, kind, lease);
|
|
2785
|
+
if (account?.authKind === "env") {
|
|
2786
|
+
return this.withLeasedAccount(channel, agentId, args, account, lease.assignment.runEpoch);
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
return (async () => {
|
|
2790
|
+
const deadline = Date.now() + LEASED_ACCOUNT_WAIT_MS;
|
|
2791
|
+
let reason = "the task's AI accounts are not available";
|
|
2792
|
+
let moved = 0;
|
|
2793
|
+
while (this.isActiveChannel(channel)) {
|
|
2794
|
+
const open = await this.leasedGateOpen(channel, deadline);
|
|
2795
|
+
if (!open) break;
|
|
2796
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2797
|
+
const account = this.leasedAccountFor(channel, agentId, kind, open);
|
|
2798
|
+
if (!account) {
|
|
2799
|
+
reason = `no ${kind} account this task can use`;
|
|
2800
|
+
break;
|
|
2801
|
+
}
|
|
2802
|
+
if (account.authKind === "env") {
|
|
2803
|
+
return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch);
|
|
2804
|
+
}
|
|
2805
|
+
// The account's home gets the task's managed config before anything
|
|
2806
|
+
// can start in it, whenever the gate's opening first reveals it.
|
|
2807
|
+
await this.ensureBrowserSetup(channel, kind === "codex");
|
|
2808
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2809
|
+
await this.ensureMcpConfig(channel);
|
|
2810
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2811
|
+
// Codex runs in a home of its own (two agents on one account never
|
|
2812
|
+
// share its state runtime); the config writers above reached it.
|
|
2813
|
+
const home = kind === "codex" ? codexPerAgentHome(account.home!, agentId) : account.home!;
|
|
2814
|
+
try {
|
|
2815
|
+
await ensureEngineHome(environment, open.assignment, account.credentialId, home);
|
|
2816
|
+
} catch (error) {
|
|
2817
|
+
// The account moved while it was written (a newer version, a
|
|
2818
|
+
// cleanup, no longer readable, the gate closing): pick again on
|
|
2819
|
+
// what the gate serves, or will once it reopens, a few times while
|
|
2820
|
+
// time remains. Anything else ends here.
|
|
2821
|
+
if (
|
|
2822
|
+
(error instanceof EngineCredentialWriteRefused || error instanceof CredentialLeaseUnavailableError) &&
|
|
2823
|
+
moved++ < 3 &&
|
|
2824
|
+
Date.now() < deadline
|
|
2825
|
+
) {
|
|
2826
|
+
continue;
|
|
2827
|
+
}
|
|
2828
|
+
reason = error instanceof CredentialLeaseUnavailableError ? error.message : "its files could not be written";
|
|
2829
|
+
break;
|
|
2830
|
+
}
|
|
2831
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2832
|
+
return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch, home);
|
|
2833
|
+
}
|
|
2834
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2835
|
+
this.leasedUnavailable(channel, agentId, reason);
|
|
2836
|
+
return null;
|
|
2837
|
+
})();
|
|
2838
|
+
}
|
|
2839
|
+
|
|
2840
|
+
/** The task's open engine lease, waited for while time remains, a newer
|
|
2841
|
+
* run's when the task moves on to one; null when none opens in time. */
|
|
2842
|
+
private async leasedGateOpen(channel: Channel, deadline: number): Promise<TaskEngineLease | null> {
|
|
2843
|
+
while (this.isActiveChannel(channel)) {
|
|
2844
|
+
const record = leasedEnginesFor(channel.taskId);
|
|
2845
|
+
if (!record) return null;
|
|
2846
|
+
try {
|
|
2847
|
+
return await waitForEngineGate(record.assignment, Math.max(0, deadline - Date.now()));
|
|
2848
|
+
} catch {
|
|
2849
|
+
// The task moved on to another run: wait for that run's gate while
|
|
2850
|
+
// time remains.
|
|
2851
|
+
if (leasedEnginesFor(channel.taskId)?.assignment.runEpoch !== record.assignment.runEpoch && Date.now() < deadline) {
|
|
2852
|
+
continue;
|
|
2853
|
+
}
|
|
2854
|
+
return null;
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
return null;
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
/**
|
|
2861
|
+
* ADR-154 §6, ADR-128: what a Managed Engine spawn on a leased task takes.
|
|
2862
|
+
* It runs on no AI account, but its agent's task token is its bearer: it
|
|
2863
|
+
* is bound to the task's open run, its launches admitted as that token on
|
|
2864
|
+
* this binding, under the engine supervisor like any leased engine's.
|
|
2865
|
+
* While the task's gate is closed it waits, bounded; null when it stays
|
|
2866
|
+
* closed, and the agent stays down with "credentials unavailable" until
|
|
2867
|
+
* the gate next opens.
|
|
2868
|
+
*/
|
|
2869
|
+
private leasedTaskTokenSpawnArgs(
|
|
2870
|
+
channel: Channel,
|
|
2871
|
+
agentId: string,
|
|
2872
|
+
args: SpawnArgs,
|
|
2873
|
+
):
|
|
2874
|
+
| { args: SpawnArgs; identity: LeasedIdentity }
|
|
2875
|
+
| null
|
|
2876
|
+
| Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
|
|
2877
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
2878
|
+
if (lease?.open) return this.withLeasedTaskToken(channel, agentId, args, lease.assignment.runEpoch);
|
|
2879
|
+
return (async () => {
|
|
2880
|
+
const open = await this.leasedGateOpen(channel, Date.now() + LEASED_ACCOUNT_WAIT_MS);
|
|
2881
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2882
|
+
if (!open) {
|
|
2883
|
+
this.leasedUnavailable(channel, agentId, "the task's credentials are not available");
|
|
2884
|
+
return null;
|
|
2885
|
+
}
|
|
2886
|
+
return this.withLeasedTaskToken(channel, agentId, args, open.assignment.runEpoch);
|
|
2887
|
+
})();
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
/** Bind the agent to the task's run `runEpoch` on its task token alone,
|
|
2891
|
+
* and give its spawn the run's marker every process it starts inherits
|
|
2892
|
+
* (its launches are known by it, and admitted on this binding). */
|
|
2893
|
+
private withLeasedTaskToken(
|
|
2894
|
+
channel: Channel,
|
|
2895
|
+
agentId: string,
|
|
2896
|
+
args: SpawnArgs,
|
|
2897
|
+
runEpoch: number,
|
|
2898
|
+
): { args: SpawnArgs; identity: LeasedIdentity } {
|
|
2899
|
+
const identity = { subject: "task-token" as const, runEpoch };
|
|
2900
|
+
channel.leasedIdentityByAgent.set(agentId, identity);
|
|
2901
|
+
return {
|
|
2902
|
+
identity,
|
|
2903
|
+
args: {
|
|
2904
|
+
...args,
|
|
2905
|
+
attachCompatibilityKey: JSON.stringify([args.attachCompatibilityKey ?? null, "lease-v1", runEpoch, "task-token"]),
|
|
2906
|
+
agentEnv: {
|
|
2907
|
+
...(args.agentEnv ?? {}),
|
|
2908
|
+
// The gateway's bearer is the agent's own token; no engine's
|
|
2909
|
+
// credential of the runner's rides beside it.
|
|
2910
|
+
...LEASED_CREDENTIAL_BLANKS,
|
|
2911
|
+
[LEASED_RUN_ENV]: String(runEpoch),
|
|
2912
|
+
},
|
|
2913
|
+
},
|
|
2914
|
+
};
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
/**
|
|
2918
|
+
* ADR-154 §6: the environment a leased engine's adapter launches through.
|
|
2919
|
+
* Each engine launch it makes (the session, each one-shot turn, a durable
|
|
2920
|
+
* runner once deployed, a classifier's exec), at whatever moment its
|
|
2921
|
+
* transport gets there, is admitted then: for an agent's spawn (`owner`),
|
|
2922
|
+
* only while that spawn is the agent's current one and, once its create
|
|
2923
|
+
* settled, the session it bound is the agent's live one, and `identity`
|
|
2924
|
+
* is still what the agent runs on and the task's open gate serves (for a
|
|
2925
|
+
* one-shot with no agent, only the latter). The admission is written into
|
|
2926
|
+
* the runtime first (its exec admitted at the provider's last moment) and
|
|
2927
|
+
* the check made again just before the launch is handed over and at the
|
|
2928
|
+
* provider's last moment. The launch then starts under the engine
|
|
2929
|
+
* supervisor, which consumes that admission. A spawn's admissions name its
|
|
2930
|
+
* slot: before its first, the agent's earlier spawns are retired in the
|
|
2931
|
+
* runtime, and once the slot is retired, nothing it was admitted for
|
|
2932
|
+
* starts there (`retireLeasedLaunch`). A cleanup unbinds first, waits for
|
|
2933
|
+
* the admissions under way (registered before their first await), and
|
|
2934
|
+
* revokes in the runtime what they wrote before it kills what the
|
|
2935
|
+
* supervisors own.
|
|
2936
|
+
*/
|
|
2937
|
+
private leasedLaunchEnvironment(
|
|
2938
|
+
channel: Channel,
|
|
2939
|
+
owner: { agentId: string; slot: LeasedLaunchSlot } | null,
|
|
2940
|
+
identity: LeasedIdentity,
|
|
2941
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
2942
|
+
): TaskEnvironmentHandle<unknown> {
|
|
2943
|
+
const agentId = owner?.agentId ?? null;
|
|
2944
|
+
const held = (): boolean =>
|
|
2945
|
+
this.isActiveChannel(channel) &&
|
|
2946
|
+
(owner === null
|
|
2947
|
+
? this.leasedMaterialServed(channel, identity)
|
|
2948
|
+
: channel.leasedLaunches.get(owner.agentId) === owner.slot &&
|
|
2949
|
+
(owner.slot.session === null
|
|
2950
|
+
? !owner.slot.created
|
|
2951
|
+
: channel.sessions.get(owner.agentId) === owner.slot.session) &&
|
|
2952
|
+
this.leasedIdentityHeld(channel, owner.agentId, identity));
|
|
2953
|
+
const admittedAs: EngineLaunchIdentity = owner?.slot.identity ?? { taskId: channel.taskId, slotId: null, ...identity };
|
|
2954
|
+
return anchoredEnvironment(environment, {
|
|
2955
|
+
held,
|
|
2956
|
+
admit: (carried) => {
|
|
2957
|
+
if (environment.anchors !== true || !held()) return Promise.reject(new EngineLaunchRefused());
|
|
2958
|
+
if (owner) owner.slot.admitted = true;
|
|
2959
|
+
const admitted = (async () => {
|
|
2960
|
+
if (owner) await this.leasedPredecessorsRetired(owner.agentId, owner.slot);
|
|
2961
|
+
return admitEngineLaunch(
|
|
2962
|
+
environment,
|
|
2963
|
+
admittedAs,
|
|
2964
|
+
() => {
|
|
2965
|
+
if (!held()) throw new EngineLaunchRefused();
|
|
2966
|
+
},
|
|
2967
|
+
carried,
|
|
2968
|
+
);
|
|
2969
|
+
})();
|
|
2970
|
+
const entry = {
|
|
2971
|
+
agentId: agentId ?? "",
|
|
2972
|
+
accountId: identity.subject === "account" ? identity.credentialId : null,
|
|
2973
|
+
runEpoch: identity.runEpoch,
|
|
2974
|
+
settled: admitted.then(
|
|
2975
|
+
() => undefined,
|
|
2976
|
+
() => undefined,
|
|
2977
|
+
),
|
|
2978
|
+
};
|
|
2979
|
+
channel.leasedCreating.add(entry);
|
|
2980
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
2981
|
+
return admitted;
|
|
2982
|
+
},
|
|
2983
|
+
});
|
|
2984
|
+
}
|
|
2985
|
+
|
|
2986
|
+
/** `session` ended (its engine's error or exit): its spawn admits no
|
|
2987
|
+
* launch from this moment, before its terminal handling awaits anything.
|
|
2988
|
+
* Every such handling retires or replaces the session. */
|
|
2989
|
+
private endLeasedLaunches(channel: Channel, agentId: string, session: AgentSession): void {
|
|
2990
|
+
const slot = channel.leasedLaunches.get(agentId);
|
|
2991
|
+
if (slot?.session === session) void this.retireLeasedLaunch(channel, agentId, slot);
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
/**
|
|
2995
|
+
* ADR-154 §6: `slot` admits nothing from this moment. Then, in its
|
|
2996
|
+
* runtime, it is retired: every admission it wrote that no launch has used
|
|
2997
|
+
* is revoked, and none of its own is written or used after, so a launch
|
|
2998
|
+
* handed to its provider before now that has not started there never
|
|
2999
|
+
* does, with the agent's old task token or anything else. The agent's
|
|
3000
|
+
* next spawn launches in that runtime only once that is done
|
|
3001
|
+
* (`leasedPredecessorsRetired`). Resolves once that is done or escalated
|
|
3002
|
+
* (see `retireInRuntime`); never rejects.
|
|
3003
|
+
*/
|
|
3004
|
+
private retireLeasedLaunch(channel: Channel, agentId: string, slot: LeasedLaunchSlot | undefined): Promise<void> {
|
|
3005
|
+
if (!slot || slot.retired) return Promise.resolve();
|
|
3006
|
+
slot.retired = true;
|
|
3007
|
+
if (channel.leasedLaunches.get(agentId) === slot) channel.leasedLaunches.delete(agentId);
|
|
3008
|
+
// It never wrote an admission: nothing of it is in its runtime.
|
|
3009
|
+
if (!slot.admitted) return Promise.resolve();
|
|
3010
|
+
const retired: RetiredLaunchSlot = { agentId, slot, attempt: null, covered: false, escalated: false };
|
|
3011
|
+
let pending = this.leasedRetired.get(channel.taskId);
|
|
3012
|
+
if (!pending) {
|
|
3013
|
+
pending = new Set();
|
|
3014
|
+
this.leasedRetired.set(channel.taskId, pending);
|
|
3015
|
+
}
|
|
3016
|
+
pending.add(retired);
|
|
3017
|
+
return this.retireInRuntime(retired).catch(() => undefined);
|
|
3018
|
+
}
|
|
3019
|
+
|
|
3020
|
+
/**
|
|
3021
|
+
* One attempt to retire `retired` in its runtime, joined while one is
|
|
3022
|
+
* under way; once done, it is no longer pending. Should it fail, the order
|
|
3023
|
+
* is lost: a launch handed over before the retirement may start before
|
|
3024
|
+
* any later one lands, and nothing a retry does stops it. So, unless a
|
|
3025
|
+
* cleanup that covers it is under way or done, the run's full credential
|
|
3026
|
+
* cleanup is asked for at once (`escalateEngineLaunchFence`): its gate
|
|
3027
|
+
* closed first, it proves no engine of the run runs (or stops the
|
|
3028
|
+
* runtime), and keeps failing closed until it can. Rejects when this
|
|
3029
|
+
* attempt failed (it is retained and retried meanwhile).
|
|
3030
|
+
*/
|
|
3031
|
+
private retireInRuntime(retired: RetiredLaunchSlot): Promise<void> {
|
|
3032
|
+
if (retired.attempt) return retired.attempt;
|
|
3033
|
+
const { taskId, runEpoch } = retired.slot.identity;
|
|
3034
|
+
const attempt = retireEngineLaunchSlot(retired.slot.environment, retired.slot.identity)
|
|
3035
|
+
.then(
|
|
3036
|
+
() => this.forgetRetired(taskId, retired),
|
|
3037
|
+
(error: unknown) => {
|
|
3038
|
+
if (!retired.covered && !retired.escalated) {
|
|
3039
|
+
retired.escalated = true;
|
|
3040
|
+
console.warn(
|
|
3041
|
+
`[orchestrator] ${taskId}/${retired.agentId}: a retired spawn could not be retired in its runtime; cleaning up its run's credentials (${
|
|
3042
|
+
error instanceof Error ? error.message : String(error)
|
|
3043
|
+
})`,
|
|
3044
|
+
);
|
|
3045
|
+
escalateEngineLaunchFence({ taskId, runEpoch });
|
|
3046
|
+
}
|
|
3047
|
+
throw error;
|
|
3048
|
+
},
|
|
3049
|
+
)
|
|
3050
|
+
.finally(() => {
|
|
3051
|
+
if (retired.attempt === attempt) retired.attempt = null;
|
|
3052
|
+
});
|
|
3053
|
+
retired.attempt = attempt;
|
|
3054
|
+
return attempt;
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
/** The task's retirements still under way (of `agentId`'s spawns alone,
|
|
3058
|
+
* when named), whichever event began them: each resolves once its slot is
|
|
3059
|
+
* retired in its runtime, or its run's cleanup has been asked for. What a
|
|
3060
|
+
* removal or a channel's close waits for. */
|
|
3061
|
+
private leasedRetirementsUnderWay(taskId: string, agentId?: string): Array<Promise<void>> {
|
|
3062
|
+
return [...(this.leasedRetired.get(taskId) ?? [])]
|
|
3063
|
+
.filter((retired) => agentId === undefined || retired.agentId === agentId)
|
|
3064
|
+
.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
private forgetRetired(taskId: string, retired: RetiredLaunchSlot): void {
|
|
3068
|
+
const pending = this.leasedRetired.get(taskId);
|
|
3069
|
+
if (!pending) return;
|
|
3070
|
+
pending.delete(retired);
|
|
3071
|
+
if (pending.size === 0) this.leasedRetired.delete(taskId);
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
/**
|
|
3075
|
+
* ADR-154 §6: before `slot` writes an admission, every earlier slot of the
|
|
3076
|
+
* same agent retired for the same run in the same runtime is retired
|
|
3077
|
+
* there: nothing the agent's earlier spawns were admitted for can start
|
|
3078
|
+
* after this one's launch. Rejects with `EngineLaunchRefused` when one
|
|
3079
|
+
* could not be now.
|
|
3080
|
+
*/
|
|
3081
|
+
private async leasedPredecessorsRetired(agentId: string, slot: LeasedLaunchSlot): Promise<void> {
|
|
3082
|
+
const earlier = [...(this.leasedRetired.get(slot.identity.taskId) ?? [])].filter(
|
|
3083
|
+
(retired) =>
|
|
3084
|
+
retired.agentId === agentId &&
|
|
3085
|
+
retired.slot.identity.runEpoch === slot.identity.runEpoch &&
|
|
3086
|
+
sameRuntime(retired.slot.environment, slot.environment),
|
|
3087
|
+
);
|
|
3088
|
+
try {
|
|
3089
|
+
await Promise.all(earlier.map((retired) => this.retireInRuntime(retired)));
|
|
3090
|
+
} catch {
|
|
3091
|
+
throw new EngineLaunchRefused("an earlier start of this agent could not be retired in its runtime yet");
|
|
3092
|
+
}
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
/** The retirements still pending that a cleanup of these runs and
|
|
3096
|
+
* accounts (of every subject, for a whole run's) covers: its stop revokes
|
|
3097
|
+
* everything they fence, wherever it was written, and kills what any
|
|
3098
|
+
* launch of theirs started. Each is marked covered: one that fails from
|
|
3099
|
+
* now on is not escalated. */
|
|
3100
|
+
private coveredLeasedRetirements(
|
|
3101
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3102
|
+
): RetiredLaunchSlot[] {
|
|
3103
|
+
const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
|
|
3104
|
+
const covered = [...(this.leasedRetired.get(request.taskId) ?? [])].filter((retired) => {
|
|
3105
|
+
const { identity } = retired.slot;
|
|
3106
|
+
return (
|
|
3107
|
+
identity.runEpoch <= request.runEpoch &&
|
|
3108
|
+
(!named || (identity.subject === "account" && named.has(identity.credentialId)))
|
|
3109
|
+
);
|
|
3110
|
+
});
|
|
3111
|
+
for (const retired of covered) retired.covered = true;
|
|
3112
|
+
return covered;
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
/** A cleanup proved every admission of these runs and accounts (of every
|
|
3116
|
+
* subject, for a whole run's) revoked wherever it was written: the
|
|
3117
|
+
* retirements still pending for them are done. */
|
|
3118
|
+
private dischargeLeasedRetirements(
|
|
3119
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3120
|
+
): void {
|
|
3121
|
+
for (const retired of this.coveredLeasedRetirements(request)) this.forgetRetired(request.taskId, retired);
|
|
3122
|
+
}
|
|
3123
|
+
|
|
3124
|
+
/** The agent stays down for want of a usable account, says why, and is
|
|
3125
|
+
* started again when the task's gate next opens. */
|
|
3126
|
+
private leasedUnavailable(channel: Channel, agentId: string, reason: string): void {
|
|
3127
|
+
channel.accountByAgent.delete(agentId);
|
|
3128
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
3129
|
+
channel.leasedBlocked.add(agentId);
|
|
3130
|
+
this.emitHost({
|
|
3131
|
+
kind: "agent.exit",
|
|
3132
|
+
taskId: channel.taskId,
|
|
3133
|
+
agentId,
|
|
3134
|
+
reason: `${agentId}: credentials unavailable (${reason}) — the agent starts again when they are.`,
|
|
3135
|
+
});
|
|
3136
|
+
}
|
|
3137
|
+
|
|
3138
|
+
/** Whether the task's open gate still serves `identity`'s account at its
|
|
3139
|
+
* revision and version, for its run, and that version is not being held
|
|
3140
|
+
* back for a newer one: what a one-shot may start on. An agent's task
|
|
3141
|
+
* token alone needs only its run's gate open. */
|
|
3142
|
+
private leasedMaterialServed(channel: Channel, identity: LeasedIdentity): boolean {
|
|
3143
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3144
|
+
if (!lease?.open || lease.assignment.runEpoch !== identity.runEpoch) return false;
|
|
3145
|
+
if (identity.subject === "task-token") return true;
|
|
3146
|
+
const account = lease.accounts.find((candidate) => candidate.credentialId === identity.credentialId);
|
|
3147
|
+
return (
|
|
3148
|
+
isLeasedEngineAccount(account) &&
|
|
3149
|
+
account.revision === identity.revision &&
|
|
3150
|
+
account.version === identity.version &&
|
|
3151
|
+
!engineVersionHeldBack(lease.assignment, identity)
|
|
3152
|
+
);
|
|
3153
|
+
}
|
|
3154
|
+
|
|
3155
|
+
/** A spawn found its lease material moved (a newer version, a cleanup):
|
|
3156
|
+
* start the agent again on what the task holds now, once its gate is
|
|
3157
|
+
* open (now, or at its next opening). */
|
|
3158
|
+
private retryLeasedStart(channel: Channel, agentId: string): void {
|
|
3159
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3160
|
+
channel.leasedRespawn.add(agentId);
|
|
3161
|
+
channel.leasedWakePending = true;
|
|
3162
|
+
if (leasedEnginesFor(channel.taskId)?.open) this.leasedEnginesChanged(channel.taskId);
|
|
3163
|
+
}
|
|
3164
|
+
|
|
3165
|
+
/** Whether the agent may start on `identity` right now: still bound to it,
|
|
3166
|
+
* and the task's open gate still serves that account at that revision and
|
|
3167
|
+
* version, for that run. */
|
|
3168
|
+
private leasedIdentityHeld(channel: Channel, agentId: string, identity: LeasedIdentity): boolean {
|
|
3169
|
+
const bound = channel.leasedIdentityByAgent.get(agentId);
|
|
3170
|
+
return bound !== undefined && sameLeasedIdentity(bound, identity) && this.leasedMaterialServed(channel, identity);
|
|
3171
|
+
}
|
|
3172
|
+
|
|
3173
|
+
/** The leased account an agent runs on: the one it is bound to while that
|
|
3174
|
+
* is served readable, else the first healthy one in try order. */
|
|
3175
|
+
private leasedAccountFor(
|
|
3176
|
+
channel: Channel,
|
|
3177
|
+
agentId: string,
|
|
3178
|
+
kind: string,
|
|
3179
|
+
lease: TaskEngineLease,
|
|
3180
|
+
): LeasedEngineAccount | undefined {
|
|
3181
|
+
const accounts = lease.accounts.filter(
|
|
3182
|
+
(account): account is LeasedEngineAccount =>
|
|
3183
|
+
account.engineKind === kind && isLeasedEngineAccount(account),
|
|
3184
|
+
);
|
|
3185
|
+
const bound = channel.accountByAgent.get(agentId);
|
|
3186
|
+
const kept = accounts.find((account) => account.credentialId === bound);
|
|
3187
|
+
if (kept) return kept;
|
|
3188
|
+
const picked = selectAccount(
|
|
3189
|
+
accounts.map((account) => ({
|
|
3190
|
+
id: account.credentialId,
|
|
3191
|
+
kind,
|
|
3192
|
+
label: account.label,
|
|
3193
|
+
isDefault: false,
|
|
3194
|
+
scope: account.scope,
|
|
3195
|
+
authKind: account.authKind,
|
|
3196
|
+
execEnv: {},
|
|
3197
|
+
})),
|
|
3198
|
+
);
|
|
3199
|
+
return accounts.find((account) => account.credentialId === picked?.id);
|
|
3200
|
+
}
|
|
3201
|
+
|
|
3202
|
+
/** Bind the agent to `account` for `runEpoch` and give its spawn the
|
|
3203
|
+
* account, and the markers every process it starts inherits (its launches
|
|
3204
|
+
* are known by them, and admitted on this binding). */
|
|
3205
|
+
private withLeasedAccount(
|
|
3206
|
+
channel: Channel,
|
|
3207
|
+
agentId: string,
|
|
3208
|
+
args: SpawnArgs,
|
|
3209
|
+
account: LeasedEngineAccount,
|
|
3210
|
+
runEpoch: number,
|
|
3211
|
+
home?: string,
|
|
3212
|
+
): { args: SpawnArgs; identity: LeasedIdentity } {
|
|
3213
|
+
const identity = {
|
|
3214
|
+
subject: "account" as const,
|
|
3215
|
+
runEpoch,
|
|
3216
|
+
credentialId: account.credentialId,
|
|
3217
|
+
revision: account.revision,
|
|
3218
|
+
version: account.version,
|
|
3219
|
+
};
|
|
3220
|
+
channel.accountByAgent.set(agentId, account.credentialId);
|
|
3221
|
+
channel.leasedIdentityByAgent.set(agentId, identity);
|
|
3222
|
+
noteEngineAccountUsed(account.credentialId);
|
|
3223
|
+
return {
|
|
3224
|
+
identity,
|
|
3225
|
+
args: {
|
|
3226
|
+
...args,
|
|
3227
|
+
// A durable runner from before a restart is attached to only when it
|
|
3228
|
+
// was launched on this exact material.
|
|
3229
|
+
attachCompatibilityKey: JSON.stringify([
|
|
3230
|
+
args.attachCompatibilityKey ?? null,
|
|
3231
|
+
"lease-v1",
|
|
3232
|
+
identity.runEpoch,
|
|
3233
|
+
identity.credentialId,
|
|
3234
|
+
identity.revision,
|
|
3235
|
+
identity.version,
|
|
3236
|
+
]),
|
|
3237
|
+
agentEnv: {
|
|
3238
|
+
...(args.agentEnv ?? {}),
|
|
3239
|
+
...LEASED_CREDENTIAL_BLANKS,
|
|
3240
|
+
...engineSpawnEnv(account, home),
|
|
3241
|
+
[LEASED_RUN_ENV]: String(runEpoch),
|
|
3242
|
+
[LEASED_ACCOUNT_ENV]: account.credentialId,
|
|
3243
|
+
},
|
|
3244
|
+
},
|
|
3245
|
+
};
|
|
3246
|
+
}
|
|
3247
|
+
|
|
3248
|
+
/**
|
|
3249
|
+
* The env of a one-shot run on the task's leased account for `kind` (the
|
|
3250
|
+
* shadow router's classifier): the agent's own account when it runs on
|
|
3251
|
+
* one of that kind, else the first healthy one; a config-directory
|
|
3252
|
+
* account's home is written first. Null when none can be used now: the
|
|
3253
|
+
* classifier does not run and the pass routes by rules.
|
|
3254
|
+
*/
|
|
3255
|
+
/** ADR-154 §6: a one-shot's env on the leased `account` chosen for it: its
|
|
3256
|
+
* home written first for a config-directory account. Null when it cannot
|
|
3257
|
+
* be given. */
|
|
3258
|
+
private async leasedOneShotEnv(
|
|
3259
|
+
channel: Channel,
|
|
3260
|
+
lease: TaskEngineLease,
|
|
3261
|
+
account: LeasedEngineAccount,
|
|
3262
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
3263
|
+
): Promise<Record<string, string> | null> {
|
|
3264
|
+
const marker = {
|
|
3265
|
+
[LEASED_RUN_ENV]: String(lease.assignment.runEpoch),
|
|
3266
|
+
[LEASED_ACCOUNT_ENV]: account.credentialId,
|
|
3267
|
+
};
|
|
3268
|
+
try {
|
|
3269
|
+
rememberLeasedEngineRuntime({
|
|
3270
|
+
taskId: channel.taskId,
|
|
3271
|
+
runEpoch: lease.assignment.runEpoch,
|
|
3272
|
+
runtime: environment.descriptor.locator,
|
|
3273
|
+
});
|
|
3274
|
+
} catch {
|
|
3275
|
+
return null;
|
|
3276
|
+
}
|
|
3277
|
+
if (account.authKind === "env") {
|
|
3278
|
+
return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
|
|
3279
|
+
}
|
|
3280
|
+
try {
|
|
3281
|
+
await ensureEngineHome(environment, lease.assignment, account.credentialId);
|
|
3282
|
+
return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
|
|
3283
|
+
} catch {
|
|
3284
|
+
return null;
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
|
|
3288
|
+
/**
|
|
3289
|
+
* ADR-154 §6, §8: what the task's AI accounts are, or whether its gate is
|
|
3290
|
+
* open, changed. Rewrite what moved in the homes this run wrote, then let
|
|
3291
|
+
* each long-lived engine running on a rewritten account (Codex's
|
|
3292
|
+
* app-server) respawn at its first idle moment; one-shot engines read the
|
|
3293
|
+
* file again at their next turn, and env-token engines take a new value at
|
|
3294
|
+
* their next spawn. Called synchronously by the engine consumer: the work
|
|
3295
|
+
* runs after, one pass at a time per channel.
|
|
3296
|
+
*/
|
|
3297
|
+
private leasedEnginesChanged(taskId: string): void {
|
|
3298
|
+
const channel = this.channels.get(taskId);
|
|
3299
|
+
if (!channel || !this.isActiveChannel(channel) || !this.isLeasedChannel(channel)) return;
|
|
3300
|
+
if (!leasedEnginesFor(taskId)?.open) return;
|
|
3301
|
+
if (channel.leasedSyncRetry) {
|
|
3302
|
+
clearTimeout(channel.leasedSyncRetry);
|
|
3303
|
+
channel.leasedSyncRetry = null;
|
|
3304
|
+
}
|
|
3305
|
+
// Agents a cleanup stopped, or that found no usable account, start again
|
|
3306
|
+
// on this view, each on its next account: one attempt per opening. The
|
|
3307
|
+
// intent outlives a pass that fails, for its retry.
|
|
3308
|
+
if (channel.leasedRespawn.size > 0 || channel.leasedBlocked.size > 0) {
|
|
3309
|
+
channel.leasedWakePending = true;
|
|
3310
|
+
}
|
|
3311
|
+
const run = (channel.leasedSync ?? Promise.resolve())
|
|
3312
|
+
.then(async () => {
|
|
3313
|
+
await this.syncLeasedHomes(channel);
|
|
3314
|
+
if (!channel.leasedWakePending || !this.isActiveChannel(channel)) return;
|
|
3315
|
+
channel.leasedWakePending = false;
|
|
3316
|
+
channel.leasedRespawn.clear();
|
|
3317
|
+
channel.leasedBlocked.clear();
|
|
3318
|
+
try {
|
|
3319
|
+
await this.ensureSessions(channel);
|
|
3320
|
+
} catch (err) {
|
|
3321
|
+
channel.leasedWakePending = true;
|
|
3322
|
+
throw err;
|
|
3323
|
+
}
|
|
3324
|
+
})
|
|
3325
|
+
.catch((err: unknown) => {
|
|
3326
|
+
console.warn(
|
|
3327
|
+
`[orchestrator] ${taskId}: rewriting AI account files failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
3328
|
+
);
|
|
3329
|
+
// A home left on an old token is written again, not left so.
|
|
3330
|
+
if (this.isActiveChannel(channel) && !channel.leasedSyncRetry) {
|
|
3331
|
+
const retry = setTimeout(() => {
|
|
3332
|
+
if (channel.leasedSyncRetry !== retry) return;
|
|
3333
|
+
channel.leasedSyncRetry = null;
|
|
3334
|
+
this.leasedEnginesChanged(taskId);
|
|
3335
|
+
}, LEASED_SYNC_RETRY_MS);
|
|
3336
|
+
retry.unref?.();
|
|
3337
|
+
channel.leasedSyncRetry = retry;
|
|
3338
|
+
}
|
|
3339
|
+
});
|
|
3340
|
+
channel.leasedSync = run;
|
|
3341
|
+
}
|
|
3342
|
+
|
|
3343
|
+
private async syncLeasedHomes(channel: Channel): Promise<void> {
|
|
3344
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3345
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3346
|
+
if (!lease?.open) return;
|
|
3347
|
+
// Nothing was written into a runtime this channel has not settled.
|
|
3348
|
+
const environment = channel.agentHandle;
|
|
3349
|
+
if (environment) await syncEngineHomes(environment, lease.assignment);
|
|
3350
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3351
|
+
// Every launch is admitted on the exact material its agent is bound to
|
|
3352
|
+
// (§6): an agent whose account moved since starts again on what the
|
|
3353
|
+
// task holds now. A long-lived engine holds the old token or file; a
|
|
3354
|
+
// one-shot one, env-token or not, would have its next turn refused.
|
|
3355
|
+
let stale = false;
|
|
3356
|
+
for (const agent of channel.roster) {
|
|
3357
|
+
const identity = channel.leasedIdentityByAgent.get(agent.id);
|
|
3358
|
+
if (!identity || this.leasedMaterialServed(channel, identity)) continue;
|
|
3359
|
+
if (!channel.sessions.has(agent.id)) continue;
|
|
3360
|
+
channel.browserStaleSessions.add(agent.id);
|
|
3361
|
+
stale = true;
|
|
3362
|
+
}
|
|
3363
|
+
// A stale idle session is recycled now; one mid-turn finishes its turn
|
|
3364
|
+
// on the material it has and is recycled at its next idle moment.
|
|
3365
|
+
if (stale) await this.reconcileIfStarted(channel);
|
|
3366
|
+
}
|
|
3367
|
+
|
|
3368
|
+
/**
|
|
3369
|
+
* ADR-154 §6: the cleanup coordinator's quiesce step for engines (its
|
|
3370
|
+
* `quiesceProcesses` hook takes this request as it is), for the run it
|
|
3371
|
+
* names and every older one, never a newer run's:
|
|
3372
|
+
* - every agent on one of the request's accounts (on any leased account,
|
|
3373
|
+
* or on its task token alone, for a whole-task cleanup) is unbound at
|
|
3374
|
+
* once, so nothing launches on them again (a launch is admitted only
|
|
3375
|
+
* while its agent is bound to its exact material), and its session is
|
|
3376
|
+
* asked to stop; a spawn creating an engine on one, or a launch being
|
|
3377
|
+
* admitted, is waited for, bounded;
|
|
3378
|
+
* - then, in every runtime those runs' engines ran in, every admission not
|
|
3379
|
+
* used yet is revoked, and everything each engine supervisor of those
|
|
3380
|
+
* runs and accounts owns is killed until the supervisor ends on its own,
|
|
3381
|
+
* having proved nothing it owned is left (`stopAnchoredEngines`). That
|
|
3382
|
+
* holds no state here, so a retried cleanup proves it again even after
|
|
3383
|
+
* this channel let the agents go. A supervisor lost without proving its
|
|
3384
|
+
* subtree empty leaves one proof: the runtime is stopped. An env-token
|
|
3385
|
+
* engine holds no credential file: stopping it moves it to its next
|
|
3386
|
+
* account.
|
|
3387
|
+
* Rejects when anything could not be proven stopped: the cleanup then
|
|
3388
|
+
* fails and is retried, never deleting under a live engine. Stopped agents
|
|
3389
|
+
* start again on their next account once the gate opens.
|
|
3390
|
+
*/
|
|
3391
|
+
async quiesceLeasedEngineAgents(
|
|
3392
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3393
|
+
): Promise<void> {
|
|
3394
|
+
const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
|
|
3395
|
+
const channel = this.channels.get(request.taskId);
|
|
3396
|
+
// Every engine being created on one of the accounts (on any subject, a
|
|
3397
|
+
// task token included, for a whole run's), this attempt's or left over
|
|
3398
|
+
// from one that timed out: never only the bound agents'.
|
|
3399
|
+
const creating = [...(channel?.leasedCreating ?? [])]
|
|
3400
|
+
.filter(
|
|
3401
|
+
(entry) =>
|
|
3402
|
+
(!named || (entry.accountId !== null && named.has(entry.accountId))) && entry.runEpoch <= request.runEpoch,
|
|
3403
|
+
)
|
|
3404
|
+
.map((entry) => entry.settled);
|
|
3405
|
+
// Every retirement still pending that this cleanup covers: its stop
|
|
3406
|
+
// proves what they fence, so one that fails from now on is not escalated.
|
|
3407
|
+
const covered = this.coveredLeasedRetirements(request);
|
|
3408
|
+
if (channel) {
|
|
3409
|
+
for (const [agentId, accountId] of [...channel.accountByAgent]) {
|
|
3410
|
+
if (named && !named.has(accountId)) continue;
|
|
3411
|
+
if ((channel.leasedIdentityByAgent.get(agentId)?.runEpoch ?? 0) > request.runEpoch) continue;
|
|
3412
|
+
channel.accountByAgent.delete(agentId);
|
|
3413
|
+
this.quiesceLeasedAgent(channel, agentId);
|
|
3414
|
+
}
|
|
3415
|
+
// A whole run's cleanup takes every agent on its task token alone too
|
|
3416
|
+
// (the Managed Engine); a cleanup of named accounts never does.
|
|
3417
|
+
if (!named) {
|
|
3418
|
+
for (const [agentId, identity] of [...channel.leasedIdentityByAgent]) {
|
|
3419
|
+
if (identity.subject !== "task-token" || identity.runEpoch > request.runEpoch) continue;
|
|
3420
|
+
this.quiesceLeasedAgent(channel, agentId);
|
|
3421
|
+
}
|
|
3422
|
+
}
|
|
3423
|
+
}
|
|
3424
|
+
if (creating.length > 0) {
|
|
3425
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
3426
|
+
const bounded = new Promise<"late">((resolve) => {
|
|
3427
|
+
timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
|
|
3428
|
+
});
|
|
3429
|
+
const outcome = await Promise.race([Promise.all(creating), bounded]);
|
|
3430
|
+
clearTimeout(timer);
|
|
3431
|
+
if (outcome === "late") {
|
|
3432
|
+
throw new Error("an engine being started on a removed AI account did not finish starting");
|
|
3433
|
+
}
|
|
3434
|
+
}
|
|
3435
|
+
// One of those under way is waited for, bounded, so nothing of it lands
|
|
3436
|
+
// after this cleanup's stop.
|
|
3437
|
+
const retiring = covered.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
|
|
3438
|
+
if (retiring.length > 0) {
|
|
3439
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
3440
|
+
const bounded = new Promise<"late">((resolve) => {
|
|
3441
|
+
timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
|
|
3442
|
+
});
|
|
3443
|
+
const outcome = await Promise.race([Promise.all(retiring), bounded]);
|
|
3444
|
+
clearTimeout(timer);
|
|
3445
|
+
if (outcome === "late") {
|
|
3446
|
+
throw new Error("an engine launch slot being retired in its runtime did not finish");
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
const runtimes = await this.leasedCleanupRuntimes(request, channel);
|
|
3450
|
+
for (const environment of runtimes.values()) {
|
|
3451
|
+
// A runtime that is not running holds no process, unless it is frozen
|
|
3452
|
+
// (paused or restarting: its processes run again without a start);
|
|
3453
|
+
// one that is frozen or cannot say cannot be proven.
|
|
3454
|
+
const status = await environment.status();
|
|
3455
|
+
if (status.state !== "unknown" && status.frozen) {
|
|
3456
|
+
throw new Error("a task runtime's engines could not be proven stopped: it is paused");
|
|
3457
|
+
}
|
|
3458
|
+
if (status.state === "stopped" || status.state === "absent") continue;
|
|
3459
|
+
if (status.state !== "running") {
|
|
3460
|
+
throw new Error("a task runtime's engines could not be proven stopped: its state is unknown");
|
|
3461
|
+
}
|
|
3462
|
+
try {
|
|
3463
|
+
await stopAnchoredEngines(environment, {
|
|
3464
|
+
taskId: request.taskId,
|
|
3465
|
+
runEpoch: request.runEpoch,
|
|
3466
|
+
...(request.credentialIds === undefined ? {} : { credentialIds: request.credentialIds }),
|
|
3467
|
+
});
|
|
3468
|
+
} catch (error) {
|
|
3469
|
+
if (!(error instanceof EngineAnchorLost)) throw error;
|
|
3470
|
+
// What a lost supervisor owned may have been reparented away from
|
|
3471
|
+
// it: only a stopped runtime proves none of it still runs.
|
|
3472
|
+
console.warn(`[orchestrator] ${request.taskId}: ${error.message}; stopping the runtime`);
|
|
3473
|
+
await environment.stop();
|
|
3474
|
+
const stopped = await environment.status();
|
|
3475
|
+
if (
|
|
3476
|
+
stopped.state === "unknown" ||
|
|
3477
|
+
stopped.frozen ||
|
|
3478
|
+
(stopped.state !== "stopped" && stopped.state !== "absent")
|
|
3479
|
+
) {
|
|
3480
|
+
throw error;
|
|
3481
|
+
}
|
|
3482
|
+
}
|
|
3483
|
+
}
|
|
3484
|
+
// Every admission of these runs not yet used is revoked in every runtime
|
|
3485
|
+
// they were written in, or that runtime is stopped or gone and its gates
|
|
3486
|
+
// with it: no revocation or slot retirement retained for them is left to
|
|
3487
|
+
// do.
|
|
3488
|
+
dischargeEngineRevocations(request);
|
|
3489
|
+
this.dischargeLeasedRetirements(request);
|
|
3490
|
+
// Every engine of these runs is proven stopped in every runtime they
|
|
3491
|
+
// ran in: a whole-task cleanup lets the record of those runtimes go.
|
|
3492
|
+
if (request.credentialIds === undefined) {
|
|
3493
|
+
forgetLeasedEngineRuntimes(request.taskId, request.runEpoch, [...runtimes.keys()]);
|
|
3494
|
+
}
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
/** A cleanup took the material `agentId` runs on: it is unbound at once,
|
|
3498
|
+
* so nothing launches on it again, its session is asked to stop, and it
|
|
3499
|
+
* starts again on what the task holds once the gate opens. */
|
|
3500
|
+
private quiesceLeasedAgent(channel: Channel, agentId: string): void {
|
|
3501
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
3502
|
+
const session = channel.sessions.get(agentId);
|
|
3503
|
+
if (this.isActiveChannel(channel)) channel.leasedRespawn.add(agentId);
|
|
3504
|
+
if (!session) return;
|
|
3505
|
+
// Delete first: the old session's last events are generation-stale.
|
|
3506
|
+
channel.sessions.delete(agentId);
|
|
3507
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3508
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3509
|
+
this.retainRetiredTurnBoundary(
|
|
3510
|
+
channel,
|
|
3511
|
+
agentId,
|
|
3512
|
+
this.needsAbortedTurnBoundary(channel, agentId),
|
|
3513
|
+
);
|
|
3514
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
3515
|
+
void session.close().catch(() => {});
|
|
3516
|
+
}
|
|
3517
|
+
|
|
3518
|
+
/**
|
|
3519
|
+
* ADR-154 §6: an account is about to be served at a newer version. In
|
|
3520
|
+
* every runtime a launch of this run could start in, its older versions
|
|
3521
|
+
* are held back first: a launch admitted on one that has not started yet
|
|
3522
|
+
* never will. Rejects, so the install fails and none of the newer version
|
|
3523
|
+
* is served, when a runtime cannot be reached or proven not running.
|
|
3524
|
+
*/
|
|
3525
|
+
private async raiseLeasedAdmissionFloors(
|
|
3526
|
+
assignment: { taskId: string; runEpoch: number },
|
|
3527
|
+
floors: readonly EngineAdmissionFloor[],
|
|
3528
|
+
): Promise<void> {
|
|
3529
|
+
const channel = this.channels.get(assignment.taskId);
|
|
3530
|
+
const runtimes = await this.leasedCleanupRuntimes(
|
|
3531
|
+
{ taskId: assignment.taskId, runEpoch: assignment.runEpoch },
|
|
3532
|
+
channel,
|
|
3533
|
+
);
|
|
3534
|
+
for (const environment of runtimes.values()) {
|
|
3535
|
+
const status = await environment.status();
|
|
3536
|
+
if (status.state !== "unknown" && status.frozen) {
|
|
3537
|
+
throw new Error("an AI account's older version could not be held back: a task runtime is paused");
|
|
3538
|
+
}
|
|
3539
|
+
if (status.state === "stopped" || status.state === "absent") continue;
|
|
3540
|
+
if (status.state !== "running") {
|
|
3541
|
+
throw new Error("an AI account's older version could not be held back: a task runtime's state is unknown");
|
|
3542
|
+
}
|
|
3543
|
+
await raiseEngineAdmissionFloors(environment, {
|
|
3544
|
+
taskId: assignment.taskId,
|
|
3545
|
+
runEpoch: assignment.runEpoch,
|
|
3546
|
+
floors,
|
|
3547
|
+
});
|
|
3548
|
+
}
|
|
3549
|
+
}
|
|
3550
|
+
|
|
3551
|
+
/**
|
|
3552
|
+
* Every runtime a cleanup reaches, by locator: the task's current one,
|
|
3553
|
+
* every one a leased engine of these runs was started in (recorded before
|
|
3554
|
+
* it started, files or none), and every one the inventory names for these
|
|
3555
|
+
* runs (an older runtime's files are deleted from that exact runtime).
|
|
3556
|
+
*/
|
|
3557
|
+
private async leasedCleanupRuntimes(
|
|
3558
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3559
|
+
channel: Channel | undefined,
|
|
3560
|
+
): Promise<Map<string, TaskEnvironmentHandle<unknown>>> {
|
|
3561
|
+
const runtimes = new Map<string, TaskEnvironmentHandle<unknown>>();
|
|
3562
|
+
const current =
|
|
3563
|
+
channel?.agentHandle ??
|
|
3564
|
+
(await (async () => {
|
|
3565
|
+
const task = getHostTask(request.taskId);
|
|
3566
|
+
return task ? reconstructPersistedTaskEnvironment(task) : null;
|
|
3567
|
+
})());
|
|
3568
|
+
if (current) runtimes.set(serializeTaskEnvironmentLocator(current.descriptor.locator), current);
|
|
3569
|
+
const recorded = [
|
|
3570
|
+
...listLeasedEngineRuntimes(request.taskId),
|
|
3571
|
+
...listCredentialCleanupPaths(request.taskId),
|
|
3572
|
+
];
|
|
3573
|
+
for (const row of recorded) {
|
|
3574
|
+
if (row.runEpoch > request.runEpoch || runtimes.has(row.runtimeLocator)) continue;
|
|
3575
|
+
const environment = await reconstructTaskEnvironment(
|
|
3576
|
+
parseTaskEnvironmentLocator(row.runtimeLocator),
|
|
3577
|
+
"loader",
|
|
3578
|
+
);
|
|
3579
|
+
if (environment.descriptor.taskId !== request.taskId) {
|
|
3580
|
+
throw new Error("a cleanup runtime belongs to a different task");
|
|
3581
|
+
}
|
|
3582
|
+
runtimes.set(row.runtimeLocator, environment);
|
|
3583
|
+
}
|
|
3584
|
+
return runtimes;
|
|
3585
|
+
}
|
|
3586
|
+
|
|
2414
3587
|
/**
|
|
2415
3588
|
* Per-agent CODEX_HOME (see the call site). Returns `args` unchanged unless
|
|
2416
3589
|
* this is a codex agent with a config-dir CODEX_HOME; then it seeds
|
|
@@ -2481,87 +3654,347 @@ export class Orchestrator {
|
|
|
2481
3654
|
channel.reconcileAgain = true;
|
|
2482
3655
|
return null;
|
|
2483
3656
|
}
|
|
2484
|
-
// ADR-
|
|
2485
|
-
//
|
|
2486
|
-
//
|
|
2487
|
-
//
|
|
2488
|
-
//
|
|
2489
|
-
//
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
const
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
3657
|
+
// ADR-154: an agent on the runner's own accounts holds them from here
|
|
3658
|
+
// until it is bound or given up. Leaving them waits for it; every process
|
|
3659
|
+
// it starts, now or at a later turn, is refused at its provider's last
|
|
3660
|
+
// moment once leaving has begun; and one created across that moment is
|
|
3661
|
+
// closed, never bound. Its next start is on the task's lease. The hold
|
|
3662
|
+
// ends in a `finally`, which adds no await (see below).
|
|
3663
|
+
const ownHold = this.isLeasedChannel(channel) ? undefined : holdOwnCredentials();
|
|
3664
|
+
if (ownHold === null) {
|
|
3665
|
+
channel.reconcileAgain = true;
|
|
3666
|
+
return null;
|
|
3667
|
+
}
|
|
3668
|
+
const onOwn = ownHold !== undefined;
|
|
3669
|
+
try {
|
|
3670
|
+
// ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
|
|
3671
|
+
// account share one CODEX_HOME, and codex's new sqlite state runtime
|
|
3672
|
+
// cannot be opened by two concurrent processes — the second agent dies
|
|
3673
|
+
// with "failed to initialize sqlite state runtime under <CODEX_HOME>"
|
|
3674
|
+
// (live 2026-09-04, a task with two codex agents on one account). Give
|
|
3675
|
+
// each codex agent its OWN CODEX_HOME, seeded once from the resolved
|
|
3676
|
+
// account's dir, so their state runtimes never collide. Single-codex
|
|
3677
|
+
// tasks are unaffected beyond a one-time in-container copy.
|
|
3678
|
+
// Only codex agents take the isolation path; everything else uses args
|
|
3679
|
+
// directly, WITHOUT an extra await — a stray microtask here reordered the
|
|
3680
|
+
// Claude re-spawn's prompt re-delivery. A leased account is applied here
|
|
3681
|
+
// too, synchronously when nothing has to be waited for or written.
|
|
3682
|
+
let boundArgs = args;
|
|
3683
|
+
let identity: LeasedIdentity | undefined;
|
|
3684
|
+
const kind = args.agent?.kind ?? "";
|
|
3685
|
+
if (!onOwn && LEASED_ENGINE_KINDS.has(kind)) {
|
|
3686
|
+
// §6: a runtime that cannot start a launch under the engine supervisor
|
|
3687
|
+
// never runs a leased engine.
|
|
3688
|
+
if (environment.anchors !== true) {
|
|
3689
|
+
this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
|
|
3690
|
+
return null;
|
|
3691
|
+
}
|
|
3692
|
+
const leased = this.leasedSpawnArgs(channel, agentId, args, environment);
|
|
3693
|
+
const applied = leased instanceof Promise ? await leased : leased;
|
|
3694
|
+
// Down until the gate next opens: never a loop.
|
|
3695
|
+
if (!applied) return null;
|
|
3696
|
+
boundArgs = applied.args;
|
|
3697
|
+
identity = applied.identity;
|
|
3698
|
+
} else if (!onOwn && kind === MANAGED_ENGINE_KIND) {
|
|
3699
|
+
// ADR-128, ADR-154 §6: no AI account, but the agent's task token is
|
|
3700
|
+
// the engine's bearer: started under the supervisor, admitted as
|
|
3701
|
+
// that token on the task's open run, or not at all.
|
|
3702
|
+
if (environment.anchors !== true) {
|
|
3703
|
+
this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
|
|
3704
|
+
return null;
|
|
3705
|
+
}
|
|
3706
|
+
const leased = this.leasedTaskTokenSpawnArgs(channel, agentId, args);
|
|
3707
|
+
const applied = leased instanceof Promise ? await leased : leased;
|
|
3708
|
+
if (!applied) return null;
|
|
3709
|
+
boundArgs = applied.args;
|
|
3710
|
+
identity = applied.identity;
|
|
3711
|
+
} else if (!onOwn && !CREDENTIAL_FREE_KINDS.has(kind)) {
|
|
3712
|
+
// ADR-154 §10: a kind with no lane (Kimi) never runs on the runner's
|
|
3713
|
+
// own sign-in for a leased task.
|
|
3714
|
+
this.leasedUnavailable(channel, agentId, `${kind} has no AI account lane on cloud credentials`);
|
|
2534
3715
|
return null;
|
|
3716
|
+
} else if (kind === "codex") {
|
|
3717
|
+
boundArgs = await this.isolateCodexHome(environment, agentId, args);
|
|
2535
3718
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
3719
|
+
while (this.isActiveChannel(channel)) {
|
|
3720
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
3721
|
+
channel.reconcileAgain = true;
|
|
3722
|
+
return null;
|
|
3723
|
+
}
|
|
3724
|
+
if (
|
|
3725
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
3726
|
+
) {
|
|
3727
|
+
channel.reconcileAgain = true;
|
|
3728
|
+
return null;
|
|
3729
|
+
}
|
|
3730
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
3731
|
+
channel.reconcileAgain = true;
|
|
3732
|
+
return null;
|
|
3733
|
+
}
|
|
3734
|
+
if (onOwn && !onOwnCredentials()) {
|
|
3735
|
+
channel.reconcileAgain = true;
|
|
3736
|
+
return null;
|
|
3737
|
+
}
|
|
3738
|
+
const generation = channel.configGeneration;
|
|
3739
|
+
// ADR-154 §6: never start an engine on lease material the task does not
|
|
3740
|
+
// hold right now (its run, account, revision and version). A cleanup
|
|
3741
|
+
// drops the binding synchronously, so a spawn whose account it took
|
|
3742
|
+
// stops here; one already creating is registered, and every cleanup
|
|
3743
|
+
// attempt waits for it.
|
|
3744
|
+
if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
|
|
3745
|
+
this.retryLeasedStart(channel, agentId);
|
|
3746
|
+
return null;
|
|
3747
|
+
}
|
|
3748
|
+
if (identity) {
|
|
3749
|
+
// Where this engine runs, before it can run there (§6): a cleanup
|
|
3750
|
+
// proves it stopped in this runtime even if it leaves no file.
|
|
3751
|
+
try {
|
|
3752
|
+
rememberLeasedEngineRuntime({
|
|
3753
|
+
taskId: channel.taskId,
|
|
3754
|
+
runEpoch: identity.runEpoch,
|
|
3755
|
+
runtime: environment.descriptor.locator,
|
|
3756
|
+
});
|
|
3757
|
+
} catch (err) {
|
|
3758
|
+
console.warn(
|
|
3759
|
+
`[orchestrator] ${channel.taskId}/${agentId}: not starting on its AI account, its runtime could not be recorded: ${err instanceof Error ? err.message : String(err)}`,
|
|
3760
|
+
);
|
|
3761
|
+
return null;
|
|
3762
|
+
}
|
|
3763
|
+
}
|
|
3764
|
+
// Every process the adapter launches for this engine (now, at each
|
|
3765
|
+
// turn, or once its durable runner is deployed) is admitted then, on
|
|
3766
|
+
// this exact material, for this spawn while it is the agent's current
|
|
3767
|
+
// one, and started under the supervisor (§6). The agent's spawn this
|
|
3768
|
+
// one replaces is retired.
|
|
3769
|
+
const slot = identity ? newLeasedLaunchSlot(channel.taskId, identity, environment) : null;
|
|
3770
|
+
if (slot) {
|
|
3771
|
+
void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
|
|
3772
|
+
channel.leasedLaunches.set(agentId, slot);
|
|
3773
|
+
}
|
|
3774
|
+
const creating = this.factory.create({
|
|
3775
|
+
...boundArgs,
|
|
3776
|
+
environment:
|
|
3777
|
+
identity && slot
|
|
3778
|
+
? this.leasedLaunchEnvironment(channel, { agentId, slot }, identity, environment)
|
|
3779
|
+
: onOwn
|
|
3780
|
+
? onOwnCredentialsEnvironment(environment)
|
|
3781
|
+
: environment,
|
|
3782
|
+
});
|
|
3783
|
+
// Settled before this spawn's own continuation: from here until it
|
|
3784
|
+
// binds, nothing launches on it.
|
|
3785
|
+
if (slot) {
|
|
3786
|
+
void creating.then(
|
|
3787
|
+
() => {
|
|
3788
|
+
slot.created = true;
|
|
3789
|
+
},
|
|
3790
|
+
() => {
|
|
3791
|
+
slot.created = true;
|
|
3792
|
+
},
|
|
2544
3793
|
);
|
|
2545
|
-
} else {
|
|
2546
|
-
await session.close().catch(() => {});
|
|
2547
3794
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
3795
|
+
if (identity) {
|
|
3796
|
+
const entry = {
|
|
3797
|
+
agentId,
|
|
3798
|
+
accountId: identity.subject === "account" ? identity.credentialId : null,
|
|
3799
|
+
runEpoch: identity.runEpoch,
|
|
3800
|
+
settled: creating.then(
|
|
3801
|
+
() => undefined,
|
|
3802
|
+
() => undefined,
|
|
3803
|
+
),
|
|
3804
|
+
};
|
|
3805
|
+
channel.leasedCreating.add(entry);
|
|
3806
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
3807
|
+
}
|
|
3808
|
+
// ADR-154 §6: a spawn that does not bind (its create failed, maybe
|
|
3809
|
+
// after it launched, or its session is closed unbound) is retired at
|
|
3810
|
+
// once, before anything else: a launch it handed over that has not
|
|
3811
|
+
// started in its runtime never does.
|
|
3812
|
+
const unbound = (): Promise<void> =>
|
|
3813
|
+
slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
|
|
3814
|
+
let session: AgentSession;
|
|
3815
|
+
try {
|
|
3816
|
+
session = await creating;
|
|
3817
|
+
} catch (error) {
|
|
3818
|
+
await unbound();
|
|
3819
|
+
throw error;
|
|
3820
|
+
}
|
|
3821
|
+
if (!this.isActiveChannel(channel)) {
|
|
3822
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3823
|
+
return null;
|
|
3824
|
+
}
|
|
3825
|
+
// ADR-154: created on the runner's own accounts across the moment
|
|
3826
|
+
// leaving them began: closed, never bound.
|
|
3827
|
+
if (onOwn && !onOwnCredentials()) {
|
|
3828
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3829
|
+
channel.reconcileAgain = true;
|
|
3830
|
+
return null;
|
|
3831
|
+
}
|
|
3832
|
+
// ADR-154 §6: a cleanup that took this account while the session was
|
|
3833
|
+
// being created quiesced the agent already; never keep a session on
|
|
3834
|
+
// material the task no longer holds.
|
|
3835
|
+
if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
|
|
3836
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3837
|
+
this.retryLeasedStart(channel, agentId);
|
|
3838
|
+
return null;
|
|
3839
|
+
}
|
|
3840
|
+
if (identity) channel.leasedBlocked.delete(agentId);
|
|
3841
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
3842
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3843
|
+
channel.reconcileAgain = true;
|
|
3844
|
+
return null;
|
|
3845
|
+
}
|
|
3846
|
+
const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
|
|
3847
|
+
if (pendingRosterClose?.state === "pending-close") {
|
|
3848
|
+
await Promise.all([
|
|
3849
|
+
unbound(),
|
|
3850
|
+
this.closeRosterChangedSession(channel, agentId, session, pendingRosterClose),
|
|
3851
|
+
]);
|
|
3852
|
+
channel.reconcileAgain = true;
|
|
3853
|
+
return null;
|
|
3854
|
+
}
|
|
3855
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
3856
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
3857
|
+
const retiring = unbound();
|
|
3858
|
+
if (change) {
|
|
3859
|
+
await this.closeRosterChangedSession(
|
|
3860
|
+
channel,
|
|
3861
|
+
agentId,
|
|
3862
|
+
session,
|
|
3863
|
+
change,
|
|
3864
|
+
);
|
|
3865
|
+
} else {
|
|
3866
|
+
await session.close().catch(() => {});
|
|
3867
|
+
}
|
|
3868
|
+
await retiring;
|
|
3869
|
+
channel.reconcileAgain = true;
|
|
3870
|
+
return null;
|
|
3871
|
+
}
|
|
3872
|
+
if (generation === channel.configGeneration) {
|
|
3873
|
+
// Keep the final generation check, stale cleanup, and map bind in one
|
|
3874
|
+
// synchronous continuation. A config writer that resumes immediately
|
|
3875
|
+
// afterward now sees this session and marks it stale; it can no longer
|
|
3876
|
+
// change the generation in the await→bind gap while the process is
|
|
3877
|
+
// invisible.
|
|
3878
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3879
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3880
|
+
if (slot) slot.session = session;
|
|
3881
|
+
this.bindSession(channel, agentId, session);
|
|
3882
|
+
return session;
|
|
3883
|
+
}
|
|
3884
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
2561
3885
|
}
|
|
2562
|
-
|
|
3886
|
+
return null;
|
|
3887
|
+
} finally {
|
|
3888
|
+
ownHold?.();
|
|
2563
3889
|
}
|
|
2564
|
-
|
|
3890
|
+
}
|
|
3891
|
+
|
|
3892
|
+
/**
|
|
3893
|
+
* ADR-154, the shadow router's one exec: the only use it makes of an AI
|
|
3894
|
+
* account, and the only place that account's material exists for it.
|
|
3895
|
+
* Null when it cannot run, and the pass routes by rules.
|
|
3896
|
+
* - A leased task's: its account is chosen, and the exec registered where
|
|
3897
|
+
* a cleanup of that account waits, in one synchronous step before any of
|
|
3898
|
+
* its material is read; the registration settles when the exec does,
|
|
3899
|
+
* and nothing the material is in outlives this call (§6). A cleanup that
|
|
3900
|
+
* took the account first leaves it unchosen. The exec starts under the
|
|
3901
|
+
* engine supervisor, only on material the task still holds at the
|
|
3902
|
+
* provider's last moment.
|
|
3903
|
+
* - On the runner's own credentials: they are held from the account's
|
|
3904
|
+
* choice to the exec's end, and the exec is refused at the provider's
|
|
3905
|
+
* last moment once leaving them has begun or the channel has closed.
|
|
3906
|
+
*/
|
|
3907
|
+
private async shadowClassify(
|
|
3908
|
+
channel: Channel,
|
|
3909
|
+
agentId: string,
|
|
3910
|
+
kind: string,
|
|
3911
|
+
handle: TaskEnvironmentHandle<unknown>,
|
|
3912
|
+
argv: [string, ...string[]],
|
|
3913
|
+
scratchDir: string,
|
|
3914
|
+
timeoutMs: number,
|
|
3915
|
+
): Promise<{ exitCode: number | null; stdout: string } | null> {
|
|
3916
|
+
const request = (env: Record<string, string>, admit: () => void) => ({
|
|
3917
|
+
argv: agentCliArgv(handle, argv),
|
|
3918
|
+
projectEnvProfile: "task" as const,
|
|
3919
|
+
cwd: scratchDir,
|
|
3920
|
+
user: "node",
|
|
3921
|
+
env,
|
|
3922
|
+
timeoutMs,
|
|
3923
|
+
maxOutputBytes: 64 * 1024,
|
|
3924
|
+
admit,
|
|
3925
|
+
});
|
|
3926
|
+
if (!this.isLeasedChannel(channel)) {
|
|
3927
|
+
const running = whileOnOwnCredentials(async () => {
|
|
3928
|
+
const env = { ...(pickEngineAccount(kind, { scope: channel.engineAccountScope })?.execEnv ?? {}) };
|
|
3929
|
+
const r = await handle.exec(
|
|
3930
|
+
request(env, () => {
|
|
3931
|
+
if (!this.isActiveChannel(channel) || !onOwnCredentials()) throw new LeftOwnCredentialsError();
|
|
3932
|
+
}),
|
|
3933
|
+
);
|
|
3934
|
+
return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
|
|
3935
|
+
});
|
|
3936
|
+
if (running === null) return null;
|
|
3937
|
+
this.trackEventOperation(channel, running);
|
|
3938
|
+
return running;
|
|
3939
|
+
}
|
|
3940
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3941
|
+
if (!lease?.open) return null;
|
|
3942
|
+
const account = this.leasedAccountFor(channel, agentId, kind, lease);
|
|
3943
|
+
if (!account) return null;
|
|
3944
|
+
const identity: LeasedIdentity = {
|
|
3945
|
+
subject: "account",
|
|
3946
|
+
runEpoch: lease.assignment.runEpoch,
|
|
3947
|
+
credentialId: account.credentialId,
|
|
3948
|
+
revision: account.revision,
|
|
3949
|
+
version: account.version,
|
|
3950
|
+
};
|
|
3951
|
+
let settle!: () => void;
|
|
3952
|
+
const entry = {
|
|
3953
|
+
agentId: `shadow:${agentId}`,
|
|
3954
|
+
accountId: account.credentialId,
|
|
3955
|
+
runEpoch: identity.runEpoch,
|
|
3956
|
+
settled: new Promise<void>((resolve) => (settle = resolve)),
|
|
3957
|
+
};
|
|
3958
|
+
channel.leasedCreating.add(entry);
|
|
3959
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
3960
|
+
this.trackEventOperation(channel, entry.settled);
|
|
3961
|
+
try {
|
|
3962
|
+
const env = await this.leasedOneShotEnv(channel, lease, account, handle);
|
|
3963
|
+
if (!env || !this.leasedMaterialServed(channel, identity)) return null;
|
|
3964
|
+
const r = await this.leasedLaunchEnvironment(channel, null, identity, handle).exec(
|
|
3965
|
+
request(env, () => {
|
|
3966
|
+
if (!this.isActiveChannel(channel) || !this.leasedMaterialServed(channel, identity)) {
|
|
3967
|
+
throw new CredentialLeaseUnavailableError("the shadow classifier's account moved");
|
|
3968
|
+
}
|
|
3969
|
+
}),
|
|
3970
|
+
);
|
|
3971
|
+
return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
|
|
3972
|
+
} finally {
|
|
3973
|
+
settle();
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
|
|
3977
|
+
/** #cb3j0z: the task's runtime, for what only it can do in the task's
|
|
3978
|
+
* workspace without following its links (macOS); null without one. */
|
|
3979
|
+
private async workspaceRuntime(taskId: string): Promise<RuntimeNamespace | null> {
|
|
3980
|
+
const channel = this.channels.get(taskId);
|
|
3981
|
+
if (!channel || !this.isActiveChannel(channel)) return null;
|
|
3982
|
+
try {
|
|
3983
|
+
return runtimeNamespaceOf(await this.agentEnvironment(channel));
|
|
3984
|
+
} catch {
|
|
3985
|
+
return null;
|
|
3986
|
+
}
|
|
3987
|
+
}
|
|
3988
|
+
|
|
3989
|
+
/** Teardown drains `operation` before the task's runtime name can be
|
|
3990
|
+
* reused. */
|
|
3991
|
+
private trackEventOperation(channel: Channel, operation: Promise<unknown>): void {
|
|
3992
|
+
const settled = operation.then(
|
|
3993
|
+
() => undefined,
|
|
3994
|
+
() => undefined,
|
|
3995
|
+
);
|
|
3996
|
+
channel.eventOperations.add(settled);
|
|
3997
|
+
void settled.then(() => channel.eventOperations.delete(settled));
|
|
2565
3998
|
}
|
|
2566
3999
|
|
|
2567
4000
|
private hasDesiredLaunchSettings(
|
|
@@ -2808,6 +4241,9 @@ export class Orchestrator {
|
|
|
2808
4241
|
),
|
|
2809
4242
|
},
|
|
2810
4243
|
);
|
|
4244
|
+
// Credentials unavailable is this agent's alone: the gate's next
|
|
4245
|
+
// opening starts it; its siblings start now.
|
|
4246
|
+
if (!session && channel.leasedBlocked.has(agent.id)) continue;
|
|
2811
4247
|
if (!session) {
|
|
2812
4248
|
// The agents that did come up are usable; the reconcile pass keeps
|
|
2813
4249
|
// retrying this one. Do not leave the page covered for a session
|
|
@@ -3203,13 +4639,16 @@ export class Orchestrator {
|
|
|
3203
4639
|
// through this same command, and classifying them would poison the very
|
|
3204
4640
|
// data the shadow pass exists to collect.
|
|
3205
4641
|
if (channel.routingShadow && origin === "human") {
|
|
3206
|
-
|
|
4642
|
+
// Its exec runs in the task's runtime: teardown drains it like any
|
|
4643
|
+
// other external operation before the runtime's name can be reused.
|
|
4644
|
+
const pass = this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
|
|
3207
4645
|
console.warn(
|
|
3208
4646
|
`[routing] shadow pass failed for ${channel.taskId}/${agentId}: ${
|
|
3209
4647
|
err instanceof Error ? err.message : String(err)
|
|
3210
4648
|
}`,
|
|
3211
4649
|
);
|
|
3212
4650
|
});
|
|
4651
|
+
this.trackEventOperation(channel, pass);
|
|
3213
4652
|
}
|
|
3214
4653
|
return { ok: true };
|
|
3215
4654
|
}
|
|
@@ -3219,7 +4658,10 @@ export class Orchestrator {
|
|
|
3219
4658
|
* one-shot executes INSIDE the task environment as the task user, with the
|
|
3220
4659
|
* same engine account the agent runs on (so the user's own subscription
|
|
3221
4660
|
* pays for routing, never Uai), then the result rides a host event. Bounded
|
|
3222
|
-
* by a hard timeout so a hung CLI
|
|
4661
|
+
* by a hard timeout, the task's own files read included, so a hung CLI or
|
|
4662
|
+
* read cannot pile up passes. ADR-154: the pass itself holds no credential
|
|
4663
|
+
* (its reads are the task's own files, which take as long as the task
|
|
4664
|
+
* makes them); only its one exec does (`shadowClassify`).
|
|
3223
4665
|
*/
|
|
3224
4666
|
private async shadowRouteDelivery(channel: Channel, agentId: string, text: string): Promise<void> {
|
|
3225
4667
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
@@ -3236,39 +4678,44 @@ export class Orchestrator {
|
|
|
3236
4678
|
// Rung 1 hosts the classifier on the agent's own engine; the Managed
|
|
3237
4679
|
// Engine delegates to claude/codex, so borrow the delegate's account.
|
|
3238
4680
|
const hostEngine = agent.kind === "uai" ? (agent.uaiProvider === "openai" ? "codex" : "claude") : agent.kind;
|
|
3239
|
-
const account = pickEngineAccount(hostEngine, { scope: channel.engineAccountScope });
|
|
3240
|
-
const env: Record<string, string> = { ...(account?.execEnv ?? {}) };
|
|
3241
|
-
const context =
|
|
3242
|
-
handle && canRead
|
|
3243
|
-
? Buffer.from((await handle.readWorkspaceFile(`${scratchDir}/chat.md`).catch(() => null)) ?? new Uint8Array())
|
|
3244
|
-
.toString("utf8")
|
|
3245
|
-
.slice(-1200)
|
|
3246
|
-
: "";
|
|
3247
4681
|
const timeoutMs = 45_000;
|
|
4682
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
4683
|
+
// Past its bound the pass is given up, and what is left of it (a read
|
|
4684
|
+
// the task held) launches nothing.
|
|
4685
|
+
let abandoned = false;
|
|
3248
4686
|
const result = await Promise.race([
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
4687
|
+
(async () => {
|
|
4688
|
+
const context =
|
|
4689
|
+
handle && canRead
|
|
4690
|
+
? Buffer.from(
|
|
4691
|
+
(await handle
|
|
4692
|
+
.readWorkspaceFile(`${scratchDir}/chat.md`, { maxBytes: SHADOW_CONTEXT_BYTES, from: "end" })
|
|
4693
|
+
.catch(() => null)) ?? new Uint8Array(),
|
|
4694
|
+
)
|
|
4695
|
+
.toString("utf8")
|
|
4696
|
+
.slice(-1200)
|
|
4697
|
+
: "";
|
|
4698
|
+
return shadowRoute(text, context, hostEngine, {
|
|
4699
|
+
scratchDir,
|
|
4700
|
+
exec: (argv) =>
|
|
4701
|
+
handle && canExec && !abandoned
|
|
4702
|
+
? this.shadowClassify(channel, agentId, hostEngine, handle, argv, scratchDir, timeoutMs)
|
|
4703
|
+
: Promise.resolve(null),
|
|
4704
|
+
readFile: async (path) => {
|
|
4705
|
+
if (!handle || !canRead) return null;
|
|
4706
|
+
const bytes = await handle.readWorkspaceFile(path, { maxBytes: SHADOW_OUTPUT_BYTES }).catch(() => null);
|
|
4707
|
+
return bytes ? Buffer.from(bytes).toString("utf8") : null;
|
|
4708
|
+
},
|
|
4709
|
+
});
|
|
4710
|
+
})(),
|
|
4711
|
+
new Promise<null>((resolve) => {
|
|
4712
|
+
timer = setTimeout(() => {
|
|
4713
|
+
abandoned = true;
|
|
4714
|
+
resolve(null);
|
|
4715
|
+
}, timeoutMs + 5_000);
|
|
3269
4716
|
}),
|
|
3270
|
-
new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs + 5_000)),
|
|
3271
4717
|
]);
|
|
4718
|
+
clearTimeout(timer);
|
|
3272
4719
|
if (!result) {
|
|
3273
4720
|
console.warn(`[routing] shadow pass timed out for ${channel.taskId}/${agentId}`);
|
|
3274
4721
|
return;
|
|
@@ -3507,14 +4954,21 @@ export class Orchestrator {
|
|
|
3507
4954
|
if (ownsBrokerSlot) channel.spawning.add(agentId);
|
|
3508
4955
|
let requested = false;
|
|
3509
4956
|
try {
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
4957
|
+
// ADR-154 §7: a leased account is refreshed by the cloud's
|
|
4958
|
+
// broker, through the task's lease, never by this runner.
|
|
4959
|
+
const lease = this.isLeasedChannel(channel) ? leasedEnginesFor(channel.taskId) : undefined;
|
|
4960
|
+
requested = this.isLeasedChannel(channel)
|
|
4961
|
+
? lease !== undefined &&
|
|
4962
|
+
accountId !== null &&
|
|
4963
|
+
(await refreshLeasedEngineAccount(lease.assignment, accountId)) === "refreshed"
|
|
4964
|
+
: await requestCredentialRefresh({
|
|
4965
|
+
kind: agent.kind,
|
|
4966
|
+
accountId:
|
|
4967
|
+
accountId && !accountId.startsWith("default:")
|
|
4968
|
+
? accountId
|
|
4969
|
+
: null,
|
|
4970
|
+
scope: channel.engineAccountScope,
|
|
4971
|
+
});
|
|
3518
4972
|
} catch (err) {
|
|
3519
4973
|
console.warn(
|
|
3520
4974
|
`[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
|
|
@@ -3825,10 +5279,16 @@ export class Orchestrator {
|
|
|
3825
5279
|
|
|
3826
5280
|
channel.spawning.add(agentId);
|
|
3827
5281
|
try {
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
5282
|
+
// ADR-154 §6: never the runner owner's own ~/.claude.json into a task on
|
|
5283
|
+
// cloud credentials; Claude writes its own again when it starts. A
|
|
5284
|
+
// repair on the runner's own credentials is counted: leaving them waits
|
|
5285
|
+
// for it, and none starts once leaving has begun.
|
|
5286
|
+
const repairing = this.isLeasedChannel(channel)
|
|
5287
|
+
? null
|
|
5288
|
+
: whileOnOwnCredentials(() =>
|
|
5289
|
+
repairClaudeConfigInContainer(channel.containerName, channel.agentHandle ?? undefined),
|
|
5290
|
+
);
|
|
5291
|
+
const restored = repairing === null ? false : await repairing;
|
|
3832
5292
|
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3833
5293
|
this.reconcileErroredRosterChange(
|
|
3834
5294
|
channel,
|
|
@@ -3896,7 +5356,7 @@ export class Orchestrator {
|
|
|
3896
5356
|
);
|
|
3897
5357
|
const boundAccountId = channel.accountByAgent.get(agentId);
|
|
3898
5358
|
const boundAccount = boundAccountId
|
|
3899
|
-
?
|
|
5359
|
+
? this.engineAccounts(channel, agent.kind).find(
|
|
3900
5360
|
(account) => account.id === boundAccountId,
|
|
3901
5361
|
)
|
|
3902
5362
|
: undefined;
|
|
@@ -3942,9 +5402,11 @@ export class Orchestrator {
|
|
|
3942
5402
|
}
|
|
3943
5403
|
this.startNextPendingPrompt(channel, agentId);
|
|
3944
5404
|
|
|
3945
|
-
const message =
|
|
3946
|
-
? `${agentId} restarted — config file was missing in the container
|
|
3947
|
-
:
|
|
5405
|
+
const message = this.isLeasedChannel(channel)
|
|
5406
|
+
? `${agentId} restarted — config file was missing in the container; Claude writes its own again.`
|
|
5407
|
+
: restored
|
|
5408
|
+
? `${agentId} restarted — config file was missing in the container, restored from the host.`
|
|
5409
|
+
: `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
|
|
3948
5410
|
channel.retiredTurnBoundaries.delete(agentId);
|
|
3949
5411
|
this.emitHost({
|
|
3950
5412
|
kind: "agent.exit",
|
|
@@ -4002,7 +5464,7 @@ export class Orchestrator {
|
|
|
4002
5464
|
// A concurrent terminal event for the same session is already being
|
|
4003
5465
|
// handled by the path that owns this spawn slot.
|
|
4004
5466
|
if (channel.spawning.has(agentId)) return true;
|
|
4005
|
-
if (
|
|
5467
|
+
if (this.engineAccounts(channel, agent.kind).length < 2) {
|
|
4006
5468
|
return false;
|
|
4007
5469
|
}
|
|
4008
5470
|
|
|
@@ -4012,10 +5474,11 @@ export class Orchestrator {
|
|
|
4012
5474
|
|
|
4013
5475
|
const current = channel.accountByAgent.get(agentId);
|
|
4014
5476
|
if (current) cooldownEngineAccount(current);
|
|
4015
|
-
const next =
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
5477
|
+
const next = this.pickAccount(
|
|
5478
|
+
channel,
|
|
5479
|
+
agent.kind,
|
|
5480
|
+
current ? new Set([current]) : undefined,
|
|
5481
|
+
);
|
|
4019
5482
|
if (!next) return false;
|
|
4020
5483
|
|
|
4021
5484
|
const task = getHostTask(channel.taskId);
|
|
@@ -4447,7 +5910,7 @@ export class Orchestrator {
|
|
|
4447
5910
|
return;
|
|
4448
5911
|
}
|
|
4449
5912
|
|
|
4450
|
-
const accounts =
|
|
5913
|
+
const accounts = this.engineAccounts(channel, agent.kind);
|
|
4451
5914
|
const account =
|
|
4452
5915
|
accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
|
|
4453
5916
|
accounts[0];
|
|
@@ -4482,14 +5945,19 @@ export class Orchestrator {
|
|
|
4482
5945
|
async refreshEngineAccountCopies(kind: string): Promise<void> {
|
|
4483
5946
|
for (const channel of this.channels.values()) {
|
|
4484
5947
|
if (!this.isActiveChannel(channel)) continue;
|
|
5948
|
+
// ADR-154: a leased task's accounts change through its lease only.
|
|
5949
|
+
if (this.isLeasedChannel(channel)) continue;
|
|
4485
5950
|
if (!channel.roster.some((agent) => agent.kind === kind)) continue;
|
|
4486
5951
|
try {
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
5952
|
+
// Only while the runner is on its own credentials, counted.
|
|
5953
|
+
await whileOnOwnCredentials(() =>
|
|
5954
|
+
provisionEngineAccounts(
|
|
5955
|
+
channel.containerName,
|
|
5956
|
+
[kind],
|
|
5957
|
+
{},
|
|
5958
|
+
channel.agentHandle ?? undefined,
|
|
5959
|
+
channel.engineAccountScope,
|
|
5960
|
+
),
|
|
4493
5961
|
);
|
|
4494
5962
|
} catch (err) {
|
|
4495
5963
|
console.warn(
|
|
@@ -4504,6 +5972,7 @@ export class Orchestrator {
|
|
|
4504
5972
|
async refreshEngineAccountAgents(kind: string): Promise<void> {
|
|
4505
5973
|
for (const channel of this.channels.values()) {
|
|
4506
5974
|
if (!this.isActiveChannel(channel)) continue;
|
|
5975
|
+
if (this.isLeasedChannel(channel)) continue;
|
|
4507
5976
|
const agentIds = channel.roster
|
|
4508
5977
|
.filter((agent) => agent.kind === kind)
|
|
4509
5978
|
.map((agent) => agent.id);
|
|
@@ -4561,7 +6030,16 @@ export class Orchestrator {
|
|
|
4561
6030
|
// can repopulate an orphan channel while its sessions close.
|
|
4562
6031
|
ch.closed = true;
|
|
4563
6032
|
this.channels.delete(taskId);
|
|
6033
|
+
// ADR-154 §6: nothing its agents' spawns were admitted for starts after:
|
|
6034
|
+
// the close ends only once every retirement of the task under way, its
|
|
6035
|
+
// own and any an earlier event began, is done or escalated.
|
|
6036
|
+
for (const [agentId, slot] of [...ch.leasedLaunches]) void this.retireLeasedLaunch(ch, agentId, slot);
|
|
6037
|
+
const retirements = this.leasedRetirementsUnderWay(taskId);
|
|
4564
6038
|
this.clearBrowserRetry(ch);
|
|
6039
|
+
if (ch.leasedSyncRetry) {
|
|
6040
|
+
clearTimeout(ch.leasedSyncRetry);
|
|
6041
|
+
ch.leasedSyncRetry = null;
|
|
6042
|
+
}
|
|
4565
6043
|
ch.browserStaleSessions.clear();
|
|
4566
6044
|
ch.rosterSessionChanges.clear();
|
|
4567
6045
|
ch.browserPendingStaleSessions.clear();
|
|
@@ -4586,6 +6064,7 @@ export class Orchestrator {
|
|
|
4586
6064
|
ch.sessionsReady,
|
|
4587
6065
|
...ch.factoryOperations,
|
|
4588
6066
|
...ch.eventOperations,
|
|
6067
|
+
...retirements,
|
|
4589
6068
|
].filter((promise): promise is Promise<unknown> => promise !== null);
|
|
4590
6069
|
const sessions = [...ch.sessions.values()];
|
|
4591
6070
|
ch.sessions.clear();
|
|
@@ -4683,28 +6162,103 @@ export class Orchestrator {
|
|
|
4683
6162
|
return this.#channelActivity.get(taskId) ?? null;
|
|
4684
6163
|
}
|
|
4685
6164
|
|
|
4686
|
-
async stopTask(
|
|
6165
|
+
async stopTask(
|
|
6166
|
+
taskId: string,
|
|
6167
|
+
options: {
|
|
6168
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6169
|
+
shouldVeto?: () => boolean;
|
|
6170
|
+
/** ADR-154 §6: the closed run's credentials cleaned up, its sessions
|
|
6171
|
+
* closed and its runtime still up for the cleanup to reach, before
|
|
6172
|
+
* the runtime stops and Stop is acknowledged. */
|
|
6173
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6174
|
+
} = {},
|
|
6175
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
4687
6176
|
return this.runTaskLifecycle(taskId, () =>
|
|
4688
|
-
this.stopTaskWithinLifecycle(taskId),
|
|
6177
|
+
this.stopTaskWithinLifecycle(taskId, options),
|
|
4689
6178
|
);
|
|
4690
6179
|
}
|
|
4691
6180
|
|
|
6181
|
+
/** `stopTask` for a caller that already holds this task's lifecycle slot
|
|
6182
|
+
* (runTaskLifecycle), so that what it checks before the stop and does
|
|
6183
|
+
* after it share the slot: ADR-154 §13's retirement re-proves the engine
|
|
6184
|
+
* first, and removes the runtime after. */
|
|
6185
|
+
async stopTaskInHeldLifecycle(
|
|
6186
|
+
taskId: string,
|
|
6187
|
+
options: {
|
|
6188
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6189
|
+
shouldVeto?: () => boolean;
|
|
6190
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6191
|
+
/** After the channel drained and before anything touches the
|
|
6192
|
+
* container runtime (previews, the runtime's stop): why the caller's
|
|
6193
|
+
* proof of the daemon no longer holds, or null. A reason stops the
|
|
6194
|
+
* Stop there, mutating nothing. */
|
|
6195
|
+
beforeRuntimeMutation?: () => Promise<string | null>;
|
|
6196
|
+
} = {},
|
|
6197
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
6198
|
+
return this.stopTaskWithinLifecycle(taskId, options);
|
|
6199
|
+
}
|
|
6200
|
+
|
|
4692
6201
|
private async stopTaskWithinLifecycle(
|
|
4693
6202
|
taskId: string,
|
|
4694
|
-
|
|
6203
|
+
options: {
|
|
6204
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6205
|
+
shouldVeto?: () => boolean;
|
|
6206
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6207
|
+
beforeRuntimeMutation?: () => Promise<string | null>;
|
|
6208
|
+
},
|
|
6209
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
6210
|
+
// The activity check belongs in the same lifecycle slot as assignment
|
|
6211
|
+
// closure and runtime stop. A turn may become active while Stop waits for
|
|
6212
|
+
// earlier lifecycle work; veto it before closing any v1 authority.
|
|
6213
|
+
if (options.shouldVeto?.()) return { ok: true, vetoed: true };
|
|
6214
|
+
let assignmentAdmission: TaskStopAssignmentAdmission = { kind: "legacy" };
|
|
6215
|
+
try {
|
|
6216
|
+
assignmentAdmission =
|
|
6217
|
+
options.admitAssignment?.() ?? ({ kind: "legacy" } as const);
|
|
6218
|
+
} catch (error) {
|
|
6219
|
+
return {
|
|
6220
|
+
ok: false,
|
|
6221
|
+
error: (error instanceof Error ? error.message : String(error)).slice(
|
|
6222
|
+
0,
|
|
6223
|
+
200,
|
|
6224
|
+
),
|
|
6225
|
+
};
|
|
6226
|
+
}
|
|
6227
|
+
if (assignmentAdmission.kind === "stale") {
|
|
6228
|
+
return {
|
|
6229
|
+
ok: false,
|
|
6230
|
+
error: "task assignment is no longer current",
|
|
6231
|
+
assignmentStale: true,
|
|
6232
|
+
};
|
|
6233
|
+
}
|
|
6234
|
+
const assignmentClosed = assignmentAdmission.kind === "closed";
|
|
4695
6235
|
if (!backgroundContainerWorkAllowed()) {
|
|
4696
6236
|
return { ok: false, error: "container runtime is not operational" };
|
|
4697
6237
|
}
|
|
4698
6238
|
const task = getHostTask(taskId);
|
|
4699
6239
|
if (!task) return { ok: false, error: "unknown task" };
|
|
4700
|
-
|
|
4701
|
-
|
|
6240
|
+
// A runtime is named by its persisted locator, a legacy row by its
|
|
6241
|
+
// Compose project. ADR-154: a start interrupted after recording its
|
|
6242
|
+
// runtime's locator keeps it, with no Compose project, for Stop to reach.
|
|
6243
|
+
if (!task.environmentLocator && !task.composeProject) {
|
|
6244
|
+
return { ok: false, error: "task has no running stack" };
|
|
6245
|
+
}
|
|
4702
6246
|
// Revoke env/proxy access before a potentially long session drain, then
|
|
4703
6247
|
// close sessions so nothing races to respawn them mid-stop.
|
|
4704
6248
|
const suspendedGateway = suspendTaskGatewayTask(taskId);
|
|
4705
6249
|
// ADR-153: no longer served; end its open proxy connections too.
|
|
4706
6250
|
activeSecretsProxy()?.runtime.stopTask(taskId);
|
|
4707
6251
|
await this.closeChannel(taskId);
|
|
6252
|
+
// ADR-154 §13: a proof of the daemon taken before the drain may have
|
|
6253
|
+
// expired during it; nothing below may reach a replaced one.
|
|
6254
|
+
const stale = await options.beforeRuntimeMutation?.();
|
|
6255
|
+
if (stale !== undefined && stale !== null) {
|
|
6256
|
+
if (!assignmentClosed) {
|
|
6257
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6258
|
+
this.allowChannel(taskId);
|
|
6259
|
+
}
|
|
6260
|
+
return { ok: false, error: stale.slice(0, 200) };
|
|
6261
|
+
}
|
|
4708
6262
|
try {
|
|
4709
6263
|
await stopPreviewSidecars(taskId);
|
|
4710
6264
|
} catch (err) {
|
|
@@ -4714,6 +6268,22 @@ export class Orchestrator {
|
|
|
4714
6268
|
}`,
|
|
4715
6269
|
);
|
|
4716
6270
|
}
|
|
6271
|
+
// ADR-154 §6: Stop is acknowledged only once the closed run's
|
|
6272
|
+
// credentials are cleaned up, while its runtime still runs for the
|
|
6273
|
+
// cleanup to reach. A failed cleanup fails the stop; the cloud retries,
|
|
6274
|
+
// and its closed run stays closed meanwhile.
|
|
6275
|
+
if (assignmentAdmission.kind === "closed" && options.closeCredentials) {
|
|
6276
|
+
try {
|
|
6277
|
+
await options.closeCredentials(assignmentAdmission.runEpoch);
|
|
6278
|
+
} catch (err) {
|
|
6279
|
+
return {
|
|
6280
|
+
ok: false,
|
|
6281
|
+
error: `the task's credentials could not be cleaned up: ${
|
|
6282
|
+
(err instanceof Error ? err.message : String(err)).slice(0, 160)
|
|
6283
|
+
}`,
|
|
6284
|
+
};
|
|
6285
|
+
}
|
|
6286
|
+
}
|
|
4717
6287
|
// Provider-neutral resumable stop. A row that reaches this point with
|
|
4718
6288
|
// runtime resources always carries a proven locator (legacy rows cross
|
|
4719
6289
|
// the adoption fence before the runtime becomes operational), so a
|
|
@@ -4724,14 +6294,18 @@ export class Orchestrator {
|
|
|
4724
6294
|
try {
|
|
4725
6295
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
4726
6296
|
if (!environment) {
|
|
4727
|
-
if (
|
|
4728
|
-
|
|
6297
|
+
if (!assignmentClosed) {
|
|
6298
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6299
|
+
this.allowChannel(taskId);
|
|
6300
|
+
}
|
|
4729
6301
|
return { ok: false, error: "task has no persisted environment" };
|
|
4730
6302
|
}
|
|
4731
6303
|
await environment.stop();
|
|
4732
6304
|
} catch (err) {
|
|
4733
|
-
if (
|
|
4734
|
-
|
|
6305
|
+
if (!assignmentClosed) {
|
|
6306
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6307
|
+
this.allowChannel(taskId);
|
|
6308
|
+
}
|
|
4735
6309
|
return {
|
|
4736
6310
|
ok: false,
|
|
4737
6311
|
error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
|
|
@@ -4757,7 +6331,9 @@ export class Orchestrator {
|
|
|
4757
6331
|
// Nothing serves the task's env now: no value of it stays in memory.
|
|
4758
6332
|
forgetTaskEnvSnapshot(taskId);
|
|
4759
6333
|
clearTaskGatewayAcl(taskId);
|
|
4760
|
-
return
|
|
6334
|
+
return assignmentAdmission.kind === "closed"
|
|
6335
|
+
? { ok: true, stoppedRunEpoch: assignmentAdmission.runEpoch }
|
|
6336
|
+
: { ok: true };
|
|
4761
6337
|
}
|
|
4762
6338
|
}
|
|
4763
6339
|
|
|
@@ -6001,8 +7577,26 @@ export type TaskRuntimeContainerInspection =
|
|
|
6001
7577
|
|
|
6002
7578
|
export interface WritableRuntimeQuarantineResult {
|
|
6003
7579
|
verdict: "ready" | "retry" | "stale";
|
|
6004
|
-
/**
|
|
6005
|
-
|
|
7580
|
+
/**
|
|
7581
|
+
* Known rows whose local `stopped` verdict must be published or replayed.
|
|
7582
|
+
* The epoch is captured while the task lifecycle slot is still held and is
|
|
7583
|
+
* deliberately absent for a legacy assignment.
|
|
7584
|
+
*/
|
|
7585
|
+
stoppedTasks: StoppedTaskStatusTarget[];
|
|
7586
|
+
}
|
|
7587
|
+
|
|
7588
|
+
export interface StoppedTaskStatusTarget {
|
|
7589
|
+
taskId: string;
|
|
7590
|
+
runEpoch?: number;
|
|
7591
|
+
}
|
|
7592
|
+
|
|
7593
|
+
function captureStoppedTaskStatusTarget(
|
|
7594
|
+
taskId: string,
|
|
7595
|
+
): StoppedTaskStatusTarget {
|
|
7596
|
+
const assignment = closeCurrentTaskRunAssignmentForStoppedRuntime(taskId);
|
|
7597
|
+
return assignment.kind === "legacy"
|
|
7598
|
+
? { taskId }
|
|
7599
|
+
: { taskId, runEpoch: assignment.runEpoch };
|
|
6006
7600
|
}
|
|
6007
7601
|
|
|
6008
7602
|
function runtimeWriterCandidate(
|
|
@@ -6390,19 +7984,19 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6390
7984
|
return quarantineWritableAppleRuntimeContainers(options);
|
|
6391
7985
|
}
|
|
6392
7986
|
const isCurrent = options.isCurrent ?? (() => true);
|
|
6393
|
-
const
|
|
6394
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
7987
|
+
const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
|
|
7988
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
6395
7989
|
|
|
6396
7990
|
const listed = await listRuntimeWriterCandidates();
|
|
6397
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
7991
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
6398
7992
|
if (listed.kind === "unreachable") {
|
|
6399
|
-
return { verdict: "retry",
|
|
7993
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
6400
7994
|
}
|
|
6401
7995
|
let resolved = !listed.ambiguous;
|
|
6402
7996
|
|
|
6403
7997
|
for (const candidate of listed.candidates) {
|
|
6404
7998
|
if (!isCurrent()) {
|
|
6405
|
-
return { verdict: "stale",
|
|
7999
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
6406
8000
|
}
|
|
6407
8001
|
await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
|
|
6408
8002
|
if (!isCurrent()) return;
|
|
@@ -6459,7 +8053,10 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6459
8053
|
// Always re-publish the stopped verdict. A prior process may have
|
|
6460
8054
|
// committed SQLite and crashed before its websocket frame left; the
|
|
6461
8055
|
// cloud could otherwise remain ghost-running indefinitely.
|
|
6462
|
-
|
|
8056
|
+
stoppedTasks.set(
|
|
8057
|
+
candidate.taskId,
|
|
8058
|
+
captureStoppedTaskStatusTarget(candidate.taskId),
|
|
8059
|
+
);
|
|
6463
8060
|
db_setStatus(candidate.taskId, "stopped", {
|
|
6464
8061
|
codeServerPort: null,
|
|
6465
8062
|
previewPorts: "[]",
|
|
@@ -6488,11 +8085,13 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6488
8085
|
}
|
|
6489
8086
|
|
|
6490
8087
|
if (!isCurrent()) {
|
|
6491
|
-
return { verdict: "stale",
|
|
8088
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
6492
8089
|
}
|
|
6493
8090
|
return {
|
|
6494
8091
|
verdict: resolved ? "ready" : "retry",
|
|
6495
|
-
|
|
8092
|
+
stoppedTasks: [...stoppedTasks.values()].sort((a, b) =>
|
|
8093
|
+
a.taskId.localeCompare(b.taskId),
|
|
8094
|
+
),
|
|
6496
8095
|
};
|
|
6497
8096
|
}
|
|
6498
8097
|
|
|
@@ -7282,25 +8881,25 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7282
8881
|
scope?: "contract" | "volume-holders";
|
|
7283
8882
|
}): Promise<WritableRuntimeQuarantineResult> {
|
|
7284
8883
|
const isCurrent = options.isCurrent ?? (() => true);
|
|
7285
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
8884
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
7286
8885
|
const binding = appleContainerRuntimeBinding();
|
|
7287
|
-
if (binding === null) return { verdict: "retry",
|
|
8886
|
+
if (binding === null) return { verdict: "retry", stoppedTasks: [] };
|
|
7288
8887
|
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
7289
8888
|
const listed = await cli(["list", "--all", "--format", "json"], {
|
|
7290
8889
|
timeoutMs: 30_000,
|
|
7291
8890
|
maxOutputBytes: 8 * 1024 * 1024,
|
|
7292
8891
|
});
|
|
7293
8892
|
if (listed.status !== 0 || listed.truncated === true) {
|
|
7294
|
-
return { verdict: "retry",
|
|
8893
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
7295
8894
|
}
|
|
7296
8895
|
let parsed: unknown;
|
|
7297
8896
|
try {
|
|
7298
8897
|
parsed = JSON.parse(listed.stdout);
|
|
7299
8898
|
} catch {
|
|
7300
|
-
return { verdict: "retry",
|
|
8899
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
7301
8900
|
}
|
|
7302
|
-
if (!Array.isArray(parsed)) return { verdict: "retry",
|
|
7303
|
-
const
|
|
8901
|
+
if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTasks: [] };
|
|
8902
|
+
const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
|
|
7304
8903
|
let resolved = true;
|
|
7305
8904
|
|
|
7306
8905
|
const stopAndProve = async (id: string): Promise<boolean> => {
|
|
@@ -7357,7 +8956,9 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7357
8956
|
}>;
|
|
7358
8957
|
};
|
|
7359
8958
|
}>) {
|
|
7360
|
-
if (!isCurrent())
|
|
8959
|
+
if (!isCurrent()) {
|
|
8960
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
8961
|
+
}
|
|
7361
8962
|
const id = item?.configuration?.id;
|
|
7362
8963
|
if (typeof id !== "string" || id.length === 0) {
|
|
7363
8964
|
resolved = false;
|
|
@@ -7410,20 +9011,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7410
9011
|
}
|
|
7411
9012
|
const task = getHostTask(taskId);
|
|
7412
9013
|
if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
|
|
9014
|
+
const stoppedTask = captureStoppedTaskStatusTarget(taskId);
|
|
7413
9015
|
db_setStatus(taskId, "stopped", {
|
|
7414
9016
|
codeServerPort: null,
|
|
7415
9017
|
previewPorts: "[]",
|
|
7416
9018
|
});
|
|
7417
|
-
|
|
9019
|
+
stoppedTasks.set(taskId, stoppedTask);
|
|
7418
9020
|
}
|
|
7419
9021
|
// Revoke live access with the environment, like the Docker sweep.
|
|
7420
9022
|
clearTaskGatewayAcl(taskId);
|
|
7421
9023
|
});
|
|
7422
9024
|
}
|
|
7423
|
-
if (!isCurrent())
|
|
9025
|
+
if (!isCurrent()) {
|
|
9026
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
9027
|
+
}
|
|
7424
9028
|
return {
|
|
7425
9029
|
verdict: resolved ? "ready" : "retry",
|
|
7426
|
-
|
|
9030
|
+
stoppedTasks: [...stoppedTasks.values()],
|
|
7427
9031
|
};
|
|
7428
9032
|
}
|
|
7429
9033
|
|
|
@@ -7477,7 +9081,7 @@ async function materializeAgentCli(
|
|
|
7477
9081
|
return false;
|
|
7478
9082
|
}
|
|
7479
9083
|
}
|
|
7480
|
-
return writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
|
|
9084
|
+
return await writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
|
|
7481
9085
|
}
|
|
7482
9086
|
|
|
7483
9087
|
async function recoverPersistedTaskEnvironment(
|
|
@@ -8604,7 +10208,9 @@ async function recoverOneDockerTask(
|
|
|
8604
10208
|
// reinject sweep only targets containers already running, so a container
|
|
8605
10209
|
// recovered here would otherwise keep whatever it held when it exited —
|
|
8606
10210
|
// possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
|
|
8607
|
-
// by the inject itself and must not abort the task's recovery.
|
|
10211
|
+
// by the inject itself and must not abort the task's recovery. ADR-154:
|
|
10212
|
+
// the runner owner's own sign-in, so only while the runner is on its own
|
|
10213
|
+
// credentials (the inject fences itself).
|
|
8608
10214
|
await injectCodexIntoContainer(containerName);
|
|
8609
10215
|
if (!isCurrent()) return true;
|
|
8610
10216
|
// Establish the task owner's GitHub transport BEFORE uai-init performs any
|