@runuai/host 0.9.150 → 0.9.152
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/db/migrations/0027_cloud_credential_foundation.sql +51 -0
- package/db/migrations/0028_leased_engine_runtimes.sql +11 -0
- package/db/migrations/meta/_journal.json +14 -0
- package/db/schema.ts +188 -0
- package/lib/agent-cli.ts +11 -13
- package/lib/agent.ts +177 -7
- package/lib/agents/durable-proc.ts +56 -44
- package/lib/agents/registry.ts +19 -3
- package/lib/agents/session-state.ts +23 -10
- package/lib/agents/transport.ts +101 -69
- package/lib/apple-one-off.ts +277 -0
- package/lib/apple-runtime-recycle.ts +4 -3
- package/lib/attachments.ts +19 -17
- package/lib/capture-flow.ts +319 -0
- package/lib/cloud-credential-cleanup-adapter.ts +123 -0
- package/lib/cloud-credential-host.ts +512 -0
- package/lib/cloud-credential-leases.ts +2544 -0
- package/lib/cloud-credential-runtime.ts +1035 -0
- package/lib/codex-auth.ts +21 -1
- package/lib/container-runtime.ts +76 -0
- package/lib/credential-cleanup-inventory.ts +405 -0
- package/lib/credential-consumers/engine-anchor.ts +1880 -0
- package/lib/credential-consumers/engines.ts +773 -0
- package/lib/credential-consumers/github.ts +594 -0
- package/lib/credential-consumers/mcp.ts +384 -0
- package/lib/credential-consumers/task-files.ts +127 -0
- package/lib/credential-store-pickup.ts +15 -2
- package/lib/credential-store.ts +361 -33
- package/lib/credentials-activation-authority.ts +546 -0
- package/lib/credentials-purge.ts +1006 -0
- package/lib/db.ts +11 -0
- package/lib/docker-exec.ts +3 -1
- package/lib/engine-accounts.ts +42 -32
- package/lib/engine-connect.ts +47 -2
- package/lib/engine-credential-env.ts +50 -0
- package/lib/engine-login.ts +83 -1
- package/lib/env-file.ts +62 -8
- package/lib/env.ts +5 -0
- package/lib/fs-aliases.ts +49 -0
- package/lib/git-clone-credentials.ts +334 -0
- package/lib/git-diff.ts +219 -177
- package/lib/git-identity.ts +59 -14
- package/lib/github-origin.ts +28 -0
- package/lib/github-tokens.ts +56 -21
- package/lib/host-env.ts +17 -4
- package/lib/launch-mode.ts +25 -0
- package/lib/legacy-runtime-retirement.ts +388 -0
- package/lib/machine-exec.ts +14 -3
- package/lib/machine-provider-aws.ts +50 -8
- package/lib/machine-provider-local.ts +4 -0
- package/lib/machine-provider.ts +4 -0
- package/lib/managed-operation-drain.ts +8 -0
- package/lib/managed-runtime.ts +371 -7
- package/lib/mcp-capture.ts +860 -0
- package/lib/mcp-connections.ts +8 -1
- package/lib/mcp-gateway.ts +168 -10
- package/lib/obs.ts +2 -6
- package/lib/orchestrator.ts +1837 -224
- package/lib/person-capture.ts +327 -0
- package/lib/project-env-legacy-tags.ts +130 -0
- package/lib/project-env-task-replica.ts +1752 -0
- package/lib/runner-boot.ts +10 -0
- package/lib/runtime-activation-tail.ts +50 -3
- package/lib/runtime-archive.ts +68 -1
- package/lib/runtime-reconnect.ts +86 -0
- package/lib/runtime-state.ts +429 -13
- package/lib/safe-http.ts +364 -0
- package/lib/secrets-proxy/task-trust.ts +34 -3
- package/lib/sidecar-resume.ts +18 -5
- package/lib/skills.ts +47 -8
- package/lib/standard-image.ts +157 -103
- package/lib/stopped-task-status-queue.ts +68 -10
- package/lib/task-credential-cleanup.ts +525 -0
- package/lib/task-diff-sandbox.ts +464 -0
- package/lib/task-diff.ts +66 -163
- package/lib/task-env-source.ts +68 -14
- package/lib/task-environment/anchor.ts +31 -0
- package/lib/task-environment/apple-container.ts +69 -28
- package/lib/task-environment/docker.ts +101 -20
- package/lib/task-environment/machine-environment.ts +61 -0
- package/lib/task-environment/machine-task-up.ts +459 -96
- package/lib/task-environment/machine.ts +101 -11
- package/lib/task-environment/self.ts +108 -50
- package/lib/task-environment/types.ts +89 -5
- package/lib/task-environment/workspace-files.ts +650 -25
- package/lib/task-environment/workspace-runtime.fixture.ts +50 -0
- package/lib/task-gateway-recovery.ts +4 -1
- package/lib/task-gateway-state.ts +15 -0
- package/lib/task-machine-swap-safety.ts +729 -0
- package/lib/token-broker-adopt.ts +344 -0
- package/lib/token-broker-lambda.ts +56 -2
- package/lib/token-broker-local.ts +28 -4
- package/lib/token-broker-revision.ts +310 -0
- package/lib/token-broker-ssm.ts +368 -28
- package/lib/token-lanes/openai-oauth.ts +7 -3
- package/lib/token-lanes/xai-oauth.ts +7 -3
- package/lib/transcript.ts +9 -5
- package/package.json +6 -2
- package/scripts/agent/task-up.sh +242 -100
- package/src/cloud-credentials.ts +2399 -0
- package/src/event-outbox.ts +8 -6
- package/src/index.ts +1386 -875
- package/src/main.ts +1061 -139
- package/src/outbound-frame.ts +31 -0
- package/src/project-env-policy.ts +125 -10
- package/src/protocol.ts +699 -23
- package/src/ui/server.ts +48 -10
package/lib/orchestrator.ts
CHANGED
|
@@ -51,6 +51,7 @@ import { isSafeHostTaskId } from "./task-identity";
|
|
|
51
51
|
import { projectEnvForTask } from "./host-env";
|
|
52
52
|
import { shadowRoute } from "./routing/shadow";
|
|
53
53
|
import {
|
|
54
|
+
closeCurrentTaskRunAssignmentForStoppedRuntime,
|
|
54
55
|
getHostTask,
|
|
55
56
|
hostTaskEnvMode,
|
|
56
57
|
hostTaskProjectIds,
|
|
@@ -79,6 +80,7 @@ import {
|
|
|
79
80
|
} from "./container-runtime";
|
|
80
81
|
import {
|
|
81
82
|
reconstructPersistedTaskEnvironment,
|
|
83
|
+
reconstructTaskEnvironment,
|
|
82
84
|
setAppleTaskEnvironmentRecoveryDriver,
|
|
83
85
|
setDockerTaskEnvironmentRecoveryDriver,
|
|
84
86
|
} from "./task-environment";
|
|
@@ -137,6 +139,14 @@ import {
|
|
|
137
139
|
type BrowserSetup,
|
|
138
140
|
} from "./browser-testing";
|
|
139
141
|
import { injectCodexIntoContainer } from "./codex-auth";
|
|
142
|
+
import { runtimeNamespaceOf, type RuntimeNamespace } from "./task-environment/workspace-files";
|
|
143
|
+
import {
|
|
144
|
+
holdOwnCredentials,
|
|
145
|
+
LeftOwnCredentialsError,
|
|
146
|
+
onOwnCredentials,
|
|
147
|
+
onOwnCredentialsEnvironment,
|
|
148
|
+
whileOnOwnCredentials,
|
|
149
|
+
} from "./cloud-credential-host";
|
|
140
150
|
import { isCredentialInvalidSignal, requestCredentialRefresh } from "./token-broker-local";
|
|
141
151
|
import {
|
|
142
152
|
codexHomeIsolatable,
|
|
@@ -147,8 +157,51 @@ import {
|
|
|
147
157
|
pickEngineAccount,
|
|
148
158
|
provisionEngineAccounts,
|
|
149
159
|
resolveEngineAccounts,
|
|
160
|
+
selectAccount,
|
|
150
161
|
type ResolvedEngineAccount,
|
|
151
162
|
} from "./engine-accounts";
|
|
163
|
+
import {
|
|
164
|
+
ENGINE_ACCOUNT_HOMES,
|
|
165
|
+
EngineCredentialWriteRefused,
|
|
166
|
+
engineSpawnEnv,
|
|
167
|
+
engineVersionHeldBack,
|
|
168
|
+
ensureEngineHome,
|
|
169
|
+
escalateEngineLaunchFence,
|
|
170
|
+
isLeasedEngineAccount,
|
|
171
|
+
leasedEnginesFor,
|
|
172
|
+
onLeasedEnginesChanged,
|
|
173
|
+
refreshLeasedEngineAccount,
|
|
174
|
+
setEngineAdmissionFloorRaiser,
|
|
175
|
+
syncEngineHomes,
|
|
176
|
+
waitForEngineGate,
|
|
177
|
+
type LeasedEngineAccount,
|
|
178
|
+
type TaskEngineLease,
|
|
179
|
+
} from "./credential-consumers/engines";
|
|
180
|
+
import {
|
|
181
|
+
admitEngineLaunch,
|
|
182
|
+
anchoredEnvironment,
|
|
183
|
+
dischargeEngineRevocations,
|
|
184
|
+
ENGINE_ACCOUNT_ENV,
|
|
185
|
+
ENGINE_RUN_ENV,
|
|
186
|
+
EngineAnchorLost,
|
|
187
|
+
EngineLaunchRefused,
|
|
188
|
+
newEngineLaunchSlotId,
|
|
189
|
+
raiseEngineAdmissionFloors,
|
|
190
|
+
retireEngineLaunchSlot,
|
|
191
|
+
sameRuntime,
|
|
192
|
+
stopAnchoredEngines,
|
|
193
|
+
type EngineAdmissionFloor,
|
|
194
|
+
type EngineLaunchIdentity,
|
|
195
|
+
} from "./credential-consumers/engine-anchor";
|
|
196
|
+
import { CredentialLeaseUnavailableError } from "./cloud-credential-leases";
|
|
197
|
+
import type { TaskCredentialCleanupRequest } from "./task-credential-cleanup";
|
|
198
|
+
import {
|
|
199
|
+
forgetLeasedEngineRuntimes,
|
|
200
|
+
listCredentialCleanupPaths,
|
|
201
|
+
listLeasedEngineRuntimes,
|
|
202
|
+
rememberLeasedEngineRuntime,
|
|
203
|
+
} from "./credential-cleanup-inventory";
|
|
204
|
+
import { ENGINE_CREDENTIAL_FILES, ENGINE_ENV_CREDENTIAL_KEYS } from "../src/cloud-credentials";
|
|
152
205
|
import { clearTaskGatewayAcl, setupMcpTaskConfig } from "./mcp-gateway";
|
|
153
206
|
import {
|
|
154
207
|
reactivateTaskGatewayTask,
|
|
@@ -179,7 +232,11 @@ import {
|
|
|
179
232
|
MAX_SECRETARY_DISPATCH_ID_CHARS,
|
|
180
233
|
SECRETARY_TYPED_DISPATCH_PROTOCOL_FEATURE,
|
|
181
234
|
} from "../src/protocol";
|
|
182
|
-
import {
|
|
235
|
+
import {
|
|
236
|
+
isEnvironmentTransportProvider,
|
|
237
|
+
parseTaskEnvironmentLocator,
|
|
238
|
+
serializeTaskEnvironmentLocator,
|
|
239
|
+
} from "./task-environment/types";
|
|
183
240
|
import { forgetTaskEnvSnapshot } from "./task-env-source";
|
|
184
241
|
|
|
185
242
|
export type HostEventSubscriber = (event: HostEvent) => void;
|
|
@@ -385,6 +442,41 @@ interface Channel {
|
|
|
385
442
|
/** ADR-076: per-agent account-rotation budget (reset on a successful turn) —
|
|
386
443
|
* bounds a rate-limit storm across accounts, mirroring the respawn budget. */
|
|
387
444
|
rotations: Map<string, number>;
|
|
445
|
+
/** ADR-154: the task's AI accounts come from its lease. Sticky for the
|
|
446
|
+
* channel: once on cloud credentials, never the runner's own accounts. */
|
|
447
|
+
leased: boolean;
|
|
448
|
+
/** ADR-154 §8: rewrites of the task's account homes after a change, one
|
|
449
|
+
* after another. */
|
|
450
|
+
leasedSync: Promise<void> | null;
|
|
451
|
+
/** ADR-154 §6: agents a cleanup stopped, started again on their next
|
|
452
|
+
* account once the task's gate opens. */
|
|
453
|
+
leasedRespawn: Set<string>;
|
|
454
|
+
/** ADR-154 §6: the exact lease material each agent's session was picked
|
|
455
|
+
* on, so a spawn starts only on what the task still holds and a cleanup
|
|
456
|
+
* of an older run never stops a newer run's agent. */
|
|
457
|
+
leasedIdentityByAgent: Map<string, LeasedIdentity>;
|
|
458
|
+
/** ADR-154 §5: agents down for want of a usable account; the gate's next
|
|
459
|
+
* opening starts them again. */
|
|
460
|
+
leasedBlocked: Set<string>;
|
|
461
|
+
/** ADR-154 §8: a failed rewrite of the account homes, retried. */
|
|
462
|
+
leasedSyncRetry: ReturnType<typeof setTimeout> | null;
|
|
463
|
+
/** ADR-154 §5, §6: agents wait to be started again (stopped by a cleanup,
|
|
464
|
+
* down for want of an account, or their spawn's material moved); kept
|
|
465
|
+
* until a start pass actually ran, so a failed pass does not lose it. */
|
|
466
|
+
leasedWakePending: boolean;
|
|
467
|
+
/** ADR-154 §6: engines being created on a leased account (null: on an
|
|
468
|
+
* agent's task token alone, the Managed Engine) right now, each until its
|
|
469
|
+
* create settles. Every cleanup attempt on the account (on every one, for
|
|
470
|
+
* a whole run's) waits for them before it proves the engines gone,
|
|
471
|
+
* whatever has become of the agent's binding meanwhile. */
|
|
472
|
+
leasedCreating: Set<{
|
|
473
|
+
agentId: string;
|
|
474
|
+
accountId: string | null;
|
|
475
|
+
runEpoch: number;
|
|
476
|
+
settled: Promise<void>;
|
|
477
|
+
}>;
|
|
478
|
+
/** ADR-154 §6: each agent's current spawn of a leased engine. */
|
|
479
|
+
leasedLaunches: Map<string, LeasedLaunchSlot>;
|
|
388
480
|
}
|
|
389
481
|
|
|
390
482
|
interface AcceptedPrompt {
|
|
@@ -405,6 +497,119 @@ interface RecoveryFence {
|
|
|
405
497
|
const MAX_RESPAWNS_PER_AGENT = 5;
|
|
406
498
|
/** ADR-076: hard cap on account rotations per agent between successful turns. */
|
|
407
499
|
const MAX_ROTATIONS_PER_AGENT = 6;
|
|
500
|
+
/** ADR-154 §5: how long a spawn waits for the task's leased AI accounts
|
|
501
|
+
* before the agent fails with "credentials unavailable". */
|
|
502
|
+
const LEASED_ACCOUNT_WAIT_MS = 60_000;
|
|
503
|
+
/** ADR-154 §6: how long the cleanup's quiesce step waits for a spawn
|
|
504
|
+
* already creating an engine on a removed account. */
|
|
505
|
+
const LEASED_QUIESCE_EXIT_MS = 20_000;
|
|
506
|
+
/** ADR-141: the shadow router reads the tail of the task's chat (it keeps
|
|
507
|
+
* 1,200 characters) and its classifier's answer, never more. */
|
|
508
|
+
const SHADOW_CONTEXT_BYTES = 8 * 1024;
|
|
509
|
+
const SHADOW_OUTPUT_BYTES = 64 * 1024;
|
|
510
|
+
/** ADR-154 §6: the markers every process of a leased engine inherits: its
|
|
511
|
+
* run and its account. A launch carrying them is an engine launch, started
|
|
512
|
+
* only under the engine supervisor. */
|
|
513
|
+
const LEASED_RUN_ENV = ENGINE_RUN_ENV;
|
|
514
|
+
const LEASED_ACCOUNT_ENV = ENGINE_ACCOUNT_ENV;
|
|
515
|
+
/**
|
|
516
|
+
* ADR-154 §6: every variable an env-token account may set, blank. A leased
|
|
517
|
+
* spawn carries these under its account's own values: an explicit blank
|
|
518
|
+
* wins over an adapter's inheritance of the runner's variable (Claude's
|
|
519
|
+
* passEnv, Grok's XAI_API_KEY, Cursor's fallback) and over the container's
|
|
520
|
+
* env, so no engine ever runs on the runner's own credential beside, or
|
|
521
|
+
* instead of, the task's.
|
|
522
|
+
*/
|
|
523
|
+
const LEASED_CREDENTIAL_BLANKS: Readonly<Record<string, string>> = Object.freeze(
|
|
524
|
+
Object.fromEntries(
|
|
525
|
+
[...new Set(Object.values(ENGINE_ENV_CREDENTIAL_KEYS).flat())].map((name) => [name, ""]),
|
|
526
|
+
),
|
|
527
|
+
);
|
|
528
|
+
/** ADR-128: the Managed Engine. It needs no AI account (it runs on Uai's
|
|
529
|
+
* gateway), but its agent's task token is its bearer: on a leased task its
|
|
530
|
+
* launches are admitted as that token (ADR-154 §6). */
|
|
531
|
+
const MANAGED_ENGINE_KIND = "uai";
|
|
532
|
+
/** ADR-154 §10: kinds whose agents carry no credential at all; every other
|
|
533
|
+
* kind outside the leased lanes (Kimi) stays down on a leased task until
|
|
534
|
+
* its lane exists. */
|
|
535
|
+
const CREDENTIAL_FREE_KINDS: ReadonlySet<string> = new Set(["mock"]);
|
|
536
|
+
/** ADR-154 §8: how soon a failed rewrite of a task's account homes is tried
|
|
537
|
+
* again. */
|
|
538
|
+
const LEASED_SYNC_RETRY_MS = 30_000;
|
|
539
|
+
|
|
540
|
+
/** ADR-154 §6: the exact lease material a spawn starts on: an AI account's,
|
|
541
|
+
* or, for the Managed Engine, the task's open run alone (its agent's task
|
|
542
|
+
* token is its credential). */
|
|
543
|
+
type LeasedIdentity =
|
|
544
|
+
| { subject: "account"; runEpoch: number; credentialId: string; revision: string; version: number }
|
|
545
|
+
| { subject: "task-token"; runEpoch: number };
|
|
546
|
+
|
|
547
|
+
type SpawnArgs = Omit<Parameters<AgentSessionFactory["create"]>[0], "environment">;
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* ADR-154 §6: one spawn of an agent's leased engine. Its launches are
|
|
551
|
+
* admitted only while it is the agent's current spawn and, once its create
|
|
552
|
+
* settled, only while the session it bound is the agent's live one: a spawn
|
|
553
|
+
* replaced (on the same material too), retired, removed, never bound, or
|
|
554
|
+
* whose session reported its end, launches nothing more, and once retired,
|
|
555
|
+
* nothing it was admitted for starts in its runtime (`retireLeasedLaunch`).
|
|
556
|
+
*/
|
|
557
|
+
interface LeasedLaunchSlot {
|
|
558
|
+
created: boolean;
|
|
559
|
+
session: AgentSession | null;
|
|
560
|
+
/** What its launches are admitted as, its slot's id included, and where. */
|
|
561
|
+
readonly identity: EngineLaunchIdentity & { slotId: string };
|
|
562
|
+
readonly environment: TaskEnvironmentHandle<unknown>;
|
|
563
|
+
/** It began writing an admission: its retirement reaches its runtime. */
|
|
564
|
+
admitted: boolean;
|
|
565
|
+
retired: boolean;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** ADR-154 §6: a retired slot not yet retired in its runtime. */
|
|
569
|
+
interface RetiredLaunchSlot {
|
|
570
|
+
readonly agentId: string;
|
|
571
|
+
readonly slot: LeasedLaunchSlot;
|
|
572
|
+
/** The attempt under way, if one is. */
|
|
573
|
+
attempt: Promise<void> | null;
|
|
574
|
+
/** A cleanup of its run that covers it proves what it fences: a failure
|
|
575
|
+
* after is not escalated. */
|
|
576
|
+
covered: boolean;
|
|
577
|
+
/** Its run's full cleanup was asked for once an attempt failed. */
|
|
578
|
+
escalated: boolean;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function newLeasedLaunchSlot(
|
|
582
|
+
taskId: string,
|
|
583
|
+
identity: LeasedIdentity,
|
|
584
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
585
|
+
): LeasedLaunchSlot {
|
|
586
|
+
return {
|
|
587
|
+
created: false,
|
|
588
|
+
session: null,
|
|
589
|
+
identity: { taskId, slotId: newEngineLaunchSlotId(), ...identity },
|
|
590
|
+
environment,
|
|
591
|
+
admitted: false,
|
|
592
|
+
retired: false,
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function sameLeasedIdentity(a: LeasedIdentity, b: LeasedIdentity): boolean {
|
|
597
|
+
if (a.subject === "account" && b.subject === "account") {
|
|
598
|
+
return (
|
|
599
|
+
a.runEpoch === b.runEpoch &&
|
|
600
|
+
a.credentialId === b.credentialId &&
|
|
601
|
+
a.revision === b.revision &&
|
|
602
|
+
a.version === b.version
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
return a.subject === b.subject && a.runEpoch === b.runEpoch;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/** The engine kinds whose agents run on an AI account from the lease. */
|
|
609
|
+
const LEASED_ENGINE_KINDS: ReadonlySet<string> = new Set([
|
|
610
|
+
...Object.keys(ENGINE_ENV_CREDENTIAL_KEYS),
|
|
611
|
+
...Object.keys(ENGINE_CREDENTIAL_FILES),
|
|
612
|
+
]);
|
|
408
613
|
/** A burned respawn budget resets after this quiet period (see reconcile). */
|
|
409
614
|
const RESPAWN_COOLDOWN_MS = 10 * 60_000;
|
|
410
615
|
/** ADR-053: gap between browser-install retries. Ensures fire every couple of
|
|
@@ -504,6 +709,20 @@ async function sshIdentityDepsFor(task: {
|
|
|
504
709
|
return { environment: () => environment };
|
|
505
710
|
}
|
|
506
711
|
|
|
712
|
+
export type TaskStopAssignmentAdmission =
|
|
713
|
+
| { kind: "legacy" }
|
|
714
|
+
| { kind: "closed"; runEpoch: number }
|
|
715
|
+
| { kind: "stale" };
|
|
716
|
+
|
|
717
|
+
export interface OrchestratorTaskStopResult {
|
|
718
|
+
ok: boolean;
|
|
719
|
+
error?: string;
|
|
720
|
+
assignmentStale?: boolean;
|
|
721
|
+
vetoed?: boolean;
|
|
722
|
+
/** Exact v1 run stopped under the lifecycle lock. Absent for legacy. */
|
|
723
|
+
stoppedRunEpoch?: number;
|
|
724
|
+
}
|
|
725
|
+
|
|
507
726
|
export class Orchestrator {
|
|
508
727
|
private readonly channels = new Map<string, Channel>();
|
|
509
728
|
private readonly channelSpecs = new Map<string, ChannelEnsureInput>();
|
|
@@ -517,8 +736,15 @@ export class Orchestrator {
|
|
|
517
736
|
* later command cannot overtake an earlier one after channel close. */
|
|
518
737
|
private readonly taskLifecycleTails = new Map<string, Promise<void>>();
|
|
519
738
|
private readonly hostSubscribers = new Set<HostEventSubscriber>();
|
|
739
|
+
/** ADR-154 §6: launch slots retired whose retirement has not yet reached
|
|
740
|
+
* their runtime, per task, whatever channel they were in: an agent's next
|
|
741
|
+
* spawn launches there only once they have (`retireLeasedLaunch`). */
|
|
742
|
+
private readonly leasedRetired = new Map<string, Set<RetiredLaunchSlot>>();
|
|
520
743
|
|
|
521
|
-
constructor(private readonly factory: AgentSessionFactory) {
|
|
744
|
+
constructor(private readonly factory: AgentSessionFactory) {
|
|
745
|
+
onLeasedEnginesChanged((taskId) => this.leasedEnginesChanged(taskId));
|
|
746
|
+
setEngineAdmissionFloorRaiser((assignment, floors) => this.raiseLeasedAdmissionFloors(assignment, floors));
|
|
747
|
+
}
|
|
522
748
|
|
|
523
749
|
// -- subscriptions --------------------------------------------------------
|
|
524
750
|
|
|
@@ -645,7 +871,7 @@ export class Orchestrator {
|
|
|
645
871
|
for (const agent of added) {
|
|
646
872
|
// Per-agent link/document skills, like getOrCreateChannel does at
|
|
647
873
|
// channel birth. Best-effort.
|
|
648
|
-
writeAgentSkills(channel.taskId, agent);
|
|
874
|
+
writeAgentSkills(channel.taskId, agent, () => this.workspaceRuntime(channel.taskId));
|
|
649
875
|
}
|
|
650
876
|
for (const { agent, reason } of changed) {
|
|
651
877
|
const recoveryAbortPublished = this.supersedeRecovery(
|
|
@@ -665,6 +891,9 @@ export class Orchestrator {
|
|
|
665
891
|
);
|
|
666
892
|
}
|
|
667
893
|
channel.accountByAgent.delete(agent.id);
|
|
894
|
+
// ADR-154 §6: the retired spawn launches nothing more on its material.
|
|
895
|
+
channel.leasedIdentityByAgent.delete(agent.id);
|
|
896
|
+
void this.retireLeasedLaunch(channel, agent.id, channel.leasedLaunches.get(agent.id));
|
|
668
897
|
channel.rotations.delete(agent.id);
|
|
669
898
|
// Crash/recovery budgets belong to the retired launch generation. A
|
|
670
899
|
// newly selected engine/model/effort must get an immediate clean start,
|
|
@@ -814,6 +1043,9 @@ export class Orchestrator {
|
|
|
814
1043
|
channel.respawnLastAt.delete(agentId);
|
|
815
1044
|
channel.everBound.delete(agentId);
|
|
816
1045
|
channel.accountByAgent.delete(agentId);
|
|
1046
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
1047
|
+
// ADR-154 §6: its spawn admits nothing from here (see below for the end).
|
|
1048
|
+
void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
|
|
817
1049
|
channel.lastPrompt.delete(agentId);
|
|
818
1050
|
channel.rotations.delete(agentId);
|
|
819
1051
|
|
|
@@ -831,6 +1063,10 @@ export class Orchestrator {
|
|
|
831
1063
|
// factory await and closes the stale process instead of binding it.
|
|
832
1064
|
await inFlightCreate.catch(() => null);
|
|
833
1065
|
}
|
|
1066
|
+
// The removal ends only once its spawns are retired in their runtime, or
|
|
1067
|
+
// their run's cleanup is asked for (retireInRuntime): the current one, and
|
|
1068
|
+
// any whose retirement an earlier event, or this removal's drain, began.
|
|
1069
|
+
await Promise.all(this.leasedRetirementsUnderWay(channel.taskId, agentId));
|
|
834
1070
|
if (!this.isActiveChannel(channel)) return;
|
|
835
1071
|
// A newer ensure can re-add the same id while the old close/create drains.
|
|
836
1072
|
// That spec owns the status now; do not stop its bound generation or emit
|
|
@@ -972,9 +1208,6 @@ export class Orchestrator {
|
|
|
972
1208
|
taskProxiedKeys(taskId),
|
|
973
1209
|
),
|
|
974
1210
|
);
|
|
975
|
-
// ADR-046: materialise this agent's skills to its per-agent SKILL.md in
|
|
976
|
-
// the workspace (bind-mounted into the container). Best-effort.
|
|
977
|
-
writeAgentSkills(taskId, agent);
|
|
978
1211
|
// ADR-022: any agent with a non-empty initialPrompt opens a first
|
|
979
1212
|
// turn at container-ready. The rest stay silent until addressed.
|
|
980
1213
|
const firstTurn = assembleFirstTurnPrompt(
|
|
@@ -1048,8 +1281,24 @@ export class Orchestrator {
|
|
|
1048
1281
|
accountByAgent: new Map(),
|
|
1049
1282
|
lastPrompt: new Map(),
|
|
1050
1283
|
rotations: new Map(),
|
|
1284
|
+
leased: false,
|
|
1285
|
+
leasedSync: null,
|
|
1286
|
+
leasedRespawn: new Set(),
|
|
1287
|
+
leasedIdentityByAgent: new Map(),
|
|
1288
|
+
leasedBlocked: new Set(),
|
|
1289
|
+
leasedSyncRetry: null,
|
|
1290
|
+
leasedWakePending: false,
|
|
1291
|
+
leasedCreating: new Set(),
|
|
1292
|
+
leasedLaunches: new Map(),
|
|
1051
1293
|
};
|
|
1052
1294
|
this.channels.set(taskId, channel);
|
|
1295
|
+
// ADR-046: materialise each agent's skills to its per-agent SKILL.md in
|
|
1296
|
+
// the workspace (bind-mounted into the container), best-effort, once the
|
|
1297
|
+
// channel can give the task's runtime: where only it can make the
|
|
1298
|
+
// agent's directory (macOS, #cb3j0z), it is asked for then.
|
|
1299
|
+
for (const agent of roster) {
|
|
1300
|
+
writeAgentSkills(taskId, agent, () => this.workspaceRuntime(taskId));
|
|
1301
|
+
}
|
|
1053
1302
|
return channel;
|
|
1054
1303
|
}
|
|
1055
1304
|
|
|
@@ -1255,22 +1504,90 @@ export class Orchestrator {
|
|
|
1255
1504
|
* copies use this gate: a false→true toggle queues migration after the older
|
|
1256
1505
|
* copy, so that copy cannot land last and erase the managed browser block.
|
|
1257
1506
|
*/
|
|
1258
|
-
private codexHomes(hasCodex: boolean,
|
|
1507
|
+
private codexHomes(hasCodex: boolean, channel: Channel): string[] {
|
|
1259
1508
|
if (!hasCodex) return [];
|
|
1260
|
-
|
|
1261
|
-
.map((account) => account.configDir?.containerDir)
|
|
1262
|
-
.filter((home): home is string => Boolean(home));
|
|
1263
|
-
return [...new Set([DEFAULT_CODEX_HOME, ...homes])];
|
|
1509
|
+
return [...new Set([DEFAULT_CODEX_HOME, ...this.accountHomes(channel, "codex")])];
|
|
1264
1510
|
}
|
|
1265
1511
|
|
|
1266
|
-
/** Every GROK_HOME a task
|
|
1267
|
-
*
|
|
1268
|
-
private grokHomes(hasGrok: boolean,
|
|
1512
|
+
/** Every GROK_HOME a task may run under: the default plus each sign-in
|
|
1513
|
+
* account's isolated home (API-key accounts use the default). */
|
|
1514
|
+
private grokHomes(hasGrok: boolean, channel: Channel): string[] {
|
|
1269
1515
|
if (!hasGrok) return [];
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1516
|
+
return [...new Set([DEFAULT_GROK_HOME, ...this.accountHomes(channel, "grok")])];
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
/** The homes `kind`'s config-directory accounts live in: the task's leased
|
|
1520
|
+
* ones, or the runner's own for a task not on cloud credentials. A leased
|
|
1521
|
+
* Codex account's home is listed with every Codex agent's own home on it,
|
|
1522
|
+
* so the task's managed config (browser, MCP) reaches the homes the
|
|
1523
|
+
* agents run in, now and whenever it changes. */
|
|
1524
|
+
private accountHomes(channel: Channel, kind: string): string[] {
|
|
1525
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1526
|
+
return resolveEngineAccounts(kind, {}, channel.engineAccountScope)
|
|
1527
|
+
.map((account) => account.configDir?.containerDir)
|
|
1528
|
+
.filter((home): home is string => Boolean(home));
|
|
1529
|
+
}
|
|
1530
|
+
const homes = (leasedEnginesFor(channel.taskId)?.accounts ?? [])
|
|
1531
|
+
.filter((account) => account.engineKind === kind && account.home !== null)
|
|
1532
|
+
.map((account) => account.home!);
|
|
1533
|
+
if (kind !== "codex") return homes;
|
|
1534
|
+
const agents = channel.roster.filter((agent) => agent.kind === "codex");
|
|
1535
|
+
return homes.flatMap((home) => [home, ...agents.map((agent) => codexPerAgentHome(home, agent.id))]);
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
/** ADR-154: whether this channel's AI accounts come from the task's lease
|
|
1539
|
+
* (the engine consumer holds a record for the task, or the runner is no
|
|
1540
|
+
* longer on its own credentials at all: no default slot, so an agent
|
|
1541
|
+
* reattached before its grant waits for it). Sticky: a channel that was
|
|
1542
|
+
* on cloud credentials never falls back to the runner's own accounts,
|
|
1543
|
+
* not even after the record is gone. */
|
|
1544
|
+
private isLeasedChannel(channel: Channel): boolean {
|
|
1545
|
+
if (!channel.leased && (leasedEnginesFor(channel.taskId) !== undefined || !onOwnCredentials())) {
|
|
1546
|
+
channel.leased = true;
|
|
1547
|
+
}
|
|
1548
|
+
return channel.leased;
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/**
|
|
1552
|
+
* The accounts `kind` may run on, in try order: the task's leased ones
|
|
1553
|
+
* that are readable now (none while its gate is closed), or the runner's
|
|
1554
|
+
* own for a task not on cloud credentials. A leased account carries no env
|
|
1555
|
+
* and no directory here: the spawn boundary applies it (leasedSpawnArgs),
|
|
1556
|
+
* and no runner-side copy ever names it.
|
|
1557
|
+
*/
|
|
1558
|
+
private engineAccounts(channel: Channel, kind: string): ResolvedEngineAccount[] {
|
|
1559
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1560
|
+
return resolveEngineAccounts(kind, {}, channel.engineAccountScope);
|
|
1561
|
+
}
|
|
1562
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
1563
|
+
if (!lease?.open) return [];
|
|
1564
|
+
return lease.accounts
|
|
1565
|
+
.filter(
|
|
1566
|
+
(account): account is LeasedEngineAccount =>
|
|
1567
|
+
account.engineKind === kind && isLeasedEngineAccount(account),
|
|
1568
|
+
)
|
|
1569
|
+
.map((account) => ({
|
|
1570
|
+
id: account.credentialId,
|
|
1571
|
+
kind,
|
|
1572
|
+
label: account.label,
|
|
1573
|
+
isDefault: false,
|
|
1574
|
+
scope: account.scope,
|
|
1575
|
+
authKind: account.authKind,
|
|
1576
|
+
execEnv: {},
|
|
1577
|
+
}));
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/** ADR-076 selection over `engineAccounts`: the first healthy account in
|
|
1581
|
+
* try order, skipping `exclude`. */
|
|
1582
|
+
private pickAccount(
|
|
1583
|
+
channel: Channel,
|
|
1584
|
+
kind: string,
|
|
1585
|
+
exclude?: Set<string>,
|
|
1586
|
+
): ResolvedEngineAccount | null {
|
|
1587
|
+
if (!this.isLeasedChannel(channel)) {
|
|
1588
|
+
return pickEngineAccount(kind, { scope: channel.engineAccountScope, exclude });
|
|
1589
|
+
}
|
|
1590
|
+
return selectAccount(this.engineAccounts(channel, kind), { exclude });
|
|
1274
1591
|
}
|
|
1275
1592
|
|
|
1276
1593
|
private noteProvisionMutation(
|
|
@@ -1299,9 +1616,12 @@ export class Orchestrator {
|
|
|
1299
1616
|
if (this.isActiveChannel(channel)) channel.configGeneration += 1;
|
|
1300
1617
|
}
|
|
1301
1618
|
|
|
1302
|
-
private provisionFingerprint(kinds: readonly string[],
|
|
1619
|
+
private provisionFingerprint(kinds: readonly string[], channel: Channel): string {
|
|
1620
|
+
// ADR-154: a leased task's accounts are never copied from the runner;
|
|
1621
|
+
// the spawn boundary writes each one's files through the engine consumer.
|
|
1622
|
+
if (this.isLeasedChannel(channel)) return "leased";
|
|
1303
1623
|
const accounts = kinds.flatMap((kind) =>
|
|
1304
|
-
resolveEngineAccounts(kind, {},
|
|
1624
|
+
resolveEngineAccounts(kind, {}, channel.engineAccountScope)
|
|
1305
1625
|
.filter((account) => !account.isDefault && account.configDir)
|
|
1306
1626
|
.map((account) => ({
|
|
1307
1627
|
kind,
|
|
@@ -1374,13 +1694,13 @@ export class Orchestrator {
|
|
|
1374
1694
|
const requestedHasCodex =
|
|
1375
1695
|
hasCodex || channel.roster.some((agent) => agent.kind === "codex");
|
|
1376
1696
|
const requestedHomes = requestedBrowser
|
|
1377
|
-
? this.codexHomes(requestedHasCodex, channel
|
|
1697
|
+
? this.codexHomes(requestedHasCodex, channel)
|
|
1378
1698
|
: [];
|
|
1379
1699
|
const requestedKinds = [
|
|
1380
1700
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1381
1701
|
];
|
|
1382
1702
|
const requestedProvisionFingerprint =
|
|
1383
|
-
this.provisionFingerprint(requestedKinds, channel
|
|
1703
|
+
this.provisionFingerprint(requestedKinds, channel);
|
|
1384
1704
|
const changedProvisionKinds = this.changedProvisionKinds(
|
|
1385
1705
|
channel.provisionAttemptedFingerprint,
|
|
1386
1706
|
requestedProvisionFingerprint,
|
|
@@ -1389,11 +1709,14 @@ export class Orchestrator {
|
|
|
1389
1709
|
const provisionKinds = forceProvision
|
|
1390
1710
|
? requestedKinds
|
|
1391
1711
|
: changedProvisionKinds;
|
|
1392
|
-
const
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
)
|
|
1396
|
-
|
|
1712
|
+
const leased = this.isLeasedChannel(channel);
|
|
1713
|
+
const mutableProvisionKinds = leased
|
|
1714
|
+
? []
|
|
1715
|
+
: provisionKinds.filter((kind) =>
|
|
1716
|
+
resolveEngineAccounts(kind, {}, channel.engineAccountScope).some(
|
|
1717
|
+
(account) => !account.isDefault && Boolean(account.configDir),
|
|
1718
|
+
),
|
|
1719
|
+
);
|
|
1397
1720
|
const mayProvisionConfig = mutableProvisionKinds.length > 0;
|
|
1398
1721
|
|
|
1399
1722
|
const inFlight = channel.browserSetup;
|
|
@@ -1410,13 +1733,13 @@ export class Orchestrator {
|
|
|
1410
1733
|
requestedHasCodex ||
|
|
1411
1734
|
channel.roster.some((agent) => agent.kind === "codex");
|
|
1412
1735
|
const latestHomes = latestBrowser
|
|
1413
|
-
? this.codexHomes(latestHasCodex, channel
|
|
1736
|
+
? this.codexHomes(latestHasCodex, channel)
|
|
1414
1737
|
: [];
|
|
1415
1738
|
const latestKinds = [
|
|
1416
1739
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1417
1740
|
];
|
|
1418
1741
|
const latestProvisionFingerprint =
|
|
1419
|
-
this.provisionFingerprint(latestKinds, channel
|
|
1742
|
+
this.provisionFingerprint(latestKinds, channel);
|
|
1420
1743
|
if (
|
|
1421
1744
|
(latestBrowser &&
|
|
1422
1745
|
coverage?.browserAttempted !== true &&
|
|
@@ -1452,7 +1775,12 @@ export class Orchestrator {
|
|
|
1452
1775
|
forceProvision ||
|
|
1453
1776
|
channel.provisionAttemptedFingerprint !==
|
|
1454
1777
|
requestedProvisionFingerprint;
|
|
1455
|
-
if (shouldProvision) {
|
|
1778
|
+
if (shouldProvision && leased) {
|
|
1779
|
+
// ADR-154: nothing is copied from the runner for a leased task.
|
|
1780
|
+
coverage.provisionAttempted = true;
|
|
1781
|
+
channel.provisionAttemptedFingerprint = requestedProvisionFingerprint;
|
|
1782
|
+
channel.provisionRetryFingerprint = undefined;
|
|
1783
|
+
} else if (shouldProvision) {
|
|
1456
1784
|
coverage.provisionAttempted = true;
|
|
1457
1785
|
if (mayProvisionConfig) {
|
|
1458
1786
|
// Fence a factory that started before the copy without declaring a
|
|
@@ -1462,21 +1790,27 @@ export class Orchestrator {
|
|
|
1462
1790
|
}
|
|
1463
1791
|
let copiedOrUnknown = false;
|
|
1464
1792
|
try {
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1793
|
+
// ADR-154: the runner's own accounts, copied only while it is on
|
|
1794
|
+
// its own credentials, counted (leaving waits for the copy);
|
|
1795
|
+
// once leaving has begun, nothing is copied.
|
|
1796
|
+
const provisioning = whileOnOwnCredentials(() =>
|
|
1797
|
+
channel.agentHandle
|
|
1798
|
+
? provisionEngineAccounts(
|
|
1799
|
+
channel.containerName,
|
|
1800
|
+
provisionKinds,
|
|
1801
|
+
{},
|
|
1802
|
+
channel.agentHandle,
|
|
1803
|
+
channel.engineAccountScope,
|
|
1804
|
+
)
|
|
1805
|
+
: provisionEngineAccounts(
|
|
1806
|
+
channel.containerName,
|
|
1807
|
+
provisionKinds,
|
|
1808
|
+
{},
|
|
1809
|
+
undefined,
|
|
1810
|
+
channel.engineAccountScope,
|
|
1811
|
+
),
|
|
1812
|
+
);
|
|
1813
|
+
const provisioned = (await provisioning) ?? { copied: 0, failed: 0, mayHaveMutated: false };
|
|
1480
1814
|
copiedOrUnknown =
|
|
1481
1815
|
provisioned.copied > 0 || provisioned.mayHaveMutated;
|
|
1482
1816
|
if (provisioned.failed > 0) {
|
|
@@ -1537,7 +1871,7 @@ export class Orchestrator {
|
|
|
1537
1871
|
requestedHomes,
|
|
1538
1872
|
channel.agentHandle,
|
|
1539
1873
|
requestedKinds,
|
|
1540
|
-
this.grokHomes(requestedKinds.includes("grok"), channel
|
|
1874
|
+
this.grokHomes(requestedKinds.includes("grok"), channel),
|
|
1541
1875
|
)
|
|
1542
1876
|
: await setupBrowserTesting(
|
|
1543
1877
|
channel.taskId,
|
|
@@ -1546,7 +1880,7 @@ export class Orchestrator {
|
|
|
1546
1880
|
requestedHomes,
|
|
1547
1881
|
undefined,
|
|
1548
1882
|
requestedKinds,
|
|
1549
|
-
this.grokHomes(requestedKinds.includes("grok"), channel
|
|
1883
|
+
this.grokHomes(requestedKinds.includes("grok"), channel),
|
|
1550
1884
|
);
|
|
1551
1885
|
} catch (err) {
|
|
1552
1886
|
console.warn(
|
|
@@ -1617,15 +1951,12 @@ export class Orchestrator {
|
|
|
1617
1951
|
requestedHasCodex ||
|
|
1618
1952
|
channel.roster.some((agent) => agent.kind === "codex");
|
|
1619
1953
|
const latestHomes = latestBrowser
|
|
1620
|
-
? this.codexHomes(latestHasCodex, channel
|
|
1954
|
+
? this.codexHomes(latestHasCodex, channel)
|
|
1621
1955
|
: [];
|
|
1622
1956
|
const latestKinds = [
|
|
1623
1957
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1624
1958
|
];
|
|
1625
|
-
const latestProvisionFingerprint = this.provisionFingerprint(
|
|
1626
|
-
latestKinds,
|
|
1627
|
-
channel.engineAccountScope,
|
|
1628
|
-
);
|
|
1959
|
+
const latestProvisionFingerprint = this.provisionFingerprint(latestKinds, channel);
|
|
1629
1960
|
if (
|
|
1630
1961
|
(latestBrowser &&
|
|
1631
1962
|
!coverage.browserAttempted &&
|
|
@@ -1714,12 +2045,9 @@ export class Orchestrator {
|
|
|
1714
2045
|
...new Set(channel.roster.map((agent) => agent.kind)),
|
|
1715
2046
|
].sort();
|
|
1716
2047
|
const codexHomes = engineKinds.includes("codex")
|
|
1717
|
-
? this.codexHomes(true, channel
|
|
2048
|
+
? this.codexHomes(true, channel)
|
|
1718
2049
|
: [];
|
|
1719
|
-
const grokHomes = this.grokHomes(
|
|
1720
|
-
engineKinds.includes("grok"),
|
|
1721
|
-
channel.engineAccountScope,
|
|
1722
|
-
);
|
|
2050
|
+
const grokHomes = this.grokHomes(engineKinds.includes("grok"), channel);
|
|
1723
2051
|
const fingerprint = JSON.stringify({
|
|
1724
2052
|
connections,
|
|
1725
2053
|
engineKinds,
|
|
@@ -2053,6 +2381,9 @@ export class Orchestrator {
|
|
|
2053
2381
|
if (channel.sessions.has(agent.id) || channel.spawning.has(agent.id)) {
|
|
2054
2382
|
return false;
|
|
2055
2383
|
}
|
|
2384
|
+
// ADR-154 §5: down for want of a usable account until the gate next
|
|
2385
|
+
// opens (which starts it again), not retried on every pass.
|
|
2386
|
+
if (channel.leasedBlocked.has(agent.id)) return false;
|
|
2056
2387
|
// Crash-loop budget: an agent whose session keeps dying stops being
|
|
2057
2388
|
// respawned after MAX_RESPAWNS_PER_AGENT (the exit paths increment) —
|
|
2058
2389
|
// but a cold budget resets so the task can heal after the cause clears.
|
|
@@ -2162,7 +2493,12 @@ export class Orchestrator {
|
|
|
2162
2493
|
),
|
|
2163
2494
|
},
|
|
2164
2495
|
);
|
|
2165
|
-
if (!session)
|
|
2496
|
+
if (!session) {
|
|
2497
|
+
// Credentials unavailable is this agent's alone: its siblings
|
|
2498
|
+
// still start.
|
|
2499
|
+
if (channel.leasedBlocked.has(agent.id)) continue;
|
|
2500
|
+
return;
|
|
2501
|
+
}
|
|
2166
2502
|
this.startNextPendingPrompt(channel, agent.id);
|
|
2167
2503
|
}
|
|
2168
2504
|
} finally {
|
|
@@ -2257,6 +2593,9 @@ export class Orchestrator {
|
|
|
2257
2593
|
channel.accountByAgent.delete(agent.id);
|
|
2258
2594
|
return { ...floor, ...base };
|
|
2259
2595
|
}
|
|
2596
|
+
// ADR-154: a leased task's account is picked and applied at the spawn
|
|
2597
|
+
// boundary (leasedSpawnArgs), which can wait for the task's gate.
|
|
2598
|
+
if (this.isLeasedChannel(channel)) return { ...floor, ...base };
|
|
2260
2599
|
const account = pickEngineAccount(agent.kind, {
|
|
2261
2600
|
scope: channel.engineAccountScope,
|
|
2262
2601
|
});
|
|
@@ -2345,6 +2684,11 @@ export class Orchestrator {
|
|
|
2345
2684
|
) {
|
|
2346
2685
|
return;
|
|
2347
2686
|
}
|
|
2687
|
+
// ADR-154 §6: an engine that ended launches nothing more from now,
|
|
2688
|
+
// whatever its terminal handling still awaits (the runtime's proof).
|
|
2689
|
+
if (event.type === "error" || event.type === "exit") {
|
|
2690
|
+
this.endLeasedLaunches(channel, agentId, session);
|
|
2691
|
+
}
|
|
2348
2692
|
const operation = eventTail
|
|
2349
2693
|
.catch(() => {})
|
|
2350
2694
|
.then(async () => {
|
|
@@ -2411,6 +2755,842 @@ export class Orchestrator {
|
|
|
2411
2755
|
return operation;
|
|
2412
2756
|
}
|
|
2413
2757
|
|
|
2758
|
+
/**
|
|
2759
|
+
* ADR-154 §5, §6, §10: what a spawn on the task's leased AI account takes,
|
|
2760
|
+
* and the exact lease material it starts on. The agent runs on the account
|
|
2761
|
+
* it is bound to while that account is still served readable, else on the
|
|
2762
|
+
* first healthy one in try order (the owner's accounts, then the
|
|
2763
|
+
* workspace's). An env-token account's values ride the spawn, synchronously
|
|
2764
|
+
* when nothing has to be waited for. A config-directory account's home gets
|
|
2765
|
+
* the task's managed config first (browser, MCP), then its files, through
|
|
2766
|
+
* the engine consumer, in the home the agent runs in (Codex's own copy,
|
|
2767
|
+
* seeded from the account's home without its credential files). While the
|
|
2768
|
+
* task's gate is closed it waits, bounded; null when no account can be
|
|
2769
|
+
* used, and the agent stays down with "credentials unavailable", never on
|
|
2770
|
+
* the runner's own accounts, until the gate next opens.
|
|
2771
|
+
*/
|
|
2772
|
+
private leasedSpawnArgs(
|
|
2773
|
+
channel: Channel,
|
|
2774
|
+
agentId: string,
|
|
2775
|
+
args: SpawnArgs,
|
|
2776
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
2777
|
+
):
|
|
2778
|
+
| { args: SpawnArgs; identity: LeasedIdentity }
|
|
2779
|
+
| null
|
|
2780
|
+
| Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
|
|
2781
|
+
const kind = args.agent?.kind ?? "";
|
|
2782
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
2783
|
+
if (lease?.open) {
|
|
2784
|
+
const account = this.leasedAccountFor(channel, agentId, kind, lease);
|
|
2785
|
+
if (account?.authKind === "env") {
|
|
2786
|
+
return this.withLeasedAccount(channel, agentId, args, account, lease.assignment.runEpoch);
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
return (async () => {
|
|
2790
|
+
const deadline = Date.now() + LEASED_ACCOUNT_WAIT_MS;
|
|
2791
|
+
let reason = "the task's AI accounts are not available";
|
|
2792
|
+
let moved = 0;
|
|
2793
|
+
while (this.isActiveChannel(channel)) {
|
|
2794
|
+
const open = await this.leasedGateOpen(channel, deadline);
|
|
2795
|
+
if (!open) break;
|
|
2796
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2797
|
+
const account = this.leasedAccountFor(channel, agentId, kind, open);
|
|
2798
|
+
if (!account) {
|
|
2799
|
+
reason = `no ${kind} account this task can use`;
|
|
2800
|
+
break;
|
|
2801
|
+
}
|
|
2802
|
+
if (account.authKind === "env") {
|
|
2803
|
+
return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch);
|
|
2804
|
+
}
|
|
2805
|
+
// The account's home gets the task's managed config before anything
|
|
2806
|
+
// can start in it, whenever the gate's opening first reveals it.
|
|
2807
|
+
await this.ensureBrowserSetup(channel, kind === "codex");
|
|
2808
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2809
|
+
await this.ensureMcpConfig(channel);
|
|
2810
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2811
|
+
// Codex runs in a home of its own (two agents on one account never
|
|
2812
|
+
// share its state runtime); the config writers above reached it.
|
|
2813
|
+
const home = kind === "codex" ? codexPerAgentHome(account.home!, agentId) : account.home!;
|
|
2814
|
+
try {
|
|
2815
|
+
await ensureEngineHome(environment, open.assignment, account.credentialId, home);
|
|
2816
|
+
} catch (error) {
|
|
2817
|
+
// The account moved while it was written (a newer version, a
|
|
2818
|
+
// cleanup, no longer readable, the gate closing): pick again on
|
|
2819
|
+
// what the gate serves, or will once it reopens, a few times while
|
|
2820
|
+
// time remains. Anything else ends here.
|
|
2821
|
+
if (
|
|
2822
|
+
(error instanceof EngineCredentialWriteRefused || error instanceof CredentialLeaseUnavailableError) &&
|
|
2823
|
+
moved++ < 3 &&
|
|
2824
|
+
Date.now() < deadline
|
|
2825
|
+
) {
|
|
2826
|
+
continue;
|
|
2827
|
+
}
|
|
2828
|
+
reason = error instanceof CredentialLeaseUnavailableError ? error.message : "its files could not be written";
|
|
2829
|
+
break;
|
|
2830
|
+
}
|
|
2831
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2832
|
+
return this.withLeasedAccount(channel, agentId, args, account, open.assignment.runEpoch, home);
|
|
2833
|
+
}
|
|
2834
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2835
|
+
this.leasedUnavailable(channel, agentId, reason);
|
|
2836
|
+
return null;
|
|
2837
|
+
})();
|
|
2838
|
+
}
|
|
2839
|
+
|
|
2840
|
+
/** The task's open engine lease, waited for while time remains, a newer
|
|
2841
|
+
* run's when the task moves on to one; null when none opens in time. */
|
|
2842
|
+
private async leasedGateOpen(channel: Channel, deadline: number): Promise<TaskEngineLease | null> {
|
|
2843
|
+
while (this.isActiveChannel(channel)) {
|
|
2844
|
+
const record = leasedEnginesFor(channel.taskId);
|
|
2845
|
+
if (!record) return null;
|
|
2846
|
+
try {
|
|
2847
|
+
return await waitForEngineGate(record.assignment, Math.max(0, deadline - Date.now()));
|
|
2848
|
+
} catch {
|
|
2849
|
+
// The task moved on to another run: wait for that run's gate while
|
|
2850
|
+
// time remains.
|
|
2851
|
+
if (leasedEnginesFor(channel.taskId)?.assignment.runEpoch !== record.assignment.runEpoch && Date.now() < deadline) {
|
|
2852
|
+
continue;
|
|
2853
|
+
}
|
|
2854
|
+
return null;
|
|
2855
|
+
}
|
|
2856
|
+
}
|
|
2857
|
+
return null;
|
|
2858
|
+
}
|
|
2859
|
+
|
|
2860
|
+
/**
|
|
2861
|
+
* ADR-154 §6, ADR-128: what a Managed Engine spawn on a leased task takes.
|
|
2862
|
+
* It runs on no AI account, but its agent's task token is its bearer: it
|
|
2863
|
+
* is bound to the task's open run, its launches admitted as that token on
|
|
2864
|
+
* this binding, under the engine supervisor like any leased engine's.
|
|
2865
|
+
* While the task's gate is closed it waits, bounded; null when it stays
|
|
2866
|
+
* closed, and the agent stays down with "credentials unavailable" until
|
|
2867
|
+
* the gate next opens.
|
|
2868
|
+
*/
|
|
2869
|
+
private leasedTaskTokenSpawnArgs(
|
|
2870
|
+
channel: Channel,
|
|
2871
|
+
agentId: string,
|
|
2872
|
+
args: SpawnArgs,
|
|
2873
|
+
):
|
|
2874
|
+
| { args: SpawnArgs; identity: LeasedIdentity }
|
|
2875
|
+
| null
|
|
2876
|
+
| Promise<{ args: SpawnArgs; identity: LeasedIdentity } | null> {
|
|
2877
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
2878
|
+
if (lease?.open) return this.withLeasedTaskToken(channel, agentId, args, lease.assignment.runEpoch);
|
|
2879
|
+
return (async () => {
|
|
2880
|
+
const open = await this.leasedGateOpen(channel, Date.now() + LEASED_ACCOUNT_WAIT_MS);
|
|
2881
|
+
if (!this.isActiveChannel(channel)) return null;
|
|
2882
|
+
if (!open) {
|
|
2883
|
+
this.leasedUnavailable(channel, agentId, "the task's credentials are not available");
|
|
2884
|
+
return null;
|
|
2885
|
+
}
|
|
2886
|
+
return this.withLeasedTaskToken(channel, agentId, args, open.assignment.runEpoch);
|
|
2887
|
+
})();
|
|
2888
|
+
}
|
|
2889
|
+
|
|
2890
|
+
/** Bind the agent to the task's run `runEpoch` on its task token alone,
|
|
2891
|
+
* and give its spawn the run's marker every process it starts inherits
|
|
2892
|
+
* (its launches are known by it, and admitted on this binding). */
|
|
2893
|
+
private withLeasedTaskToken(
|
|
2894
|
+
channel: Channel,
|
|
2895
|
+
agentId: string,
|
|
2896
|
+
args: SpawnArgs,
|
|
2897
|
+
runEpoch: number,
|
|
2898
|
+
): { args: SpawnArgs; identity: LeasedIdentity } {
|
|
2899
|
+
const identity = { subject: "task-token" as const, runEpoch };
|
|
2900
|
+
channel.leasedIdentityByAgent.set(agentId, identity);
|
|
2901
|
+
return {
|
|
2902
|
+
identity,
|
|
2903
|
+
args: {
|
|
2904
|
+
...args,
|
|
2905
|
+
attachCompatibilityKey: JSON.stringify([args.attachCompatibilityKey ?? null, "lease-v1", runEpoch, "task-token"]),
|
|
2906
|
+
agentEnv: {
|
|
2907
|
+
...(args.agentEnv ?? {}),
|
|
2908
|
+
// The gateway's bearer is the agent's own token; no engine's
|
|
2909
|
+
// credential of the runner's rides beside it.
|
|
2910
|
+
...LEASED_CREDENTIAL_BLANKS,
|
|
2911
|
+
[LEASED_RUN_ENV]: String(runEpoch),
|
|
2912
|
+
},
|
|
2913
|
+
},
|
|
2914
|
+
};
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
/**
|
|
2918
|
+
* ADR-154 §6: the environment a leased engine's adapter launches through.
|
|
2919
|
+
* Each engine launch it makes (the session, each one-shot turn, a durable
|
|
2920
|
+
* runner once deployed, a classifier's exec), at whatever moment its
|
|
2921
|
+
* transport gets there, is admitted then: for an agent's spawn (`owner`),
|
|
2922
|
+
* only while that spawn is the agent's current one and, once its create
|
|
2923
|
+
* settled, the session it bound is the agent's live one, and `identity`
|
|
2924
|
+
* is still what the agent runs on and the task's open gate serves (for a
|
|
2925
|
+
* one-shot with no agent, only the latter). The admission is written into
|
|
2926
|
+
* the runtime first (its exec admitted at the provider's last moment) and
|
|
2927
|
+
* the check made again just before the launch is handed over and at the
|
|
2928
|
+
* provider's last moment. The launch then starts under the engine
|
|
2929
|
+
* supervisor, which consumes that admission. A spawn's admissions name its
|
|
2930
|
+
* slot: before its first, the agent's earlier spawns are retired in the
|
|
2931
|
+
* runtime, and once the slot is retired, nothing it was admitted for
|
|
2932
|
+
* starts there (`retireLeasedLaunch`). A cleanup unbinds first, waits for
|
|
2933
|
+
* the admissions under way (registered before their first await), and
|
|
2934
|
+
* revokes in the runtime what they wrote before it kills what the
|
|
2935
|
+
* supervisors own.
|
|
2936
|
+
*/
|
|
2937
|
+
private leasedLaunchEnvironment(
|
|
2938
|
+
channel: Channel,
|
|
2939
|
+
owner: { agentId: string; slot: LeasedLaunchSlot } | null,
|
|
2940
|
+
identity: LeasedIdentity,
|
|
2941
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
2942
|
+
): TaskEnvironmentHandle<unknown> {
|
|
2943
|
+
const agentId = owner?.agentId ?? null;
|
|
2944
|
+
const held = (): boolean =>
|
|
2945
|
+
this.isActiveChannel(channel) &&
|
|
2946
|
+
(owner === null
|
|
2947
|
+
? this.leasedMaterialServed(channel, identity)
|
|
2948
|
+
: channel.leasedLaunches.get(owner.agentId) === owner.slot &&
|
|
2949
|
+
(owner.slot.session === null
|
|
2950
|
+
? !owner.slot.created
|
|
2951
|
+
: channel.sessions.get(owner.agentId) === owner.slot.session) &&
|
|
2952
|
+
this.leasedIdentityHeld(channel, owner.agentId, identity));
|
|
2953
|
+
const admittedAs: EngineLaunchIdentity = owner?.slot.identity ?? { taskId: channel.taskId, slotId: null, ...identity };
|
|
2954
|
+
return anchoredEnvironment(environment, {
|
|
2955
|
+
held,
|
|
2956
|
+
admit: (carried) => {
|
|
2957
|
+
if (environment.anchors !== true || !held()) return Promise.reject(new EngineLaunchRefused());
|
|
2958
|
+
if (owner) owner.slot.admitted = true;
|
|
2959
|
+
const admitted = (async () => {
|
|
2960
|
+
if (owner) await this.leasedPredecessorsRetired(owner.agentId, owner.slot);
|
|
2961
|
+
return admitEngineLaunch(
|
|
2962
|
+
environment,
|
|
2963
|
+
admittedAs,
|
|
2964
|
+
() => {
|
|
2965
|
+
if (!held()) throw new EngineLaunchRefused();
|
|
2966
|
+
},
|
|
2967
|
+
carried,
|
|
2968
|
+
);
|
|
2969
|
+
})();
|
|
2970
|
+
const entry = {
|
|
2971
|
+
agentId: agentId ?? "",
|
|
2972
|
+
accountId: identity.subject === "account" ? identity.credentialId : null,
|
|
2973
|
+
runEpoch: identity.runEpoch,
|
|
2974
|
+
settled: admitted.then(
|
|
2975
|
+
() => undefined,
|
|
2976
|
+
() => undefined,
|
|
2977
|
+
),
|
|
2978
|
+
};
|
|
2979
|
+
channel.leasedCreating.add(entry);
|
|
2980
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
2981
|
+
return admitted;
|
|
2982
|
+
},
|
|
2983
|
+
});
|
|
2984
|
+
}
|
|
2985
|
+
|
|
2986
|
+
/** `session` ended (its engine's error or exit): its spawn admits no
|
|
2987
|
+
* launch from this moment, before its terminal handling awaits anything.
|
|
2988
|
+
* Every such handling retires or replaces the session. */
|
|
2989
|
+
private endLeasedLaunches(channel: Channel, agentId: string, session: AgentSession): void {
|
|
2990
|
+
const slot = channel.leasedLaunches.get(agentId);
|
|
2991
|
+
if (slot?.session === session) void this.retireLeasedLaunch(channel, agentId, slot);
|
|
2992
|
+
}
|
|
2993
|
+
|
|
2994
|
+
/**
|
|
2995
|
+
* ADR-154 §6: `slot` admits nothing from this moment. Then, in its
|
|
2996
|
+
* runtime, it is retired: every admission it wrote that no launch has used
|
|
2997
|
+
* is revoked, and none of its own is written or used after, so a launch
|
|
2998
|
+
* handed to its provider before now that has not started there never
|
|
2999
|
+
* does, with the agent's old task token or anything else. The agent's
|
|
3000
|
+
* next spawn launches in that runtime only once that is done
|
|
3001
|
+
* (`leasedPredecessorsRetired`). Resolves once that is done or escalated
|
|
3002
|
+
* (see `retireInRuntime`); never rejects.
|
|
3003
|
+
*/
|
|
3004
|
+
private retireLeasedLaunch(channel: Channel, agentId: string, slot: LeasedLaunchSlot | undefined): Promise<void> {
|
|
3005
|
+
if (!slot || slot.retired) return Promise.resolve();
|
|
3006
|
+
slot.retired = true;
|
|
3007
|
+
if (channel.leasedLaunches.get(agentId) === slot) channel.leasedLaunches.delete(agentId);
|
|
3008
|
+
// It never wrote an admission: nothing of it is in its runtime.
|
|
3009
|
+
if (!slot.admitted) return Promise.resolve();
|
|
3010
|
+
const retired: RetiredLaunchSlot = { agentId, slot, attempt: null, covered: false, escalated: false };
|
|
3011
|
+
let pending = this.leasedRetired.get(channel.taskId);
|
|
3012
|
+
if (!pending) {
|
|
3013
|
+
pending = new Set();
|
|
3014
|
+
this.leasedRetired.set(channel.taskId, pending);
|
|
3015
|
+
}
|
|
3016
|
+
pending.add(retired);
|
|
3017
|
+
return this.retireInRuntime(retired).catch(() => undefined);
|
|
3018
|
+
}
|
|
3019
|
+
|
|
3020
|
+
/**
|
|
3021
|
+
* One attempt to retire `retired` in its runtime, joined while one is
|
|
3022
|
+
* under way; once done, it is no longer pending. Should it fail, the order
|
|
3023
|
+
* is lost: a launch handed over before the retirement may start before
|
|
3024
|
+
* any later one lands, and nothing a retry does stops it. So, unless a
|
|
3025
|
+
* cleanup that covers it is under way or done, the run's full credential
|
|
3026
|
+
* cleanup is asked for at once (`escalateEngineLaunchFence`): its gate
|
|
3027
|
+
* closed first, it proves no engine of the run runs (or stops the
|
|
3028
|
+
* runtime), and keeps failing closed until it can. Rejects when this
|
|
3029
|
+
* attempt failed (it is retained and retried meanwhile).
|
|
3030
|
+
*/
|
|
3031
|
+
private retireInRuntime(retired: RetiredLaunchSlot): Promise<void> {
|
|
3032
|
+
if (retired.attempt) return retired.attempt;
|
|
3033
|
+
const { taskId, runEpoch } = retired.slot.identity;
|
|
3034
|
+
const attempt = retireEngineLaunchSlot(retired.slot.environment, retired.slot.identity)
|
|
3035
|
+
.then(
|
|
3036
|
+
() => this.forgetRetired(taskId, retired),
|
|
3037
|
+
(error: unknown) => {
|
|
3038
|
+
if (!retired.covered && !retired.escalated) {
|
|
3039
|
+
retired.escalated = true;
|
|
3040
|
+
console.warn(
|
|
3041
|
+
`[orchestrator] ${taskId}/${retired.agentId}: a retired spawn could not be retired in its runtime; cleaning up its run's credentials (${
|
|
3042
|
+
error instanceof Error ? error.message : String(error)
|
|
3043
|
+
})`,
|
|
3044
|
+
);
|
|
3045
|
+
escalateEngineLaunchFence({ taskId, runEpoch });
|
|
3046
|
+
}
|
|
3047
|
+
throw error;
|
|
3048
|
+
},
|
|
3049
|
+
)
|
|
3050
|
+
.finally(() => {
|
|
3051
|
+
if (retired.attempt === attempt) retired.attempt = null;
|
|
3052
|
+
});
|
|
3053
|
+
retired.attempt = attempt;
|
|
3054
|
+
return attempt;
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
/** The task's retirements still under way (of `agentId`'s spawns alone,
|
|
3058
|
+
* when named), whichever event began them: each resolves once its slot is
|
|
3059
|
+
* retired in its runtime, or its run's cleanup has been asked for. What a
|
|
3060
|
+
* removal or a channel's close waits for. */
|
|
3061
|
+
private leasedRetirementsUnderWay(taskId: string, agentId?: string): Array<Promise<void>> {
|
|
3062
|
+
return [...(this.leasedRetired.get(taskId) ?? [])]
|
|
3063
|
+
.filter((retired) => agentId === undefined || retired.agentId === agentId)
|
|
3064
|
+
.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
private forgetRetired(taskId: string, retired: RetiredLaunchSlot): void {
|
|
3068
|
+
const pending = this.leasedRetired.get(taskId);
|
|
3069
|
+
if (!pending) return;
|
|
3070
|
+
pending.delete(retired);
|
|
3071
|
+
if (pending.size === 0) this.leasedRetired.delete(taskId);
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
/**
|
|
3075
|
+
* ADR-154 §6: before `slot` writes an admission, every earlier slot of the
|
|
3076
|
+
* same agent retired for the same run in the same runtime is retired
|
|
3077
|
+
* there: nothing the agent's earlier spawns were admitted for can start
|
|
3078
|
+
* after this one's launch. Rejects with `EngineLaunchRefused` when one
|
|
3079
|
+
* could not be now.
|
|
3080
|
+
*/
|
|
3081
|
+
private async leasedPredecessorsRetired(agentId: string, slot: LeasedLaunchSlot): Promise<void> {
|
|
3082
|
+
const earlier = [...(this.leasedRetired.get(slot.identity.taskId) ?? [])].filter(
|
|
3083
|
+
(retired) =>
|
|
3084
|
+
retired.agentId === agentId &&
|
|
3085
|
+
retired.slot.identity.runEpoch === slot.identity.runEpoch &&
|
|
3086
|
+
sameRuntime(retired.slot.environment, slot.environment),
|
|
3087
|
+
);
|
|
3088
|
+
try {
|
|
3089
|
+
await Promise.all(earlier.map((retired) => this.retireInRuntime(retired)));
|
|
3090
|
+
} catch (error) {
|
|
3091
|
+
// Live 2026-10-11: this refusal was silent, so agents that never started
|
|
3092
|
+
// left nothing in the log to say why.
|
|
3093
|
+
console.warn(
|
|
3094
|
+
`[orchestrator] ${slot.identity.taskId}/${agentId}: not starting the agent, an earlier start could not be retired in its runtime (${
|
|
3095
|
+
error instanceof Error ? error.message : String(error)
|
|
3096
|
+
})`,
|
|
3097
|
+
);
|
|
3098
|
+
throw new EngineLaunchRefused("an earlier start of this agent could not be retired in its runtime yet");
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
/** The retirements still pending that a cleanup of these runs and
|
|
3103
|
+
* accounts (of every subject, for a whole run's) covers: its stop revokes
|
|
3104
|
+
* everything they fence, wherever it was written, and kills what any
|
|
3105
|
+
* launch of theirs started. Each is marked covered: one that fails from
|
|
3106
|
+
* now on is not escalated. */
|
|
3107
|
+
private coveredLeasedRetirements(
|
|
3108
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3109
|
+
): RetiredLaunchSlot[] {
|
|
3110
|
+
const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
|
|
3111
|
+
const covered = [...(this.leasedRetired.get(request.taskId) ?? [])].filter((retired) => {
|
|
3112
|
+
const { identity } = retired.slot;
|
|
3113
|
+
return (
|
|
3114
|
+
identity.runEpoch <= request.runEpoch &&
|
|
3115
|
+
(!named || (identity.subject === "account" && named.has(identity.credentialId)))
|
|
3116
|
+
);
|
|
3117
|
+
});
|
|
3118
|
+
for (const retired of covered) retired.covered = true;
|
|
3119
|
+
return covered;
|
|
3120
|
+
}
|
|
3121
|
+
|
|
3122
|
+
/** A cleanup proved every admission of these runs and accounts (of every
|
|
3123
|
+
* subject, for a whole run's) revoked wherever it was written: the
|
|
3124
|
+
* retirements still pending for them are done. */
|
|
3125
|
+
private dischargeLeasedRetirements(
|
|
3126
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3127
|
+
): void {
|
|
3128
|
+
for (const retired of this.coveredLeasedRetirements(request)) this.forgetRetired(request.taskId, retired);
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
/** The agent stays down for want of a usable account, says why, and is
|
|
3132
|
+
* started again when the task's gate next opens. */
|
|
3133
|
+
private leasedUnavailable(channel: Channel, agentId: string, reason: string): void {
|
|
3134
|
+
channel.accountByAgent.delete(agentId);
|
|
3135
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
3136
|
+
channel.leasedBlocked.add(agentId);
|
|
3137
|
+
this.emitHost({
|
|
3138
|
+
kind: "agent.exit",
|
|
3139
|
+
taskId: channel.taskId,
|
|
3140
|
+
agentId,
|
|
3141
|
+
reason: `${agentId}: credentials unavailable (${reason}) — the agent starts again when they are.`,
|
|
3142
|
+
});
|
|
3143
|
+
}
|
|
3144
|
+
|
|
3145
|
+
/** Whether the task's open gate still serves `identity`'s account at its
|
|
3146
|
+
* revision and version, for its run, and that version is not being held
|
|
3147
|
+
* back for a newer one: what a one-shot may start on. An agent's task
|
|
3148
|
+
* token alone needs only its run's gate open. */
|
|
3149
|
+
private leasedMaterialServed(channel: Channel, identity: LeasedIdentity): boolean {
|
|
3150
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3151
|
+
if (!lease?.open || lease.assignment.runEpoch !== identity.runEpoch) return false;
|
|
3152
|
+
if (identity.subject === "task-token") return true;
|
|
3153
|
+
const account = lease.accounts.find((candidate) => candidate.credentialId === identity.credentialId);
|
|
3154
|
+
return (
|
|
3155
|
+
isLeasedEngineAccount(account) &&
|
|
3156
|
+
account.revision === identity.revision &&
|
|
3157
|
+
account.version === identity.version &&
|
|
3158
|
+
!engineVersionHeldBack(lease.assignment, identity)
|
|
3159
|
+
);
|
|
3160
|
+
}
|
|
3161
|
+
|
|
3162
|
+
/** A spawn found its lease material moved (a newer version, a cleanup):
|
|
3163
|
+
* start the agent again on what the task holds now, once its gate is
|
|
3164
|
+
* open (now, or at its next opening). */
|
|
3165
|
+
private retryLeasedStart(channel: Channel, agentId: string): void {
|
|
3166
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3167
|
+
channel.leasedRespawn.add(agentId);
|
|
3168
|
+
channel.leasedWakePending = true;
|
|
3169
|
+
if (leasedEnginesFor(channel.taskId)?.open) this.leasedEnginesChanged(channel.taskId);
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
/** Whether the agent may start on `identity` right now: still bound to it,
|
|
3173
|
+
* and the task's open gate still serves that account at that revision and
|
|
3174
|
+
* version, for that run. */
|
|
3175
|
+
private leasedIdentityHeld(channel: Channel, agentId: string, identity: LeasedIdentity): boolean {
|
|
3176
|
+
const bound = channel.leasedIdentityByAgent.get(agentId);
|
|
3177
|
+
return bound !== undefined && sameLeasedIdentity(bound, identity) && this.leasedMaterialServed(channel, identity);
|
|
3178
|
+
}
|
|
3179
|
+
|
|
3180
|
+
/** The leased account an agent runs on: the one it is bound to while that
|
|
3181
|
+
* is served readable, else the first healthy one in try order. */
|
|
3182
|
+
private leasedAccountFor(
|
|
3183
|
+
channel: Channel,
|
|
3184
|
+
agentId: string,
|
|
3185
|
+
kind: string,
|
|
3186
|
+
lease: TaskEngineLease,
|
|
3187
|
+
): LeasedEngineAccount | undefined {
|
|
3188
|
+
const accounts = lease.accounts.filter(
|
|
3189
|
+
(account): account is LeasedEngineAccount =>
|
|
3190
|
+
account.engineKind === kind && isLeasedEngineAccount(account),
|
|
3191
|
+
);
|
|
3192
|
+
const bound = channel.accountByAgent.get(agentId);
|
|
3193
|
+
const kept = accounts.find((account) => account.credentialId === bound);
|
|
3194
|
+
if (kept) return kept;
|
|
3195
|
+
const picked = selectAccount(
|
|
3196
|
+
accounts.map((account) => ({
|
|
3197
|
+
id: account.credentialId,
|
|
3198
|
+
kind,
|
|
3199
|
+
label: account.label,
|
|
3200
|
+
isDefault: false,
|
|
3201
|
+
scope: account.scope,
|
|
3202
|
+
authKind: account.authKind,
|
|
3203
|
+
execEnv: {},
|
|
3204
|
+
})),
|
|
3205
|
+
);
|
|
3206
|
+
return accounts.find((account) => account.credentialId === picked?.id);
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
/** Bind the agent to `account` for `runEpoch` and give its spawn the
|
|
3210
|
+
* account, and the markers every process it starts inherits (its launches
|
|
3211
|
+
* are known by them, and admitted on this binding). */
|
|
3212
|
+
private withLeasedAccount(
|
|
3213
|
+
channel: Channel,
|
|
3214
|
+
agentId: string,
|
|
3215
|
+
args: SpawnArgs,
|
|
3216
|
+
account: LeasedEngineAccount,
|
|
3217
|
+
runEpoch: number,
|
|
3218
|
+
home?: string,
|
|
3219
|
+
): { args: SpawnArgs; identity: LeasedIdentity } {
|
|
3220
|
+
const identity = {
|
|
3221
|
+
subject: "account" as const,
|
|
3222
|
+
runEpoch,
|
|
3223
|
+
credentialId: account.credentialId,
|
|
3224
|
+
revision: account.revision,
|
|
3225
|
+
version: account.version,
|
|
3226
|
+
};
|
|
3227
|
+
channel.accountByAgent.set(agentId, account.credentialId);
|
|
3228
|
+
channel.leasedIdentityByAgent.set(agentId, identity);
|
|
3229
|
+
noteEngineAccountUsed(account.credentialId);
|
|
3230
|
+
return {
|
|
3231
|
+
identity,
|
|
3232
|
+
args: {
|
|
3233
|
+
...args,
|
|
3234
|
+
// A durable runner from before a restart is attached to only when it
|
|
3235
|
+
// was launched on this exact material.
|
|
3236
|
+
attachCompatibilityKey: JSON.stringify([
|
|
3237
|
+
args.attachCompatibilityKey ?? null,
|
|
3238
|
+
"lease-v1",
|
|
3239
|
+
identity.runEpoch,
|
|
3240
|
+
identity.credentialId,
|
|
3241
|
+
identity.revision,
|
|
3242
|
+
identity.version,
|
|
3243
|
+
]),
|
|
3244
|
+
agentEnv: {
|
|
3245
|
+
...(args.agentEnv ?? {}),
|
|
3246
|
+
...LEASED_CREDENTIAL_BLANKS,
|
|
3247
|
+
...engineSpawnEnv(account, home),
|
|
3248
|
+
[LEASED_RUN_ENV]: String(runEpoch),
|
|
3249
|
+
[LEASED_ACCOUNT_ENV]: account.credentialId,
|
|
3250
|
+
},
|
|
3251
|
+
},
|
|
3252
|
+
};
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
/**
|
|
3256
|
+
* The env of a one-shot run on the task's leased account for `kind` (the
|
|
3257
|
+
* shadow router's classifier): the agent's own account when it runs on
|
|
3258
|
+
* one of that kind, else the first healthy one; a config-directory
|
|
3259
|
+
* account's home is written first. Null when none can be used now: the
|
|
3260
|
+
* classifier does not run and the pass routes by rules.
|
|
3261
|
+
*/
|
|
3262
|
+
/** ADR-154 §6: a one-shot's env on the leased `account` chosen for it: its
|
|
3263
|
+
* home written first for a config-directory account. Null when it cannot
|
|
3264
|
+
* be given. */
|
|
3265
|
+
private async leasedOneShotEnv(
|
|
3266
|
+
channel: Channel,
|
|
3267
|
+
lease: TaskEngineLease,
|
|
3268
|
+
account: LeasedEngineAccount,
|
|
3269
|
+
environment: TaskEnvironmentHandle<unknown>,
|
|
3270
|
+
): Promise<Record<string, string> | null> {
|
|
3271
|
+
const marker = {
|
|
3272
|
+
[LEASED_RUN_ENV]: String(lease.assignment.runEpoch),
|
|
3273
|
+
[LEASED_ACCOUNT_ENV]: account.credentialId,
|
|
3274
|
+
};
|
|
3275
|
+
try {
|
|
3276
|
+
rememberLeasedEngineRuntime({
|
|
3277
|
+
taskId: channel.taskId,
|
|
3278
|
+
runEpoch: lease.assignment.runEpoch,
|
|
3279
|
+
runtime: environment.descriptor.locator,
|
|
3280
|
+
});
|
|
3281
|
+
} catch {
|
|
3282
|
+
return null;
|
|
3283
|
+
}
|
|
3284
|
+
if (account.authKind === "env") {
|
|
3285
|
+
return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
|
|
3286
|
+
}
|
|
3287
|
+
try {
|
|
3288
|
+
await ensureEngineHome(environment, lease.assignment, account.credentialId);
|
|
3289
|
+
return { ...LEASED_CREDENTIAL_BLANKS, ...engineSpawnEnv(account), ...marker };
|
|
3290
|
+
} catch {
|
|
3291
|
+
return null;
|
|
3292
|
+
}
|
|
3293
|
+
}
|
|
3294
|
+
|
|
3295
|
+
/**
|
|
3296
|
+
* ADR-154 §6, §8: what the task's AI accounts are, or whether its gate is
|
|
3297
|
+
* open, changed. Rewrite what moved in the homes this run wrote, then let
|
|
3298
|
+
* each long-lived engine running on a rewritten account (Codex's
|
|
3299
|
+
* app-server) respawn at its first idle moment; one-shot engines read the
|
|
3300
|
+
* file again at their next turn, and env-token engines take a new value at
|
|
3301
|
+
* their next spawn. Called synchronously by the engine consumer: the work
|
|
3302
|
+
* runs after, one pass at a time per channel.
|
|
3303
|
+
*/
|
|
3304
|
+
private leasedEnginesChanged(taskId: string): void {
|
|
3305
|
+
const channel = this.channels.get(taskId);
|
|
3306
|
+
if (!channel || !this.isActiveChannel(channel) || !this.isLeasedChannel(channel)) return;
|
|
3307
|
+
if (!leasedEnginesFor(taskId)?.open) return;
|
|
3308
|
+
if (channel.leasedSyncRetry) {
|
|
3309
|
+
clearTimeout(channel.leasedSyncRetry);
|
|
3310
|
+
channel.leasedSyncRetry = null;
|
|
3311
|
+
}
|
|
3312
|
+
// Agents a cleanup stopped, or that found no usable account, start again
|
|
3313
|
+
// on this view, each on its next account: one attempt per opening. The
|
|
3314
|
+
// intent outlives a pass that fails, for its retry.
|
|
3315
|
+
if (channel.leasedRespawn.size > 0 || channel.leasedBlocked.size > 0) {
|
|
3316
|
+
channel.leasedWakePending = true;
|
|
3317
|
+
}
|
|
3318
|
+
const run = (channel.leasedSync ?? Promise.resolve())
|
|
3319
|
+
.then(async () => {
|
|
3320
|
+
await this.syncLeasedHomes(channel);
|
|
3321
|
+
if (!channel.leasedWakePending || !this.isActiveChannel(channel)) return;
|
|
3322
|
+
channel.leasedWakePending = false;
|
|
3323
|
+
channel.leasedRespawn.clear();
|
|
3324
|
+
channel.leasedBlocked.clear();
|
|
3325
|
+
try {
|
|
3326
|
+
await this.ensureSessions(channel);
|
|
3327
|
+
} catch (err) {
|
|
3328
|
+
channel.leasedWakePending = true;
|
|
3329
|
+
throw err;
|
|
3330
|
+
}
|
|
3331
|
+
})
|
|
3332
|
+
.catch((err: unknown) => {
|
|
3333
|
+
console.warn(
|
|
3334
|
+
`[orchestrator] ${taskId}: rewriting AI account files failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
3335
|
+
);
|
|
3336
|
+
// A home left on an old token is written again, not left so.
|
|
3337
|
+
if (this.isActiveChannel(channel) && !channel.leasedSyncRetry) {
|
|
3338
|
+
const retry = setTimeout(() => {
|
|
3339
|
+
if (channel.leasedSyncRetry !== retry) return;
|
|
3340
|
+
channel.leasedSyncRetry = null;
|
|
3341
|
+
this.leasedEnginesChanged(taskId);
|
|
3342
|
+
}, LEASED_SYNC_RETRY_MS);
|
|
3343
|
+
retry.unref?.();
|
|
3344
|
+
channel.leasedSyncRetry = retry;
|
|
3345
|
+
}
|
|
3346
|
+
});
|
|
3347
|
+
channel.leasedSync = run;
|
|
3348
|
+
}
|
|
3349
|
+
|
|
3350
|
+
private async syncLeasedHomes(channel: Channel): Promise<void> {
|
|
3351
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3352
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3353
|
+
if (!lease?.open) return;
|
|
3354
|
+
// Nothing was written into a runtime this channel has not settled.
|
|
3355
|
+
const environment = channel.agentHandle;
|
|
3356
|
+
if (environment) await syncEngineHomes(environment, lease.assignment);
|
|
3357
|
+
if (!this.isActiveChannel(channel)) return;
|
|
3358
|
+
// Every launch is admitted on the exact material its agent is bound to
|
|
3359
|
+
// (§6): an agent whose account moved since starts again on what the
|
|
3360
|
+
// task holds now. A long-lived engine holds the old token or file; a
|
|
3361
|
+
// one-shot one, env-token or not, would have its next turn refused.
|
|
3362
|
+
let stale = false;
|
|
3363
|
+
for (const agent of channel.roster) {
|
|
3364
|
+
const identity = channel.leasedIdentityByAgent.get(agent.id);
|
|
3365
|
+
if (!identity || this.leasedMaterialServed(channel, identity)) continue;
|
|
3366
|
+
if (!channel.sessions.has(agent.id)) continue;
|
|
3367
|
+
channel.browserStaleSessions.add(agent.id);
|
|
3368
|
+
stale = true;
|
|
3369
|
+
}
|
|
3370
|
+
// A stale idle session is recycled now; one mid-turn finishes its turn
|
|
3371
|
+
// on the material it has and is recycled at its next idle moment.
|
|
3372
|
+
if (stale) await this.reconcileIfStarted(channel);
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
/**
|
|
3376
|
+
* ADR-154 §6: the cleanup coordinator's quiesce step for engines (its
|
|
3377
|
+
* `quiesceProcesses` hook takes this request as it is), for the run it
|
|
3378
|
+
* names and every older one, never a newer run's:
|
|
3379
|
+
* - every agent on one of the request's accounts (on any leased account,
|
|
3380
|
+
* or on its task token alone, for a whole-task cleanup) is unbound at
|
|
3381
|
+
* once, so nothing launches on them again (a launch is admitted only
|
|
3382
|
+
* while its agent is bound to its exact material), and its session is
|
|
3383
|
+
* asked to stop; a spawn creating an engine on one, or a launch being
|
|
3384
|
+
* admitted, is waited for, bounded;
|
|
3385
|
+
* - then, in every runtime those runs' engines ran in, every admission not
|
|
3386
|
+
* used yet is revoked, and everything each engine supervisor of those
|
|
3387
|
+
* runs and accounts owns is killed until the supervisor ends on its own,
|
|
3388
|
+
* having proved nothing it owned is left (`stopAnchoredEngines`). That
|
|
3389
|
+
* holds no state here, so a retried cleanup proves it again even after
|
|
3390
|
+
* this channel let the agents go. A supervisor lost without proving its
|
|
3391
|
+
* subtree empty leaves one proof: the runtime is stopped. An env-token
|
|
3392
|
+
* engine holds no credential file: stopping it moves it to its next
|
|
3393
|
+
* account.
|
|
3394
|
+
* Rejects when anything could not be proven stopped: the cleanup then
|
|
3395
|
+
* fails and is retried, never deleting under a live engine. Stopped agents
|
|
3396
|
+
* start again on their next account once the gate opens.
|
|
3397
|
+
*/
|
|
3398
|
+
async quiesceLeasedEngineAgents(
|
|
3399
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3400
|
+
): Promise<void> {
|
|
3401
|
+
const named = request.credentialIds === undefined ? null : new Set(request.credentialIds);
|
|
3402
|
+
const channel = this.channels.get(request.taskId);
|
|
3403
|
+
// Every engine being created on one of the accounts (on any subject, a
|
|
3404
|
+
// task token included, for a whole run's), this attempt's or left over
|
|
3405
|
+
// from one that timed out: never only the bound agents'.
|
|
3406
|
+
const creating = [...(channel?.leasedCreating ?? [])]
|
|
3407
|
+
.filter(
|
|
3408
|
+
(entry) =>
|
|
3409
|
+
(!named || (entry.accountId !== null && named.has(entry.accountId))) && entry.runEpoch <= request.runEpoch,
|
|
3410
|
+
)
|
|
3411
|
+
.map((entry) => entry.settled);
|
|
3412
|
+
// Every retirement still pending that this cleanup covers: its stop
|
|
3413
|
+
// proves what they fence, so one that fails from now on is not escalated.
|
|
3414
|
+
const covered = this.coveredLeasedRetirements(request);
|
|
3415
|
+
if (channel) {
|
|
3416
|
+
for (const [agentId, accountId] of [...channel.accountByAgent]) {
|
|
3417
|
+
if (named && !named.has(accountId)) continue;
|
|
3418
|
+
if ((channel.leasedIdentityByAgent.get(agentId)?.runEpoch ?? 0) > request.runEpoch) continue;
|
|
3419
|
+
channel.accountByAgent.delete(agentId);
|
|
3420
|
+
this.quiesceLeasedAgent(channel, agentId);
|
|
3421
|
+
}
|
|
3422
|
+
// A whole run's cleanup takes every agent on its task token alone too
|
|
3423
|
+
// (the Managed Engine); a cleanup of named accounts never does.
|
|
3424
|
+
if (!named) {
|
|
3425
|
+
for (const [agentId, identity] of [...channel.leasedIdentityByAgent]) {
|
|
3426
|
+
if (identity.subject !== "task-token" || identity.runEpoch > request.runEpoch) continue;
|
|
3427
|
+
this.quiesceLeasedAgent(channel, agentId);
|
|
3428
|
+
}
|
|
3429
|
+
}
|
|
3430
|
+
}
|
|
3431
|
+
if (creating.length > 0) {
|
|
3432
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
3433
|
+
const bounded = new Promise<"late">((resolve) => {
|
|
3434
|
+
timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
|
|
3435
|
+
});
|
|
3436
|
+
const outcome = await Promise.race([Promise.all(creating), bounded]);
|
|
3437
|
+
clearTimeout(timer);
|
|
3438
|
+
if (outcome === "late") {
|
|
3439
|
+
throw new Error("an engine being started on a removed AI account did not finish starting");
|
|
3440
|
+
}
|
|
3441
|
+
}
|
|
3442
|
+
// One of those under way is waited for, bounded, so nothing of it lands
|
|
3443
|
+
// after this cleanup's stop.
|
|
3444
|
+
const retiring = covered.flatMap((retired) => (retired.attempt ? [retired.attempt.catch(() => undefined)] : []));
|
|
3445
|
+
if (retiring.length > 0) {
|
|
3446
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
3447
|
+
const bounded = new Promise<"late">((resolve) => {
|
|
3448
|
+
timer = setTimeout(() => resolve("late"), LEASED_QUIESCE_EXIT_MS);
|
|
3449
|
+
});
|
|
3450
|
+
const outcome = await Promise.race([Promise.all(retiring), bounded]);
|
|
3451
|
+
clearTimeout(timer);
|
|
3452
|
+
if (outcome === "late") {
|
|
3453
|
+
throw new Error("an engine launch slot being retired in its runtime did not finish");
|
|
3454
|
+
}
|
|
3455
|
+
}
|
|
3456
|
+
const runtimes = await this.leasedCleanupRuntimes(request, channel);
|
|
3457
|
+
for (const environment of runtimes.values()) {
|
|
3458
|
+
// A runtime that is not running holds no process, unless it is frozen
|
|
3459
|
+
// (paused or restarting: its processes run again without a start);
|
|
3460
|
+
// one that is frozen or cannot say cannot be proven.
|
|
3461
|
+
const status = await environment.status();
|
|
3462
|
+
if (status.state !== "unknown" && status.frozen) {
|
|
3463
|
+
throw new Error("a task runtime's engines could not be proven stopped: it is paused");
|
|
3464
|
+
}
|
|
3465
|
+
if (status.state === "stopped" || status.state === "absent") continue;
|
|
3466
|
+
if (status.state !== "running") {
|
|
3467
|
+
throw new Error("a task runtime's engines could not be proven stopped: its state is unknown");
|
|
3468
|
+
}
|
|
3469
|
+
try {
|
|
3470
|
+
await stopAnchoredEngines(environment, {
|
|
3471
|
+
taskId: request.taskId,
|
|
3472
|
+
runEpoch: request.runEpoch,
|
|
3473
|
+
...(request.credentialIds === undefined ? {} : { credentialIds: request.credentialIds }),
|
|
3474
|
+
});
|
|
3475
|
+
} catch (error) {
|
|
3476
|
+
if (!(error instanceof EngineAnchorLost)) throw error;
|
|
3477
|
+
// What a lost supervisor owned may have been reparented away from
|
|
3478
|
+
// it: only a stopped runtime proves none of it still runs.
|
|
3479
|
+
console.warn(`[orchestrator] ${request.taskId}: ${error.message}; stopping the runtime`);
|
|
3480
|
+
await environment.stop();
|
|
3481
|
+
const stopped = await environment.status();
|
|
3482
|
+
if (
|
|
3483
|
+
stopped.state === "unknown" ||
|
|
3484
|
+
stopped.frozen ||
|
|
3485
|
+
(stopped.state !== "stopped" && stopped.state !== "absent")
|
|
3486
|
+
) {
|
|
3487
|
+
throw error;
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
// Every admission of these runs not yet used is revoked in every runtime
|
|
3492
|
+
// they were written in, or that runtime is stopped or gone and its gates
|
|
3493
|
+
// with it: no revocation or slot retirement retained for them is left to
|
|
3494
|
+
// do.
|
|
3495
|
+
dischargeEngineRevocations(request);
|
|
3496
|
+
this.dischargeLeasedRetirements(request);
|
|
3497
|
+
// Every engine of these runs is proven stopped in every runtime they
|
|
3498
|
+
// ran in: a whole-task cleanup lets the record of those runtimes go.
|
|
3499
|
+
if (request.credentialIds === undefined) {
|
|
3500
|
+
forgetLeasedEngineRuntimes(request.taskId, request.runEpoch, [...runtimes.keys()]);
|
|
3501
|
+
}
|
|
3502
|
+
}
|
|
3503
|
+
|
|
3504
|
+
/** A cleanup took the material `agentId` runs on: it is unbound at once,
|
|
3505
|
+
* so nothing launches on it again, its session is asked to stop, and it
|
|
3506
|
+
* starts again on what the task holds once the gate opens. */
|
|
3507
|
+
private quiesceLeasedAgent(channel: Channel, agentId: string): void {
|
|
3508
|
+
channel.leasedIdentityByAgent.delete(agentId);
|
|
3509
|
+
const session = channel.sessions.get(agentId);
|
|
3510
|
+
if (this.isActiveChannel(channel)) channel.leasedRespawn.add(agentId);
|
|
3511
|
+
if (!session) return;
|
|
3512
|
+
// Delete first: the old session's last events are generation-stale.
|
|
3513
|
+
channel.sessions.delete(agentId);
|
|
3514
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3515
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3516
|
+
this.retainRetiredTurnBoundary(
|
|
3517
|
+
channel,
|
|
3518
|
+
agentId,
|
|
3519
|
+
this.needsAbortedTurnBoundary(channel, agentId),
|
|
3520
|
+
);
|
|
3521
|
+
this.retireRunnerGeneration(channel, agentId);
|
|
3522
|
+
void session.close().catch(() => {});
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
/**
|
|
3526
|
+
* ADR-154 §6: an account is about to be served at a newer version. In
|
|
3527
|
+
* every runtime a launch of this run could start in, its older versions
|
|
3528
|
+
* are held back first: a launch admitted on one that has not started yet
|
|
3529
|
+
* never will. Rejects, so the install fails and none of the newer version
|
|
3530
|
+
* is served, when a runtime cannot be reached or proven not running.
|
|
3531
|
+
*/
|
|
3532
|
+
private async raiseLeasedAdmissionFloors(
|
|
3533
|
+
assignment: { taskId: string; runEpoch: number },
|
|
3534
|
+
floors: readonly EngineAdmissionFloor[],
|
|
3535
|
+
): Promise<void> {
|
|
3536
|
+
const channel = this.channels.get(assignment.taskId);
|
|
3537
|
+
const runtimes = await this.leasedCleanupRuntimes(
|
|
3538
|
+
{ taskId: assignment.taskId, runEpoch: assignment.runEpoch },
|
|
3539
|
+
channel,
|
|
3540
|
+
);
|
|
3541
|
+
for (const environment of runtimes.values()) {
|
|
3542
|
+
const status = await environment.status();
|
|
3543
|
+
if (status.state !== "unknown" && status.frozen) {
|
|
3544
|
+
throw new Error("an AI account's older version could not be held back: a task runtime is paused");
|
|
3545
|
+
}
|
|
3546
|
+
if (status.state === "stopped" || status.state === "absent") continue;
|
|
3547
|
+
if (status.state !== "running") {
|
|
3548
|
+
throw new Error("an AI account's older version could not be held back: a task runtime's state is unknown");
|
|
3549
|
+
}
|
|
3550
|
+
await raiseEngineAdmissionFloors(environment, {
|
|
3551
|
+
taskId: assignment.taskId,
|
|
3552
|
+
runEpoch: assignment.runEpoch,
|
|
3553
|
+
floors,
|
|
3554
|
+
});
|
|
3555
|
+
}
|
|
3556
|
+
}
|
|
3557
|
+
|
|
3558
|
+
/**
|
|
3559
|
+
* Every runtime a cleanup reaches, by locator: the task's current one,
|
|
3560
|
+
* every one a leased engine of these runs was started in (recorded before
|
|
3561
|
+
* it started, files or none), and every one the inventory names for these
|
|
3562
|
+
* runs (an older runtime's files are deleted from that exact runtime).
|
|
3563
|
+
*/
|
|
3564
|
+
private async leasedCleanupRuntimes(
|
|
3565
|
+
request: Pick<TaskCredentialCleanupRequest, "taskId" | "runEpoch" | "credentialIds">,
|
|
3566
|
+
channel: Channel | undefined,
|
|
3567
|
+
): Promise<Map<string, TaskEnvironmentHandle<unknown>>> {
|
|
3568
|
+
const runtimes = new Map<string, TaskEnvironmentHandle<unknown>>();
|
|
3569
|
+
const current =
|
|
3570
|
+
channel?.agentHandle ??
|
|
3571
|
+
(await (async () => {
|
|
3572
|
+
const task = getHostTask(request.taskId);
|
|
3573
|
+
return task ? reconstructPersistedTaskEnvironment(task) : null;
|
|
3574
|
+
})());
|
|
3575
|
+
if (current) runtimes.set(serializeTaskEnvironmentLocator(current.descriptor.locator), current);
|
|
3576
|
+
const recorded = [
|
|
3577
|
+
...listLeasedEngineRuntimes(request.taskId),
|
|
3578
|
+
...listCredentialCleanupPaths(request.taskId),
|
|
3579
|
+
];
|
|
3580
|
+
for (const row of recorded) {
|
|
3581
|
+
if (row.runEpoch > request.runEpoch || runtimes.has(row.runtimeLocator)) continue;
|
|
3582
|
+
const environment = await reconstructTaskEnvironment(
|
|
3583
|
+
parseTaskEnvironmentLocator(row.runtimeLocator),
|
|
3584
|
+
"loader",
|
|
3585
|
+
);
|
|
3586
|
+
if (environment.descriptor.taskId !== request.taskId) {
|
|
3587
|
+
throw new Error("a cleanup runtime belongs to a different task");
|
|
3588
|
+
}
|
|
3589
|
+
runtimes.set(row.runtimeLocator, environment);
|
|
3590
|
+
}
|
|
3591
|
+
return runtimes;
|
|
3592
|
+
}
|
|
3593
|
+
|
|
2414
3594
|
/**
|
|
2415
3595
|
* Per-agent CODEX_HOME (see the call site). Returns `args` unchanged unless
|
|
2416
3596
|
* this is a codex agent with a config-dir CODEX_HOME; then it seeds
|
|
@@ -2481,87 +3661,347 @@ export class Orchestrator {
|
|
|
2481
3661
|
channel.reconcileAgain = true;
|
|
2482
3662
|
return null;
|
|
2483
3663
|
}
|
|
2484
|
-
// ADR-
|
|
2485
|
-
//
|
|
2486
|
-
//
|
|
2487
|
-
//
|
|
2488
|
-
//
|
|
2489
|
-
//
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
const
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
3664
|
+
// ADR-154: an agent on the runner's own accounts holds them from here
|
|
3665
|
+
// until it is bound or given up. Leaving them waits for it; every process
|
|
3666
|
+
// it starts, now or at a later turn, is refused at its provider's last
|
|
3667
|
+
// moment once leaving has begun; and one created across that moment is
|
|
3668
|
+
// closed, never bound. Its next start is on the task's lease. The hold
|
|
3669
|
+
// ends in a `finally`, which adds no await (see below).
|
|
3670
|
+
const ownHold = this.isLeasedChannel(channel) ? undefined : holdOwnCredentials();
|
|
3671
|
+
if (ownHold === null) {
|
|
3672
|
+
channel.reconcileAgain = true;
|
|
3673
|
+
return null;
|
|
3674
|
+
}
|
|
3675
|
+
const onOwn = ownHold !== undefined;
|
|
3676
|
+
try {
|
|
3677
|
+
// ADR-076 + codex-cli 0.152.0: two codex agents that resolve to the SAME
|
|
3678
|
+
// account share one CODEX_HOME, and codex's new sqlite state runtime
|
|
3679
|
+
// cannot be opened by two concurrent processes — the second agent dies
|
|
3680
|
+
// with "failed to initialize sqlite state runtime under <CODEX_HOME>"
|
|
3681
|
+
// (live 2026-09-04, a task with two codex agents on one account). Give
|
|
3682
|
+
// each codex agent its OWN CODEX_HOME, seeded once from the resolved
|
|
3683
|
+
// account's dir, so their state runtimes never collide. Single-codex
|
|
3684
|
+
// tasks are unaffected beyond a one-time in-container copy.
|
|
3685
|
+
// Only codex agents take the isolation path; everything else uses args
|
|
3686
|
+
// directly, WITHOUT an extra await — a stray microtask here reordered the
|
|
3687
|
+
// Claude re-spawn's prompt re-delivery. A leased account is applied here
|
|
3688
|
+
// too, synchronously when nothing has to be waited for or written.
|
|
3689
|
+
let boundArgs = args;
|
|
3690
|
+
let identity: LeasedIdentity | undefined;
|
|
3691
|
+
const kind = args.agent?.kind ?? "";
|
|
3692
|
+
if (!onOwn && LEASED_ENGINE_KINDS.has(kind)) {
|
|
3693
|
+
// §6: a runtime that cannot start a launch under the engine supervisor
|
|
3694
|
+
// never runs a leased engine.
|
|
3695
|
+
if (environment.anchors !== true) {
|
|
3696
|
+
this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
|
|
3697
|
+
return null;
|
|
3698
|
+
}
|
|
3699
|
+
const leased = this.leasedSpawnArgs(channel, agentId, args, environment);
|
|
3700
|
+
const applied = leased instanceof Promise ? await leased : leased;
|
|
3701
|
+
// Down until the gate next opens: never a loop.
|
|
3702
|
+
if (!applied) return null;
|
|
3703
|
+
boundArgs = applied.args;
|
|
3704
|
+
identity = applied.identity;
|
|
3705
|
+
} else if (!onOwn && kind === MANAGED_ENGINE_KIND) {
|
|
3706
|
+
// ADR-128, ADR-154 §6: no AI account, but the agent's task token is
|
|
3707
|
+
// the engine's bearer: started under the supervisor, admitted as
|
|
3708
|
+
// that token on the task's open run, or not at all.
|
|
3709
|
+
if (environment.anchors !== true) {
|
|
3710
|
+
this.leasedUnavailable(channel, agentId, "its runtime cannot supervise an engine");
|
|
3711
|
+
return null;
|
|
3712
|
+
}
|
|
3713
|
+
const leased = this.leasedTaskTokenSpawnArgs(channel, agentId, args);
|
|
3714
|
+
const applied = leased instanceof Promise ? await leased : leased;
|
|
3715
|
+
if (!applied) return null;
|
|
3716
|
+
boundArgs = applied.args;
|
|
3717
|
+
identity = applied.identity;
|
|
3718
|
+
} else if (!onOwn && !CREDENTIAL_FREE_KINDS.has(kind)) {
|
|
3719
|
+
// ADR-154 §10: a kind with no lane (Kimi) never runs on the runner's
|
|
3720
|
+
// own sign-in for a leased task.
|
|
3721
|
+
this.leasedUnavailable(channel, agentId, `${kind} has no AI account lane on cloud credentials`);
|
|
2534
3722
|
return null;
|
|
3723
|
+
} else if (kind === "codex") {
|
|
3724
|
+
boundArgs = await this.isolateCodexHome(environment, agentId, args);
|
|
2535
3725
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
3726
|
+
while (this.isActiveChannel(channel)) {
|
|
3727
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
3728
|
+
channel.reconcileAgain = true;
|
|
3729
|
+
return null;
|
|
3730
|
+
}
|
|
3731
|
+
if (
|
|
3732
|
+
channel.rosterSessionChanges.get(agentId)?.state === "pending-close"
|
|
3733
|
+
) {
|
|
3734
|
+
channel.reconcileAgain = true;
|
|
3735
|
+
return null;
|
|
3736
|
+
}
|
|
3737
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
3738
|
+
channel.reconcileAgain = true;
|
|
3739
|
+
return null;
|
|
3740
|
+
}
|
|
3741
|
+
if (onOwn && !onOwnCredentials()) {
|
|
3742
|
+
channel.reconcileAgain = true;
|
|
3743
|
+
return null;
|
|
3744
|
+
}
|
|
3745
|
+
const generation = channel.configGeneration;
|
|
3746
|
+
// ADR-154 §6: never start an engine on lease material the task does not
|
|
3747
|
+
// hold right now (its run, account, revision and version). A cleanup
|
|
3748
|
+
// drops the binding synchronously, so a spawn whose account it took
|
|
3749
|
+
// stops here; one already creating is registered, and every cleanup
|
|
3750
|
+
// attempt waits for it.
|
|
3751
|
+
if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
|
|
3752
|
+
this.retryLeasedStart(channel, agentId);
|
|
3753
|
+
return null;
|
|
3754
|
+
}
|
|
3755
|
+
if (identity) {
|
|
3756
|
+
// Where this engine runs, before it can run there (§6): a cleanup
|
|
3757
|
+
// proves it stopped in this runtime even if it leaves no file.
|
|
3758
|
+
try {
|
|
3759
|
+
rememberLeasedEngineRuntime({
|
|
3760
|
+
taskId: channel.taskId,
|
|
3761
|
+
runEpoch: identity.runEpoch,
|
|
3762
|
+
runtime: environment.descriptor.locator,
|
|
3763
|
+
});
|
|
3764
|
+
} catch (err) {
|
|
3765
|
+
console.warn(
|
|
3766
|
+
`[orchestrator] ${channel.taskId}/${agentId}: not starting on its AI account, its runtime could not be recorded: ${err instanceof Error ? err.message : String(err)}`,
|
|
3767
|
+
);
|
|
3768
|
+
return null;
|
|
3769
|
+
}
|
|
3770
|
+
}
|
|
3771
|
+
// Every process the adapter launches for this engine (now, at each
|
|
3772
|
+
// turn, or once its durable runner is deployed) is admitted then, on
|
|
3773
|
+
// this exact material, for this spawn while it is the agent's current
|
|
3774
|
+
// one, and started under the supervisor (§6). The agent's spawn this
|
|
3775
|
+
// one replaces is retired.
|
|
3776
|
+
const slot = identity ? newLeasedLaunchSlot(channel.taskId, identity, environment) : null;
|
|
3777
|
+
if (slot) {
|
|
3778
|
+
void this.retireLeasedLaunch(channel, agentId, channel.leasedLaunches.get(agentId));
|
|
3779
|
+
channel.leasedLaunches.set(agentId, slot);
|
|
3780
|
+
}
|
|
3781
|
+
const creating = this.factory.create({
|
|
3782
|
+
...boundArgs,
|
|
3783
|
+
environment:
|
|
3784
|
+
identity && slot
|
|
3785
|
+
? this.leasedLaunchEnvironment(channel, { agentId, slot }, identity, environment)
|
|
3786
|
+
: onOwn
|
|
3787
|
+
? onOwnCredentialsEnvironment(environment)
|
|
3788
|
+
: environment,
|
|
3789
|
+
});
|
|
3790
|
+
// Settled before this spawn's own continuation: from here until it
|
|
3791
|
+
// binds, nothing launches on it.
|
|
3792
|
+
if (slot) {
|
|
3793
|
+
void creating.then(
|
|
3794
|
+
() => {
|
|
3795
|
+
slot.created = true;
|
|
3796
|
+
},
|
|
3797
|
+
() => {
|
|
3798
|
+
slot.created = true;
|
|
3799
|
+
},
|
|
2544
3800
|
);
|
|
2545
|
-
} else {
|
|
2546
|
-
await session.close().catch(() => {});
|
|
2547
3801
|
}
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
3802
|
+
if (identity) {
|
|
3803
|
+
const entry = {
|
|
3804
|
+
agentId,
|
|
3805
|
+
accountId: identity.subject === "account" ? identity.credentialId : null,
|
|
3806
|
+
runEpoch: identity.runEpoch,
|
|
3807
|
+
settled: creating.then(
|
|
3808
|
+
() => undefined,
|
|
3809
|
+
() => undefined,
|
|
3810
|
+
),
|
|
3811
|
+
};
|
|
3812
|
+
channel.leasedCreating.add(entry);
|
|
3813
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
3814
|
+
}
|
|
3815
|
+
// ADR-154 §6: a spawn that does not bind (its create failed, maybe
|
|
3816
|
+
// after it launched, or its session is closed unbound) is retired at
|
|
3817
|
+
// once, before anything else: a launch it handed over that has not
|
|
3818
|
+
// started in its runtime never does.
|
|
3819
|
+
const unbound = (): Promise<void> =>
|
|
3820
|
+
slot ? this.retireLeasedLaunch(channel, agentId, slot) : Promise.resolve();
|
|
3821
|
+
let session: AgentSession;
|
|
3822
|
+
try {
|
|
3823
|
+
session = await creating;
|
|
3824
|
+
} catch (error) {
|
|
3825
|
+
await unbound();
|
|
3826
|
+
throw error;
|
|
3827
|
+
}
|
|
3828
|
+
if (!this.isActiveChannel(channel)) {
|
|
3829
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3830
|
+
return null;
|
|
3831
|
+
}
|
|
3832
|
+
// ADR-154: created on the runner's own accounts across the moment
|
|
3833
|
+
// leaving them began: closed, never bound.
|
|
3834
|
+
if (onOwn && !onOwnCredentials()) {
|
|
3835
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3836
|
+
channel.reconcileAgain = true;
|
|
3837
|
+
return null;
|
|
3838
|
+
}
|
|
3839
|
+
// ADR-154 §6: a cleanup that took this account while the session was
|
|
3840
|
+
// being created quiesced the agent already; never keep a session on
|
|
3841
|
+
// material the task no longer holds.
|
|
3842
|
+
if (identity && !this.leasedIdentityHeld(channel, agentId, identity)) {
|
|
3843
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3844
|
+
this.retryLeasedStart(channel, agentId);
|
|
3845
|
+
return null;
|
|
3846
|
+
}
|
|
3847
|
+
if (identity) channel.leasedBlocked.delete(agentId);
|
|
3848
|
+
if (recovery && !this.recoveryIsCurrent(channel, agentId, recovery)) {
|
|
3849
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
3850
|
+
channel.reconcileAgain = true;
|
|
3851
|
+
return null;
|
|
3852
|
+
}
|
|
3853
|
+
const pendingRosterClose = channel.rosterSessionChanges.get(agentId);
|
|
3854
|
+
if (pendingRosterClose?.state === "pending-close") {
|
|
3855
|
+
await Promise.all([
|
|
3856
|
+
unbound(),
|
|
3857
|
+
this.closeRosterChangedSession(channel, agentId, session, pendingRosterClose),
|
|
3858
|
+
]);
|
|
3859
|
+
channel.reconcileAgain = true;
|
|
3860
|
+
return null;
|
|
3861
|
+
}
|
|
3862
|
+
if (!this.hasDesiredLaunchSettings(channel, agentId, boundArgs.agent)) {
|
|
3863
|
+
const change = channel.rosterSessionChanges.get(agentId);
|
|
3864
|
+
const retiring = unbound();
|
|
3865
|
+
if (change) {
|
|
3866
|
+
await this.closeRosterChangedSession(
|
|
3867
|
+
channel,
|
|
3868
|
+
agentId,
|
|
3869
|
+
session,
|
|
3870
|
+
change,
|
|
3871
|
+
);
|
|
3872
|
+
} else {
|
|
3873
|
+
await session.close().catch(() => {});
|
|
3874
|
+
}
|
|
3875
|
+
await retiring;
|
|
3876
|
+
channel.reconcileAgain = true;
|
|
3877
|
+
return null;
|
|
3878
|
+
}
|
|
3879
|
+
if (generation === channel.configGeneration) {
|
|
3880
|
+
// Keep the final generation check, stale cleanup, and map bind in one
|
|
3881
|
+
// synchronous continuation. A config writer that resumes immediately
|
|
3882
|
+
// afterward now sees this session and marks it stale; it can no longer
|
|
3883
|
+
// change the generation in the await→bind gap while the process is
|
|
3884
|
+
// invisible.
|
|
3885
|
+
channel.browserStaleSessions.delete(agentId);
|
|
3886
|
+
channel.browserPendingStaleSessions.delete(agentId);
|
|
3887
|
+
if (slot) slot.session = session;
|
|
3888
|
+
this.bindSession(channel, agentId, session);
|
|
3889
|
+
return session;
|
|
3890
|
+
}
|
|
3891
|
+
await Promise.all([unbound(), session.close().catch(() => {})]);
|
|
2561
3892
|
}
|
|
2562
|
-
|
|
3893
|
+
return null;
|
|
3894
|
+
} finally {
|
|
3895
|
+
ownHold?.();
|
|
2563
3896
|
}
|
|
2564
|
-
|
|
3897
|
+
}
|
|
3898
|
+
|
|
3899
|
+
/**
|
|
3900
|
+
* ADR-154, the shadow router's one exec: the only use it makes of an AI
|
|
3901
|
+
* account, and the only place that account's material exists for it.
|
|
3902
|
+
* Null when it cannot run, and the pass routes by rules.
|
|
3903
|
+
* - A leased task's: its account is chosen, and the exec registered where
|
|
3904
|
+
* a cleanup of that account waits, in one synchronous step before any of
|
|
3905
|
+
* its material is read; the registration settles when the exec does,
|
|
3906
|
+
* and nothing the material is in outlives this call (§6). A cleanup that
|
|
3907
|
+
* took the account first leaves it unchosen. The exec starts under the
|
|
3908
|
+
* engine supervisor, only on material the task still holds at the
|
|
3909
|
+
* provider's last moment.
|
|
3910
|
+
* - On the runner's own credentials: they are held from the account's
|
|
3911
|
+
* choice to the exec's end, and the exec is refused at the provider's
|
|
3912
|
+
* last moment once leaving them has begun or the channel has closed.
|
|
3913
|
+
*/
|
|
3914
|
+
private async shadowClassify(
|
|
3915
|
+
channel: Channel,
|
|
3916
|
+
agentId: string,
|
|
3917
|
+
kind: string,
|
|
3918
|
+
handle: TaskEnvironmentHandle<unknown>,
|
|
3919
|
+
argv: [string, ...string[]],
|
|
3920
|
+
scratchDir: string,
|
|
3921
|
+
timeoutMs: number,
|
|
3922
|
+
): Promise<{ exitCode: number | null; stdout: string } | null> {
|
|
3923
|
+
const request = (env: Record<string, string>, admit: () => void) => ({
|
|
3924
|
+
argv: agentCliArgv(handle, argv),
|
|
3925
|
+
projectEnvProfile: "task" as const,
|
|
3926
|
+
cwd: scratchDir,
|
|
3927
|
+
user: "node",
|
|
3928
|
+
env,
|
|
3929
|
+
timeoutMs,
|
|
3930
|
+
maxOutputBytes: 64 * 1024,
|
|
3931
|
+
admit,
|
|
3932
|
+
});
|
|
3933
|
+
if (!this.isLeasedChannel(channel)) {
|
|
3934
|
+
const running = whileOnOwnCredentials(async () => {
|
|
3935
|
+
const env = { ...(pickEngineAccount(kind, { scope: channel.engineAccountScope })?.execEnv ?? {}) };
|
|
3936
|
+
const r = await handle.exec(
|
|
3937
|
+
request(env, () => {
|
|
3938
|
+
if (!this.isActiveChannel(channel) || !onOwnCredentials()) throw new LeftOwnCredentialsError();
|
|
3939
|
+
}),
|
|
3940
|
+
);
|
|
3941
|
+
return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
|
|
3942
|
+
});
|
|
3943
|
+
if (running === null) return null;
|
|
3944
|
+
this.trackEventOperation(channel, running);
|
|
3945
|
+
return running;
|
|
3946
|
+
}
|
|
3947
|
+
const lease = leasedEnginesFor(channel.taskId);
|
|
3948
|
+
if (!lease?.open) return null;
|
|
3949
|
+
const account = this.leasedAccountFor(channel, agentId, kind, lease);
|
|
3950
|
+
if (!account) return null;
|
|
3951
|
+
const identity: LeasedIdentity = {
|
|
3952
|
+
subject: "account",
|
|
3953
|
+
runEpoch: lease.assignment.runEpoch,
|
|
3954
|
+
credentialId: account.credentialId,
|
|
3955
|
+
revision: account.revision,
|
|
3956
|
+
version: account.version,
|
|
3957
|
+
};
|
|
3958
|
+
let settle!: () => void;
|
|
3959
|
+
const entry = {
|
|
3960
|
+
agentId: `shadow:${agentId}`,
|
|
3961
|
+
accountId: account.credentialId,
|
|
3962
|
+
runEpoch: identity.runEpoch,
|
|
3963
|
+
settled: new Promise<void>((resolve) => (settle = resolve)),
|
|
3964
|
+
};
|
|
3965
|
+
channel.leasedCreating.add(entry);
|
|
3966
|
+
void entry.settled.then(() => channel.leasedCreating.delete(entry));
|
|
3967
|
+
this.trackEventOperation(channel, entry.settled);
|
|
3968
|
+
try {
|
|
3969
|
+
const env = await this.leasedOneShotEnv(channel, lease, account, handle);
|
|
3970
|
+
if (!env || !this.leasedMaterialServed(channel, identity)) return null;
|
|
3971
|
+
const r = await this.leasedLaunchEnvironment(channel, null, identity, handle).exec(
|
|
3972
|
+
request(env, () => {
|
|
3973
|
+
if (!this.isActiveChannel(channel) || !this.leasedMaterialServed(channel, identity)) {
|
|
3974
|
+
throw new CredentialLeaseUnavailableError("the shadow classifier's account moved");
|
|
3975
|
+
}
|
|
3976
|
+
}),
|
|
3977
|
+
);
|
|
3978
|
+
return { exitCode: r.exitCode, stdout: Buffer.from(r.stdout).toString("utf8") };
|
|
3979
|
+
} finally {
|
|
3980
|
+
settle();
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
|
|
3984
|
+
/** #cb3j0z: the task's runtime, for what only it can do in the task's
|
|
3985
|
+
* workspace without following its links (macOS); null without one. */
|
|
3986
|
+
private async workspaceRuntime(taskId: string): Promise<RuntimeNamespace | null> {
|
|
3987
|
+
const channel = this.channels.get(taskId);
|
|
3988
|
+
if (!channel || !this.isActiveChannel(channel)) return null;
|
|
3989
|
+
try {
|
|
3990
|
+
return runtimeNamespaceOf(await this.agentEnvironment(channel));
|
|
3991
|
+
} catch {
|
|
3992
|
+
return null;
|
|
3993
|
+
}
|
|
3994
|
+
}
|
|
3995
|
+
|
|
3996
|
+
/** Teardown drains `operation` before the task's runtime name can be
|
|
3997
|
+
* reused. */
|
|
3998
|
+
private trackEventOperation(channel: Channel, operation: Promise<unknown>): void {
|
|
3999
|
+
const settled = operation.then(
|
|
4000
|
+
() => undefined,
|
|
4001
|
+
() => undefined,
|
|
4002
|
+
);
|
|
4003
|
+
channel.eventOperations.add(settled);
|
|
4004
|
+
void settled.then(() => channel.eventOperations.delete(settled));
|
|
2565
4005
|
}
|
|
2566
4006
|
|
|
2567
4007
|
private hasDesiredLaunchSettings(
|
|
@@ -2808,6 +4248,9 @@ export class Orchestrator {
|
|
|
2808
4248
|
),
|
|
2809
4249
|
},
|
|
2810
4250
|
);
|
|
4251
|
+
// Credentials unavailable is this agent's alone: the gate's next
|
|
4252
|
+
// opening starts it; its siblings start now.
|
|
4253
|
+
if (!session && channel.leasedBlocked.has(agent.id)) continue;
|
|
2811
4254
|
if (!session) {
|
|
2812
4255
|
// The agents that did come up are usable; the reconcile pass keeps
|
|
2813
4256
|
// retrying this one. Do not leave the page covered for a session
|
|
@@ -3203,13 +4646,16 @@ export class Orchestrator {
|
|
|
3203
4646
|
// through this same command, and classifying them would poison the very
|
|
3204
4647
|
// data the shadow pass exists to collect.
|
|
3205
4648
|
if (channel.routingShadow && origin === "human") {
|
|
3206
|
-
|
|
4649
|
+
// Its exec runs in the task's runtime: teardown drains it like any
|
|
4650
|
+
// other external operation before the runtime's name can be reused.
|
|
4651
|
+
const pass = this.shadowRouteDelivery(channel, agentId, text).catch((err) => {
|
|
3207
4652
|
console.warn(
|
|
3208
4653
|
`[routing] shadow pass failed for ${channel.taskId}/${agentId}: ${
|
|
3209
4654
|
err instanceof Error ? err.message : String(err)
|
|
3210
4655
|
}`,
|
|
3211
4656
|
);
|
|
3212
4657
|
});
|
|
4658
|
+
this.trackEventOperation(channel, pass);
|
|
3213
4659
|
}
|
|
3214
4660
|
return { ok: true };
|
|
3215
4661
|
}
|
|
@@ -3219,7 +4665,10 @@ export class Orchestrator {
|
|
|
3219
4665
|
* one-shot executes INSIDE the task environment as the task user, with the
|
|
3220
4666
|
* same engine account the agent runs on (so the user's own subscription
|
|
3221
4667
|
* pays for routing, never Uai), then the result rides a host event. Bounded
|
|
3222
|
-
* by a hard timeout so a hung CLI
|
|
4668
|
+
* by a hard timeout, the task's own files read included, so a hung CLI or
|
|
4669
|
+
* read cannot pile up passes. ADR-154: the pass itself holds no credential
|
|
4670
|
+
* (its reads are the task's own files, which take as long as the task
|
|
4671
|
+
* makes them); only its one exec does (`shadowClassify`).
|
|
3223
4672
|
*/
|
|
3224
4673
|
private async shadowRouteDelivery(channel: Channel, agentId: string, text: string): Promise<void> {
|
|
3225
4674
|
const agent = channel.roster.find((a) => a.id === agentId);
|
|
@@ -3236,39 +4685,44 @@ export class Orchestrator {
|
|
|
3236
4685
|
// Rung 1 hosts the classifier on the agent's own engine; the Managed
|
|
3237
4686
|
// Engine delegates to claude/codex, so borrow the delegate's account.
|
|
3238
4687
|
const hostEngine = agent.kind === "uai" ? (agent.uaiProvider === "openai" ? "codex" : "claude") : agent.kind;
|
|
3239
|
-
const account = pickEngineAccount(hostEngine, { scope: channel.engineAccountScope });
|
|
3240
|
-
const env: Record<string, string> = { ...(account?.execEnv ?? {}) };
|
|
3241
|
-
const context =
|
|
3242
|
-
handle && canRead
|
|
3243
|
-
? Buffer.from((await handle.readWorkspaceFile(`${scratchDir}/chat.md`).catch(() => null)) ?? new Uint8Array())
|
|
3244
|
-
.toString("utf8")
|
|
3245
|
-
.slice(-1200)
|
|
3246
|
-
: "";
|
|
3247
4688
|
const timeoutMs = 45_000;
|
|
4689
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
4690
|
+
// Past its bound the pass is given up, and what is left of it (a read
|
|
4691
|
+
// the task held) launches nothing.
|
|
4692
|
+
let abandoned = false;
|
|
3248
4693
|
const result = await Promise.race([
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
4694
|
+
(async () => {
|
|
4695
|
+
const context =
|
|
4696
|
+
handle && canRead
|
|
4697
|
+
? Buffer.from(
|
|
4698
|
+
(await handle
|
|
4699
|
+
.readWorkspaceFile(`${scratchDir}/chat.md`, { maxBytes: SHADOW_CONTEXT_BYTES, from: "end" })
|
|
4700
|
+
.catch(() => null)) ?? new Uint8Array(),
|
|
4701
|
+
)
|
|
4702
|
+
.toString("utf8")
|
|
4703
|
+
.slice(-1200)
|
|
4704
|
+
: "";
|
|
4705
|
+
return shadowRoute(text, context, hostEngine, {
|
|
4706
|
+
scratchDir,
|
|
4707
|
+
exec: (argv) =>
|
|
4708
|
+
handle && canExec && !abandoned
|
|
4709
|
+
? this.shadowClassify(channel, agentId, hostEngine, handle, argv, scratchDir, timeoutMs)
|
|
4710
|
+
: Promise.resolve(null),
|
|
4711
|
+
readFile: async (path) => {
|
|
4712
|
+
if (!handle || !canRead) return null;
|
|
4713
|
+
const bytes = await handle.readWorkspaceFile(path, { maxBytes: SHADOW_OUTPUT_BYTES }).catch(() => null);
|
|
4714
|
+
return bytes ? Buffer.from(bytes).toString("utf8") : null;
|
|
4715
|
+
},
|
|
4716
|
+
});
|
|
4717
|
+
})(),
|
|
4718
|
+
new Promise<null>((resolve) => {
|
|
4719
|
+
timer = setTimeout(() => {
|
|
4720
|
+
abandoned = true;
|
|
4721
|
+
resolve(null);
|
|
4722
|
+
}, timeoutMs + 5_000);
|
|
3269
4723
|
}),
|
|
3270
|
-
new Promise<null>((resolve) => setTimeout(() => resolve(null), timeoutMs + 5_000)),
|
|
3271
4724
|
]);
|
|
4725
|
+
clearTimeout(timer);
|
|
3272
4726
|
if (!result) {
|
|
3273
4727
|
console.warn(`[routing] shadow pass timed out for ${channel.taskId}/${agentId}`);
|
|
3274
4728
|
return;
|
|
@@ -3507,14 +4961,21 @@ export class Orchestrator {
|
|
|
3507
4961
|
if (ownsBrokerSlot) channel.spawning.add(agentId);
|
|
3508
4962
|
let requested = false;
|
|
3509
4963
|
try {
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
4964
|
+
// ADR-154 §7: a leased account is refreshed by the cloud's
|
|
4965
|
+
// broker, through the task's lease, never by this runner.
|
|
4966
|
+
const lease = this.isLeasedChannel(channel) ? leasedEnginesFor(channel.taskId) : undefined;
|
|
4967
|
+
requested = this.isLeasedChannel(channel)
|
|
4968
|
+
? lease !== undefined &&
|
|
4969
|
+
accountId !== null &&
|
|
4970
|
+
(await refreshLeasedEngineAccount(lease.assignment, accountId)) === "refreshed"
|
|
4971
|
+
: await requestCredentialRefresh({
|
|
4972
|
+
kind: agent.kind,
|
|
4973
|
+
accountId:
|
|
4974
|
+
accountId && !accountId.startsWith("default:")
|
|
4975
|
+
? accountId
|
|
4976
|
+
: null,
|
|
4977
|
+
scope: channel.engineAccountScope,
|
|
4978
|
+
});
|
|
3518
4979
|
} catch (err) {
|
|
3519
4980
|
console.warn(
|
|
3520
4981
|
`[orchestrator] ${channel.taskId}/${agentId}: credential refresh request failed: ${
|
|
@@ -3825,10 +5286,16 @@ export class Orchestrator {
|
|
|
3825
5286
|
|
|
3826
5287
|
channel.spawning.add(agentId);
|
|
3827
5288
|
try {
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
5289
|
+
// ADR-154 §6: never the runner owner's own ~/.claude.json into a task on
|
|
5290
|
+
// cloud credentials; Claude writes its own again when it starts. A
|
|
5291
|
+
// repair on the runner's own credentials is counted: leaving them waits
|
|
5292
|
+
// for it, and none starts once leaving has begun.
|
|
5293
|
+
const repairing = this.isLeasedChannel(channel)
|
|
5294
|
+
? null
|
|
5295
|
+
: whileOnOwnCredentials(() =>
|
|
5296
|
+
repairClaudeConfigInContainer(channel.containerName, channel.agentHandle ?? undefined),
|
|
5297
|
+
);
|
|
5298
|
+
const restored = repairing === null ? false : await repairing;
|
|
3832
5299
|
if (channel.rosterSessionChanges.has(agentId)) {
|
|
3833
5300
|
this.reconcileErroredRosterChange(
|
|
3834
5301
|
channel,
|
|
@@ -3896,7 +5363,7 @@ export class Orchestrator {
|
|
|
3896
5363
|
);
|
|
3897
5364
|
const boundAccountId = channel.accountByAgent.get(agentId);
|
|
3898
5365
|
const boundAccount = boundAccountId
|
|
3899
|
-
?
|
|
5366
|
+
? this.engineAccounts(channel, agent.kind).find(
|
|
3900
5367
|
(account) => account.id === boundAccountId,
|
|
3901
5368
|
)
|
|
3902
5369
|
: undefined;
|
|
@@ -3942,9 +5409,11 @@ export class Orchestrator {
|
|
|
3942
5409
|
}
|
|
3943
5410
|
this.startNextPendingPrompt(channel, agentId);
|
|
3944
5411
|
|
|
3945
|
-
const message =
|
|
3946
|
-
? `${agentId} restarted — config file was missing in the container
|
|
3947
|
-
:
|
|
5412
|
+
const message = this.isLeasedChannel(channel)
|
|
5413
|
+
? `${agentId} restarted — config file was missing in the container; Claude writes its own again.`
|
|
5414
|
+
: restored
|
|
5415
|
+
? `${agentId} restarted — config file was missing in the container, restored from the host.`
|
|
5416
|
+
: `${agentId} restarted — config file was missing in the container (host copy not found; spawned anyway).`;
|
|
3948
5417
|
channel.retiredTurnBoundaries.delete(agentId);
|
|
3949
5418
|
this.emitHost({
|
|
3950
5419
|
kind: "agent.exit",
|
|
@@ -4002,7 +5471,7 @@ export class Orchestrator {
|
|
|
4002
5471
|
// A concurrent terminal event for the same session is already being
|
|
4003
5472
|
// handled by the path that owns this spawn slot.
|
|
4004
5473
|
if (channel.spawning.has(agentId)) return true;
|
|
4005
|
-
if (
|
|
5474
|
+
if (this.engineAccounts(channel, agent.kind).length < 2) {
|
|
4006
5475
|
return false;
|
|
4007
5476
|
}
|
|
4008
5477
|
|
|
@@ -4012,10 +5481,11 @@ export class Orchestrator {
|
|
|
4012
5481
|
|
|
4013
5482
|
const current = channel.accountByAgent.get(agentId);
|
|
4014
5483
|
if (current) cooldownEngineAccount(current);
|
|
4015
|
-
const next =
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
5484
|
+
const next = this.pickAccount(
|
|
5485
|
+
channel,
|
|
5486
|
+
agent.kind,
|
|
5487
|
+
current ? new Set([current]) : undefined,
|
|
5488
|
+
);
|
|
4019
5489
|
if (!next) return false;
|
|
4020
5490
|
|
|
4021
5491
|
const task = getHostTask(channel.taskId);
|
|
@@ -4447,7 +5917,7 @@ export class Orchestrator {
|
|
|
4447
5917
|
return;
|
|
4448
5918
|
}
|
|
4449
5919
|
|
|
4450
|
-
const accounts =
|
|
5920
|
+
const accounts = this.engineAccounts(channel, agent.kind);
|
|
4451
5921
|
const account =
|
|
4452
5922
|
accounts.find((a) => a.id === channel.accountByAgent.get(agentId)) ??
|
|
4453
5923
|
accounts[0];
|
|
@@ -4482,14 +5952,19 @@ export class Orchestrator {
|
|
|
4482
5952
|
async refreshEngineAccountCopies(kind: string): Promise<void> {
|
|
4483
5953
|
for (const channel of this.channels.values()) {
|
|
4484
5954
|
if (!this.isActiveChannel(channel)) continue;
|
|
5955
|
+
// ADR-154: a leased task's accounts change through its lease only.
|
|
5956
|
+
if (this.isLeasedChannel(channel)) continue;
|
|
4485
5957
|
if (!channel.roster.some((agent) => agent.kind === kind)) continue;
|
|
4486
5958
|
try {
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
5959
|
+
// Only while the runner is on its own credentials, counted.
|
|
5960
|
+
await whileOnOwnCredentials(() =>
|
|
5961
|
+
provisionEngineAccounts(
|
|
5962
|
+
channel.containerName,
|
|
5963
|
+
[kind],
|
|
5964
|
+
{},
|
|
5965
|
+
channel.agentHandle ?? undefined,
|
|
5966
|
+
channel.engineAccountScope,
|
|
5967
|
+
),
|
|
4493
5968
|
);
|
|
4494
5969
|
} catch (err) {
|
|
4495
5970
|
console.warn(
|
|
@@ -4504,6 +5979,7 @@ export class Orchestrator {
|
|
|
4504
5979
|
async refreshEngineAccountAgents(kind: string): Promise<void> {
|
|
4505
5980
|
for (const channel of this.channels.values()) {
|
|
4506
5981
|
if (!this.isActiveChannel(channel)) continue;
|
|
5982
|
+
if (this.isLeasedChannel(channel)) continue;
|
|
4507
5983
|
const agentIds = channel.roster
|
|
4508
5984
|
.filter((agent) => agent.kind === kind)
|
|
4509
5985
|
.map((agent) => agent.id);
|
|
@@ -4561,7 +6037,16 @@ export class Orchestrator {
|
|
|
4561
6037
|
// can repopulate an orphan channel while its sessions close.
|
|
4562
6038
|
ch.closed = true;
|
|
4563
6039
|
this.channels.delete(taskId);
|
|
6040
|
+
// ADR-154 §6: nothing its agents' spawns were admitted for starts after:
|
|
6041
|
+
// the close ends only once every retirement of the task under way, its
|
|
6042
|
+
// own and any an earlier event began, is done or escalated.
|
|
6043
|
+
for (const [agentId, slot] of [...ch.leasedLaunches]) void this.retireLeasedLaunch(ch, agentId, slot);
|
|
6044
|
+
const retirements = this.leasedRetirementsUnderWay(taskId);
|
|
4564
6045
|
this.clearBrowserRetry(ch);
|
|
6046
|
+
if (ch.leasedSyncRetry) {
|
|
6047
|
+
clearTimeout(ch.leasedSyncRetry);
|
|
6048
|
+
ch.leasedSyncRetry = null;
|
|
6049
|
+
}
|
|
4565
6050
|
ch.browserStaleSessions.clear();
|
|
4566
6051
|
ch.rosterSessionChanges.clear();
|
|
4567
6052
|
ch.browserPendingStaleSessions.clear();
|
|
@@ -4586,6 +6071,7 @@ export class Orchestrator {
|
|
|
4586
6071
|
ch.sessionsReady,
|
|
4587
6072
|
...ch.factoryOperations,
|
|
4588
6073
|
...ch.eventOperations,
|
|
6074
|
+
...retirements,
|
|
4589
6075
|
].filter((promise): promise is Promise<unknown> => promise !== null);
|
|
4590
6076
|
const sessions = [...ch.sessions.values()];
|
|
4591
6077
|
ch.sessions.clear();
|
|
@@ -4683,28 +6169,103 @@ export class Orchestrator {
|
|
|
4683
6169
|
return this.#channelActivity.get(taskId) ?? null;
|
|
4684
6170
|
}
|
|
4685
6171
|
|
|
4686
|
-
async stopTask(
|
|
6172
|
+
async stopTask(
|
|
6173
|
+
taskId: string,
|
|
6174
|
+
options: {
|
|
6175
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6176
|
+
shouldVeto?: () => boolean;
|
|
6177
|
+
/** ADR-154 §6: the closed run's credentials cleaned up, its sessions
|
|
6178
|
+
* closed and its runtime still up for the cleanup to reach, before
|
|
6179
|
+
* the runtime stops and Stop is acknowledged. */
|
|
6180
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6181
|
+
} = {},
|
|
6182
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
4687
6183
|
return this.runTaskLifecycle(taskId, () =>
|
|
4688
|
-
this.stopTaskWithinLifecycle(taskId),
|
|
6184
|
+
this.stopTaskWithinLifecycle(taskId, options),
|
|
4689
6185
|
);
|
|
4690
6186
|
}
|
|
4691
6187
|
|
|
6188
|
+
/** `stopTask` for a caller that already holds this task's lifecycle slot
|
|
6189
|
+
* (runTaskLifecycle), so that what it checks before the stop and does
|
|
6190
|
+
* after it share the slot: ADR-154 §13's retirement re-proves the engine
|
|
6191
|
+
* first, and removes the runtime after. */
|
|
6192
|
+
async stopTaskInHeldLifecycle(
|
|
6193
|
+
taskId: string,
|
|
6194
|
+
options: {
|
|
6195
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6196
|
+
shouldVeto?: () => boolean;
|
|
6197
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6198
|
+
/** After the channel drained and before anything touches the
|
|
6199
|
+
* container runtime (previews, the runtime's stop): why the caller's
|
|
6200
|
+
* proof of the daemon no longer holds, or null. A reason stops the
|
|
6201
|
+
* Stop there, mutating nothing. */
|
|
6202
|
+
beforeRuntimeMutation?: () => Promise<string | null>;
|
|
6203
|
+
} = {},
|
|
6204
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
6205
|
+
return this.stopTaskWithinLifecycle(taskId, options);
|
|
6206
|
+
}
|
|
6207
|
+
|
|
4692
6208
|
private async stopTaskWithinLifecycle(
|
|
4693
6209
|
taskId: string,
|
|
4694
|
-
|
|
6210
|
+
options: {
|
|
6211
|
+
admitAssignment?: () => TaskStopAssignmentAdmission;
|
|
6212
|
+
shouldVeto?: () => boolean;
|
|
6213
|
+
closeCredentials?: (runEpoch: number) => Promise<void>;
|
|
6214
|
+
beforeRuntimeMutation?: () => Promise<string | null>;
|
|
6215
|
+
},
|
|
6216
|
+
): Promise<OrchestratorTaskStopResult> {
|
|
6217
|
+
// The activity check belongs in the same lifecycle slot as assignment
|
|
6218
|
+
// closure and runtime stop. A turn may become active while Stop waits for
|
|
6219
|
+
// earlier lifecycle work; veto it before closing any v1 authority.
|
|
6220
|
+
if (options.shouldVeto?.()) return { ok: true, vetoed: true };
|
|
6221
|
+
let assignmentAdmission: TaskStopAssignmentAdmission = { kind: "legacy" };
|
|
6222
|
+
try {
|
|
6223
|
+
assignmentAdmission =
|
|
6224
|
+
options.admitAssignment?.() ?? ({ kind: "legacy" } as const);
|
|
6225
|
+
} catch (error) {
|
|
6226
|
+
return {
|
|
6227
|
+
ok: false,
|
|
6228
|
+
error: (error instanceof Error ? error.message : String(error)).slice(
|
|
6229
|
+
0,
|
|
6230
|
+
200,
|
|
6231
|
+
),
|
|
6232
|
+
};
|
|
6233
|
+
}
|
|
6234
|
+
if (assignmentAdmission.kind === "stale") {
|
|
6235
|
+
return {
|
|
6236
|
+
ok: false,
|
|
6237
|
+
error: "task assignment is no longer current",
|
|
6238
|
+
assignmentStale: true,
|
|
6239
|
+
};
|
|
6240
|
+
}
|
|
6241
|
+
const assignmentClosed = assignmentAdmission.kind === "closed";
|
|
4695
6242
|
if (!backgroundContainerWorkAllowed()) {
|
|
4696
6243
|
return { ok: false, error: "container runtime is not operational" };
|
|
4697
6244
|
}
|
|
4698
6245
|
const task = getHostTask(taskId);
|
|
4699
6246
|
if (!task) return { ok: false, error: "unknown task" };
|
|
4700
|
-
|
|
4701
|
-
|
|
6247
|
+
// A runtime is named by its persisted locator, a legacy row by its
|
|
6248
|
+
// Compose project. ADR-154: a start interrupted after recording its
|
|
6249
|
+
// runtime's locator keeps it, with no Compose project, for Stop to reach.
|
|
6250
|
+
if (!task.environmentLocator && !task.composeProject) {
|
|
6251
|
+
return { ok: false, error: "task has no running stack" };
|
|
6252
|
+
}
|
|
4702
6253
|
// Revoke env/proxy access before a potentially long session drain, then
|
|
4703
6254
|
// close sessions so nothing races to respawn them mid-stop.
|
|
4704
6255
|
const suspendedGateway = suspendTaskGatewayTask(taskId);
|
|
4705
6256
|
// ADR-153: no longer served; end its open proxy connections too.
|
|
4706
6257
|
activeSecretsProxy()?.runtime.stopTask(taskId);
|
|
4707
6258
|
await this.closeChannel(taskId);
|
|
6259
|
+
// ADR-154 §13: a proof of the daemon taken before the drain may have
|
|
6260
|
+
// expired during it; nothing below may reach a replaced one.
|
|
6261
|
+
const stale = await options.beforeRuntimeMutation?.();
|
|
6262
|
+
if (stale !== undefined && stale !== null) {
|
|
6263
|
+
if (!assignmentClosed) {
|
|
6264
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6265
|
+
this.allowChannel(taskId);
|
|
6266
|
+
}
|
|
6267
|
+
return { ok: false, error: stale.slice(0, 200) };
|
|
6268
|
+
}
|
|
4708
6269
|
try {
|
|
4709
6270
|
await stopPreviewSidecars(taskId);
|
|
4710
6271
|
} catch (err) {
|
|
@@ -4714,6 +6275,22 @@ export class Orchestrator {
|
|
|
4714
6275
|
}`,
|
|
4715
6276
|
);
|
|
4716
6277
|
}
|
|
6278
|
+
// ADR-154 §6: Stop is acknowledged only once the closed run's
|
|
6279
|
+
// credentials are cleaned up, while its runtime still runs for the
|
|
6280
|
+
// cleanup to reach. A failed cleanup fails the stop; the cloud retries,
|
|
6281
|
+
// and its closed run stays closed meanwhile.
|
|
6282
|
+
if (assignmentAdmission.kind === "closed" && options.closeCredentials) {
|
|
6283
|
+
try {
|
|
6284
|
+
await options.closeCredentials(assignmentAdmission.runEpoch);
|
|
6285
|
+
} catch (err) {
|
|
6286
|
+
return {
|
|
6287
|
+
ok: false,
|
|
6288
|
+
error: `the task's credentials could not be cleaned up: ${
|
|
6289
|
+
(err instanceof Error ? err.message : String(err)).slice(0, 160)
|
|
6290
|
+
}`,
|
|
6291
|
+
};
|
|
6292
|
+
}
|
|
6293
|
+
}
|
|
4717
6294
|
// Provider-neutral resumable stop. A row that reaches this point with
|
|
4718
6295
|
// runtime resources always carries a proven locator (legacy rows cross
|
|
4719
6296
|
// the adoption fence before the runtime becomes operational), so a
|
|
@@ -4724,14 +6301,18 @@ export class Orchestrator {
|
|
|
4724
6301
|
try {
|
|
4725
6302
|
const environment = await reconstructPersistedTaskEnvironment(task);
|
|
4726
6303
|
if (!environment) {
|
|
4727
|
-
if (
|
|
4728
|
-
|
|
6304
|
+
if (!assignmentClosed) {
|
|
6305
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6306
|
+
this.allowChannel(taskId);
|
|
6307
|
+
}
|
|
4729
6308
|
return { ok: false, error: "task has no persisted environment" };
|
|
4730
6309
|
}
|
|
4731
6310
|
await environment.stop();
|
|
4732
6311
|
} catch (err) {
|
|
4733
|
-
if (
|
|
4734
|
-
|
|
6312
|
+
if (!assignmentClosed) {
|
|
6313
|
+
if (suspendedGateway) reactivateTaskGatewayTask(suspendedGateway);
|
|
6314
|
+
this.allowChannel(taskId);
|
|
6315
|
+
}
|
|
4735
6316
|
return {
|
|
4736
6317
|
ok: false,
|
|
4737
6318
|
error: (err instanceof Error ? err.message : String(err)).slice(0, 200),
|
|
@@ -4757,7 +6338,9 @@ export class Orchestrator {
|
|
|
4757
6338
|
// Nothing serves the task's env now: no value of it stays in memory.
|
|
4758
6339
|
forgetTaskEnvSnapshot(taskId);
|
|
4759
6340
|
clearTaskGatewayAcl(taskId);
|
|
4760
|
-
return
|
|
6341
|
+
return assignmentAdmission.kind === "closed"
|
|
6342
|
+
? { ok: true, stoppedRunEpoch: assignmentAdmission.runEpoch }
|
|
6343
|
+
: { ok: true };
|
|
4761
6344
|
}
|
|
4762
6345
|
}
|
|
4763
6346
|
|
|
@@ -6001,8 +7584,26 @@ export type TaskRuntimeContainerInspection =
|
|
|
6001
7584
|
|
|
6002
7585
|
export interface WritableRuntimeQuarantineResult {
|
|
6003
7586
|
verdict: "ready" | "retry" | "stale";
|
|
6004
|
-
/**
|
|
6005
|
-
|
|
7587
|
+
/**
|
|
7588
|
+
* Known rows whose local `stopped` verdict must be published or replayed.
|
|
7589
|
+
* The epoch is captured while the task lifecycle slot is still held and is
|
|
7590
|
+
* deliberately absent for a legacy assignment.
|
|
7591
|
+
*/
|
|
7592
|
+
stoppedTasks: StoppedTaskStatusTarget[];
|
|
7593
|
+
}
|
|
7594
|
+
|
|
7595
|
+
export interface StoppedTaskStatusTarget {
|
|
7596
|
+
taskId: string;
|
|
7597
|
+
runEpoch?: number;
|
|
7598
|
+
}
|
|
7599
|
+
|
|
7600
|
+
function captureStoppedTaskStatusTarget(
|
|
7601
|
+
taskId: string,
|
|
7602
|
+
): StoppedTaskStatusTarget {
|
|
7603
|
+
const assignment = closeCurrentTaskRunAssignmentForStoppedRuntime(taskId);
|
|
7604
|
+
return assignment.kind === "legacy"
|
|
7605
|
+
? { taskId }
|
|
7606
|
+
: { taskId, runEpoch: assignment.runEpoch };
|
|
6006
7607
|
}
|
|
6007
7608
|
|
|
6008
7609
|
function runtimeWriterCandidate(
|
|
@@ -6390,19 +7991,19 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6390
7991
|
return quarantineWritableAppleRuntimeContainers(options);
|
|
6391
7992
|
}
|
|
6392
7993
|
const isCurrent = options.isCurrent ?? (() => true);
|
|
6393
|
-
const
|
|
6394
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
7994
|
+
const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
|
|
7995
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
6395
7996
|
|
|
6396
7997
|
const listed = await listRuntimeWriterCandidates();
|
|
6397
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
7998
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
6398
7999
|
if (listed.kind === "unreachable") {
|
|
6399
|
-
return { verdict: "retry",
|
|
8000
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
6400
8001
|
}
|
|
6401
8002
|
let resolved = !listed.ambiguous;
|
|
6402
8003
|
|
|
6403
8004
|
for (const candidate of listed.candidates) {
|
|
6404
8005
|
if (!isCurrent()) {
|
|
6405
|
-
return { verdict: "stale",
|
|
8006
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
6406
8007
|
}
|
|
6407
8008
|
await options.lifecycle.runTaskLifecycle(candidate.taskId, async () => {
|
|
6408
8009
|
if (!isCurrent()) return;
|
|
@@ -6459,7 +8060,10 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6459
8060
|
// Always re-publish the stopped verdict. A prior process may have
|
|
6460
8061
|
// committed SQLite and crashed before its websocket frame left; the
|
|
6461
8062
|
// cloud could otherwise remain ghost-running indefinitely.
|
|
6462
|
-
|
|
8063
|
+
stoppedTasks.set(
|
|
8064
|
+
candidate.taskId,
|
|
8065
|
+
captureStoppedTaskStatusTarget(candidate.taskId),
|
|
8066
|
+
);
|
|
6463
8067
|
db_setStatus(candidate.taskId, "stopped", {
|
|
6464
8068
|
codeServerPort: null,
|
|
6465
8069
|
previewPorts: "[]",
|
|
@@ -6488,11 +8092,13 @@ export async function quarantineWritableRuntimeContainers(options: {
|
|
|
6488
8092
|
}
|
|
6489
8093
|
|
|
6490
8094
|
if (!isCurrent()) {
|
|
6491
|
-
return { verdict: "stale",
|
|
8095
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
6492
8096
|
}
|
|
6493
8097
|
return {
|
|
6494
8098
|
verdict: resolved ? "ready" : "retry",
|
|
6495
|
-
|
|
8099
|
+
stoppedTasks: [...stoppedTasks.values()].sort((a, b) =>
|
|
8100
|
+
a.taskId.localeCompare(b.taskId),
|
|
8101
|
+
),
|
|
6496
8102
|
};
|
|
6497
8103
|
}
|
|
6498
8104
|
|
|
@@ -7282,25 +8888,25 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7282
8888
|
scope?: "contract" | "volume-holders";
|
|
7283
8889
|
}): Promise<WritableRuntimeQuarantineResult> {
|
|
7284
8890
|
const isCurrent = options.isCurrent ?? (() => true);
|
|
7285
|
-
if (!isCurrent()) return { verdict: "stale",
|
|
8891
|
+
if (!isCurrent()) return { verdict: "stale", stoppedTasks: [] };
|
|
7286
8892
|
const binding = appleContainerRuntimeBinding();
|
|
7287
|
-
if (binding === null) return { verdict: "retry",
|
|
8893
|
+
if (binding === null) return { verdict: "retry", stoppedTasks: [] };
|
|
7288
8894
|
const cli = appleCliRunner(() => binding.containerCliPath);
|
|
7289
8895
|
const listed = await cli(["list", "--all", "--format", "json"], {
|
|
7290
8896
|
timeoutMs: 30_000,
|
|
7291
8897
|
maxOutputBytes: 8 * 1024 * 1024,
|
|
7292
8898
|
});
|
|
7293
8899
|
if (listed.status !== 0 || listed.truncated === true) {
|
|
7294
|
-
return { verdict: "retry",
|
|
8900
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
7295
8901
|
}
|
|
7296
8902
|
let parsed: unknown;
|
|
7297
8903
|
try {
|
|
7298
8904
|
parsed = JSON.parse(listed.stdout);
|
|
7299
8905
|
} catch {
|
|
7300
|
-
return { verdict: "retry",
|
|
8906
|
+
return { verdict: "retry", stoppedTasks: [] };
|
|
7301
8907
|
}
|
|
7302
|
-
if (!Array.isArray(parsed)) return { verdict: "retry",
|
|
7303
|
-
const
|
|
8908
|
+
if (!Array.isArray(parsed)) return { verdict: "retry", stoppedTasks: [] };
|
|
8909
|
+
const stoppedTasks = new Map<string, StoppedTaskStatusTarget>();
|
|
7304
8910
|
let resolved = true;
|
|
7305
8911
|
|
|
7306
8912
|
const stopAndProve = async (id: string): Promise<boolean> => {
|
|
@@ -7357,7 +8963,9 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7357
8963
|
}>;
|
|
7358
8964
|
};
|
|
7359
8965
|
}>) {
|
|
7360
|
-
if (!isCurrent())
|
|
8966
|
+
if (!isCurrent()) {
|
|
8967
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
8968
|
+
}
|
|
7361
8969
|
const id = item?.configuration?.id;
|
|
7362
8970
|
if (typeof id !== "string" || id.length === 0) {
|
|
7363
8971
|
resolved = false;
|
|
@@ -7410,20 +9018,23 @@ async function quarantineWritableAppleRuntimeContainers(options: {
|
|
|
7410
9018
|
}
|
|
7411
9019
|
const task = getHostTask(taskId);
|
|
7412
9020
|
if (task?.statusMirror && isOnDashboard(task.statusMirror)) {
|
|
9021
|
+
const stoppedTask = captureStoppedTaskStatusTarget(taskId);
|
|
7413
9022
|
db_setStatus(taskId, "stopped", {
|
|
7414
9023
|
codeServerPort: null,
|
|
7415
9024
|
previewPorts: "[]",
|
|
7416
9025
|
});
|
|
7417
|
-
|
|
9026
|
+
stoppedTasks.set(taskId, stoppedTask);
|
|
7418
9027
|
}
|
|
7419
9028
|
// Revoke live access with the environment, like the Docker sweep.
|
|
7420
9029
|
clearTaskGatewayAcl(taskId);
|
|
7421
9030
|
});
|
|
7422
9031
|
}
|
|
7423
|
-
if (!isCurrent())
|
|
9032
|
+
if (!isCurrent()) {
|
|
9033
|
+
return { verdict: "stale", stoppedTasks: [...stoppedTasks.values()] };
|
|
9034
|
+
}
|
|
7424
9035
|
return {
|
|
7425
9036
|
verdict: resolved ? "ready" : "retry",
|
|
7426
|
-
|
|
9037
|
+
stoppedTasks: [...stoppedTasks.values()],
|
|
7427
9038
|
};
|
|
7428
9039
|
}
|
|
7429
9040
|
|
|
@@ -7477,7 +9088,7 @@ async function materializeAgentCli(
|
|
|
7477
9088
|
return false;
|
|
7478
9089
|
}
|
|
7479
9090
|
}
|
|
7480
|
-
return writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
|
|
9091
|
+
return await writeAgentCli(task.taskId, roster, apiUrl, allowPermissionless);
|
|
7481
9092
|
}
|
|
7482
9093
|
|
|
7483
9094
|
async function recoverPersistedTaskEnvironment(
|
|
@@ -8604,7 +10215,9 @@ async function recoverOneDockerTask(
|
|
|
8604
10215
|
// reinject sweep only targets containers already running, so a container
|
|
8605
10216
|
// recovered here would otherwise keep whatever it held when it exited —
|
|
8606
10217
|
// possibly host-uid-owned 0600 files Codex can't read. Failure is surfaced
|
|
8607
|
-
// by the inject itself and must not abort the task's recovery.
|
|
10218
|
+
// by the inject itself and must not abort the task's recovery. ADR-154:
|
|
10219
|
+
// the runner owner's own sign-in, so only while the runner is on its own
|
|
10220
|
+
// credentials (the inject fences itself).
|
|
8608
10221
|
await injectCodexIntoContainer(containerName);
|
|
8609
10222
|
if (!isCurrent()) return true;
|
|
8610
10223
|
// Establish the task owner's GitHub transport BEFORE uai-init performs any
|