@opengeni/worker-bundle 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +24 -1
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +5 -3
- package/dist/activities/types.d.ts +27 -0
- package/dist/{activities-control-XURE4VKN.js → activities-control-SMKF2OBE.js} +54 -7
- package/dist/{activities-control-XURE4VKN.js.map → activities-control-SMKF2OBE.js.map} +1 -1
- package/dist/activities-control.d.ts +2 -2
- package/dist/{activities-turn-6EBXGGLS.js → activities-turn-IDS3WEHA.js} +1471 -975
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +3 -3
- package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +37 -0
- package/dist/workflow-bundle.js +174 -4
- package/dist/workflows/session.d.ts +17 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1675 -1175
- package/src/activities/run-input.ts +137 -45
- package/src/activities/session-state.ts +69 -5
- package/src/activities/types.ts +27 -0
- package/src/activities.ts +2 -0
- package/src/observability-metrics.ts +184 -3
- package/src/workflows/session.ts +199 -8
- package/dist/activities-turn-6EBXGGLS.js.map +0 -1
- package/dist/chunk-TYNPCG4U.js.map +0 -1
- /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
|
@@ -87,6 +87,7 @@ import {
|
|
|
87
87
|
SandboxImageConflictError,
|
|
88
88
|
ActiveSessionHistoryLimitExceededError,
|
|
89
89
|
ApprovalRunStateLimitExceededError,
|
|
90
|
+
isRetryableDatabaseTransportFailure,
|
|
90
91
|
isSessionEventPersistenceError,
|
|
91
92
|
getEnrollment,
|
|
92
93
|
abandonRecordingForTurnAttempt,
|
|
@@ -344,6 +345,7 @@ import {
|
|
|
344
345
|
import type {
|
|
345
346
|
TurnActivityServices as ActivityServices,
|
|
346
347
|
EscapedMcpTimeoutRecoveryDetail,
|
|
348
|
+
PreClaimFailureDetail,
|
|
347
349
|
RunAgentTurnInput,
|
|
348
350
|
RunAgentTurnResult,
|
|
349
351
|
SessionAttemptQuiescenceProof,
|
|
@@ -351,6 +353,8 @@ import type {
|
|
|
351
353
|
import {
|
|
352
354
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
353
355
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
356
|
+
PRE_CLAIM_FAILURE_MESSAGE,
|
|
357
|
+
PRE_CLAIM_FAILURE_TYPE,
|
|
354
358
|
} from "./types";
|
|
355
359
|
import {
|
|
356
360
|
resumeBoxForTurn,
|
|
@@ -381,10 +385,14 @@ import {
|
|
|
381
385
|
recordCompanyBrainContributions,
|
|
382
386
|
recordSessionEventAppendLatency,
|
|
383
387
|
recordSessionEventPublishLatency,
|
|
388
|
+
recordTurnSandboxEstablishPolicy,
|
|
389
|
+
recordTurnStartupPhase,
|
|
390
|
+
recordTurnWorkerPreparationTotal,
|
|
384
391
|
runtimeMetricsHooksForObservability,
|
|
385
392
|
StreamTimingMetrics,
|
|
386
393
|
turnLifecycleMetricsFor,
|
|
387
394
|
type TurnOutcome,
|
|
395
|
+
type TurnSandboxEstablishReason,
|
|
388
396
|
} from "../observability-metrics";
|
|
389
397
|
import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
|
|
390
398
|
import {
|
|
@@ -704,6 +712,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
704
712
|
});
|
|
705
713
|
}
|
|
706
714
|
|
|
715
|
+
/**
|
|
716
|
+
* Convert the atomic claim transaction's failure into a small, stable
|
|
717
|
+
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
718
|
+
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
719
|
+
* workflow history. Contention and operational database unavailability are
|
|
720
|
+
* safe to re-read after backoff because the claim transaction contains no
|
|
721
|
+
* model/tool effects; permanent database and state failures require terminal
|
|
722
|
+
* settlement.
|
|
723
|
+
*/
|
|
724
|
+
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
725
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
726
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
727
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
728
|
+
// The database transaction itself retries only the two contention failures
|
|
729
|
+
// proven safe for immediate replay. Once the activity has failed, the
|
|
730
|
+
// workflow may also retry operational outages after a durable re-read and
|
|
731
|
+
// bounded delay. Unknown driver/database failures stay recoverable because
|
|
732
|
+
// they commonly represent a lost connection; known constraint, auth, and
|
|
733
|
+
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
734
|
+
const retryable = Boolean(
|
|
735
|
+
rawTransportFailure ||
|
|
736
|
+
(persistenceFailure &&
|
|
737
|
+
(sqlState === null ||
|
|
738
|
+
sqlState.startsWith("08") ||
|
|
739
|
+
sqlState.startsWith("40") ||
|
|
740
|
+
sqlState.startsWith("53") ||
|
|
741
|
+
sqlState === "55P03" ||
|
|
742
|
+
sqlState === "57014" ||
|
|
743
|
+
sqlState === "57P01" ||
|
|
744
|
+
sqlState === "57P02" ||
|
|
745
|
+
sqlState === "57P03" ||
|
|
746
|
+
sqlState.startsWith("58"))),
|
|
747
|
+
);
|
|
748
|
+
const detail: PreClaimFailureDetail = {
|
|
749
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
750
|
+
code:
|
|
751
|
+
persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
|
|
752
|
+
};
|
|
753
|
+
return ApplicationFailure.create({
|
|
754
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
755
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
756
|
+
nonRetryable: true,
|
|
757
|
+
details: [detail],
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
|
|
707
761
|
/**
|
|
708
762
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
709
763
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -729,6 +783,12 @@ export function filterUnmaterializedSandboxFileDownloads(
|
|
|
729
783
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
730
784
|
}
|
|
731
785
|
|
|
786
|
+
export function sandboxFileMaterializationOutcome(
|
|
787
|
+
failures: readonly SandboxFileDownloadFailure[],
|
|
788
|
+
): "completed" | "failed" {
|
|
789
|
+
return failures.length === 0 ? "completed" : "failed";
|
|
790
|
+
}
|
|
791
|
+
|
|
732
792
|
/** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
|
|
733
793
|
export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
734
794
|
return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
@@ -1998,6 +2058,33 @@ export function shouldEstablishSandboxForTurn(
|
|
|
1998
2058
|
return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
|
|
1999
2059
|
}
|
|
2000
2060
|
|
|
2061
|
+
export function sandboxEstablishPolicyDecision(input: {
|
|
2062
|
+
lazyEnabled: boolean;
|
|
2063
|
+
machinePrimary: boolean;
|
|
2064
|
+
sandboxBackend: Settings["sandboxBackend"];
|
|
2065
|
+
hasInitialRunCredentialMaterial: boolean;
|
|
2066
|
+
generatedVideoFileCount: number;
|
|
2067
|
+
hasSignedFileResources: boolean;
|
|
2068
|
+
}): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
|
|
2069
|
+
if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
|
|
2070
|
+
if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
|
|
2071
|
+
if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
|
|
2072
|
+
if (input.hasInitialRunCredentialMaterial) {
|
|
2073
|
+
return { policy: "eager", reason: "initial_run_credentials" };
|
|
2074
|
+
}
|
|
2075
|
+
if (input.generatedVideoFileCount > 0) {
|
|
2076
|
+
return { policy: "eager", reason: "generated_video_files" };
|
|
2077
|
+
}
|
|
2078
|
+
// Signed file downloads must finish before the first model boundary so any
|
|
2079
|
+
// integrity/download failure is present in the prepared input. Deferring the
|
|
2080
|
+
// box until the first tool call would make that first request advertise a
|
|
2081
|
+
// path whose materialization outcome is not known yet.
|
|
2082
|
+
if (input.hasSignedFileResources) {
|
|
2083
|
+
return { policy: "eager", reason: "signed_file_resources" };
|
|
2084
|
+
}
|
|
2085
|
+
return { policy: "on-demand", reason: "eligible" };
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2001
2088
|
/**
|
|
2002
2089
|
* Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
|
|
2003
2090
|
* invariant B). Returns the typed reason to RESET the pointer to the session HOME,
|
|
@@ -4073,7 +4160,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4073
4160
|
// only exists in the RunState blob), never through a swapped trigger.
|
|
4074
4161
|
let triggerType: string | null = null;
|
|
4075
4162
|
try {
|
|
4076
|
-
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
4077
4163
|
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
4078
4164
|
sessionId: input.sessionId,
|
|
4079
4165
|
workflowId: input.workflowId,
|
|
@@ -4088,6 +4174,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4088
4174
|
}
|
|
4089
4175
|
const turn = claim.turn;
|
|
4090
4176
|
turnId = turn.id;
|
|
4177
|
+
// Establish durable attempt ownership before any later read can fail.
|
|
4178
|
+
// Therefore every failure with no turnId came from the one atomic claim
|
|
4179
|
+
// transaction and can be classified without conflating ordinary runtime
|
|
4180
|
+
// or transport failures with admission failures.
|
|
4181
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
4091
4182
|
executionGeneration = turn.executionGeneration;
|
|
4092
4183
|
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
4093
4184
|
let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
|
|
@@ -4410,6 +4501,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4410
4501
|
// on a healthy worker.
|
|
4411
4502
|
throwIfWorkerShuttingDown();
|
|
4412
4503
|
throwIfTurnCancelled();
|
|
4504
|
+
recordTurnStartupPhase(observability, {
|
|
4505
|
+
phase: "claim_and_policy",
|
|
4506
|
+
provider: turnExecutionPolicy.providerId,
|
|
4507
|
+
backend: turn.sandboxBackend,
|
|
4508
|
+
outcome: "completed",
|
|
4509
|
+
durationSeconds: (performance.now() - activityStarted) / 1_000,
|
|
4510
|
+
});
|
|
4511
|
+
const turnStartSettlementStartedAt = performance.now();
|
|
4413
4512
|
if (
|
|
4414
4513
|
!(await settle({
|
|
4415
4514
|
events: [
|
|
@@ -4426,7 +4525,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4426
4525
|
) {
|
|
4427
4526
|
return claimedResult({ status: "cancelled" });
|
|
4428
4527
|
}
|
|
4528
|
+
recordTurnStartupPhase(observability, {
|
|
4529
|
+
phase: "turn_start_settlement",
|
|
4530
|
+
provider: turnExecutionPolicy.providerId,
|
|
4531
|
+
backend: turn.sandboxBackend,
|
|
4532
|
+
outcome: "completed",
|
|
4533
|
+
durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
|
|
4534
|
+
count: 2,
|
|
4535
|
+
});
|
|
4429
4536
|
turnStartedPublished = true;
|
|
4537
|
+
const workerPreparationStartedAt = performance.now();
|
|
4430
4538
|
|
|
4431
4539
|
// Multi-account (P1): resolve the effective Codex account for this turn
|
|
4432
4540
|
// (session-pin > workspace active) and stamp it on the session so the
|
|
@@ -4434,80 +4542,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4434
4542
|
// when it changed from the prior run's account so the pill flips live.
|
|
4435
4543
|
// Gated on the codex-billed predicate — non-codex turns never touch this.
|
|
4436
4544
|
if (isCodexTurn) {
|
|
4437
|
-
const
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4545
|
+
const credentialSelectionStartedAt = performance.now();
|
|
4546
|
+
let credentialSelectionOutcome: "completed" | "failed" = "completed";
|
|
4547
|
+
try {
|
|
4548
|
+
const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
|
|
4549
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
4550
|
+
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
4551
|
+
const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
|
|
4552
|
+
selectCodexCredentialLeaseForTurn({
|
|
4553
|
+
context,
|
|
4554
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
4555
|
+
sessionId: input.sessionId,
|
|
4556
|
+
sessionPinnedCredentialId: sessionPin,
|
|
4557
|
+
sessionPinSource,
|
|
4558
|
+
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
4559
|
+
now: new Date(),
|
|
4560
|
+
});
|
|
4450
4561
|
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
4457
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
4458
|
-
leased = await acquireCodexCredentialLease(
|
|
4459
|
-
db,
|
|
4460
|
-
{
|
|
4461
|
-
accountId: input.accountId,
|
|
4462
|
-
workspaceId: input.workspaceId,
|
|
4463
|
-
turnId,
|
|
4464
|
-
holderId: codexLeaseHolderId,
|
|
4465
|
-
advanceActivePointer: sessionPin === null,
|
|
4466
|
-
},
|
|
4467
|
-
selectForTurn,
|
|
4468
|
-
);
|
|
4469
|
-
} else {
|
|
4470
|
-
const [rotation, accounts] = await Promise.all([
|
|
4471
|
-
getCodexRotationSettings(db, input.workspaceId),
|
|
4472
|
-
listCodexAccountStatuses(db, input.workspaceId),
|
|
4473
|
-
]);
|
|
4474
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
4475
|
-
...account,
|
|
4476
|
-
activeLeaseCount: 0,
|
|
4477
|
-
selectionCount: 0,
|
|
4478
|
-
lastSelectedAt: null,
|
|
4479
|
-
}));
|
|
4480
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4481
|
-
const selected = selectForTurn({
|
|
4482
|
-
accounts: leaseAccounts,
|
|
4483
|
-
activeCredentialId,
|
|
4484
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4485
|
-
leaseRotationEnabled: false,
|
|
4486
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4487
|
-
existingCredentialId: null,
|
|
4488
|
-
policyScope: null,
|
|
4489
|
-
unavailableDiagnostics: [],
|
|
4490
|
-
});
|
|
4491
|
-
leased = {
|
|
4492
|
-
...selected,
|
|
4493
|
-
accounts: leaseAccounts,
|
|
4494
|
-
activeCredentialId,
|
|
4495
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4496
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4497
|
-
reused: false,
|
|
4498
|
-
holderId: null,
|
|
4499
|
-
generation: null,
|
|
4500
|
-
leasedUntil: null,
|
|
4501
|
-
unavailableDiagnostics: [],
|
|
4502
|
-
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4503
|
-
};
|
|
4504
|
-
}
|
|
4505
|
-
if (leased.decision.kind === "allCapped") {
|
|
4506
|
-
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4507
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4508
|
-
signalCodexCapacityWorkflow,
|
|
4509
|
-
wakeSessionWorkflow,
|
|
4510
|
-
});
|
|
4562
|
+
// Rollout/rollback path is intentionally table-inert. With the flag off,
|
|
4563
|
+
// old and new workers both use legacy pin > active-pointer selection and
|
|
4564
|
+
// neither reads nor writes the additive lease/cursor schema.
|
|
4565
|
+
let leased: CodexCredentialLeaseResult<RotationDecision>;
|
|
4566
|
+
let leaseAcquisitionStartedAtMs: number | null = null;
|
|
4511
4567
|
if (settings.codexCredentialLeasingEnabled) {
|
|
4512
4568
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
4513
4569
|
leased = await acquireCodexCredentialLease(
|
|
@@ -4532,9 +4588,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4532
4588
|
selectionCount: 0,
|
|
4533
4589
|
lastSelectedAt: null,
|
|
4534
4590
|
}));
|
|
4591
|
+
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4535
4592
|
const selected = selectForTurn({
|
|
4536
4593
|
accounts: leaseAccounts,
|
|
4537
|
-
activeCredentialId
|
|
4594
|
+
activeCredentialId,
|
|
4538
4595
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4539
4596
|
leaseRotationEnabled: false,
|
|
4540
4597
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
@@ -4545,7 +4602,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4545
4602
|
leased = {
|
|
4546
4603
|
...selected,
|
|
4547
4604
|
accounts: leaseAccounts,
|
|
4548
|
-
activeCredentialId
|
|
4605
|
+
activeCredentialId,
|
|
4549
4606
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4550
4607
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4551
4608
|
reused: false,
|
|
@@ -4556,212 +4613,334 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4556
4613
|
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4557
4614
|
};
|
|
4558
4615
|
}
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4570
|
-
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4571
|
-
if (
|
|
4572
|
-
selectedPinDisposition === "sharded" &&
|
|
4573
|
-
leased.credentialId !== null &&
|
|
4574
|
-
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4575
|
-
) {
|
|
4576
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4577
|
-
db,
|
|
4578
|
-
{
|
|
4579
|
-
workspaceId: input.workspaceId,
|
|
4580
|
-
reason: "codex_policy_pin_changed",
|
|
4581
|
-
},
|
|
4582
|
-
async (tx) => {
|
|
4583
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
4584
|
-
tx,
|
|
4585
|
-
input.workspaceId,
|
|
4586
|
-
input.sessionId,
|
|
4587
|
-
leased.credentialId,
|
|
4588
|
-
"policy",
|
|
4616
|
+
if (leased.decision.kind === "allCapped") {
|
|
4617
|
+
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4618
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4619
|
+
signalCodexCapacityWorkflow,
|
|
4620
|
+
wakeSessionWorkflow,
|
|
4621
|
+
});
|
|
4622
|
+
if (settings.codexCredentialLeasingEnabled) {
|
|
4623
|
+
leaseAcquisitionStartedAtMs = performance.now();
|
|
4624
|
+
leased = await acquireCodexCredentialLease(
|
|
4625
|
+
db,
|
|
4589
4626
|
{
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4627
|
+
accountId: input.accountId,
|
|
4628
|
+
workspaceId: input.workspaceId,
|
|
4629
|
+
turnId,
|
|
4630
|
+
holderId: codexLeaseHolderId,
|
|
4631
|
+
advanceActivePointer: sessionPin === null,
|
|
4594
4632
|
},
|
|
4633
|
+
selectForTurn,
|
|
4595
4634
|
);
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4635
|
+
} else {
|
|
4636
|
+
const [rotation, accounts] = await Promise.all([
|
|
4637
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
4638
|
+
listCodexAccountStatuses(db, input.workspaceId),
|
|
4639
|
+
]);
|
|
4640
|
+
const leaseAccounts = accounts.map((account) => ({
|
|
4641
|
+
...account,
|
|
4642
|
+
activeLeaseCount: 0,
|
|
4643
|
+
selectionCount: 0,
|
|
4644
|
+
lastSelectedAt: null,
|
|
4645
|
+
}));
|
|
4646
|
+
const selected = selectForTurn({
|
|
4647
|
+
accounts: leaseAccounts,
|
|
4648
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4649
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4650
|
+
leaseRotationEnabled: false,
|
|
4651
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4652
|
+
existingCredentialId: null,
|
|
4653
|
+
policyScope: null,
|
|
4654
|
+
unavailableDiagnostics: [],
|
|
4655
|
+
});
|
|
4656
|
+
leased = {
|
|
4657
|
+
...selected,
|
|
4658
|
+
accounts: leaseAccounts,
|
|
4659
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4660
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4661
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4662
|
+
reused: false,
|
|
4663
|
+
holderId: null,
|
|
4664
|
+
generation: null,
|
|
4665
|
+
leasedUntil: null,
|
|
4666
|
+
unavailableDiagnostics: [],
|
|
4667
|
+
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4668
|
+
};
|
|
4669
|
+
}
|
|
4670
|
+
}
|
|
4671
|
+
const rotationDecision = leased.decision;
|
|
4672
|
+
const selectedPinDisposition = classifyCodexPin({
|
|
4673
|
+
pinnedCredentialId: sessionPin,
|
|
4674
|
+
pinSource: sessionPinSource,
|
|
4675
|
+
strategy: leased.rotationStrategy as CodexRotationStrategy,
|
|
4676
|
+
rotationEnabled: leased.rotationEnabled,
|
|
4677
|
+
});
|
|
4678
|
+
// pin policy pin persistence follows the atomic credential allocator selection. The selector
|
|
4679
|
+
// already ran exact-turn reuse before policy filtering and vetoed pointer
|
|
4680
|
+
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4681
|
+
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4682
|
+
if (
|
|
4683
|
+
selectedPinDisposition === "sharded" &&
|
|
4684
|
+
leased.credentialId !== null &&
|
|
4685
|
+
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4686
|
+
) {
|
|
4687
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4688
|
+
db,
|
|
4689
|
+
{
|
|
4690
|
+
workspaceId: input.workspaceId,
|
|
4691
|
+
reason: "codex_policy_pin_changed",
|
|
4692
|
+
},
|
|
4693
|
+
async (tx) => {
|
|
4694
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4695
|
+
tx,
|
|
4696
|
+
input.workspaceId,
|
|
4697
|
+
input.sessionId,
|
|
4698
|
+
leased.credentialId,
|
|
4699
|
+
"policy",
|
|
4700
|
+
{
|
|
4701
|
+
expected: {
|
|
4702
|
+
pinnedCredentialId: sessionPin,
|
|
4703
|
+
pinSource: sessionPinSource,
|
|
4704
|
+
},
|
|
4621
4705
|
},
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4706
|
+
);
|
|
4707
|
+
return { result: changed, changed };
|
|
4708
|
+
},
|
|
4709
|
+
);
|
|
4710
|
+
await signalCodexCapacityWakeTargets(
|
|
4711
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4712
|
+
pinMutation.wakeTargets,
|
|
4713
|
+
);
|
|
4714
|
+
} else if (selectedPinDisposition === "clearStale") {
|
|
4715
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4716
|
+
db,
|
|
4717
|
+
{
|
|
4718
|
+
workspaceId: input.workspaceId,
|
|
4719
|
+
reason: "codex_stale_policy_pin_cleared",
|
|
4720
|
+
},
|
|
4721
|
+
async (tx) => {
|
|
4722
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4723
|
+
tx,
|
|
4724
|
+
input.workspaceId,
|
|
4725
|
+
input.sessionId,
|
|
4726
|
+
null,
|
|
4727
|
+
"policy",
|
|
4728
|
+
{
|
|
4729
|
+
expected: {
|
|
4730
|
+
pinnedCredentialId: sessionPin,
|
|
4731
|
+
pinSource: sessionPinSource,
|
|
4732
|
+
},
|
|
4733
|
+
},
|
|
4734
|
+
);
|
|
4735
|
+
return { result: changed, changed };
|
|
4736
|
+
},
|
|
4737
|
+
);
|
|
4738
|
+
await signalCodexCapacityWakeTargets(
|
|
4739
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4740
|
+
pinMutation.wakeTargets,
|
|
4741
|
+
);
|
|
4742
|
+
}
|
|
4743
|
+
if (
|
|
4744
|
+
!settings.codexCredentialLeasingEnabled &&
|
|
4745
|
+
leased.advanceActivePointer &&
|
|
4746
|
+
sessionPin === null &&
|
|
4747
|
+
rotationDecision.kind === "active" &&
|
|
4748
|
+
rotationDecision.moved
|
|
4749
|
+
) {
|
|
4750
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
4751
|
+
}
|
|
4752
|
+
effectiveCodexCredentialId = leased.credentialId;
|
|
4753
|
+
codexLeaseGeneration = leased.generation;
|
|
4754
|
+
codexLeaseConfirmedUntilMs =
|
|
4755
|
+
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
4756
|
+
? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
|
|
4757
|
+
: null;
|
|
4758
|
+
codexLeaseHeld =
|
|
4759
|
+
effectiveCodexCredentialId !== null &&
|
|
4760
|
+
leased.holderId !== null &&
|
|
4761
|
+
leased.generation !== null &&
|
|
4762
|
+
codexLeaseConfirmedUntilMs !== null;
|
|
4763
|
+
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
4764
|
+
|
|
4765
|
+
const actualOutcome = effectiveCodexCredentialId
|
|
4766
|
+
? "selected"
|
|
4767
|
+
: rotationDecision.kind === "allCapped"
|
|
4768
|
+
? "waiting"
|
|
4769
|
+
: "none";
|
|
4770
|
+
const actualReason = effectiveCodexCredentialId
|
|
4771
|
+
? leased.reused
|
|
4772
|
+
? "lease_reused"
|
|
4773
|
+
: sessionPin === effectiveCodexCredentialId
|
|
4774
|
+
? "pin"
|
|
4775
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
4776
|
+
? "rotation"
|
|
4777
|
+
: "active"
|
|
4778
|
+
: rotationDecision.kind === "allCapped"
|
|
4779
|
+
? "all_capped"
|
|
4780
|
+
: "none";
|
|
4781
|
+
const fencedInFlight = leased.reused;
|
|
4782
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4783
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4784
|
+
decision: {
|
|
4785
|
+
accounts: leased.accounts,
|
|
4786
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
4787
|
+
actualOutcome,
|
|
4788
|
+
actualReason,
|
|
4789
|
+
affinityCredentialId: fencedInFlight
|
|
4790
|
+
? effectiveCodexCredentialId
|
|
4791
|
+
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4792
|
+
fencedInFlight,
|
|
4793
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4794
|
+
now: new Date(),
|
|
4795
|
+
aliasSeed: randomUUID(),
|
|
4625
4796
|
},
|
|
4626
|
-
|
|
4627
|
-
await signalCodexCapacityWakeTargets(
|
|
4628
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4629
|
-
pinMutation.wakeTargets,
|
|
4630
|
-
);
|
|
4631
|
-
}
|
|
4632
|
-
if (
|
|
4633
|
-
!settings.codexCredentialLeasingEnabled &&
|
|
4634
|
-
leased.advanceActivePointer &&
|
|
4635
|
-
sessionPin === null &&
|
|
4636
|
-
rotationDecision.kind === "active" &&
|
|
4637
|
-
rotationDecision.moved
|
|
4638
|
-
) {
|
|
4639
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
4640
|
-
}
|
|
4641
|
-
effectiveCodexCredentialId = leased.credentialId;
|
|
4642
|
-
codexLeaseGeneration = leased.generation;
|
|
4643
|
-
codexLeaseConfirmedUntilMs =
|
|
4644
|
-
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
4645
|
-
? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
|
|
4646
|
-
: null;
|
|
4647
|
-
codexLeaseHeld =
|
|
4648
|
-
effectiveCodexCredentialId !== null &&
|
|
4649
|
-
leased.holderId !== null &&
|
|
4650
|
-
leased.generation !== null &&
|
|
4651
|
-
codexLeaseConfirmedUntilMs !== null;
|
|
4652
|
-
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
4653
|
-
|
|
4654
|
-
const actualOutcome = effectiveCodexCredentialId
|
|
4655
|
-
? "selected"
|
|
4656
|
-
: rotationDecision.kind === "allCapped"
|
|
4657
|
-
? "waiting"
|
|
4658
|
-
: "none";
|
|
4659
|
-
const actualReason = effectiveCodexCredentialId
|
|
4660
|
-
? leased.reused
|
|
4661
|
-
? "lease_reused"
|
|
4662
|
-
: sessionPin === effectiveCodexCredentialId
|
|
4663
|
-
? "pin"
|
|
4664
|
-
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
4665
|
-
? "rotation"
|
|
4666
|
-
: "active"
|
|
4667
|
-
: rotationDecision.kind === "allCapped"
|
|
4668
|
-
? "all_capped"
|
|
4669
|
-
: "none";
|
|
4670
|
-
const fencedInFlight = leased.reused;
|
|
4671
|
-
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4672
|
-
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4673
|
-
decision: {
|
|
4674
|
-
accounts: leased.accounts,
|
|
4675
|
-
actualCredentialId: effectiveCodexCredentialId,
|
|
4676
|
-
actualOutcome,
|
|
4677
|
-
actualReason,
|
|
4678
|
-
affinityCredentialId: fencedInFlight
|
|
4679
|
-
? effectiveCodexCredentialId
|
|
4680
|
-
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4681
|
-
fencedInFlight,
|
|
4682
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4683
|
-
now: new Date(),
|
|
4684
|
-
aliasSeed: randomUUID(),
|
|
4685
|
-
},
|
|
4686
|
-
publish,
|
|
4687
|
-
});
|
|
4688
|
-
if (shadowResult.outcome === "published") {
|
|
4689
|
-
const shadowPayload = shadowResult.payload;
|
|
4690
|
-
observability.incrementCounter({
|
|
4691
|
-
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4692
|
-
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4693
|
-
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4694
|
-
});
|
|
4695
|
-
observability.info("Codex adaptive fleet shadow decision", {
|
|
4696
|
-
workspaceId: input.workspaceId,
|
|
4697
|
-
policyVersion: shadowPayload.replay.policyVersion,
|
|
4698
|
-
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4699
|
-
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4700
|
-
actualOutcome: shadowPayload.actual.outcome,
|
|
4701
|
-
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4702
|
-
comparison: shadowPayload.comparison,
|
|
4703
|
-
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4704
|
-
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4705
|
-
payloadBytes: shadowResult.payloadBytes,
|
|
4797
|
+
publish,
|
|
4706
4798
|
});
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4799
|
+
if (shadowResult.outcome === "published") {
|
|
4800
|
+
const shadowPayload = shadowResult.payload;
|
|
4801
|
+
observability.incrementCounter({
|
|
4802
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4803
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4804
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4805
|
+
});
|
|
4806
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
4807
|
+
workspaceId: input.workspaceId,
|
|
4808
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
4809
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4810
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4811
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
4812
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4813
|
+
comparison: shadowPayload.comparison,
|
|
4814
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4815
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4816
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4817
|
+
});
|
|
4818
|
+
} else if (shadowResult.outcome === "failed") {
|
|
4819
|
+
// Shadow observability is explicitly non-authoritative. A malformed
|
|
4820
|
+
// snapshot or event-write fault must never change the authoritative lease,
|
|
4821
|
+
// capacity wait, failover, or the account serving this fenced turn.
|
|
4822
|
+
observability.incrementCounter({
|
|
4823
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
4824
|
+
help: "Shadow decision build/publication failures.",
|
|
4825
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
4826
|
+
});
|
|
4827
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
4828
|
+
stage: shadowResult.stage,
|
|
4829
|
+
reason: shadowResult.reason,
|
|
4830
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
4831
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
4832
|
+
origin: "worker",
|
|
4833
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4834
|
+
});
|
|
4835
|
+
}
|
|
4725
4836
|
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
observability.incrementCounter({
|
|
4731
|
-
name: "opengeni_codex_pool_observations_total",
|
|
4732
|
-
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4733
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4734
|
-
});
|
|
4735
|
-
if (eligibleCount <= 1) {
|
|
4837
|
+
const eligibleCount = leased.accounts.filter((account) =>
|
|
4838
|
+
isCodexCredentialEligible(account, new Date()),
|
|
4839
|
+
).length;
|
|
4840
|
+
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
4736
4841
|
observability.incrementCounter({
|
|
4737
|
-
name: "
|
|
4738
|
-
help: "
|
|
4842
|
+
name: "opengeni_codex_pool_observations_total",
|
|
4843
|
+
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4739
4844
|
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4740
4845
|
});
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4846
|
+
if (eligibleCount <= 1) {
|
|
4847
|
+
observability.incrementCounter({
|
|
4848
|
+
name: "opengeni_codex_pool_low_total",
|
|
4849
|
+
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
4850
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4851
|
+
});
|
|
4852
|
+
observability.warn("Codex eligible credential pool is low", {
|
|
4853
|
+
workspaceId: input.workspaceId,
|
|
4854
|
+
eligibleCount,
|
|
4855
|
+
connectedCount: leased.accounts.length,
|
|
4856
|
+
depth: poolDepth,
|
|
4857
|
+
});
|
|
4858
|
+
}
|
|
4748
4859
|
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4860
|
+
if (
|
|
4861
|
+
effectiveCodexCredentialId === null &&
|
|
4862
|
+
leased.accounts.length > 0 &&
|
|
4863
|
+
leased.accounts.every((account) => !account.allocatorEnabled) &&
|
|
4864
|
+
turnId
|
|
4865
|
+
) {
|
|
4866
|
+
if (turn.source === "compaction") {
|
|
4867
|
+
if (
|
|
4868
|
+
!(await settle!({
|
|
4869
|
+
events: [
|
|
4870
|
+
{
|
|
4871
|
+
type: "turn.cancelled",
|
|
4872
|
+
payload: {
|
|
4873
|
+
maintenance: "context_compaction",
|
|
4874
|
+
reason: "codex_allocator_disabled",
|
|
4875
|
+
requestPreserved: true,
|
|
4876
|
+
},
|
|
4877
|
+
},
|
|
4878
|
+
{
|
|
4879
|
+
type: "session.status.changed",
|
|
4880
|
+
payload: { status: "idle" },
|
|
4881
|
+
},
|
|
4882
|
+
],
|
|
4883
|
+
turnStatus: "cancelled",
|
|
4884
|
+
sessionStatus: "idle",
|
|
4885
|
+
activeTurnId: null,
|
|
4886
|
+
}))
|
|
4887
|
+
) {
|
|
4888
|
+
return claimedResult({ status: "cancelled" });
|
|
4889
|
+
}
|
|
4890
|
+
turnMetricOutcome = "cancelled";
|
|
4891
|
+
activityStatus = "idle";
|
|
4892
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4893
|
+
}
|
|
4894
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4895
|
+
() => null,
|
|
4896
|
+
);
|
|
4897
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
4898
|
+
const armed = await armCodexCapacityWait(db, {
|
|
4899
|
+
accountId: input.accountId,
|
|
4900
|
+
workspaceId: input.workspaceId,
|
|
4901
|
+
sessionId: input.sessionId,
|
|
4902
|
+
turnId,
|
|
4903
|
+
attemptId: input.attemptId,
|
|
4904
|
+
workflowId: input.workflowId,
|
|
4905
|
+
goalId: activeGoal?.id ?? null,
|
|
4906
|
+
goalVersion: activeGoal?.version ?? null,
|
|
4907
|
+
earliestResetAt: null,
|
|
4908
|
+
resetKind: "bounded_refresh",
|
|
4909
|
+
failurePayload: {
|
|
4910
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4911
|
+
code: "codex_allocator_disabled",
|
|
4912
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added",
|
|
4913
|
+
},
|
|
4914
|
+
});
|
|
4915
|
+
if (armed.action === "waiting") {
|
|
4916
|
+
await publishDurableSessionEvents(
|
|
4917
|
+
bus,
|
|
4918
|
+
input.workspaceId,
|
|
4919
|
+
input.sessionId,
|
|
4920
|
+
armed.events,
|
|
4921
|
+
);
|
|
4922
|
+
turnMetricOutcome = "recovering";
|
|
4923
|
+
activityStatus = "waiting_capacity";
|
|
4924
|
+
return claimedResult({
|
|
4925
|
+
status: "waiting_capacity",
|
|
4926
|
+
capacityWait: {
|
|
4927
|
+
waiterId: armed.waiter.id,
|
|
4928
|
+
generation: armed.waiter.generation,
|
|
4929
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
4930
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
4931
|
+
},
|
|
4932
|
+
});
|
|
4933
|
+
}
|
|
4756
4934
|
if (
|
|
4757
4935
|
!(await settle!({
|
|
4758
4936
|
events: [
|
|
4759
4937
|
{
|
|
4760
|
-
type: "turn.
|
|
4938
|
+
type: "turn.failed",
|
|
4761
4939
|
payload: {
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4940
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4941
|
+
code: "codex_allocator_disabled",
|
|
4942
|
+
retryable: false,
|
|
4943
|
+
recovery: "user_message",
|
|
4765
4944
|
},
|
|
4766
4945
|
},
|
|
4767
4946
|
{
|
|
@@ -4769,94 +4948,119 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4769
4948
|
payload: { status: "idle" },
|
|
4770
4949
|
},
|
|
4771
4950
|
],
|
|
4772
|
-
turnStatus: "
|
|
4951
|
+
turnStatus: "failed",
|
|
4773
4952
|
sessionStatus: "idle",
|
|
4774
4953
|
activeTurnId: null,
|
|
4775
4954
|
}))
|
|
4776
4955
|
) {
|
|
4777
4956
|
return claimedResult({ status: "cancelled" });
|
|
4778
4957
|
}
|
|
4779
|
-
turnMetricOutcome = "
|
|
4958
|
+
turnMetricOutcome = "failed";
|
|
4780
4959
|
activityStatus = "idle";
|
|
4781
|
-
return claimedResult({ status: "idle"
|
|
4960
|
+
return claimedResult({ status: "idle" });
|
|
4782
4961
|
}
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4962
|
+
|
|
4963
|
+
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
4964
|
+
if (turn.source === "compaction") {
|
|
4965
|
+
if (
|
|
4966
|
+
!(await settle!({
|
|
4967
|
+
events: [
|
|
4968
|
+
{
|
|
4969
|
+
type: "turn.cancelled",
|
|
4970
|
+
payload: {
|
|
4971
|
+
maintenance: "context_compaction",
|
|
4972
|
+
reason: "codex_capacity_unavailable",
|
|
4973
|
+
requestPreserved: true,
|
|
4974
|
+
},
|
|
4975
|
+
},
|
|
4976
|
+
{
|
|
4977
|
+
type: "session.status.changed",
|
|
4978
|
+
payload: { status: "idle" },
|
|
4979
|
+
},
|
|
4980
|
+
],
|
|
4981
|
+
turnStatus: "cancelled",
|
|
4982
|
+
sessionStatus: "idle",
|
|
4983
|
+
activeTurnId: null,
|
|
4984
|
+
}))
|
|
4985
|
+
) {
|
|
4986
|
+
return claimedResult({ status: "cancelled" });
|
|
4987
|
+
}
|
|
4988
|
+
turnMetricOutcome = "cancelled";
|
|
4989
|
+
activityStatus = "idle";
|
|
4990
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4991
|
+
}
|
|
4992
|
+
// Every eligible account is capped/cooling (and a usage refresh did NOT
|
|
4993
|
+
// surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
|
|
4994
|
+
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
4995
|
+
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
4996
|
+
// no model ran, nothing to freeze.
|
|
4997
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4998
|
+
() => null,
|
|
4810
4999
|
);
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
5000
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
5001
|
+
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
5002
|
+
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
5003
|
+
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
5004
|
+
const resumeMs = computeIdleDelayMs(
|
|
5005
|
+
rotationDecision.earliestResetAt,
|
|
5006
|
+
new Date(),
|
|
5007
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
5008
|
+
);
|
|
5009
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
5010
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
5011
|
+
"all connected Codex subscriptions are rate-limited",
|
|
5012
|
+
{ allAccounts: true },
|
|
5013
|
+
);
|
|
5014
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
5015
|
+
leased.accounts,
|
|
5016
|
+
new Date(),
|
|
5017
|
+
);
|
|
5018
|
+
const armed = await armCodexCapacityWait(db, {
|
|
5019
|
+
accountId: input.accountId,
|
|
5020
|
+
workspaceId: input.workspaceId,
|
|
5021
|
+
sessionId: input.sessionId,
|
|
5022
|
+
turnId,
|
|
5023
|
+
attemptId: input.attemptId,
|
|
5024
|
+
workflowId: input.workflowId,
|
|
5025
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
5026
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
5027
|
+
earliestResetAt: authoritativeResetAt,
|
|
5028
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
5029
|
+
failurePayload,
|
|
4821
5030
|
});
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
5031
|
+
if (armed.action === "waiting") {
|
|
5032
|
+
await publishDurableSessionEvents(
|
|
5033
|
+
bus,
|
|
5034
|
+
input.workspaceId,
|
|
5035
|
+
input.sessionId,
|
|
5036
|
+
armed.events,
|
|
5037
|
+
);
|
|
5038
|
+
turnMetricOutcome = "recovering";
|
|
5039
|
+
activityStatus = "waiting_capacity";
|
|
5040
|
+
return claimedResult({
|
|
5041
|
+
status: "waiting_capacity",
|
|
5042
|
+
capacityWait: {
|
|
5043
|
+
waiterId: armed.waiter.id,
|
|
5044
|
+
generation: armed.waiter.generation,
|
|
5045
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
5046
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
4834
5047
|
},
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
turnStatus: "failed",
|
|
4838
|
-
sessionStatus: "idle",
|
|
4839
|
-
activeTurnId: null,
|
|
4840
|
-
}))
|
|
4841
|
-
) {
|
|
4842
|
-
return claimedResult({ status: "cancelled" });
|
|
4843
|
-
}
|
|
4844
|
-
turnMetricOutcome = "failed";
|
|
4845
|
-
activityStatus = "idle";
|
|
4846
|
-
return claimedResult({ status: "idle" });
|
|
4847
|
-
}
|
|
4848
|
-
|
|
4849
|
-
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
4850
|
-
if (turn.source === "compaction") {
|
|
5048
|
+
});
|
|
5049
|
+
}
|
|
4851
5050
|
if (
|
|
4852
5051
|
!(await settle!({
|
|
4853
5052
|
events: [
|
|
5053
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
5054
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
5055
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
5056
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
5057
|
+
// the proactive path while the reactive path spares it.
|
|
4854
5058
|
{
|
|
4855
|
-
type: "turn.
|
|
5059
|
+
type: "turn.failed",
|
|
4856
5060
|
payload: {
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
5061
|
+
...failurePayload,
|
|
5062
|
+
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
5063
|
+
rotated: true,
|
|
4860
5064
|
},
|
|
4861
5065
|
},
|
|
4862
5066
|
{
|
|
@@ -4864,165 +5068,90 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4864
5068
|
payload: { status: "idle" },
|
|
4865
5069
|
},
|
|
4866
5070
|
],
|
|
4867
|
-
turnStatus: "
|
|
5071
|
+
turnStatus: "failed",
|
|
4868
5072
|
sessionStatus: "idle",
|
|
4869
5073
|
activeTurnId: null,
|
|
4870
5074
|
}))
|
|
4871
5075
|
) {
|
|
4872
5076
|
return claimedResult({ status: "cancelled" });
|
|
4873
5077
|
}
|
|
4874
|
-
turnMetricOutcome = "
|
|
5078
|
+
turnMetricOutcome = "failed";
|
|
4875
5079
|
activityStatus = "idle";
|
|
4876
|
-
|
|
5080
|
+
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
5081
|
+
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
5082
|
+
return claimedResult(
|
|
5083
|
+
goalActive
|
|
5084
|
+
? {
|
|
5085
|
+
status: "idle",
|
|
5086
|
+
continueDelayMs: resumeMs,
|
|
5087
|
+
idleUntilReset: true,
|
|
5088
|
+
}
|
|
5089
|
+
: { status: "idle" },
|
|
5090
|
+
);
|
|
4877
5091
|
}
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
// no model ran, nothing to freeze.
|
|
4883
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4884
|
-
() => null,
|
|
4885
|
-
);
|
|
4886
|
-
const goalActive = Boolean(goal && goal.status === "active");
|
|
4887
|
-
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
4888
|
-
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
4889
|
-
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
4890
|
-
const resumeMs = computeIdleDelayMs(
|
|
4891
|
-
rotationDecision.earliestResetAt,
|
|
4892
|
-
new Date(),
|
|
4893
|
-
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
4894
|
-
);
|
|
4895
|
-
const failurePayload = codexUsageLimitFailurePayload(
|
|
4896
|
-
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
4897
|
-
"all connected Codex subscriptions are rate-limited",
|
|
4898
|
-
{ allAccounts: true },
|
|
4899
|
-
);
|
|
4900
|
-
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4901
|
-
leased.accounts,
|
|
4902
|
-
new Date(),
|
|
4903
|
-
);
|
|
4904
|
-
const armed = await armCodexCapacityWait(db, {
|
|
4905
|
-
accountId: input.accountId,
|
|
4906
|
-
workspaceId: input.workspaceId,
|
|
4907
|
-
sessionId: input.sessionId,
|
|
4908
|
-
turnId,
|
|
4909
|
-
attemptId: input.attemptId,
|
|
4910
|
-
workflowId: input.workflowId,
|
|
4911
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
4912
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
4913
|
-
earliestResetAt: authoritativeResetAt,
|
|
4914
|
-
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
4915
|
-
failurePayload,
|
|
4916
|
-
});
|
|
4917
|
-
if (armed.action === "waiting") {
|
|
4918
|
-
await publishDurableSessionEvents(
|
|
4919
|
-
bus,
|
|
5092
|
+
if (effectiveCodexCredentialId) {
|
|
5093
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
5094
|
+
await recordSessionActiveCodexCredential(
|
|
5095
|
+
db,
|
|
4920
5096
|
input.workspaceId,
|
|
4921
5097
|
input.sessionId,
|
|
4922
|
-
|
|
5098
|
+
effectiveCodexCredentialId,
|
|
4923
5099
|
);
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
status: "waiting_capacity",
|
|
4928
|
-
capacityWait: {
|
|
4929
|
-
waiterId: armed.waiter.id,
|
|
4930
|
-
generation: armed.waiter.generation,
|
|
4931
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
4932
|
-
wakeRevision: armed.waiter.wakeRevision,
|
|
4933
|
-
},
|
|
4934
|
-
});
|
|
4935
|
-
}
|
|
4936
|
-
if (
|
|
4937
|
-
!(await settle!({
|
|
4938
|
-
events: [
|
|
4939
|
-
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
4940
|
-
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
4941
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
4942
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
4943
|
-
// the proactive path while the reactive path spares it.
|
|
5100
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
5101
|
+
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
5102
|
+
await publish([
|
|
4944
5103
|
{
|
|
4945
|
-
type: "
|
|
5104
|
+
type: "codex.account.switched",
|
|
4946
5105
|
payload: {
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
rotated:
|
|
5106
|
+
fromAccountId: priorAccountId,
|
|
5107
|
+
toAccountId: effectiveCodexCredentialId,
|
|
5108
|
+
reason: rotated ? "rotation" : "manual",
|
|
4950
5109
|
},
|
|
4951
5110
|
},
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
}
|
|
4972
|
-
: { status: "idle" },
|
|
4973
|
-
);
|
|
4974
|
-
}
|
|
4975
|
-
if (effectiveCodexCredentialId) {
|
|
4976
|
-
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
4977
|
-
await recordSessionActiveCodexCredential(
|
|
4978
|
-
db,
|
|
4979
|
-
input.workspaceId,
|
|
4980
|
-
input.sessionId,
|
|
4981
|
-
effectiveCodexCredentialId,
|
|
4982
|
-
);
|
|
4983
|
-
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
4984
|
-
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
5111
|
+
]);
|
|
5112
|
+
}
|
|
5113
|
+
|
|
5114
|
+
const selectionReason = leased.reused
|
|
5115
|
+
? "lease_reused"
|
|
5116
|
+
: sessionPin === effectiveCodexCredentialId
|
|
5117
|
+
? "pin"
|
|
5118
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
5119
|
+
? "rotation"
|
|
5120
|
+
: "active";
|
|
5121
|
+
observability.incrementCounter({
|
|
5122
|
+
name: "opengeni_codex_credential_selections_total",
|
|
5123
|
+
help: "Codex credential selections by strategy and reason.",
|
|
5124
|
+
labels: {
|
|
5125
|
+
workspace_key: codexWorkspaceKey,
|
|
5126
|
+
strategy: leased.rotationStrategy,
|
|
5127
|
+
reason: selectionReason,
|
|
5128
|
+
},
|
|
5129
|
+
});
|
|
4985
5130
|
await publish([
|
|
4986
5131
|
{
|
|
4987
|
-
type: "codex.
|
|
5132
|
+
type: "codex.credential.selected",
|
|
4988
5133
|
payload: {
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
reason:
|
|
5134
|
+
credentialId: effectiveCodexCredentialId,
|
|
5135
|
+
strategy: leased.rotationStrategy,
|
|
5136
|
+
reason: selectionReason,
|
|
5137
|
+
eligibleCount,
|
|
5138
|
+
connectedCount: leased.accounts.length,
|
|
5139
|
+
reused: leased.reused,
|
|
4992
5140
|
},
|
|
4993
5141
|
},
|
|
4994
5142
|
]);
|
|
4995
5143
|
}
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
help: "Codex credential selections by strategy and reason.",
|
|
5007
|
-
labels: {
|
|
5008
|
-
workspace_key: codexWorkspaceKey,
|
|
5009
|
-
strategy: leased.rotationStrategy,
|
|
5010
|
-
reason: selectionReason,
|
|
5011
|
-
},
|
|
5144
|
+
} catch (error) {
|
|
5145
|
+
credentialSelectionOutcome = "failed";
|
|
5146
|
+
throw error;
|
|
5147
|
+
} finally {
|
|
5148
|
+
recordTurnStartupPhase(observability, {
|
|
5149
|
+
phase: "credential_selection",
|
|
5150
|
+
provider: "codex-subscription",
|
|
5151
|
+
backend: turn.sandboxBackend,
|
|
5152
|
+
outcome: credentialSelectionOutcome,
|
|
5153
|
+
durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
|
|
5012
5154
|
});
|
|
5013
|
-
await publish([
|
|
5014
|
-
{
|
|
5015
|
-
type: "codex.credential.selected",
|
|
5016
|
-
payload: {
|
|
5017
|
-
credentialId: effectiveCodexCredentialId,
|
|
5018
|
-
strategy: leased.rotationStrategy,
|
|
5019
|
-
reason: selectionReason,
|
|
5020
|
-
eligibleCount,
|
|
5021
|
-
connectedCount: leased.accounts.length,
|
|
5022
|
-
reused: leased.reused,
|
|
5023
|
-
},
|
|
5024
|
-
},
|
|
5025
|
-
]);
|
|
5026
5155
|
}
|
|
5027
5156
|
}
|
|
5028
5157
|
|
|
@@ -5232,6 +5361,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5232
5361
|
});
|
|
5233
5362
|
}
|
|
5234
5363
|
|
|
5364
|
+
const runtimePreparationStartedAt = performance.now();
|
|
5365
|
+
|
|
5235
5366
|
// Pack-scoped runtime: enabled packs may declare the sandbox image this
|
|
5236
5367
|
// workspace's sessions run in and skills for the sandbox skill index.
|
|
5237
5368
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
@@ -5469,6 +5600,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5469
5600
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
5470
5601
|
// would hit the codex backend unauthenticated.
|
|
5471
5602
|
let codexModelRequestSequence = 0;
|
|
5603
|
+
let firstModelRequestPreparationStartedAt: number | null = null;
|
|
5604
|
+
let firstModelRequestPreparationRecorded = false;
|
|
5605
|
+
let firstModelRequestAuditRecorded = false;
|
|
5606
|
+
let firstModelRequestCheckpointAt: number | null = null;
|
|
5607
|
+
const firstModelRequestCheckpoints = new Set<string>();
|
|
5472
5608
|
const codexContext: CodexRequestContext | null =
|
|
5473
5609
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
5474
5610
|
? ((): CodexRequestContext => {
|
|
@@ -5499,10 +5635,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5499
5635
|
onUsageHeaders: (snapshot) => {
|
|
5500
5636
|
latestCodexUsage = snapshot;
|
|
5501
5637
|
}, // latest wins; flushed once in finally
|
|
5638
|
+
onRequestPreparationDiagnostic: (phase) => {
|
|
5639
|
+
if (
|
|
5640
|
+
firstModelRequestCheckpointAt === null ||
|
|
5641
|
+
firstModelRequestCheckpoints.has(phase)
|
|
5642
|
+
) {
|
|
5643
|
+
return;
|
|
5644
|
+
}
|
|
5645
|
+
const now = performance.now();
|
|
5646
|
+
const metricPhase =
|
|
5647
|
+
phase === "transport_entry"
|
|
5648
|
+
? "model_sdk_serialization"
|
|
5649
|
+
: phase === "credential_ready"
|
|
5650
|
+
? "model_credential_resolution"
|
|
5651
|
+
: "model_wire_normalization";
|
|
5652
|
+
firstModelRequestCheckpoints.add(phase);
|
|
5653
|
+
recordTurnStartupPhase(observability, {
|
|
5654
|
+
phase: metricPhase,
|
|
5655
|
+
provider: turnExecutionPolicy.providerId,
|
|
5656
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5657
|
+
outcome: "completed",
|
|
5658
|
+
durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
|
|
5659
|
+
count: turnTools.length,
|
|
5660
|
+
});
|
|
5661
|
+
firstModelRequestCheckpointAt = now;
|
|
5662
|
+
},
|
|
5502
5663
|
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
5503
5664
|
lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
5504
5665
|
},
|
|
5505
5666
|
onModelRequestDiagnostic: (event) => {
|
|
5667
|
+
if (
|
|
5668
|
+
event.phase === "started" &&
|
|
5669
|
+
firstModelRequestPreparationStartedAt !== null &&
|
|
5670
|
+
!firstModelRequestPreparationRecorded
|
|
5671
|
+
) {
|
|
5672
|
+
firstModelRequestPreparationRecorded = true;
|
|
5673
|
+
recordTurnStartupPhase(observability, {
|
|
5674
|
+
phase: "model_request_preparation",
|
|
5675
|
+
provider: turnExecutionPolicy.providerId,
|
|
5676
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5677
|
+
outcome: "completed",
|
|
5678
|
+
durationSeconds:
|
|
5679
|
+
(performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
|
|
5680
|
+
count: turnTools.length,
|
|
5681
|
+
});
|
|
5682
|
+
}
|
|
5506
5683
|
const phase =
|
|
5507
5684
|
event.phase === "headers"
|
|
5508
5685
|
? "headers"
|
|
@@ -5532,19 +5709,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5532
5709
|
if (!publish || !turnId) {
|
|
5533
5710
|
throw new Error("Codex model request started before the turn event producer");
|
|
5534
5711
|
}
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5712
|
+
const shouldRecordStartedAudit =
|
|
5713
|
+
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
5714
|
+
if (shouldRecordStartedAudit) {
|
|
5715
|
+
firstModelRequestAuditRecorded = true;
|
|
5716
|
+
}
|
|
5717
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
5718
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
5719
|
+
try {
|
|
5720
|
+
await publish([
|
|
5721
|
+
{
|
|
5722
|
+
type: "agent.model.request",
|
|
5723
|
+
payload: {
|
|
5724
|
+
...event,
|
|
5725
|
+
provider: "codex-subscription",
|
|
5726
|
+
turnId,
|
|
5727
|
+
attemptId: input.attemptId,
|
|
5728
|
+
dispatchId,
|
|
5729
|
+
executionGeneration,
|
|
5730
|
+
},
|
|
5545
5731
|
},
|
|
5546
|
-
|
|
5547
|
-
|
|
5732
|
+
]);
|
|
5733
|
+
} catch (error) {
|
|
5734
|
+
auditOutcome = "failed";
|
|
5735
|
+
throw error;
|
|
5736
|
+
} finally {
|
|
5737
|
+
if (auditStartedAt !== null) {
|
|
5738
|
+
recordTurnStartupPhase(observability, {
|
|
5739
|
+
phase: "model_request_audit",
|
|
5740
|
+
provider: turnExecutionPolicy.providerId,
|
|
5741
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5742
|
+
outcome: auditOutcome,
|
|
5743
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
5744
|
+
count: turnTools.length,
|
|
5745
|
+
});
|
|
5746
|
+
}
|
|
5747
|
+
}
|
|
5548
5748
|
},
|
|
5549
5749
|
};
|
|
5550
5750
|
})()
|
|
@@ -6085,17 +6285,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6085
6285
|
cancellationSignal,
|
|
6086
6286
|
undefined,
|
|
6087
6287
|
);
|
|
6088
|
-
const
|
|
6089
|
-
lazyProvisionEnabled(settings)
|
|
6090
|
-
|
|
6091
|
-
runSettings.sandboxBackend
|
|
6288
|
+
const establishDecision = sandboxEstablishPolicyDecision({
|
|
6289
|
+
lazyEnabled: lazyProvisionEnabled(settings),
|
|
6290
|
+
machinePrimary,
|
|
6291
|
+
sandboxBackend: runSettings.sandboxBackend,
|
|
6092
6292
|
// Resolved run credentials must be written to one exact leased sandbox
|
|
6093
6293
|
// before agent execution. A warm active pointer can bypass the lazy
|
|
6094
6294
|
// provisioner entirely, which previously skipped materialization.
|
|
6095
|
-
|
|
6096
|
-
requiredGeneratedVideoFiles.length
|
|
6097
|
-
|
|
6098
|
-
|
|
6295
|
+
hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
|
|
6296
|
+
generatedVideoFileCount: requiredGeneratedVideoFiles.length,
|
|
6297
|
+
hasSignedFileResources:
|
|
6298
|
+
requiresSignedFileResourceDownloads(
|
|
6299
|
+
runSettings,
|
|
6300
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
6301
|
+
) && turnResources.some((resource) => resource.kind === "file"),
|
|
6302
|
+
});
|
|
6303
|
+
const establishPolicy = establishDecision.policy;
|
|
6304
|
+
recordTurnSandboxEstablishPolicy(observability, {
|
|
6305
|
+
policy: establishDecision.policy,
|
|
6306
|
+
reason: establishDecision.reason,
|
|
6307
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
6308
|
+
});
|
|
6099
6309
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
6100
6310
|
// (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
|
|
6101
6311
|
// agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
|
|
@@ -6568,6 +6778,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6568
6778
|
// applies the agent's manifest to this provided session and throws on ANY
|
|
6569
6779
|
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
6570
6780
|
// here makes current==target so the delta is empty.
|
|
6781
|
+
recordTurnStartupPhase(observability, {
|
|
6782
|
+
phase: "runtime_preparation",
|
|
6783
|
+
provider: turnExecutionPolicy.providerId,
|
|
6784
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
6785
|
+
outcome: "completed",
|
|
6786
|
+
durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
|
|
6787
|
+
});
|
|
6571
6788
|
if (
|
|
6572
6789
|
shouldEstablishSandboxForTurn(
|
|
6573
6790
|
settings.sandboxOwnershipEnabled,
|
|
@@ -6575,242 +6792,275 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6575
6792
|
machinePrimary,
|
|
6576
6793
|
)
|
|
6577
6794
|
) {
|
|
6578
|
-
const
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
// STAGE D honest-label guard: a machine-home session carries
|
|
6586
|
-
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
6587
|
-
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
6588
|
-
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
6589
|
-
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6590
|
-
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6591
|
-
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6592
|
-
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6593
|
-
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6594
|
-
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6595
|
-
if (machinePrimary) {
|
|
6596
|
-
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6597
|
-
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6598
|
-
// the machine route's own epoch fence; the managed-home lease separately
|
|
6599
|
-
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6600
|
-
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6601
|
-
// healthy explicitly selected machine.
|
|
6602
|
-
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6603
|
-
// (refreshed on every connect). Read only when the server flag is on —
|
|
6604
|
-
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6605
|
-
const machineOpStream =
|
|
6606
|
-
settings.agentOpStreamEnabled === true
|
|
6607
|
-
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6608
|
-
?.opStream === true
|
|
6609
|
-
: false;
|
|
6610
|
-
const established = await establishSelfhostedTurnSession(
|
|
6611
|
-
{
|
|
6612
|
-
db,
|
|
6613
|
-
settings,
|
|
6614
|
-
bus,
|
|
6615
|
-
onOp: machineOpObserver.observer,
|
|
6616
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6617
|
-
opJournal,
|
|
6618
|
-
},
|
|
6619
|
-
{
|
|
6620
|
-
workspaceId: input.workspaceId,
|
|
6621
|
-
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6622
|
-
opStream: machineOpStream,
|
|
6623
|
-
epoch: activeSandboxPointer!.activeEpoch,
|
|
6624
|
-
environment: sandboxEnvironment,
|
|
6625
|
-
...(transientCodemodeEnvironment
|
|
6626
|
-
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6627
|
-
: {}),
|
|
6628
|
-
workingDir: activeSandboxPointer!.workingDir,
|
|
6629
|
-
},
|
|
6795
|
+
const sandboxEstablishStartedAt = performance.now();
|
|
6796
|
+
let sandboxEstablishOutcome: "completed" | "failed" = "completed";
|
|
6797
|
+
try {
|
|
6798
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
6799
|
+
machinePrimary,
|
|
6800
|
+
input.attemptId,
|
|
6801
|
+
session.sandboxGroupId,
|
|
6630
6802
|
);
|
|
6631
|
-
|
|
6632
|
-
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6803
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
6804
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
6805
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
6806
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
6807
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
6808
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
6809
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6810
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6811
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6812
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6813
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6814
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6815
|
+
if (machinePrimary) {
|
|
6816
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6817
|
+
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6818
|
+
// the machine route's own epoch fence; the managed-home lease separately
|
|
6819
|
+
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6820
|
+
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6821
|
+
// healthy explicitly selected machine.
|
|
6822
|
+
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6823
|
+
// (refreshed on every connect). Read only when the server flag is on —
|
|
6824
|
+
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6825
|
+
const machineOpStream =
|
|
6826
|
+
settings.agentOpStreamEnabled === true
|
|
6827
|
+
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6828
|
+
?.opStream === true
|
|
6829
|
+
: false;
|
|
6830
|
+
const established = await establishSelfhostedTurnSession(
|
|
6643
6831
|
{
|
|
6644
6832
|
db,
|
|
6645
6833
|
settings,
|
|
6646
6834
|
bus,
|
|
6647
|
-
opJournal,
|
|
6648
6835
|
onOp: machineOpObserver.observer,
|
|
6649
6836
|
onSandboxOperation: sandboxOperationObserver,
|
|
6650
|
-
|
|
6651
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6837
|
+
opJournal,
|
|
6652
6838
|
},
|
|
6653
6839
|
{
|
|
6654
6840
|
workspaceId: input.workspaceId,
|
|
6655
|
-
|
|
6841
|
+
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6842
|
+
opStream: machineOpStream,
|
|
6843
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6656
6844
|
environment: sandboxEnvironment,
|
|
6657
6845
|
...(transientCodemodeEnvironment
|
|
6658
6846
|
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6659
6847
|
: {}),
|
|
6660
|
-
|
|
6661
|
-
|
|
6662
|
-
|
|
6663
|
-
|
|
6664
|
-
|
|
6848
|
+
workingDir: activeSandboxPointer!.workingDir,
|
|
6849
|
+
},
|
|
6850
|
+
);
|
|
6851
|
+
// The machine-primary establish narrows `session` to SelfhostedSession
|
|
6852
|
+
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
6853
|
+
machinePrimarySession =
|
|
6854
|
+
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
6855
|
+
setupBoxSession = established.session;
|
|
6856
|
+
resolvedSandbox = {
|
|
6857
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
6858
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
6859
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
6860
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
6861
|
+
// instance — no two-instance manifest divergence.
|
|
6862
|
+
established: wrapTurnBoxWithRouting(
|
|
6863
|
+
{
|
|
6864
|
+
db,
|
|
6865
|
+
settings,
|
|
6866
|
+
bus,
|
|
6867
|
+
opJournal,
|
|
6868
|
+
onOp: machineOpObserver.observer,
|
|
6869
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
6870
|
+
onHomeSandboxRebound,
|
|
6871
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6665
6872
|
},
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6873
|
+
{
|
|
6874
|
+
workspaceId: input.workspaceId,
|
|
6875
|
+
sessionId: input.sessionId,
|
|
6876
|
+
environment: sandboxEnvironment,
|
|
6877
|
+
...(transientCodemodeEnvironment
|
|
6878
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6879
|
+
: {}),
|
|
6880
|
+
workspaceMutationFence: {
|
|
6881
|
+
accountId: input.accountId,
|
|
6882
|
+
turnId: turn.id,
|
|
6883
|
+
executionGeneration,
|
|
6884
|
+
attemptId: input.attemptId,
|
|
6885
|
+
},
|
|
6886
|
+
pinnedSelfhosted: {
|
|
6887
|
+
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
6888
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6889
|
+
},
|
|
6890
|
+
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
6891
|
+
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
6892
|
+
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
6893
|
+
// session merely PINNED to a machine this turn never established its group
|
|
6894
|
+
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
6895
|
+
// (`home_unavailable_this_turn`) rather than silently serving the machine;
|
|
6896
|
+
// the detach takes effect next turn. (Lazy home-box establishment on such a
|
|
6897
|
+
// clear is a deferred follow-up; issue #341.)
|
|
6898
|
+
defaultIsHome: session.sandboxBackend === "selfhosted",
|
|
6669
6899
|
},
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
//
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
} else {
|
|
6694
|
-
resolvedSandbox = await waitForTurnOperation(
|
|
6695
|
-
resumeBoxForTurn(
|
|
6696
|
-
{
|
|
6697
|
-
db,
|
|
6698
|
-
settings: runSettings,
|
|
6699
|
-
logicalFallbackSettings: logicalSandboxSettings,
|
|
6700
|
-
cancellationSignal: sandboxResumeSignal,
|
|
6701
|
-
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6702
|
-
onSandboxLost: publishSandboxLost,
|
|
6703
|
-
},
|
|
6704
|
-
{
|
|
6705
|
-
accountId: input.accountId,
|
|
6706
|
-
workspaceId: input.workspaceId,
|
|
6707
|
-
sandboxGroupId: session.sandboxGroupId,
|
|
6708
|
-
sessionId: input.sessionId,
|
|
6709
|
-
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
6710
|
-
// is not machine-primary resumes a real cloud group box (the
|
|
6711
|
-
// deployment default), never a "selfhosted" box (which would throw
|
|
6712
|
-
// for lack of a bound agentId).
|
|
6713
|
-
backend: groupBoxBackend,
|
|
6714
|
-
os: session.sandboxOs,
|
|
6715
|
-
environment: sandboxEnvironment,
|
|
6716
|
-
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
6717
|
-
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
6718
|
-
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6719
|
-
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6720
|
-
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6721
|
-
// (establishSelfhostedTurnSession) NEVER passes
|
|
6722
|
-
// an image — B3 lives only on this Modal else-branch.
|
|
6723
|
-
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6724
|
-
? {
|
|
6725
|
-
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6726
|
-
}
|
|
6727
|
-
: {}),
|
|
6728
|
-
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
6729
|
-
// conflicts on a live shared box set up under a different rig (solo
|
|
6730
|
-
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6731
|
-
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6732
|
-
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6733
|
-
},
|
|
6734
|
-
"turn",
|
|
6735
|
-
managedOwnership!.holderId,
|
|
6736
|
-
),
|
|
6737
|
-
sandboxResumeSignal,
|
|
6738
|
-
releaseLateSandbox,
|
|
6739
|
-
);
|
|
6740
|
-
setupBoxSession = resolvedSandbox.established.session;
|
|
6741
|
-
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6742
|
-
// record every box transition in session_events so the NEXT box loss is
|
|
6743
|
-
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6744
|
-
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6745
|
-
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6746
|
-
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6747
|
-
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6748
|
-
// into the run. The SDK binds to this ONE object once and calls its
|
|
6749
|
-
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
6750
|
-
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
6751
|
-
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
6752
|
-
// the flag off the established group box is injected unchanged (today's
|
|
6753
|
-
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
6754
|
-
// routing veneer, not an owner.
|
|
6755
|
-
resolvedSandbox = {
|
|
6756
|
-
...resolvedSandbox,
|
|
6757
|
-
established: wrapTurnBoxWithRouting(
|
|
6758
|
-
{
|
|
6759
|
-
db,
|
|
6760
|
-
settings,
|
|
6761
|
-
bus,
|
|
6762
|
-
opJournal,
|
|
6763
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6764
|
-
onHomeSandboxLost: publishSandboxLost,
|
|
6765
|
-
onHomeSandboxRebound,
|
|
6766
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6767
|
-
},
|
|
6768
|
-
// Thread the SAME declared environment the group box was created with
|
|
6769
|
-
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
6770
|
-
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
6771
|
-
// (no "cannot change manifest environment variables" throw).
|
|
6772
|
-
{
|
|
6773
|
-
workspaceId: input.workspaceId,
|
|
6774
|
-
sessionId: input.sessionId,
|
|
6775
|
-
environment: sandboxEnvironment,
|
|
6776
|
-
...(transientCodemodeEnvironment
|
|
6777
|
-
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6778
|
-
: {}),
|
|
6779
|
-
workspaceMutationFence: {
|
|
6780
|
-
accountId: input.accountId,
|
|
6781
|
-
turnId: turn.id,
|
|
6782
|
-
executionGeneration,
|
|
6783
|
-
attemptId: input.attemptId,
|
|
6900
|
+
established,
|
|
6901
|
+
),
|
|
6902
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
6903
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
6904
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
6905
|
+
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
6906
|
+
release: async () => undefined,
|
|
6907
|
+
};
|
|
6908
|
+
} else if (establishPolicy === "on-demand") {
|
|
6909
|
+
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
6910
|
+
// but the lease acquire + box establish + setup move behind the routing
|
|
6911
|
+
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
6912
|
+
// no lease row, no provider box, no warm-meter interval.
|
|
6913
|
+
} else {
|
|
6914
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
6915
|
+
resumeBoxForTurn(
|
|
6916
|
+
{
|
|
6917
|
+
db,
|
|
6918
|
+
settings: runSettings,
|
|
6919
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
6920
|
+
cancellationSignal: sandboxResumeSignal,
|
|
6921
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6922
|
+
onSandboxLost: publishSandboxLost,
|
|
6784
6923
|
},
|
|
6785
|
-
|
|
6924
|
+
{
|
|
6786
6925
|
accountId: input.accountId,
|
|
6926
|
+
workspaceId: input.workspaceId,
|
|
6787
6927
|
sandboxGroupId: session.sandboxGroupId,
|
|
6788
|
-
|
|
6789
|
-
|
|
6928
|
+
sessionId: input.sessionId,
|
|
6929
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
6930
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
6931
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
6932
|
+
// for lack of a bound agentId).
|
|
6790
6933
|
backend: groupBoxBackend,
|
|
6934
|
+
os: session.sandboxOs,
|
|
6935
|
+
environment: sandboxEnvironment,
|
|
6936
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
6937
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
6938
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6939
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6940
|
+
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6941
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
6942
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
6943
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6944
|
+
? {
|
|
6945
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6946
|
+
}
|
|
6947
|
+
: {}),
|
|
6948
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
6949
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
6950
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6951
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6952
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6791
6953
|
},
|
|
6792
|
-
|
|
6793
|
-
|
|
6794
|
-
|
|
6795
|
-
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6954
|
+
"turn",
|
|
6955
|
+
managedOwnership!.holderId,
|
|
6956
|
+
),
|
|
6957
|
+
sandboxResumeSignal,
|
|
6958
|
+
releaseLateSandbox,
|
|
6959
|
+
);
|
|
6960
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
6961
|
+
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6962
|
+
// record every box transition in session_events so the NEXT box loss is
|
|
6963
|
+
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6964
|
+
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6965
|
+
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6966
|
+
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6967
|
+
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6968
|
+
// into the run. The SDK binds to this ONE object once and calls its
|
|
6969
|
+
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
6970
|
+
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
6971
|
+
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
6972
|
+
// the flag off the established group box is injected unchanged (today's
|
|
6973
|
+
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
6974
|
+
// routing veneer, not an owner.
|
|
6975
|
+
resolvedSandbox = {
|
|
6976
|
+
...resolvedSandbox,
|
|
6977
|
+
established: wrapTurnBoxWithRouting(
|
|
6978
|
+
{
|
|
6979
|
+
db,
|
|
6980
|
+
settings,
|
|
6981
|
+
bus,
|
|
6982
|
+
opJournal,
|
|
6983
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
6984
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
6985
|
+
onHomeSandboxRebound,
|
|
6986
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6987
|
+
},
|
|
6988
|
+
// Thread the SAME declared environment the group box was created with
|
|
6989
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
6990
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
6991
|
+
// (no "cannot change manifest environment variables" throw).
|
|
6992
|
+
{
|
|
6993
|
+
workspaceId: input.workspaceId,
|
|
6994
|
+
sessionId: input.sessionId,
|
|
6995
|
+
environment: sandboxEnvironment,
|
|
6996
|
+
...(transientCodemodeEnvironment
|
|
6997
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6998
|
+
: {}),
|
|
6999
|
+
workspaceMutationFence: {
|
|
7000
|
+
accountId: input.accountId,
|
|
7001
|
+
turnId: turn.id,
|
|
7002
|
+
executionGeneration,
|
|
7003
|
+
attemptId: input.attemptId,
|
|
7004
|
+
},
|
|
7005
|
+
homeLease: {
|
|
7006
|
+
accountId: input.accountId,
|
|
7007
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
7008
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
7009
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
7010
|
+
backend: groupBoxBackend,
|
|
7011
|
+
},
|
|
7012
|
+
},
|
|
7013
|
+
resolvedSandbox.established,
|
|
7014
|
+
),
|
|
7015
|
+
};
|
|
7016
|
+
}
|
|
7017
|
+
if (resolvedSandbox) {
|
|
7018
|
+
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
7019
|
+
}
|
|
7020
|
+
} catch (error) {
|
|
7021
|
+
sandboxEstablishOutcome = "failed";
|
|
7022
|
+
throw error;
|
|
7023
|
+
} finally {
|
|
7024
|
+
recordTurnStartupPhase(observability, {
|
|
7025
|
+
phase: "sandbox_establish",
|
|
7026
|
+
provider: turnExecutionPolicy.providerId,
|
|
7027
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
7028
|
+
outcome: sandboxEstablishOutcome,
|
|
7029
|
+
durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
|
|
7030
|
+
});
|
|
6799
7031
|
}
|
|
6800
7032
|
}
|
|
6801
7033
|
|
|
6802
|
-
const ordinaryFileResourceDownloads = await
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
7034
|
+
const ordinaryFileResourceDownloads = await (async () => {
|
|
7035
|
+
const fileResolutionStartedAt = performance.now();
|
|
7036
|
+
let fileResolutionOutcome: "completed" | "failed" = "completed";
|
|
7037
|
+
try {
|
|
7038
|
+
return await waitForTurnOperation(
|
|
7039
|
+
sandboxFileDownloadsForRun(
|
|
7040
|
+
runSettings,
|
|
7041
|
+
db,
|
|
7042
|
+
objectStorage,
|
|
7043
|
+
input.workspaceId,
|
|
7044
|
+
turnResources,
|
|
7045
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
7046
|
+
),
|
|
7047
|
+
cancellationSignal,
|
|
7048
|
+
undefined,
|
|
7049
|
+
);
|
|
7050
|
+
} catch (error) {
|
|
7051
|
+
fileResolutionOutcome = "failed";
|
|
7052
|
+
throw error;
|
|
7053
|
+
} finally {
|
|
7054
|
+
recordTurnStartupPhase(observability, {
|
|
7055
|
+
phase: "file_resolution",
|
|
7056
|
+
provider: turnExecutionPolicy.providerId,
|
|
7057
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7058
|
+
outcome: fileResolutionOutcome,
|
|
7059
|
+
durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
|
|
7060
|
+
count: turnResources.length,
|
|
7061
|
+
});
|
|
7062
|
+
}
|
|
7063
|
+
})();
|
|
6814
7064
|
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
6815
7065
|
throw new Error("Generated video materialization requires object storage");
|
|
6816
7066
|
}
|
|
@@ -6840,6 +7090,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6840
7090
|
]),
|
|
6841
7091
|
).values(),
|
|
6842
7092
|
];
|
|
7093
|
+
const toolContextPreparationStartedAt = performance.now();
|
|
6843
7094
|
throwIfWorkerShuttingDown();
|
|
6844
7095
|
throwIfTurnCancelled();
|
|
6845
7096
|
const mcpCredentialRootSessionId =
|
|
@@ -6938,59 +7189,94 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6938
7189
|
runSettings,
|
|
6939
7190
|
session.firstPartyMcpTools,
|
|
6940
7191
|
);
|
|
6941
|
-
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
|
|
6949
|
-
|
|
6950
|
-
|
|
6951
|
-
|
|
6952
|
-
|
|
6953
|
-
|
|
6954
|
-
|
|
6955
|
-
|
|
6956
|
-
|
|
6957
|
-
|
|
6958
|
-
|
|
6959
|
-
|
|
6960
|
-
|
|
6961
|
-
|
|
6962
|
-
|
|
6963
|
-
// for their first-party MCP token; null keeps the fixed default.
|
|
6964
|
-
...(session.firstPartyMcpPermissions?.length
|
|
6965
|
-
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
6966
|
-
: {}),
|
|
6967
|
-
firstPartyTools: selectedFirstPartyMcpTools,
|
|
6968
|
-
nestedAgentDepth: session.nestedAgentDepth,
|
|
6969
|
-
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
6970
|
-
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
6971
|
-
settings: runSettings,
|
|
6972
|
-
scope: {
|
|
6973
|
-
accountId: input.accountId,
|
|
6974
|
-
workspaceId: input.workspaceId,
|
|
6975
|
-
sessionId: input.sessionId,
|
|
6976
|
-
turnId: turn.id,
|
|
6977
|
-
attemptId: input.attemptId,
|
|
6978
|
-
executionGeneration,
|
|
6979
|
-
},
|
|
6980
|
-
...(session.firstPartyMcpPermissions?.length
|
|
6981
|
-
? { permissions: session.firstPartyMcpPermissions }
|
|
6982
|
-
: {}),
|
|
6983
|
-
selectedTools: selectedFirstPartyMcpTools,
|
|
7192
|
+
recordTurnStartupPhase(observability, {
|
|
7193
|
+
phase: "tool_context_preparation",
|
|
7194
|
+
provider: turnExecutionPolicy.providerId,
|
|
7195
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7196
|
+
outcome: "completed",
|
|
7197
|
+
durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
|
|
7198
|
+
count: turnTools.length,
|
|
7199
|
+
});
|
|
7200
|
+
const toolPreparationStartedAt = performance.now();
|
|
7201
|
+
let toolPreparationOutcome: "completed" | "failed" = "completed";
|
|
7202
|
+
try {
|
|
7203
|
+
preparedTools = await waitForTurnOperation(
|
|
7204
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
7205
|
+
accountId: input.accountId,
|
|
7206
|
+
workspaceId: input.workspaceId,
|
|
7207
|
+
sessionId: input.sessionId,
|
|
7208
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
7209
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
7210
|
+
// live active pointer.
|
|
7211
|
+
...(turnId ? { turnId } : {}),
|
|
7212
|
+
attemptId: input.attemptId,
|
|
7213
|
+
executionGeneration,
|
|
6984
7214
|
subjectId: "worker:first-party-mcp",
|
|
6985
7215
|
subjectLabel: "OpenGeni worker",
|
|
6986
|
-
...(
|
|
6987
|
-
|
|
7216
|
+
...(credentialSubjectId ? { credentialSubjectId } : {}),
|
|
7217
|
+
...(codexAppsAuth ? { codexAppsAuth } : {}),
|
|
7218
|
+
resolveCredential,
|
|
7219
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
7220
|
+
localMcpServers,
|
|
7221
|
+
onPreparationPhase: (measurement) => {
|
|
7222
|
+
recordTurnStartupPhase(observability, {
|
|
7223
|
+
phase: `tool_${measurement.phase}`,
|
|
7224
|
+
provider: turnExecutionPolicy.providerId,
|
|
7225
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7226
|
+
outcome: measurement.outcome,
|
|
7227
|
+
durationSeconds: measurement.durationSeconds,
|
|
7228
|
+
count: turnTools.length,
|
|
7229
|
+
});
|
|
7230
|
+
},
|
|
7231
|
+
onAttemptToolCatalog: async (catalog) => {
|
|
7232
|
+
await persistAttemptToolCatalog(db, catalog);
|
|
7233
|
+
},
|
|
7234
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
7235
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
7236
|
+
...(session.firstPartyMcpPermissions?.length
|
|
7237
|
+
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
6988
7238
|
: {}),
|
|
7239
|
+
firstPartyTools: selectedFirstPartyMcpTools,
|
|
7240
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
7241
|
+
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
7242
|
+
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
7243
|
+
settings: runSettings,
|
|
7244
|
+
scope: {
|
|
7245
|
+
accountId: input.accountId,
|
|
7246
|
+
workspaceId: input.workspaceId,
|
|
7247
|
+
sessionId: input.sessionId,
|
|
7248
|
+
turnId: turn.id,
|
|
7249
|
+
attemptId: input.attemptId,
|
|
7250
|
+
executionGeneration,
|
|
7251
|
+
},
|
|
7252
|
+
...(session.firstPartyMcpPermissions?.length
|
|
7253
|
+
? { permissions: session.firstPartyMcpPermissions }
|
|
7254
|
+
: {}),
|
|
7255
|
+
selectedTools: selectedFirstPartyMcpTools,
|
|
7256
|
+
subjectId: "worker:first-party-mcp",
|
|
7257
|
+
subjectLabel: "OpenGeni worker",
|
|
7258
|
+
...(interactionInterventionResume
|
|
7259
|
+
? { interventionResume: interactionInterventionResume }
|
|
7260
|
+
: {}),
|
|
7261
|
+
}),
|
|
6989
7262
|
}),
|
|
6990
|
-
|
|
6991
|
-
|
|
6992
|
-
|
|
6993
|
-
)
|
|
7263
|
+
cancellationSignal,
|
|
7264
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
7265
|
+
);
|
|
7266
|
+
} catch (error) {
|
|
7267
|
+
toolPreparationOutcome = "failed";
|
|
7268
|
+
throw error;
|
|
7269
|
+
} finally {
|
|
7270
|
+
recordTurnStartupPhase(observability, {
|
|
7271
|
+
phase: "tool_preparation",
|
|
7272
|
+
provider: turnExecutionPolicy.providerId,
|
|
7273
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7274
|
+
outcome: toolPreparationOutcome,
|
|
7275
|
+
durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
|
|
7276
|
+
count: turnTools.length,
|
|
7277
|
+
});
|
|
7278
|
+
}
|
|
7279
|
+
const postToolPreparationStartedAt = performance.now();
|
|
6994
7280
|
if (turnId && preparedTools.attemptToolEnvironment) {
|
|
6995
7281
|
codemodeDispatcher = new CodemodeAttemptDispatcher(
|
|
6996
7282
|
db,
|
|
@@ -7378,203 +7664,228 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7378
7664
|
prepare: async (call) =>
|
|
7379
7665
|
await prepareConnectorActionApproval(db, connectorActionIdentity, call),
|
|
7380
7666
|
begin: async (call) =>
|
|
7381
|
-
await beginConnectorActionExecution(db, connectorActionIdentity, call),
|
|
7382
|
-
complete: async ({ requestId, outcome }) =>
|
|
7383
|
-
await completeConnectorActionExecution(db, {
|
|
7384
|
-
accountId: input.accountId,
|
|
7385
|
-
workspaceId: input.workspaceId,
|
|
7386
|
-
requestId,
|
|
7387
|
-
attemptId: input.attemptId,
|
|
7388
|
-
outcome,
|
|
7389
|
-
}),
|
|
7390
|
-
};
|
|
7391
|
-
const runtimeSkillActivations = [
|
|
7392
|
-
...installedSkillRuntime.activations,
|
|
7393
|
-
...packRuntime.skillActivations,
|
|
7394
|
-
...session.skills.map((skill) => ({
|
|
7395
|
-
source: "session" as const,
|
|
7396
|
-
id: `session:${session.id}:${skill.name}`,
|
|
7397
|
-
artifact: {
|
|
7398
|
-
name: skill.name,
|
|
7399
|
-
description: skill.description ?? null,
|
|
7400
|
-
files: skill.files.map((file) => ({
|
|
7401
|
-
path: file.path,
|
|
7402
|
-
content: file.content,
|
|
7403
|
-
})),
|
|
7404
|
-
},
|
|
7405
|
-
reason: "attached to session",
|
|
7406
|
-
})),
|
|
7407
|
-
];
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
genesisTitleHint: isGenesisTurn,
|
|
7415
|
-
persistentSessionSettings: {
|
|
7416
|
-
titleIsSet: Boolean(session.title?.trim()),
|
|
7417
|
-
},
|
|
7418
|
-
sandboxEnvironment,
|
|
7419
|
-
...(preparedTools.attemptToolCatalog
|
|
7420
|
-
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
7421
|
-
: {}),
|
|
7422
|
-
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
7423
|
-
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
7424
|
-
onToolCancellationFence: (fence) => {
|
|
7425
|
-
toolCancellationFenceRef.current = fence;
|
|
7426
|
-
},
|
|
7427
|
-
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
7428
|
-
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
7429
|
-
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
7430
|
-
// AND the mint actually produced a token (repo resources present). The runtime
|
|
7431
|
-
// seeds it to the box's token file before the repository-clone runs; it never
|
|
7432
|
-
// touches the box/agent manifest env.
|
|
7433
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
7434
|
-
? { gitTokenSeeds: sandboxGitTokens }
|
|
7435
|
-
: {}),
|
|
7436
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
7437
|
-
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
7438
|
-
: {}),
|
|
7439
|
-
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
7440
|
-
? { gitTokenSeed: sandboxGitToken }
|
|
7441
|
-
: {}),
|
|
7442
|
-
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7443
|
-
// Managed boxes receive the bearer through their protected per-session
|
|
7444
|
-
// token file. Connected Machines use transient per-exec delivery above,
|
|
7445
|
-
// so they must not run the file-seeding lifecycle hook.
|
|
7446
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
7447
|
-
? {
|
|
7448
|
-
codemodeTokenSeed: sandboxCodemodeToken,
|
|
7449
|
-
codemodeTokenSessionId: input.sessionId,
|
|
7450
|
-
}
|
|
7451
|
-
: {}),
|
|
7452
|
-
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
7453
|
-
fileResourceDownloads,
|
|
7454
|
-
mcpServers: preparedTools.mcpServers,
|
|
7455
|
-
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
7456
|
-
connectorActionPolicy,
|
|
7457
|
-
// LIVE by-reference connector namespaces (fills during this turn's
|
|
7458
|
-
// codex_apps tools/list): the codex tool_search description reads it per
|
|
7459
|
-
// model call so the model sees the account's real connected sources.
|
|
7460
|
-
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
7461
|
-
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
7462
|
-
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
7463
|
-
// NOT a Model instance: an instance only survives the in-process
|
|
7464
|
-
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
7465
|
-
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
7466
|
-
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
7467
|
-
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
7468
|
-
// client. The gating still comes from the resolved provider: server-side
|
|
7469
|
-
// store/compaction follow the provider's compaction mode (registry
|
|
7470
|
-
// providers resolve to "client"); encrypted reasoning is only
|
|
7471
|
-
// round-tripped on the Responses wire API; hosted web search is attached
|
|
7472
|
-
// whenever the provider declares it runnable and is independent of the
|
|
7473
|
-
// session's MCP allow-list; the effective context window drives the
|
|
7474
|
-
// compaction threshold.
|
|
7475
|
-
hostedWebSearch,
|
|
7476
|
-
...imageGenerationOption,
|
|
7477
|
-
...videoGenerationOption,
|
|
7478
|
-
lazyToolTransport,
|
|
7479
|
-
supportsImageInput,
|
|
7480
|
-
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
7481
|
-
...(resolvedModel
|
|
7482
|
-
? {
|
|
7483
|
-
encryptedReasoning:
|
|
7484
|
-
resolvedModel.provider.api === "responses" &&
|
|
7485
|
-
runSettings.openaiReasoningEncryptedContent,
|
|
7486
|
-
contextWindowTokens:
|
|
7487
|
-
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
7488
|
-
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
7489
|
-
// tool. Gateway Responses routes likewise expose ordinary function
|
|
7490
|
-
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
7491
|
-
// function apply_patch and wrap successful view_image results as
|
|
7492
|
-
// typed input_image content. Chat wires have no proven typed image
|
|
7493
|
-
// result transport and therefore receive no view_image tool.
|
|
7494
|
-
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
7495
|
-
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
7496
|
-
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
7497
|
-
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
7498
|
-
// constructor name. See {@link computerToolModeForTurn}.
|
|
7499
|
-
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
7500
|
-
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
7501
|
-
}
|
|
7502
|
-
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
7503
|
-
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
7504
|
-
// client). That backend has real hosted support, so pin computerToolMode to
|
|
7505
|
-
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
7506
|
-
{
|
|
7507
|
-
computerToolMode: computerToolModeForTurn(null),
|
|
7508
|
-
promptCacheKey: input.sessionId,
|
|
7509
|
-
}),
|
|
7510
|
-
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
7511
|
-
// selects a computer tool, then this hook begins the optional proof
|
|
7512
|
-
// recording. Shell/filesystem turns never invoke either operation.
|
|
7513
|
-
onComputerUseReady: async () => {
|
|
7514
|
-
if (!resolvedSandbox) {
|
|
7515
|
-
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
7516
|
-
}
|
|
7517
|
-
// This callback is the authoritative execution boundary. Record the
|
|
7518
|
-
// action before async ffmpeg startup so transport-event ordering cannot
|
|
7519
|
-
// make settlement misclassify a real computer turn as unused.
|
|
7520
|
-
didComputerUse = true;
|
|
7521
|
-
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
7522
|
-
},
|
|
7523
|
-
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
7524
|
-
...(runtimeSkillActivations.length > 0
|
|
7525
|
-
? { skillActivations: runtimeSkillActivations }
|
|
7526
|
-
: {}),
|
|
7527
|
-
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
7528
|
-
? { instructionsTemplate: workspaceAgentInstructions }
|
|
7529
|
-
: {}),
|
|
7530
|
-
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
7531
|
-
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7532
|
-
goalSnapshot: turn.goalSnapshot,
|
|
7533
|
-
// Per-session persona tier (session > workspace > deployment default).
|
|
7534
|
-
// Composed system-level AFTER the workspace persona so it refines it for
|
|
7535
|
-
// this one session; absent ⇒ byte-identical to today's composition.
|
|
7536
|
-
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
7537
|
-
// Exact host context captured when this turn was accepted. It is
|
|
7538
|
-
// system-level and disappears with the turn rather than entering chat.
|
|
7539
|
-
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
7540
|
-
? {
|
|
7541
|
-
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
7542
|
-
.filter((value): value is string => Boolean(value))
|
|
7543
|
-
.join(" "),
|
|
7544
|
-
}
|
|
7545
|
-
: {}),
|
|
7546
|
-
...workspaceEnvironmentOption,
|
|
7547
|
-
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
7548
|
-
// the frozen version carries a non-empty script), and the rig credential
|
|
7549
|
-
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
7550
|
-
...(rigVersion && rigName
|
|
7551
|
-
? {
|
|
7552
|
-
rig: { name: rigName, version: rigVersion.version },
|
|
7553
|
-
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
7554
|
-
? {
|
|
7555
|
-
rigSetup: {
|
|
7556
|
-
rigId: session.rigId!,
|
|
7557
|
-
versionId: rigVersion.id,
|
|
7558
|
-
rigName,
|
|
7559
|
-
script: rigVersion.setupScript,
|
|
7560
|
-
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
7561
|
-
contentHash: rigProviderImageContentHash({
|
|
7562
|
-
backend: turn.sandboxBackend,
|
|
7563
|
-
sourceImage: rigProviderImageSourceImage(
|
|
7564
|
-
logicalSandboxSettings,
|
|
7565
|
-
turn.sandboxBackend,
|
|
7566
|
-
),
|
|
7567
|
-
definition: rigVersion,
|
|
7568
|
-
}),
|
|
7569
|
-
},
|
|
7570
|
-
}
|
|
7571
|
-
: {}),
|
|
7572
|
-
...(rigVersion.credentialHooks.length > 0
|
|
7573
|
-
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
7574
|
-
: {}),
|
|
7575
|
-
}
|
|
7576
|
-
: {}),
|
|
7667
|
+
await beginConnectorActionExecution(db, connectorActionIdentity, call),
|
|
7668
|
+
complete: async ({ requestId, outcome }) =>
|
|
7669
|
+
await completeConnectorActionExecution(db, {
|
|
7670
|
+
accountId: input.accountId,
|
|
7671
|
+
workspaceId: input.workspaceId,
|
|
7672
|
+
requestId,
|
|
7673
|
+
attemptId: input.attemptId,
|
|
7674
|
+
outcome,
|
|
7675
|
+
}),
|
|
7676
|
+
};
|
|
7677
|
+
const runtimeSkillActivations = [
|
|
7678
|
+
...installedSkillRuntime.activations,
|
|
7679
|
+
...packRuntime.skillActivations,
|
|
7680
|
+
...session.skills.map((skill) => ({
|
|
7681
|
+
source: "session" as const,
|
|
7682
|
+
id: `session:${session.id}:${skill.name}`,
|
|
7683
|
+
artifact: {
|
|
7684
|
+
name: skill.name,
|
|
7685
|
+
description: skill.description ?? null,
|
|
7686
|
+
files: skill.files.map((file) => ({
|
|
7687
|
+
path: file.path,
|
|
7688
|
+
content: file.content,
|
|
7689
|
+
})),
|
|
7690
|
+
},
|
|
7691
|
+
reason: "attached to session",
|
|
7692
|
+
})),
|
|
7693
|
+
];
|
|
7694
|
+
recordTurnStartupPhase(observability, {
|
|
7695
|
+
phase: "post_tool_preparation",
|
|
7696
|
+
provider: turnExecutionPolicy.providerId,
|
|
7697
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7698
|
+
outcome: "completed",
|
|
7699
|
+
durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
|
|
7577
7700
|
});
|
|
7701
|
+
const agent = (() => {
|
|
7702
|
+
const agentConstructionStartedAt = performance.now();
|
|
7703
|
+
let agentConstructionOutcome: "completed" | "failed" = "completed";
|
|
7704
|
+
try {
|
|
7705
|
+
return runtime.buildAgent(modelRunSettings, turnResources, {
|
|
7706
|
+
reasoningEffort: turn.reasoningEffort,
|
|
7707
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
7708
|
+
...(serviceTier ? { serviceTier } : {}),
|
|
7709
|
+
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
7710
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
7711
|
+
genesisTitleHint: isGenesisTurn,
|
|
7712
|
+
persistentSessionSettings: {
|
|
7713
|
+
titleIsSet: Boolean(session.title?.trim()),
|
|
7714
|
+
},
|
|
7715
|
+
sandboxEnvironment,
|
|
7716
|
+
...(preparedTools.attemptToolCatalog
|
|
7717
|
+
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
7718
|
+
: {}),
|
|
7719
|
+
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
7720
|
+
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
7721
|
+
onToolCancellationFence: (fence) => {
|
|
7722
|
+
toolCancellationFenceRef.current = fence;
|
|
7723
|
+
},
|
|
7724
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
7725
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
7726
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
7727
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
7728
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
7729
|
+
// touches the box/agent manifest env.
|
|
7730
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
7731
|
+
? { gitTokenSeeds: sandboxGitTokens }
|
|
7732
|
+
: {}),
|
|
7733
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
7734
|
+
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
7735
|
+
: {}),
|
|
7736
|
+
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
7737
|
+
? { gitTokenSeed: sandboxGitToken }
|
|
7738
|
+
: {}),
|
|
7739
|
+
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7740
|
+
// Managed boxes receive the bearer through their protected per-session
|
|
7741
|
+
// token file. Connected Machines use transient per-exec delivery above,
|
|
7742
|
+
// so they must not run the file-seeding lifecycle hook.
|
|
7743
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
7744
|
+
? {
|
|
7745
|
+
codemodeTokenSeed: sandboxCodemodeToken,
|
|
7746
|
+
codemodeTokenSessionId: input.sessionId,
|
|
7747
|
+
}
|
|
7748
|
+
: {}),
|
|
7749
|
+
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
7750
|
+
fileResourceDownloads,
|
|
7751
|
+
mcpServers: preparedTools.mcpServers,
|
|
7752
|
+
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
7753
|
+
connectorActionPolicy,
|
|
7754
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
7755
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
7756
|
+
// model call so the model sees the account's real connected sources.
|
|
7757
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
7758
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
7759
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
7760
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
7761
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
7762
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
7763
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
7764
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
7765
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
7766
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
7767
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
7768
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
7769
|
+
// whenever the provider declares it runnable and is independent of the
|
|
7770
|
+
// session's MCP allow-list; the effective context window drives the
|
|
7771
|
+
// compaction threshold.
|
|
7772
|
+
hostedWebSearch,
|
|
7773
|
+
...imageGenerationOption,
|
|
7774
|
+
...videoGenerationOption,
|
|
7775
|
+
lazyToolTransport,
|
|
7776
|
+
supportsImageInput,
|
|
7777
|
+
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
7778
|
+
...(resolvedModel
|
|
7779
|
+
? {
|
|
7780
|
+
encryptedReasoning:
|
|
7781
|
+
resolvedModel.provider.api === "responses" &&
|
|
7782
|
+
runSettings.openaiReasoningEncryptedContent,
|
|
7783
|
+
contextWindowTokens:
|
|
7784
|
+
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
7785
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
7786
|
+
// tool. Gateway Responses routes likewise expose ordinary function
|
|
7787
|
+
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
7788
|
+
// function apply_patch and wrap successful view_image results as
|
|
7789
|
+
// typed input_image content. Chat wires have no proven typed image
|
|
7790
|
+
// result transport and therefore receive no view_image tool.
|
|
7791
|
+
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
7792
|
+
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
7793
|
+
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
7794
|
+
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
7795
|
+
// constructor name. See {@link computerToolModeForTurn}.
|
|
7796
|
+
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
7797
|
+
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
7798
|
+
}
|
|
7799
|
+
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
7800
|
+
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
7801
|
+
// client). That backend has real hosted support, so pin computerToolMode to
|
|
7802
|
+
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
7803
|
+
{
|
|
7804
|
+
computerToolMode: computerToolModeForTurn(null),
|
|
7805
|
+
promptCacheKey: input.sessionId,
|
|
7806
|
+
}),
|
|
7807
|
+
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
7808
|
+
// selects a computer tool, then this hook begins the optional proof
|
|
7809
|
+
// recording. Shell/filesystem turns never invoke either operation.
|
|
7810
|
+
onComputerUseReady: async () => {
|
|
7811
|
+
if (!resolvedSandbox) {
|
|
7812
|
+
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
7813
|
+
}
|
|
7814
|
+
// This callback is the authoritative execution boundary. Record the
|
|
7815
|
+
// action before async ffmpeg startup so transport-event ordering cannot
|
|
7816
|
+
// make settlement misclassify a real computer turn as unused.
|
|
7817
|
+
didComputerUse = true;
|
|
7818
|
+
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
7819
|
+
},
|
|
7820
|
+
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
7821
|
+
...(runtimeSkillActivations.length > 0
|
|
7822
|
+
? { skillActivations: runtimeSkillActivations }
|
|
7823
|
+
: {}),
|
|
7824
|
+
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
7825
|
+
? { instructionsTemplate: workspaceAgentInstructions }
|
|
7826
|
+
: {}),
|
|
7827
|
+
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
7828
|
+
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7829
|
+
goalSnapshot: turn.goalSnapshot,
|
|
7830
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
7831
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
7832
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
7833
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
7834
|
+
// Exact host context captured when this turn was accepted. It is
|
|
7835
|
+
// system-level and disappears with the turn rather than entering chat.
|
|
7836
|
+
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
7837
|
+
? {
|
|
7838
|
+
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
7839
|
+
.filter((value): value is string => Boolean(value))
|
|
7840
|
+
.join(" "),
|
|
7841
|
+
}
|
|
7842
|
+
: {}),
|
|
7843
|
+
...workspaceEnvironmentOption,
|
|
7844
|
+
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
7845
|
+
// the frozen version carries a non-empty script), and the rig credential
|
|
7846
|
+
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
7847
|
+
...(rigVersion && rigName
|
|
7848
|
+
? {
|
|
7849
|
+
rig: { name: rigName, version: rigVersion.version },
|
|
7850
|
+
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
7851
|
+
? {
|
|
7852
|
+
rigSetup: {
|
|
7853
|
+
rigId: session.rigId!,
|
|
7854
|
+
versionId: rigVersion.id,
|
|
7855
|
+
rigName,
|
|
7856
|
+
script: rigVersion.setupScript,
|
|
7857
|
+
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
7858
|
+
contentHash: rigProviderImageContentHash({
|
|
7859
|
+
backend: turn.sandboxBackend,
|
|
7860
|
+
sourceImage: rigProviderImageSourceImage(
|
|
7861
|
+
logicalSandboxSettings,
|
|
7862
|
+
turn.sandboxBackend,
|
|
7863
|
+
),
|
|
7864
|
+
definition: rigVersion,
|
|
7865
|
+
}),
|
|
7866
|
+
},
|
|
7867
|
+
}
|
|
7868
|
+
: {}),
|
|
7869
|
+
...(rigVersion.credentialHooks.length > 0
|
|
7870
|
+
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
7871
|
+
: {}),
|
|
7872
|
+
}
|
|
7873
|
+
: {}),
|
|
7874
|
+
});
|
|
7875
|
+
} catch (error) {
|
|
7876
|
+
agentConstructionOutcome = "failed";
|
|
7877
|
+
throw error;
|
|
7878
|
+
} finally {
|
|
7879
|
+
recordTurnStartupPhase(observability, {
|
|
7880
|
+
phase: "agent_construction",
|
|
7881
|
+
provider: turnExecutionPolicy.providerId,
|
|
7882
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7883
|
+
outcome: agentConstructionOutcome,
|
|
7884
|
+
durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
|
|
7885
|
+
});
|
|
7886
|
+
}
|
|
7887
|
+
})();
|
|
7888
|
+
const postAgentPreparationStartedAt = performance.now();
|
|
7578
7889
|
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
7579
7890
|
throw new Error(
|
|
7580
7891
|
"Sandbox agent construction did not install the mandatory turn tool cancellation fence",
|
|
@@ -7629,6 +7940,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7629
7940
|
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
7630
7941
|
}
|
|
7631
7942
|
: {}),
|
|
7943
|
+
// The lazy acquire must enforce the same frozen rig authority
|
|
7944
|
+
// as the eager acquire; otherwise a warm box for another rig
|
|
7945
|
+
// can bypass the shared-state conflict/rotation fence.
|
|
7946
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
7632
7947
|
},
|
|
7633
7948
|
"turn",
|
|
7634
7949
|
lazyHolderId,
|
|
@@ -8083,69 +8398,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8083
8398
|
return claimedResult({ status: "idle" });
|
|
8084
8399
|
}
|
|
8085
8400
|
}
|
|
8401
|
+
recordTurnStartupPhase(observability, {
|
|
8402
|
+
phase: "post_agent_preparation",
|
|
8403
|
+
provider: turnExecutionPolicy.providerId,
|
|
8404
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8405
|
+
outcome: "completed",
|
|
8406
|
+
durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
|
|
8407
|
+
});
|
|
8086
8408
|
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
8087
8409
|
let fileDownloadsMaterializedForRun = false;
|
|
8088
8410
|
if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
|
|
8089
|
-
const
|
|
8090
|
-
|
|
8091
|
-
// memoize successful downloads. A connected machine is user-owned: files
|
|
8092
|
-
// can be changed or removed between turns, so verify/materialize every
|
|
8093
|
-
// attached file each turn instead of trusting the managed-box cache.
|
|
8411
|
+
const fileMaterializationStartedAt = performance.now();
|
|
8412
|
+
let fileMaterializationOutcome: "completed" | "failed" = "completed";
|
|
8094
8413
|
const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
|
|
8095
|
-
|
|
8096
|
-
?
|
|
8097
|
-
|
|
8098
|
-
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
|
|
8104
|
-
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
downloadsToMaterialize,
|
|
8117
|
-
{
|
|
8118
|
-
onRuntimeEvent: async (event) => {
|
|
8119
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8120
|
-
},
|
|
8121
|
-
...(runAs ? { runAs } : {}),
|
|
8122
|
-
...(toolCancellationFenceRef.current
|
|
8123
|
-
? {
|
|
8124
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8125
|
-
toolCancellationFenceRef.current,
|
|
8126
|
-
),
|
|
8127
|
-
}
|
|
8128
|
-
: {}),
|
|
8129
|
-
},
|
|
8130
|
-
),
|
|
8414
|
+
let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
|
|
8415
|
+
? "miss"
|
|
8416
|
+
: "disabled";
|
|
8417
|
+
try {
|
|
8418
|
+
const boxInstanceId = resolvedSandbox.established.instanceId;
|
|
8419
|
+
// Managed boxes are immutable platform state, so their durable lease can
|
|
8420
|
+
// memoize successful downloads. A connected machine is user-owned: files
|
|
8421
|
+
// can be changed or removed between turns, so verify/materialize every
|
|
8422
|
+
// attached file each turn instead of trusting the managed-box cache.
|
|
8423
|
+
const alreadyMaterialized = cacheMaterialization
|
|
8424
|
+
? await getMaterializedSandboxFileResources(db, {
|
|
8425
|
+
accountId: input.accountId,
|
|
8426
|
+
workspaceId: input.workspaceId,
|
|
8427
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8428
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8429
|
+
instanceId: boxInstanceId,
|
|
8430
|
+
})
|
|
8431
|
+
: new Set<string>();
|
|
8432
|
+
const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
|
|
8433
|
+
fileResourceDownloads,
|
|
8434
|
+
alreadyMaterialized,
|
|
8131
8435
|
);
|
|
8132
|
-
|
|
8133
|
-
|
|
8134
|
-
|
|
8135
|
-
|
|
8136
|
-
|
|
8137
|
-
|
|
8138
|
-
|
|
8139
|
-
|
|
8140
|
-
|
|
8141
|
-
|
|
8142
|
-
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8436
|
+
if (cacheMaterialization && downloadsToMaterialize.length === 0) {
|
|
8437
|
+
fileMaterializationCache = "hit";
|
|
8438
|
+
}
|
|
8439
|
+
const runAs = sandboxRunAs(runSettings);
|
|
8440
|
+
if (downloadsToMaterialize.length > 0) {
|
|
8441
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
8442
|
+
resolvedSandbox,
|
|
8443
|
+
"fileMaterialization",
|
|
8444
|
+
async () =>
|
|
8445
|
+
await materializeSandboxFileDownloads(
|
|
8446
|
+
setupBoxSession as CodemodeTokenWriterSession,
|
|
8447
|
+
downloadsToMaterialize,
|
|
8448
|
+
{
|
|
8449
|
+
onRuntimeEvent: async (event) => {
|
|
8450
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8451
|
+
},
|
|
8452
|
+
...(runAs ? { runAs } : {}),
|
|
8453
|
+
...(toolCancellationFenceRef.current
|
|
8454
|
+
? {
|
|
8455
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8456
|
+
toolCancellationFenceRef.current,
|
|
8457
|
+
),
|
|
8458
|
+
}
|
|
8459
|
+
: {}),
|
|
8460
|
+
},
|
|
8461
|
+
),
|
|
8462
|
+
);
|
|
8463
|
+
fileMaterializationFailures = materialized.failures;
|
|
8464
|
+
fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
|
|
8465
|
+
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
8466
|
+
const succeededFileIds = downloadsToMaterialize
|
|
8467
|
+
.map((download) => download.fileId)
|
|
8468
|
+
.filter((fileId) => !failedFileIds.has(fileId));
|
|
8469
|
+
if (cacheMaterialization && succeededFileIds.length > 0) {
|
|
8470
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
8471
|
+
accountId: input.accountId,
|
|
8472
|
+
workspaceId: input.workspaceId,
|
|
8473
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8474
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8475
|
+
instanceId: boxInstanceId,
|
|
8476
|
+
fileIds: succeededFileIds,
|
|
8477
|
+
});
|
|
8478
|
+
}
|
|
8146
8479
|
}
|
|
8480
|
+
fileDownloadsMaterializedForRun = true;
|
|
8481
|
+
} catch (error) {
|
|
8482
|
+
fileMaterializationOutcome = "failed";
|
|
8483
|
+
throw error;
|
|
8484
|
+
} finally {
|
|
8485
|
+
recordTurnStartupPhase(observability, {
|
|
8486
|
+
phase: "file_materialization",
|
|
8487
|
+
provider: turnExecutionPolicy.providerId,
|
|
8488
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8489
|
+
outcome: fileMaterializationOutcome,
|
|
8490
|
+
durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
|
|
8491
|
+
count: fileResourceDownloads.length,
|
|
8492
|
+
cache: fileMaterializationCache,
|
|
8493
|
+
});
|
|
8147
8494
|
}
|
|
8148
|
-
|
|
8495
|
+
} else {
|
|
8496
|
+
recordTurnStartupPhase(observability, {
|
|
8497
|
+
phase: "file_materialization",
|
|
8498
|
+
provider: turnExecutionPolicy.providerId,
|
|
8499
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8500
|
+
outcome: "completed",
|
|
8501
|
+
durationSeconds: 0,
|
|
8502
|
+
count: 0,
|
|
8503
|
+
});
|
|
8149
8504
|
}
|
|
8150
8505
|
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
8151
8506
|
const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
|
|
@@ -8190,52 +8545,110 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8190
8545
|
}
|
|
8191
8546
|
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
8192
8547
|
const prepareRunAttemptInput = async () => {
|
|
8193
|
-
const
|
|
8194
|
-
|
|
8195
|
-
|
|
8196
|
-
|
|
8197
|
-
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8203
|
-
|
|
8204
|
-
|
|
8205
|
-
|
|
8548
|
+
const historyPreparationStartedAt = performance.now();
|
|
8549
|
+
let historyPreparationOutcome: "completed" | "failed" = "completed";
|
|
8550
|
+
let preparedHistoryCount: number | null = null;
|
|
8551
|
+
try {
|
|
8552
|
+
const prepared = await turnInput(db, runtime, agent, trigger, {
|
|
8553
|
+
turnId: activeTurnId,
|
|
8554
|
+
recovering: turn.executionGeneration > 1,
|
|
8555
|
+
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
8556
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
8557
|
+
providerApi,
|
|
8558
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
8559
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
8560
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
8561
|
+
projectModelHistory: modelHistoryProjector,
|
|
8562
|
+
onPreparationPhase: (measurement) => {
|
|
8563
|
+
recordTurnStartupPhase(observability, {
|
|
8564
|
+
phase: `history_${measurement.phase}`,
|
|
8565
|
+
provider: turnExecutionPolicy.providerId,
|
|
8566
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8567
|
+
outcome: measurement.outcome,
|
|
8568
|
+
durationSeconds: measurement.durationSeconds,
|
|
8569
|
+
});
|
|
8570
|
+
},
|
|
8571
|
+
});
|
|
8572
|
+
const generatedImageMaterializationStartedAt = performance.now();
|
|
8573
|
+
let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
|
|
8574
|
+
try {
|
|
8575
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8576
|
+
await materializeGeneratedImage(receipt);
|
|
8577
|
+
}
|
|
8578
|
+
} catch (error) {
|
|
8579
|
+
generatedImageMaterializationOutcome = "failed";
|
|
8580
|
+
throw error;
|
|
8581
|
+
} finally {
|
|
8582
|
+
recordTurnStartupPhase(observability, {
|
|
8583
|
+
phase: "history_generated_image_materialization",
|
|
8584
|
+
provider: turnExecutionPolicy.providerId,
|
|
8585
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8586
|
+
outcome: generatedImageMaterializationOutcome,
|
|
8587
|
+
durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
|
|
8588
|
+
count: generatedImageReceiptsByArtifactId.size,
|
|
8589
|
+
});
|
|
8590
|
+
}
|
|
8591
|
+
runInput = prepared.input;
|
|
8592
|
+
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8593
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
8594
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
8595
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
8596
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
8597
|
+
// absolute write position is tracked separately (next whole number past
|
|
8598
|
+
// the max existing position) because the fractional summary row means
|
|
8599
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
8600
|
+
// the old total-count value, so the common path is unchanged.
|
|
8601
|
+
//
|
|
8602
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
8603
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
8604
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
8605
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
8606
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
8607
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
8608
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
8609
|
+
// `function_call` left in that skipped region can later have its
|
|
8610
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
8611
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
8612
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
8613
|
+
// slice begins exactly at the first genuinely-new item.
|
|
8614
|
+
// prepareInput already sanitized the exact durable prefix represented
|
|
8615
|
+
// by state.history. Carry its count forward instead of loading and
|
|
8616
|
+
// retaining the full active transcript a second time beside runInput.
|
|
8617
|
+
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
8618
|
+
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
8619
|
+
const historyPositionStartedAt = performance.now();
|
|
8620
|
+
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
8621
|
+
try {
|
|
8622
|
+
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
8623
|
+
db,
|
|
8624
|
+
input.workspaceId,
|
|
8625
|
+
input.sessionId,
|
|
8626
|
+
);
|
|
8627
|
+
} catch (error) {
|
|
8628
|
+
historyPositionOutcome = "failed";
|
|
8629
|
+
throw error;
|
|
8630
|
+
} finally {
|
|
8631
|
+
recordTurnStartupPhase(observability, {
|
|
8632
|
+
phase: "history_position_load",
|
|
8633
|
+
provider: turnExecutionPolicy.providerId,
|
|
8634
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8635
|
+
outcome: historyPositionOutcome,
|
|
8636
|
+
durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
|
|
8637
|
+
});
|
|
8638
|
+
}
|
|
8639
|
+
} catch (error) {
|
|
8640
|
+
historyPreparationOutcome = "failed";
|
|
8641
|
+
throw error;
|
|
8642
|
+
} finally {
|
|
8643
|
+
recordTurnStartupPhase(observability, {
|
|
8644
|
+
phase: "history_preparation",
|
|
8645
|
+
provider: turnExecutionPolicy.providerId,
|
|
8646
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8647
|
+
outcome: historyPreparationOutcome,
|
|
8648
|
+
durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
|
|
8649
|
+
count: preparedHistoryCount,
|
|
8650
|
+
});
|
|
8206
8651
|
}
|
|
8207
|
-
runInput = prepared.input;
|
|
8208
|
-
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8209
|
-
// Slice index = the length of the model-facing (active) history this turn
|
|
8210
|
-
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
8211
|
-
// generated items) are the ones to persist. After a compaction this is the
|
|
8212
|
-
// short [summary, ...tail] active set, NOT the total row count. The
|
|
8213
|
-
// absolute write position is tracked separately (next whole number past
|
|
8214
|
-
// the max existing position) because the fractional summary row means
|
|
8215
|
-
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
8216
|
-
// the old total-count value, so the common path is unchanged.
|
|
8217
|
-
//
|
|
8218
|
-
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
8219
|
-
// count. `prepareRunInput` builds `state.history` from
|
|
8220
|
-
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
8221
|
-
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
8222
|
-
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
8223
|
-
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
8224
|
-
// the raw count (K too high) skips K genuinely-new items, and a
|
|
8225
|
-
// `function_call` left in that skipped region can later have its
|
|
8226
|
-
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
8227
|
-
// and bricks the session (issue-61). The repaired seed is already
|
|
8228
|
-
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
8229
|
-
// slice begins exactly at the first genuinely-new item.
|
|
8230
|
-
// prepareInput already sanitized the exact durable prefix represented
|
|
8231
|
-
// by state.history. Carry its count forward instead of loading and
|
|
8232
|
-
// retaining the full active transcript a second time beside runInput.
|
|
8233
|
-
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
8234
|
-
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
8235
|
-
db,
|
|
8236
|
-
input.workspaceId,
|
|
8237
|
-
input.sessionId,
|
|
8238
|
-
);
|
|
8239
8652
|
};
|
|
8240
8653
|
|
|
8241
8654
|
const forceContextCompaction = async (
|
|
@@ -8287,6 +8700,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8287
8700
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
8288
8701
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
8289
8702
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
8703
|
+
let workerPreparationTotalRecorded = false;
|
|
8290
8704
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
8291
8705
|
if (!runInput) {
|
|
8292
8706
|
throw new Error("Run input was not prepared");
|
|
@@ -8313,139 +8727,203 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8313
8727
|
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
8314
8728
|
// its first-operation provisioner above.
|
|
8315
8729
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
8316
|
-
const
|
|
8317
|
-
|
|
8318
|
-
|
|
8319
|
-
|
|
8320
|
-
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8324
|
-
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8337
|
-
|
|
8338
|
-
|
|
8339
|
-
|
|
8340
|
-
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
8346
|
-
|
|
8347
|
-
|
|
8348
|
-
|
|
8349
|
-
|
|
8350
|
-
|
|
8351
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8352
|
-
toolCancellationFenceRef.current,
|
|
8353
|
-
),
|
|
8730
|
+
const ownedSandboxSetupStartedAt = performance.now();
|
|
8731
|
+
let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
|
|
8732
|
+
try {
|
|
8733
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
8734
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
8735
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
8736
|
+
// session callback. Materialize host-managed run credentials here
|
|
8737
|
+
// before any setup command (including provider login hooks), then
|
|
8738
|
+
// decorate setup commands so they source the active generation.
|
|
8739
|
+
await attachRunCredentialRenewal(
|
|
8740
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
8741
|
+
resolvedSandbox,
|
|
8742
|
+
);
|
|
8743
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
8744
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
8745
|
+
: eagerSetupSession;
|
|
8746
|
+
await runWorkspaceMutationForSandbox(
|
|
8747
|
+
resolvedSandbox,
|
|
8748
|
+
"eagerOwnedSandboxSetup",
|
|
8749
|
+
async () =>
|
|
8750
|
+
await runOwnedSandboxSetup(
|
|
8751
|
+
agent,
|
|
8752
|
+
ownedEstablished.session as never,
|
|
8753
|
+
eagerCredentialSetupSession as never,
|
|
8754
|
+
{
|
|
8755
|
+
settings: runSettings,
|
|
8756
|
+
environment: sandboxEnvironment,
|
|
8757
|
+
preparedInput: runInput!,
|
|
8758
|
+
...(fileDownloadsMaterializedForRun
|
|
8759
|
+
? { fileDownloadsMaterialized: true }
|
|
8760
|
+
: {}),
|
|
8761
|
+
onRuntimeEvent: async (event) => {
|
|
8762
|
+
await renewServingCredentialLease("runtime_event");
|
|
8763
|
+
if (servingCredentialLeaseLost()) {
|
|
8764
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
8354
8765
|
}
|
|
8355
|
-
|
|
8356
|
-
|
|
8357
|
-
|
|
8358
|
-
|
|
8359
|
-
|
|
8360
|
-
|
|
8361
|
-
|
|
8362
|
-
|
|
8766
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8767
|
+
},
|
|
8768
|
+
...(toolCancellationFenceRef.current
|
|
8769
|
+
? {
|
|
8770
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8771
|
+
toolCancellationFenceRef.current,
|
|
8772
|
+
),
|
|
8773
|
+
}
|
|
8774
|
+
: {}),
|
|
8775
|
+
},
|
|
8776
|
+
),
|
|
8777
|
+
);
|
|
8778
|
+
await attachGitCredentialRenewal(
|
|
8779
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
8780
|
+
initialGitCredentials,
|
|
8781
|
+
);
|
|
8782
|
+
} catch (error) {
|
|
8783
|
+
ownedSandboxSetupOutcome = "failed";
|
|
8784
|
+
throw error;
|
|
8785
|
+
} finally {
|
|
8786
|
+
recordTurnStartupPhase(observability, {
|
|
8787
|
+
phase: "owned_sandbox_setup",
|
|
8788
|
+
provider: turnExecutionPolicy.providerId,
|
|
8789
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8790
|
+
outcome: ownedSandboxSetupOutcome,
|
|
8791
|
+
durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
|
|
8792
|
+
count: fileResourceDownloads.length,
|
|
8793
|
+
});
|
|
8794
|
+
}
|
|
8363
8795
|
}
|
|
8364
8796
|
// Conservative request boundary: once control enters runStream, the
|
|
8365
8797
|
// provider may have accepted work even if no response/event follows.
|
|
8366
8798
|
// Escaped MCP setup timeouts are automatically recoverable only
|
|
8367
8799
|
// before this line.
|
|
8368
8800
|
recordCompanyBrainContributionReceiptOnce();
|
|
8369
|
-
|
|
8370
|
-
|
|
8371
|
-
|
|
8372
|
-
|
|
8373
|
-
|
|
8374
|
-
|
|
8375
|
-
|
|
8376
|
-
|
|
8377
|
-
|
|
8378
|
-
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8801
|
+
const providerDispatchStartedAt = performance.now();
|
|
8802
|
+
let providerDispatchOutcome: "completed" | "failed" = "completed";
|
|
8803
|
+
try {
|
|
8804
|
+
// This histogram describes worker preparation until the first entry
|
|
8805
|
+
// into the runtime. Lazy SDK request preparation and the durable
|
|
8806
|
+
// model-request audit happen after this boundary and have their own
|
|
8807
|
+
// phase metrics. Context-compaction recovery re-enters
|
|
8808
|
+
// runStreamAttempt; recording again would fold the prior request and
|
|
8809
|
+
// compaction into a bogus second startup sample.
|
|
8810
|
+
if (!workerPreparationTotalRecorded) {
|
|
8811
|
+
workerPreparationTotalRecorded = true;
|
|
8812
|
+
recordTurnWorkerPreparationTotal(observability, {
|
|
8813
|
+
provider: turnExecutionPolicy.providerId,
|
|
8814
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8815
|
+
outcome: "completed",
|
|
8816
|
+
durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
|
|
8817
|
+
});
|
|
8818
|
+
}
|
|
8819
|
+
modelRequestStarted = true;
|
|
8820
|
+
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
8821
|
+
signal: runtimeCancellationSignal,
|
|
8822
|
+
sandboxEnvironment,
|
|
8823
|
+
onRuntimeEvent: async (event) => {
|
|
8824
|
+
await renewServingCredentialLease("runtime_event");
|
|
8825
|
+
if (servingCredentialLeaseLost()) {
|
|
8826
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
8827
|
+
}
|
|
8828
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8829
|
+
},
|
|
8830
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
8831
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
8832
|
+
...(ownedEstablished
|
|
8833
|
+
? {
|
|
8834
|
+
ownedSandbox: {
|
|
8835
|
+
client: ownedEstablished.client,
|
|
8836
|
+
session: ownedEstablished.session,
|
|
8837
|
+
...(resolvedSandbox?.established.sessionState
|
|
8838
|
+
? {
|
|
8839
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
8840
|
+
}
|
|
8841
|
+
: {}),
|
|
8842
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
8843
|
+
// established box — never through the routing proxy, which would
|
|
8844
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
8845
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
8846
|
+
...(fileDownloadsMaterializedForRun
|
|
8847
|
+
? { fileDownloadsMaterialized: true }
|
|
8848
|
+
: {}),
|
|
8849
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
8850
|
+
// above under its exact admission, lazy in the provisioner.
|
|
8851
|
+
deferredSetup: true,
|
|
8852
|
+
},
|
|
8853
|
+
}
|
|
8854
|
+
: {}),
|
|
8855
|
+
...(activeSandboxBackend !== "selfhosted" &&
|
|
8856
|
+
sandboxCodemodeToken &&
|
|
8857
|
+
sandboxCodemodeTokenExpiresAt &&
|
|
8858
|
+
!lazyOwnedSandbox
|
|
8859
|
+
? {
|
|
8860
|
+
onCodemodeTokenSessionReady: async (
|
|
8861
|
+
tokenSession: CodemodeTokenWriterSession,
|
|
8862
|
+
) => {
|
|
8863
|
+
const renewalSession =
|
|
8864
|
+
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
8865
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
8866
|
+
},
|
|
8867
|
+
}
|
|
8868
|
+
: {}),
|
|
8869
|
+
...(runCredentialResolver
|
|
8870
|
+
? {
|
|
8871
|
+
runCredentialSessionId: input.sessionId,
|
|
8872
|
+
...(!lazyOwnedSandbox
|
|
8388
8873
|
? {
|
|
8389
|
-
|
|
8874
|
+
onRunCredentialSessionReady: async (
|
|
8875
|
+
credentialSession: RunCredentialCommandSession,
|
|
8876
|
+
) => {
|
|
8877
|
+
const pinnedCredentialSession = setupBoxSession
|
|
8878
|
+
? (setupBoxSession as RunCredentialCommandSession)
|
|
8879
|
+
: credentialSession;
|
|
8880
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
8881
|
+
},
|
|
8390
8882
|
}
|
|
8391
8883
|
: {}),
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
|
|
8398
|
-
|
|
8399
|
-
|
|
8400
|
-
},
|
|
8401
|
-
}
|
|
8402
|
-
: {}),
|
|
8403
|
-
...(activeSandboxBackend !== "selfhosted" &&
|
|
8404
|
-
sandboxCodemodeToken &&
|
|
8405
|
-
sandboxCodemodeTokenExpiresAt &&
|
|
8406
|
-
!lazyOwnedSandbox
|
|
8407
|
-
? {
|
|
8408
|
-
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
8409
|
-
const renewalSession =
|
|
8410
|
-
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
8411
|
-
await attachCodemodeTokenRenewal(renewalSession);
|
|
8412
|
-
},
|
|
8413
|
-
}
|
|
8414
|
-
: {}),
|
|
8415
|
-
...(runCredentialResolver
|
|
8416
|
-
? {
|
|
8417
|
-
runCredentialSessionId: input.sessionId,
|
|
8418
|
-
...(!lazyOwnedSandbox
|
|
8419
|
-
? {
|
|
8420
|
-
onRunCredentialSessionReady: async (
|
|
8421
|
-
credentialSession: RunCredentialCommandSession,
|
|
8422
|
-
) => {
|
|
8423
|
-
const pinnedCredentialSession = setupBoxSession
|
|
8424
|
-
? (setupBoxSession as RunCredentialCommandSession)
|
|
8425
|
-
: credentialSession;
|
|
8426
|
-
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
8427
|
-
},
|
|
8884
|
+
}
|
|
8885
|
+
: {}),
|
|
8886
|
+
...(modelRunSettings.sandboxBackend !== "none"
|
|
8887
|
+
? {
|
|
8888
|
+
onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
|
|
8889
|
+
sdkOwnedSandboxSession = sandboxSession;
|
|
8890
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8891
|
+
await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
8428
8892
|
}
|
|
8429
|
-
|
|
8430
|
-
|
|
8431
|
-
|
|
8432
|
-
|
|
8433
|
-
|
|
8434
|
-
|
|
8435
|
-
|
|
8436
|
-
|
|
8437
|
-
|
|
8438
|
-
|
|
8439
|
-
|
|
8440
|
-
|
|
8441
|
-
|
|
8442
|
-
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8448
|
-
|
|
8893
|
+
},
|
|
8894
|
+
}
|
|
8895
|
+
: {}),
|
|
8896
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
8897
|
+
contextCompactionRequested: () =>
|
|
8898
|
+
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
8899
|
+
onModelPreparationPhase: (measurement) => {
|
|
8900
|
+
recordTurnStartupPhase(observability, {
|
|
8901
|
+
phase: `model_prepare_${measurement.phase}`,
|
|
8902
|
+
provider: turnExecutionPolicy.providerId,
|
|
8903
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8904
|
+
outcome: measurement.outcome,
|
|
8905
|
+
durationSeconds: measurement.durationSeconds,
|
|
8906
|
+
...(measurement.count === undefined ? {} : { count: measurement.count }),
|
|
8907
|
+
});
|
|
8908
|
+
},
|
|
8909
|
+
...(toolCancellationFenceRef.current
|
|
8910
|
+
? {
|
|
8911
|
+
turnToolCancellationFence: toolCancellationFenceRef.current,
|
|
8912
|
+
}
|
|
8913
|
+
: {}),
|
|
8914
|
+
});
|
|
8915
|
+
} catch (error) {
|
|
8916
|
+
providerDispatchOutcome = "failed";
|
|
8917
|
+
throw error;
|
|
8918
|
+
} finally {
|
|
8919
|
+
recordTurnStartupPhase(observability, {
|
|
8920
|
+
phase: "provider_dispatch",
|
|
8921
|
+
provider: turnExecutionPolicy.providerId,
|
|
8922
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8923
|
+
outcome: providerDispatchOutcome,
|
|
8924
|
+
durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
|
|
8925
|
+
});
|
|
8926
|
+
}
|
|
8449
8927
|
};
|
|
8450
8928
|
if (codexLeaseLost) {
|
|
8451
8929
|
throw new Error("Codex credential lease expired before the model run");
|
|
@@ -8471,11 +8949,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8471
8949
|
},
|
|
8472
8950
|
);
|
|
8473
8951
|
|
|
8952
|
+
const streamBootstrapStartedAt = performance.now();
|
|
8953
|
+
let streamBootstrapRecorded = false;
|
|
8954
|
+
const recordStreamBootstrap = (): void => {
|
|
8955
|
+
if (streamBootstrapRecorded) return;
|
|
8956
|
+
streamBootstrapRecorded = true;
|
|
8957
|
+
recordTurnStartupPhase(observability, {
|
|
8958
|
+
phase: "stream_bootstrap",
|
|
8959
|
+
provider: turnExecutionPolicy.providerId,
|
|
8960
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8961
|
+
outcome: "completed",
|
|
8962
|
+
durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
|
|
8963
|
+
count: turnTools.length,
|
|
8964
|
+
});
|
|
8965
|
+
};
|
|
8966
|
+
if (!firstModelRequestPreparationRecorded) {
|
|
8967
|
+
firstModelRequestPreparationStartedAt = performance.now();
|
|
8968
|
+
firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
|
|
8969
|
+
}
|
|
8474
8970
|
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
8475
8971
|
let streamDone = false;
|
|
8476
8972
|
try {
|
|
8477
8973
|
while (true) {
|
|
8478
8974
|
const next = await nextStreamEvent(iterator, activityContext);
|
|
8975
|
+
recordStreamBootstrap();
|
|
8479
8976
|
if (next.done) {
|
|
8480
8977
|
streamDone = true;
|
|
8481
8978
|
break;
|
|
@@ -10437,7 +10934,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
10437
10934
|
}
|
|
10438
10935
|
activityStatus = "failed";
|
|
10439
10936
|
activityError = error;
|
|
10440
|
-
if (!
|
|
10937
|
+
if (!turnId) {
|
|
10938
|
+
throw preClaimAdmissionFailure(error);
|
|
10939
|
+
}
|
|
10940
|
+
if (!publish || !turnStartedPublished) {
|
|
10441
10941
|
throw error;
|
|
10442
10942
|
}
|
|
10443
10943
|
// A partial/malformed stream may have emitted assistant/tool items (and
|