@opengeni/worker-bundle 0.20.0 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +25 -2
- package/dist/activities/google-drive-publication.d.ts +53 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +29 -1
- package/dist/activities/run-input.d.ts +20 -2
- package/dist/activities/session-state.d.ts +5 -3
- package/dist/activities/types.d.ts +27 -0
- package/dist/{activities-control-XURE4VKN.js → activities-control-BJSDRSIP.js} +369 -20
- package/dist/activities-control-BJSDRSIP.js.map +1 -0
- package/dist/activities-control.d.ts +2 -2
- package/dist/{activities-turn-6EBXGGLS.js → activities-turn-J5YO7XXX.js} +2281 -1074
- package/dist/activities-turn-J5YO7XXX.js.map +1 -0
- package/dist/activities.d.ts +3 -3
- package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +37 -0
- package/dist/workflow-bundle.js +174 -4
- package/dist/workflows/session.d.ts +17 -1
- package/package.json +13 -13
- package/src/activities/agent-turn.ts +1801 -1175
- package/src/activities/documents.ts +6 -4
- package/src/activities/google-drive-publication.ts +840 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +384 -9
- package/src/activities/run-input.ts +175 -51
- package/src/activities/session-state.ts +69 -5
- package/src/activities/types.ts +27 -0
- package/src/activities.ts +2 -0
- package/src/observability-metrics.ts +184 -3
- package/src/workflows/session.ts +199 -8
- package/dist/activities-control-XURE4VKN.js.map +0 -1
- package/dist/activities-turn-6EBXGGLS.js.map +0 -1
- package/dist/chunk-TYNPCG4U.js.map +0 -1
- /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
|
@@ -9,7 +9,9 @@ import {
|
|
|
9
9
|
getRigName,
|
|
10
10
|
getRigVersion,
|
|
11
11
|
getSandbox,
|
|
12
|
+
getFilesForSubject,
|
|
12
13
|
readActiveSandbox,
|
|
14
|
+
requireFileForSubject,
|
|
13
15
|
setActiveSandbox,
|
|
14
16
|
requireFile,
|
|
15
17
|
getSessionEvent,
|
|
@@ -87,6 +89,7 @@ import {
|
|
|
87
89
|
SandboxImageConflictError,
|
|
88
90
|
ActiveSessionHistoryLimitExceededError,
|
|
89
91
|
ApprovalRunStateLimitExceededError,
|
|
92
|
+
isRetryableDatabaseTransportFailure,
|
|
90
93
|
isSessionEventPersistenceError,
|
|
91
94
|
getEnrollment,
|
|
92
95
|
abandonRecordingForTurnAttempt,
|
|
@@ -167,6 +170,7 @@ import {
|
|
|
167
170
|
type ModelCallUsageInput,
|
|
168
171
|
type ModelCallUsageNormalization,
|
|
169
172
|
type BuildAgentOptions,
|
|
173
|
+
type AttemptConnectorActionBinding,
|
|
170
174
|
type ConnectorActionPolicyHooks,
|
|
171
175
|
type TurnToolCancellationFence,
|
|
172
176
|
type BackendUnresolvableCode,
|
|
@@ -180,6 +184,12 @@ import {
|
|
|
180
184
|
deleteRecordingArtifacts,
|
|
181
185
|
stopRecording as stopRecordingOnBox,
|
|
182
186
|
} from "@opengeni/runtime";
|
|
187
|
+
import {
|
|
188
|
+
authorizeGoogleDrivePublicationAttempt,
|
|
189
|
+
createGoogleDrivePublicationAttemptTool,
|
|
190
|
+
googleDrivePublicationConnectorCall,
|
|
191
|
+
resolveGoogleDrivePublicationTarget,
|
|
192
|
+
} from "./google-drive-publication";
|
|
183
193
|
import { connectionTokenResolverForTurn } from "./mcp-credentials";
|
|
184
194
|
import { buildApiIntegrationServersForTurn } from "./api-integrations";
|
|
185
195
|
import {
|
|
@@ -344,6 +354,7 @@ import {
|
|
|
344
354
|
import type {
|
|
345
355
|
TurnActivityServices as ActivityServices,
|
|
346
356
|
EscapedMcpTimeoutRecoveryDetail,
|
|
357
|
+
PreClaimFailureDetail,
|
|
347
358
|
RunAgentTurnInput,
|
|
348
359
|
RunAgentTurnResult,
|
|
349
360
|
SessionAttemptQuiescenceProof,
|
|
@@ -351,6 +362,8 @@ import type {
|
|
|
351
362
|
import {
|
|
352
363
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
353
364
|
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
365
|
+
PRE_CLAIM_FAILURE_MESSAGE,
|
|
366
|
+
PRE_CLAIM_FAILURE_TYPE,
|
|
354
367
|
} from "./types";
|
|
355
368
|
import {
|
|
356
369
|
resumeBoxForTurn,
|
|
@@ -381,10 +394,14 @@ import {
|
|
|
381
394
|
recordCompanyBrainContributions,
|
|
382
395
|
recordSessionEventAppendLatency,
|
|
383
396
|
recordSessionEventPublishLatency,
|
|
397
|
+
recordTurnSandboxEstablishPolicy,
|
|
398
|
+
recordTurnStartupPhase,
|
|
399
|
+
recordTurnWorkerPreparationTotal,
|
|
384
400
|
runtimeMetricsHooksForObservability,
|
|
385
401
|
StreamTimingMetrics,
|
|
386
402
|
turnLifecycleMetricsFor,
|
|
387
403
|
type TurnOutcome,
|
|
404
|
+
type TurnSandboxEstablishReason,
|
|
388
405
|
} from "../observability-metrics";
|
|
389
406
|
import { buildCompanyBrainContributionReceipt } from "../model-context-contributions";
|
|
390
407
|
import {
|
|
@@ -660,7 +677,7 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
|
|
|
660
677
|
* this notice prevents a graceful runtime drop from becoming a silent source-
|
|
661
678
|
* of-truth substitution or a false claim that the disconnected system was read.
|
|
662
679
|
*/
|
|
663
|
-
export function
|
|
680
|
+
export function unavailableMcpOperationalContext(input: {
|
|
664
681
|
droppedIds: readonly string[];
|
|
665
682
|
droppedCount: number;
|
|
666
683
|
}): string | undefined {
|
|
@@ -704,6 +721,52 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
704
721
|
});
|
|
705
722
|
}
|
|
706
723
|
|
|
724
|
+
/**
|
|
725
|
+
* Convert the atomic claim transaction's failure into a small, stable
|
|
726
|
+
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
727
|
+
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
728
|
+
* workflow history. Contention and operational database unavailability are
|
|
729
|
+
* safe to re-read after backoff because the claim transaction contains no
|
|
730
|
+
* model/tool effects; permanent database and state failures require terminal
|
|
731
|
+
* settlement.
|
|
732
|
+
*/
|
|
733
|
+
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
734
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
735
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
736
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
737
|
+
// The database transaction itself retries only the two contention failures
|
|
738
|
+
// proven safe for immediate replay. Once the activity has failed, the
|
|
739
|
+
// workflow may also retry operational outages after a durable re-read and
|
|
740
|
+
// bounded delay. Unknown driver/database failures stay recoverable because
|
|
741
|
+
// they commonly represent a lost connection; known constraint, auth, and
|
|
742
|
+
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
743
|
+
const retryable = Boolean(
|
|
744
|
+
rawTransportFailure ||
|
|
745
|
+
(persistenceFailure &&
|
|
746
|
+
(sqlState === null ||
|
|
747
|
+
sqlState.startsWith("08") ||
|
|
748
|
+
sqlState.startsWith("40") ||
|
|
749
|
+
sqlState.startsWith("53") ||
|
|
750
|
+
sqlState === "55P03" ||
|
|
751
|
+
sqlState === "57014" ||
|
|
752
|
+
sqlState === "57P01" ||
|
|
753
|
+
sqlState === "57P02" ||
|
|
754
|
+
sqlState === "57P03" ||
|
|
755
|
+
sqlState.startsWith("58"))),
|
|
756
|
+
);
|
|
757
|
+
const detail: PreClaimFailureDetail = {
|
|
758
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
759
|
+
code:
|
|
760
|
+
persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
|
|
761
|
+
};
|
|
762
|
+
return ApplicationFailure.create({
|
|
763
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
764
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
765
|
+
nonRetryable: true,
|
|
766
|
+
details: [detail],
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
|
|
707
770
|
/**
|
|
708
771
|
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
709
772
|
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
@@ -729,6 +792,12 @@ export function filterUnmaterializedSandboxFileDownloads(
|
|
|
729
792
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
730
793
|
}
|
|
731
794
|
|
|
795
|
+
export function sandboxFileMaterializationOutcome(
|
|
796
|
+
failures: readonly SandboxFileDownloadFailure[],
|
|
797
|
+
): "completed" | "failed" {
|
|
798
|
+
return failures.length === 0 ? "completed" : "failed";
|
|
799
|
+
}
|
|
800
|
+
|
|
732
801
|
/** Fixed-length one-way tenant correlation for metrics/alerts; never a raw id. */
|
|
733
802
|
export function codexWorkspaceMetricKey(workspaceId: string): string {
|
|
734
803
|
return createHash("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
@@ -1998,6 +2067,33 @@ export function shouldEstablishSandboxForTurn(
|
|
|
1998
2067
|
return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
|
|
1999
2068
|
}
|
|
2000
2069
|
|
|
2070
|
+
export function sandboxEstablishPolicyDecision(input: {
|
|
2071
|
+
lazyEnabled: boolean;
|
|
2072
|
+
machinePrimary: boolean;
|
|
2073
|
+
sandboxBackend: Settings["sandboxBackend"];
|
|
2074
|
+
hasInitialRunCredentialMaterial: boolean;
|
|
2075
|
+
generatedVideoFileCount: number;
|
|
2076
|
+
hasSignedFileResources: boolean;
|
|
2077
|
+
}): { policy: "eager" | "on-demand"; reason: TurnSandboxEstablishReason } {
|
|
2078
|
+
if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
|
|
2079
|
+
if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
|
|
2080
|
+
if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
|
|
2081
|
+
if (input.hasInitialRunCredentialMaterial) {
|
|
2082
|
+
return { policy: "eager", reason: "initial_run_credentials" };
|
|
2083
|
+
}
|
|
2084
|
+
if (input.generatedVideoFileCount > 0) {
|
|
2085
|
+
return { policy: "eager", reason: "generated_video_files" };
|
|
2086
|
+
}
|
|
2087
|
+
// Signed file downloads must finish before the first model boundary so any
|
|
2088
|
+
// integrity/download failure is present in the prepared input. Deferring the
|
|
2089
|
+
// box until the first tool call would make that first request advertise a
|
|
2090
|
+
// path whose materialization outcome is not known yet.
|
|
2091
|
+
if (input.hasSignedFileResources) {
|
|
2092
|
+
return { policy: "eager", reason: "signed_file_resources" };
|
|
2093
|
+
}
|
|
2094
|
+
return { policy: "on-demand", reason: "eligible" };
|
|
2095
|
+
}
|
|
2096
|
+
|
|
2001
2097
|
/**
|
|
2002
2098
|
* Classify a persisted active-sandbox pointer for TURN-START RECONCILE (issue #341
|
|
2003
2099
|
* invariant B). Returns the typed reason to RESET the pointer to the session HOME,
|
|
@@ -4073,7 +4169,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4073
4169
|
// only exists in the RunState blob), never through a swapped trigger.
|
|
4074
4170
|
let triggerType: string | null = null;
|
|
4075
4171
|
try {
|
|
4076
|
-
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
4077
4172
|
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
4078
4173
|
sessionId: input.sessionId,
|
|
4079
4174
|
workflowId: input.workflowId,
|
|
@@ -4088,10 +4183,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4088
4183
|
}
|
|
4089
4184
|
const turn = claim.turn;
|
|
4090
4185
|
turnId = turn.id;
|
|
4186
|
+
// Establish durable attempt ownership before any later read can fail.
|
|
4187
|
+
// Therefore every failure with no turnId came from the one atomic claim
|
|
4188
|
+
// transaction and can be classified without conflating ordinary runtime
|
|
4189
|
+
// or transport failures with admission failures.
|
|
4190
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
4091
4191
|
executionGeneration = turn.executionGeneration;
|
|
4092
4192
|
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
4093
4193
|
let installedApiIntegrations: readonly ApiIntegrationRuntime[] = [];
|
|
4094
4194
|
const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
|
|
4195
|
+
const fileAuthoritySubjectId = turn.initiatingHumanSubjectId ?? null;
|
|
4095
4196
|
const mcpSettings = await settingsWithEnabledCapabilityMcpServers(
|
|
4096
4197
|
db,
|
|
4097
4198
|
input.workspaceId,
|
|
@@ -4410,6 +4511,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4410
4511
|
// on a healthy worker.
|
|
4411
4512
|
throwIfWorkerShuttingDown();
|
|
4412
4513
|
throwIfTurnCancelled();
|
|
4514
|
+
recordTurnStartupPhase(observability, {
|
|
4515
|
+
phase: "claim_and_policy",
|
|
4516
|
+
provider: turnExecutionPolicy.providerId,
|
|
4517
|
+
backend: turn.sandboxBackend,
|
|
4518
|
+
outcome: "completed",
|
|
4519
|
+
durationSeconds: (performance.now() - activityStarted) / 1_000,
|
|
4520
|
+
});
|
|
4521
|
+
const turnStartSettlementStartedAt = performance.now();
|
|
4413
4522
|
if (
|
|
4414
4523
|
!(await settle({
|
|
4415
4524
|
events: [
|
|
@@ -4426,7 +4535,16 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4426
4535
|
) {
|
|
4427
4536
|
return claimedResult({ status: "cancelled" });
|
|
4428
4537
|
}
|
|
4538
|
+
recordTurnStartupPhase(observability, {
|
|
4539
|
+
phase: "turn_start_settlement",
|
|
4540
|
+
provider: turnExecutionPolicy.providerId,
|
|
4541
|
+
backend: turn.sandboxBackend,
|
|
4542
|
+
outcome: "completed",
|
|
4543
|
+
durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1_000,
|
|
4544
|
+
count: 2,
|
|
4545
|
+
});
|
|
4429
4546
|
turnStartedPublished = true;
|
|
4547
|
+
const workerPreparationStartedAt = performance.now();
|
|
4430
4548
|
|
|
4431
4549
|
// Multi-account (P1): resolve the effective Codex account for this turn
|
|
4432
4550
|
// (session-pin > workspace active) and stamp it on the session so the
|
|
@@ -4434,80 +4552,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4434
4552
|
// when it changed from the prior run's account so the pill flips live.
|
|
4435
4553
|
// Gated on the codex-billed predicate — non-codex turns never touch this.
|
|
4436
4554
|
if (isCodexTurn) {
|
|
4437
|
-
const
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4555
|
+
const credentialSelectionStartedAt = performance.now();
|
|
4556
|
+
let credentialSelectionOutcome: "completed" | "failed" = "completed";
|
|
4557
|
+
try {
|
|
4558
|
+
const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
|
|
4559
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
4560
|
+
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
4561
|
+
const selectForTurn = (context: CodexCredentialLeaseSelectionContext) =>
|
|
4562
|
+
selectCodexCredentialLeaseForTurn({
|
|
4563
|
+
context,
|
|
4564
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
4565
|
+
sessionId: input.sessionId,
|
|
4566
|
+
sessionPinnedCredentialId: sessionPin,
|
|
4567
|
+
sessionPinSource,
|
|
4568
|
+
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
4569
|
+
now: new Date(),
|
|
4570
|
+
});
|
|
4450
4571
|
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
4457
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
4458
|
-
leased = await acquireCodexCredentialLease(
|
|
4459
|
-
db,
|
|
4460
|
-
{
|
|
4461
|
-
accountId: input.accountId,
|
|
4462
|
-
workspaceId: input.workspaceId,
|
|
4463
|
-
turnId,
|
|
4464
|
-
holderId: codexLeaseHolderId,
|
|
4465
|
-
advanceActivePointer: sessionPin === null,
|
|
4466
|
-
},
|
|
4467
|
-
selectForTurn,
|
|
4468
|
-
);
|
|
4469
|
-
} else {
|
|
4470
|
-
const [rotation, accounts] = await Promise.all([
|
|
4471
|
-
getCodexRotationSettings(db, input.workspaceId),
|
|
4472
|
-
listCodexAccountStatuses(db, input.workspaceId),
|
|
4473
|
-
]);
|
|
4474
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
4475
|
-
...account,
|
|
4476
|
-
activeLeaseCount: 0,
|
|
4477
|
-
selectionCount: 0,
|
|
4478
|
-
lastSelectedAt: null,
|
|
4479
|
-
}));
|
|
4480
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4481
|
-
const selected = selectForTurn({
|
|
4482
|
-
accounts: leaseAccounts,
|
|
4483
|
-
activeCredentialId,
|
|
4484
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4485
|
-
leaseRotationEnabled: false,
|
|
4486
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4487
|
-
existingCredentialId: null,
|
|
4488
|
-
policyScope: null,
|
|
4489
|
-
unavailableDiagnostics: [],
|
|
4490
|
-
});
|
|
4491
|
-
leased = {
|
|
4492
|
-
...selected,
|
|
4493
|
-
accounts: leaseAccounts,
|
|
4494
|
-
activeCredentialId,
|
|
4495
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4496
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4497
|
-
reused: false,
|
|
4498
|
-
holderId: null,
|
|
4499
|
-
generation: null,
|
|
4500
|
-
leasedUntil: null,
|
|
4501
|
-
unavailableDiagnostics: [],
|
|
4502
|
-
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4503
|
-
};
|
|
4504
|
-
}
|
|
4505
|
-
if (leased.decision.kind === "allCapped") {
|
|
4506
|
-
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4507
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4508
|
-
signalCodexCapacityWorkflow,
|
|
4509
|
-
wakeSessionWorkflow,
|
|
4510
|
-
});
|
|
4572
|
+
// Rollout/rollback path is intentionally table-inert. With the flag off,
|
|
4573
|
+
// old and new workers both use legacy pin > active-pointer selection and
|
|
4574
|
+
// neither reads nor writes the additive lease/cursor schema.
|
|
4575
|
+
let leased: CodexCredentialLeaseResult<RotationDecision>;
|
|
4576
|
+
let leaseAcquisitionStartedAtMs: number | null = null;
|
|
4511
4577
|
if (settings.codexCredentialLeasingEnabled) {
|
|
4512
4578
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
4513
4579
|
leased = await acquireCodexCredentialLease(
|
|
@@ -4532,9 +4598,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4532
4598
|
selectionCount: 0,
|
|
4533
4599
|
lastSelectedAt: null,
|
|
4534
4600
|
}));
|
|
4601
|
+
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
4535
4602
|
const selected = selectForTurn({
|
|
4536
4603
|
accounts: leaseAccounts,
|
|
4537
|
-
activeCredentialId
|
|
4604
|
+
activeCredentialId,
|
|
4538
4605
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4539
4606
|
leaseRotationEnabled: false,
|
|
4540
4607
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
@@ -4545,7 +4612,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4545
4612
|
leased = {
|
|
4546
4613
|
...selected,
|
|
4547
4614
|
accounts: leaseAccounts,
|
|
4548
|
-
activeCredentialId
|
|
4615
|
+
activeCredentialId,
|
|
4549
4616
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4550
4617
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4551
4618
|
reused: false,
|
|
@@ -4556,212 +4623,334 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4556
4623
|
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4557
4624
|
};
|
|
4558
4625
|
}
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4570
|
-
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4571
|
-
if (
|
|
4572
|
-
selectedPinDisposition === "sharded" &&
|
|
4573
|
-
leased.credentialId !== null &&
|
|
4574
|
-
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4575
|
-
) {
|
|
4576
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4577
|
-
db,
|
|
4578
|
-
{
|
|
4579
|
-
workspaceId: input.workspaceId,
|
|
4580
|
-
reason: "codex_policy_pin_changed",
|
|
4581
|
-
},
|
|
4582
|
-
async (tx) => {
|
|
4583
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
4584
|
-
tx,
|
|
4585
|
-
input.workspaceId,
|
|
4586
|
-
input.sessionId,
|
|
4587
|
-
leased.credentialId,
|
|
4588
|
-
"policy",
|
|
4626
|
+
if (leased.decision.kind === "allCapped") {
|
|
4627
|
+
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
4628
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
4629
|
+
signalCodexCapacityWorkflow,
|
|
4630
|
+
wakeSessionWorkflow,
|
|
4631
|
+
});
|
|
4632
|
+
if (settings.codexCredentialLeasingEnabled) {
|
|
4633
|
+
leaseAcquisitionStartedAtMs = performance.now();
|
|
4634
|
+
leased = await acquireCodexCredentialLease(
|
|
4635
|
+
db,
|
|
4589
4636
|
{
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4637
|
+
accountId: input.accountId,
|
|
4638
|
+
workspaceId: input.workspaceId,
|
|
4639
|
+
turnId,
|
|
4640
|
+
holderId: codexLeaseHolderId,
|
|
4641
|
+
advanceActivePointer: sessionPin === null,
|
|
4594
4642
|
},
|
|
4643
|
+
selectForTurn,
|
|
4595
4644
|
);
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4645
|
+
} else {
|
|
4646
|
+
const [rotation, accounts] = await Promise.all([
|
|
4647
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
4648
|
+
listCodexAccountStatuses(db, input.workspaceId),
|
|
4649
|
+
]);
|
|
4650
|
+
const leaseAccounts = accounts.map((account) => ({
|
|
4651
|
+
...account,
|
|
4652
|
+
activeLeaseCount: 0,
|
|
4653
|
+
selectionCount: 0,
|
|
4654
|
+
lastSelectedAt: null,
|
|
4655
|
+
}));
|
|
4656
|
+
const selected = selectForTurn({
|
|
4657
|
+
accounts: leaseAccounts,
|
|
4658
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4659
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4660
|
+
leaseRotationEnabled: false,
|
|
4661
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4662
|
+
existingCredentialId: null,
|
|
4663
|
+
policyScope: null,
|
|
4664
|
+
unavailableDiagnostics: [],
|
|
4665
|
+
});
|
|
4666
|
+
leased = {
|
|
4667
|
+
...selected,
|
|
4668
|
+
accounts: leaseAccounts,
|
|
4669
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
4670
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
4671
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
4672
|
+
reused: false,
|
|
4673
|
+
holderId: null,
|
|
4674
|
+
generation: null,
|
|
4675
|
+
leasedUntil: null,
|
|
4676
|
+
unavailableDiagnostics: [],
|
|
4677
|
+
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
4678
|
+
};
|
|
4679
|
+
}
|
|
4680
|
+
}
|
|
4681
|
+
const rotationDecision = leased.decision;
|
|
4682
|
+
const selectedPinDisposition = classifyCodexPin({
|
|
4683
|
+
pinnedCredentialId: sessionPin,
|
|
4684
|
+
pinSource: sessionPinSource,
|
|
4685
|
+
strategy: leased.rotationStrategy as CodexRotationStrategy,
|
|
4686
|
+
rotationEnabled: leased.rotationEnabled,
|
|
4687
|
+
});
|
|
4688
|
+
// pin policy pin persistence follows the atomic credential allocator selection. The selector
|
|
4689
|
+
// already ran exact-turn reuse before policy filtering and vetoed pointer
|
|
4690
|
+
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
4691
|
+
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
4692
|
+
if (
|
|
4693
|
+
selectedPinDisposition === "sharded" &&
|
|
4694
|
+
leased.credentialId !== null &&
|
|
4695
|
+
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
4696
|
+
) {
|
|
4697
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4698
|
+
db,
|
|
4699
|
+
{
|
|
4700
|
+
workspaceId: input.workspaceId,
|
|
4701
|
+
reason: "codex_policy_pin_changed",
|
|
4702
|
+
},
|
|
4703
|
+
async (tx) => {
|
|
4704
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4705
|
+
tx,
|
|
4706
|
+
input.workspaceId,
|
|
4707
|
+
input.sessionId,
|
|
4708
|
+
leased.credentialId,
|
|
4709
|
+
"policy",
|
|
4710
|
+
{
|
|
4711
|
+
expected: {
|
|
4712
|
+
pinnedCredentialId: sessionPin,
|
|
4713
|
+
pinSource: sessionPinSource,
|
|
4714
|
+
},
|
|
4621
4715
|
},
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4716
|
+
);
|
|
4717
|
+
return { result: changed, changed };
|
|
4718
|
+
},
|
|
4719
|
+
);
|
|
4720
|
+
await signalCodexCapacityWakeTargets(
|
|
4721
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4722
|
+
pinMutation.wakeTargets,
|
|
4723
|
+
);
|
|
4724
|
+
} else if (selectedPinDisposition === "clearStale") {
|
|
4725
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
4726
|
+
db,
|
|
4727
|
+
{
|
|
4728
|
+
workspaceId: input.workspaceId,
|
|
4729
|
+
reason: "codex_stale_policy_pin_cleared",
|
|
4730
|
+
},
|
|
4731
|
+
async (tx) => {
|
|
4732
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
4733
|
+
tx,
|
|
4734
|
+
input.workspaceId,
|
|
4735
|
+
input.sessionId,
|
|
4736
|
+
null,
|
|
4737
|
+
"policy",
|
|
4738
|
+
{
|
|
4739
|
+
expected: {
|
|
4740
|
+
pinnedCredentialId: sessionPin,
|
|
4741
|
+
pinSource: sessionPinSource,
|
|
4742
|
+
},
|
|
4743
|
+
},
|
|
4744
|
+
);
|
|
4745
|
+
return { result: changed, changed };
|
|
4746
|
+
},
|
|
4747
|
+
);
|
|
4748
|
+
await signalCodexCapacityWakeTargets(
|
|
4749
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4750
|
+
pinMutation.wakeTargets,
|
|
4751
|
+
);
|
|
4752
|
+
}
|
|
4753
|
+
if (
|
|
4754
|
+
!settings.codexCredentialLeasingEnabled &&
|
|
4755
|
+
leased.advanceActivePointer &&
|
|
4756
|
+
sessionPin === null &&
|
|
4757
|
+
rotationDecision.kind === "active" &&
|
|
4758
|
+
rotationDecision.moved
|
|
4759
|
+
) {
|
|
4760
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
4761
|
+
}
|
|
4762
|
+
effectiveCodexCredentialId = leased.credentialId;
|
|
4763
|
+
codexLeaseGeneration = leased.generation;
|
|
4764
|
+
codexLeaseConfirmedUntilMs =
|
|
4765
|
+
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
4766
|
+
? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
|
|
4767
|
+
: null;
|
|
4768
|
+
codexLeaseHeld =
|
|
4769
|
+
effectiveCodexCredentialId !== null &&
|
|
4770
|
+
leased.holderId !== null &&
|
|
4771
|
+
leased.generation !== null &&
|
|
4772
|
+
codexLeaseConfirmedUntilMs !== null;
|
|
4773
|
+
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
4774
|
+
|
|
4775
|
+
const actualOutcome = effectiveCodexCredentialId
|
|
4776
|
+
? "selected"
|
|
4777
|
+
: rotationDecision.kind === "allCapped"
|
|
4778
|
+
? "waiting"
|
|
4779
|
+
: "none";
|
|
4780
|
+
const actualReason = effectiveCodexCredentialId
|
|
4781
|
+
? leased.reused
|
|
4782
|
+
? "lease_reused"
|
|
4783
|
+
: sessionPin === effectiveCodexCredentialId
|
|
4784
|
+
? "pin"
|
|
4785
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
4786
|
+
? "rotation"
|
|
4787
|
+
: "active"
|
|
4788
|
+
: rotationDecision.kind === "allCapped"
|
|
4789
|
+
? "all_capped"
|
|
4790
|
+
: "none";
|
|
4791
|
+
const fencedInFlight = leased.reused;
|
|
4792
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4793
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4794
|
+
decision: {
|
|
4795
|
+
accounts: leased.accounts,
|
|
4796
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
4797
|
+
actualOutcome,
|
|
4798
|
+
actualReason,
|
|
4799
|
+
affinityCredentialId: fencedInFlight
|
|
4800
|
+
? effectiveCodexCredentialId
|
|
4801
|
+
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4802
|
+
fencedInFlight,
|
|
4803
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4804
|
+
now: new Date(),
|
|
4805
|
+
aliasSeed: randomUUID(),
|
|
4625
4806
|
},
|
|
4626
|
-
|
|
4627
|
-
await signalCodexCapacityWakeTargets(
|
|
4628
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
4629
|
-
pinMutation.wakeTargets,
|
|
4630
|
-
);
|
|
4631
|
-
}
|
|
4632
|
-
if (
|
|
4633
|
-
!settings.codexCredentialLeasingEnabled &&
|
|
4634
|
-
leased.advanceActivePointer &&
|
|
4635
|
-
sessionPin === null &&
|
|
4636
|
-
rotationDecision.kind === "active" &&
|
|
4637
|
-
rotationDecision.moved
|
|
4638
|
-
) {
|
|
4639
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
4640
|
-
}
|
|
4641
|
-
effectiveCodexCredentialId = leased.credentialId;
|
|
4642
|
-
codexLeaseGeneration = leased.generation;
|
|
4643
|
-
codexLeaseConfirmedUntilMs =
|
|
4644
|
-
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
4645
|
-
? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS
|
|
4646
|
-
: null;
|
|
4647
|
-
codexLeaseHeld =
|
|
4648
|
-
effectiveCodexCredentialId !== null &&
|
|
4649
|
-
leased.holderId !== null &&
|
|
4650
|
-
leased.generation !== null &&
|
|
4651
|
-
codexLeaseConfirmedUntilMs !== null;
|
|
4652
|
-
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
4653
|
-
|
|
4654
|
-
const actualOutcome = effectiveCodexCredentialId
|
|
4655
|
-
? "selected"
|
|
4656
|
-
: rotationDecision.kind === "allCapped"
|
|
4657
|
-
? "waiting"
|
|
4658
|
-
: "none";
|
|
4659
|
-
const actualReason = effectiveCodexCredentialId
|
|
4660
|
-
? leased.reused
|
|
4661
|
-
? "lease_reused"
|
|
4662
|
-
: sessionPin === effectiveCodexCredentialId
|
|
4663
|
-
? "pin"
|
|
4664
|
-
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
4665
|
-
? "rotation"
|
|
4666
|
-
: "active"
|
|
4667
|
-
: rotationDecision.kind === "allCapped"
|
|
4668
|
-
? "all_capped"
|
|
4669
|
-
: "none";
|
|
4670
|
-
const fencedInFlight = leased.reused;
|
|
4671
|
-
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
4672
|
-
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
4673
|
-
decision: {
|
|
4674
|
-
accounts: leased.accounts,
|
|
4675
|
-
actualCredentialId: effectiveCodexCredentialId,
|
|
4676
|
-
actualOutcome,
|
|
4677
|
-
actualReason,
|
|
4678
|
-
affinityCredentialId: fencedInFlight
|
|
4679
|
-
? effectiveCodexCredentialId
|
|
4680
|
-
: (sessionPin ?? sessionCodex?.lastCredentialId ?? null),
|
|
4681
|
-
fencedInFlight,
|
|
4682
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
4683
|
-
now: new Date(),
|
|
4684
|
-
aliasSeed: randomUUID(),
|
|
4685
|
-
},
|
|
4686
|
-
publish,
|
|
4687
|
-
});
|
|
4688
|
-
if (shadowResult.outcome === "published") {
|
|
4689
|
-
const shadowPayload = shadowResult.payload;
|
|
4690
|
-
observability.incrementCounter({
|
|
4691
|
-
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4692
|
-
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4693
|
-
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4694
|
-
});
|
|
4695
|
-
observability.info("Codex adaptive fleet shadow decision", {
|
|
4696
|
-
workspaceId: input.workspaceId,
|
|
4697
|
-
policyVersion: shadowPayload.replay.policyVersion,
|
|
4698
|
-
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4699
|
-
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4700
|
-
actualOutcome: shadowPayload.actual.outcome,
|
|
4701
|
-
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4702
|
-
comparison: shadowPayload.comparison,
|
|
4703
|
-
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4704
|
-
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4705
|
-
payloadBytes: shadowResult.payloadBytes,
|
|
4706
|
-
});
|
|
4707
|
-
} else if (shadowResult.outcome === "failed") {
|
|
4708
|
-
// Shadow observability is explicitly non-authoritative. A malformed
|
|
4709
|
-
// snapshot or event-write fault must never change the authoritative lease,
|
|
4710
|
-
// capacity wait, failover, or the account serving this fenced turn.
|
|
4711
|
-
observability.incrementCounter({
|
|
4712
|
-
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
4713
|
-
help: "Shadow decision build/publication failures.",
|
|
4714
|
-
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
4715
|
-
});
|
|
4716
|
-
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
4717
|
-
stage: shadowResult.stage,
|
|
4718
|
-
reason: shadowResult.reason,
|
|
4719
|
-
errorClass: "CodexFleetShadowOperationError",
|
|
4720
|
-
errorCode: "codex_fleet_shadow_failed",
|
|
4721
|
-
origin: "worker",
|
|
4722
|
-
payloadBytes: shadowResult.payloadBytes,
|
|
4807
|
+
publish,
|
|
4723
4808
|
});
|
|
4724
|
-
|
|
4809
|
+
if (shadowResult.outcome === "published") {
|
|
4810
|
+
const shadowPayload = shadowResult.payload;
|
|
4811
|
+
observability.incrementCounter({
|
|
4812
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
4813
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
4814
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload),
|
|
4815
|
+
});
|
|
4816
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
4817
|
+
workspaceId: input.workspaceId,
|
|
4818
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
4819
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
4820
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
4821
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
4822
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
4823
|
+
comparison: shadowPayload.comparison,
|
|
4824
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
4825
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
4826
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4827
|
+
});
|
|
4828
|
+
} else if (shadowResult.outcome === "failed") {
|
|
4829
|
+
// Shadow observability is explicitly non-authoritative. A malformed
|
|
4830
|
+
// snapshot or event-write fault must never change the authoritative lease,
|
|
4831
|
+
// capacity wait, failover, or the account serving this fenced turn.
|
|
4832
|
+
observability.incrementCounter({
|
|
4833
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
4834
|
+
help: "Shadow decision build/publication failures.",
|
|
4835
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult),
|
|
4836
|
+
});
|
|
4837
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
4838
|
+
stage: shadowResult.stage,
|
|
4839
|
+
reason: shadowResult.reason,
|
|
4840
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
4841
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
4842
|
+
origin: "worker",
|
|
4843
|
+
payloadBytes: shadowResult.payloadBytes,
|
|
4844
|
+
});
|
|
4845
|
+
}
|
|
4725
4846
|
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
observability.incrementCounter({
|
|
4731
|
-
name: "opengeni_codex_pool_observations_total",
|
|
4732
|
-
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4733
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4734
|
-
});
|
|
4735
|
-
if (eligibleCount <= 1) {
|
|
4847
|
+
const eligibleCount = leased.accounts.filter((account) =>
|
|
4848
|
+
isCodexCredentialEligible(account, new Date()),
|
|
4849
|
+
).length;
|
|
4850
|
+
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
4736
4851
|
observability.incrementCounter({
|
|
4737
|
-
name: "
|
|
4738
|
-
help: "
|
|
4852
|
+
name: "opengeni_codex_pool_observations_total",
|
|
4853
|
+
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
4739
4854
|
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4740
4855
|
});
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
|
|
4856
|
+
if (eligibleCount <= 1) {
|
|
4857
|
+
observability.incrementCounter({
|
|
4858
|
+
name: "opengeni_codex_pool_low_total",
|
|
4859
|
+
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
4860
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth },
|
|
4861
|
+
});
|
|
4862
|
+
observability.warn("Codex eligible credential pool is low", {
|
|
4863
|
+
workspaceId: input.workspaceId,
|
|
4864
|
+
eligibleCount,
|
|
4865
|
+
connectedCount: leased.accounts.length,
|
|
4866
|
+
depth: poolDepth,
|
|
4867
|
+
});
|
|
4868
|
+
}
|
|
4748
4869
|
|
|
4749
|
-
|
|
4750
|
-
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4870
|
+
if (
|
|
4871
|
+
effectiveCodexCredentialId === null &&
|
|
4872
|
+
leased.accounts.length > 0 &&
|
|
4873
|
+
leased.accounts.every((account) => !account.allocatorEnabled) &&
|
|
4874
|
+
turnId
|
|
4875
|
+
) {
|
|
4876
|
+
if (turn.source === "compaction") {
|
|
4877
|
+
if (
|
|
4878
|
+
!(await settle!({
|
|
4879
|
+
events: [
|
|
4880
|
+
{
|
|
4881
|
+
type: "turn.cancelled",
|
|
4882
|
+
payload: {
|
|
4883
|
+
maintenance: "context_compaction",
|
|
4884
|
+
reason: "codex_allocator_disabled",
|
|
4885
|
+
requestPreserved: true,
|
|
4886
|
+
},
|
|
4887
|
+
},
|
|
4888
|
+
{
|
|
4889
|
+
type: "session.status.changed",
|
|
4890
|
+
payload: { status: "idle" },
|
|
4891
|
+
},
|
|
4892
|
+
],
|
|
4893
|
+
turnStatus: "cancelled",
|
|
4894
|
+
sessionStatus: "idle",
|
|
4895
|
+
activeTurnId: null,
|
|
4896
|
+
}))
|
|
4897
|
+
) {
|
|
4898
|
+
return claimedResult({ status: "cancelled" });
|
|
4899
|
+
}
|
|
4900
|
+
turnMetricOutcome = "cancelled";
|
|
4901
|
+
activityStatus = "idle";
|
|
4902
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
4903
|
+
}
|
|
4904
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4905
|
+
() => null,
|
|
4906
|
+
);
|
|
4907
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
4908
|
+
const armed = await armCodexCapacityWait(db, {
|
|
4909
|
+
accountId: input.accountId,
|
|
4910
|
+
workspaceId: input.workspaceId,
|
|
4911
|
+
sessionId: input.sessionId,
|
|
4912
|
+
turnId,
|
|
4913
|
+
attemptId: input.attemptId,
|
|
4914
|
+
workflowId: input.workflowId,
|
|
4915
|
+
goalId: activeGoal?.id ?? null,
|
|
4916
|
+
goalVersion: activeGoal?.version ?? null,
|
|
4917
|
+
earliestResetAt: null,
|
|
4918
|
+
resetKind: "bounded_refresh",
|
|
4919
|
+
failurePayload: {
|
|
4920
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4921
|
+
code: "codex_allocator_disabled",
|
|
4922
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added",
|
|
4923
|
+
},
|
|
4924
|
+
});
|
|
4925
|
+
if (armed.action === "waiting") {
|
|
4926
|
+
await publishDurableSessionEvents(
|
|
4927
|
+
bus,
|
|
4928
|
+
input.workspaceId,
|
|
4929
|
+
input.sessionId,
|
|
4930
|
+
armed.events,
|
|
4931
|
+
);
|
|
4932
|
+
turnMetricOutcome = "recovering";
|
|
4933
|
+
activityStatus = "waiting_capacity";
|
|
4934
|
+
return claimedResult({
|
|
4935
|
+
status: "waiting_capacity",
|
|
4936
|
+
capacityWait: {
|
|
4937
|
+
waiterId: armed.waiter.id,
|
|
4938
|
+
generation: armed.waiter.generation,
|
|
4939
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
4940
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
4941
|
+
},
|
|
4942
|
+
});
|
|
4943
|
+
}
|
|
4756
4944
|
if (
|
|
4757
4945
|
!(await settle!({
|
|
4758
4946
|
events: [
|
|
4759
4947
|
{
|
|
4760
|
-
type: "turn.
|
|
4948
|
+
type: "turn.failed",
|
|
4761
4949
|
payload: {
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4950
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
4951
|
+
code: "codex_allocator_disabled",
|
|
4952
|
+
retryable: false,
|
|
4953
|
+
recovery: "user_message",
|
|
4765
4954
|
},
|
|
4766
4955
|
},
|
|
4767
4956
|
{
|
|
@@ -4769,94 +4958,119 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4769
4958
|
payload: { status: "idle" },
|
|
4770
4959
|
},
|
|
4771
4960
|
],
|
|
4772
|
-
turnStatus: "
|
|
4961
|
+
turnStatus: "failed",
|
|
4773
4962
|
sessionStatus: "idle",
|
|
4774
4963
|
activeTurnId: null,
|
|
4775
4964
|
}))
|
|
4776
4965
|
) {
|
|
4777
4966
|
return claimedResult({ status: "cancelled" });
|
|
4778
4967
|
}
|
|
4779
|
-
turnMetricOutcome = "
|
|
4968
|
+
turnMetricOutcome = "failed";
|
|
4780
4969
|
activityStatus = "idle";
|
|
4781
|
-
return claimedResult({ status: "idle"
|
|
4970
|
+
return claimedResult({ status: "idle" });
|
|
4782
4971
|
}
|
|
4783
|
-
|
|
4784
|
-
|
|
4785
|
-
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
|
|
4793
|
-
|
|
4794
|
-
|
|
4795
|
-
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
4799
|
-
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4972
|
+
|
|
4973
|
+
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
4974
|
+
if (turn.source === "compaction") {
|
|
4975
|
+
if (
|
|
4976
|
+
!(await settle!({
|
|
4977
|
+
events: [
|
|
4978
|
+
{
|
|
4979
|
+
type: "turn.cancelled",
|
|
4980
|
+
payload: {
|
|
4981
|
+
maintenance: "context_compaction",
|
|
4982
|
+
reason: "codex_capacity_unavailable",
|
|
4983
|
+
requestPreserved: true,
|
|
4984
|
+
},
|
|
4985
|
+
},
|
|
4986
|
+
{
|
|
4987
|
+
type: "session.status.changed",
|
|
4988
|
+
payload: { status: "idle" },
|
|
4989
|
+
},
|
|
4990
|
+
],
|
|
4991
|
+
turnStatus: "cancelled",
|
|
4992
|
+
sessionStatus: "idle",
|
|
4993
|
+
activeTurnId: null,
|
|
4994
|
+
}))
|
|
4995
|
+
) {
|
|
4996
|
+
return claimedResult({ status: "cancelled" });
|
|
4997
|
+
}
|
|
4998
|
+
turnMetricOutcome = "cancelled";
|
|
4999
|
+
activityStatus = "idle";
|
|
5000
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
5001
|
+
}
|
|
5002
|
+
// Every eligible account is capped/cooling (and a usage refresh did NOT
|
|
5003
|
+
// surface a reset): idle the turn AT THE BOUNDARY (no wasted model/sandbox
|
|
5004
|
+
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
5005
|
+
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
5006
|
+
// no model ran, nothing to freeze.
|
|
5007
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
5008
|
+
() => null,
|
|
4810
5009
|
);
|
|
4811
|
-
|
|
4812
|
-
|
|
4813
|
-
|
|
4814
|
-
|
|
4815
|
-
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
5010
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
5011
|
+
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
5012
|
+
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
5013
|
+
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
5014
|
+
const resumeMs = computeIdleDelayMs(
|
|
5015
|
+
rotationDecision.earliestResetAt,
|
|
5016
|
+
new Date(),
|
|
5017
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
5018
|
+
);
|
|
5019
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
5020
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
5021
|
+
"all connected Codex subscriptions are rate-limited",
|
|
5022
|
+
{ allAccounts: true },
|
|
5023
|
+
);
|
|
5024
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
5025
|
+
leased.accounts,
|
|
5026
|
+
new Date(),
|
|
5027
|
+
);
|
|
5028
|
+
const armed = await armCodexCapacityWait(db, {
|
|
5029
|
+
accountId: input.accountId,
|
|
5030
|
+
workspaceId: input.workspaceId,
|
|
5031
|
+
sessionId: input.sessionId,
|
|
5032
|
+
turnId,
|
|
5033
|
+
attemptId: input.attemptId,
|
|
5034
|
+
workflowId: input.workflowId,
|
|
5035
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
5036
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
5037
|
+
earliestResetAt: authoritativeResetAt,
|
|
5038
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
5039
|
+
failurePayload,
|
|
4821
5040
|
});
|
|
4822
|
-
|
|
4823
|
-
|
|
4824
|
-
|
|
4825
|
-
|
|
4826
|
-
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
5041
|
+
if (armed.action === "waiting") {
|
|
5042
|
+
await publishDurableSessionEvents(
|
|
5043
|
+
bus,
|
|
5044
|
+
input.workspaceId,
|
|
5045
|
+
input.sessionId,
|
|
5046
|
+
armed.events,
|
|
5047
|
+
);
|
|
5048
|
+
turnMetricOutcome = "recovering";
|
|
5049
|
+
activityStatus = "waiting_capacity";
|
|
5050
|
+
return claimedResult({
|
|
5051
|
+
status: "waiting_capacity",
|
|
5052
|
+
capacityWait: {
|
|
5053
|
+
waiterId: armed.waiter.id,
|
|
5054
|
+
generation: armed.waiter.generation,
|
|
5055
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
5056
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
4834
5057
|
},
|
|
4835
|
-
|
|
4836
|
-
|
|
4837
|
-
turnStatus: "failed",
|
|
4838
|
-
sessionStatus: "idle",
|
|
4839
|
-
activeTurnId: null,
|
|
4840
|
-
}))
|
|
4841
|
-
) {
|
|
4842
|
-
return claimedResult({ status: "cancelled" });
|
|
4843
|
-
}
|
|
4844
|
-
turnMetricOutcome = "failed";
|
|
4845
|
-
activityStatus = "idle";
|
|
4846
|
-
return claimedResult({ status: "idle" });
|
|
4847
|
-
}
|
|
4848
|
-
|
|
4849
|
-
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
4850
|
-
if (turn.source === "compaction") {
|
|
5058
|
+
});
|
|
5059
|
+
}
|
|
4851
5060
|
if (
|
|
4852
5061
|
!(await settle!({
|
|
4853
5062
|
events: [
|
|
5063
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
5064
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
5065
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
5066
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
5067
|
+
// the proactive path while the reactive path spares it.
|
|
4854
5068
|
{
|
|
4855
|
-
type: "turn.
|
|
5069
|
+
type: "turn.failed",
|
|
4856
5070
|
payload: {
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
5071
|
+
...failurePayload,
|
|
5072
|
+
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
5073
|
+
rotated: true,
|
|
4860
5074
|
},
|
|
4861
5075
|
},
|
|
4862
5076
|
{
|
|
@@ -4864,165 +5078,90 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
4864
5078
|
payload: { status: "idle" },
|
|
4865
5079
|
},
|
|
4866
5080
|
],
|
|
4867
|
-
turnStatus: "
|
|
5081
|
+
turnStatus: "failed",
|
|
4868
5082
|
sessionStatus: "idle",
|
|
4869
5083
|
activeTurnId: null,
|
|
4870
5084
|
}))
|
|
4871
5085
|
) {
|
|
4872
5086
|
return claimedResult({ status: "cancelled" });
|
|
4873
5087
|
}
|
|
4874
|
-
turnMetricOutcome = "
|
|
5088
|
+
turnMetricOutcome = "failed";
|
|
4875
5089
|
activityStatus = "idle";
|
|
4876
|
-
|
|
5090
|
+
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
5091
|
+
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
5092
|
+
return claimedResult(
|
|
5093
|
+
goalActive
|
|
5094
|
+
? {
|
|
5095
|
+
status: "idle",
|
|
5096
|
+
continueDelayMs: resumeMs,
|
|
5097
|
+
idleUntilReset: true,
|
|
5098
|
+
}
|
|
5099
|
+
: { status: "idle" },
|
|
5100
|
+
);
|
|
4877
5101
|
}
|
|
4878
|
-
|
|
4879
|
-
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
// no model ran, nothing to freeze.
|
|
4883
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
4884
|
-
() => null,
|
|
4885
|
-
);
|
|
4886
|
-
const goalActive = Boolean(goal && goal.status === "active");
|
|
4887
|
-
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
4888
|
-
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
4889
|
-
// re-entering this path in a tight CPU/DB-hammering loop).
|
|
4890
|
-
const resumeMs = computeIdleDelayMs(
|
|
4891
|
-
rotationDecision.earliestResetAt,
|
|
4892
|
-
new Date(),
|
|
4893
|
-
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
4894
|
-
);
|
|
4895
|
-
const failurePayload = codexUsageLimitFailurePayload(
|
|
4896
|
-
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
4897
|
-
"all connected Codex subscriptions are rate-limited",
|
|
4898
|
-
{ allAccounts: true },
|
|
4899
|
-
);
|
|
4900
|
-
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
4901
|
-
leased.accounts,
|
|
4902
|
-
new Date(),
|
|
4903
|
-
);
|
|
4904
|
-
const armed = await armCodexCapacityWait(db, {
|
|
4905
|
-
accountId: input.accountId,
|
|
4906
|
-
workspaceId: input.workspaceId,
|
|
4907
|
-
sessionId: input.sessionId,
|
|
4908
|
-
turnId,
|
|
4909
|
-
attemptId: input.attemptId,
|
|
4910
|
-
workflowId: input.workflowId,
|
|
4911
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
4912
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
4913
|
-
earliestResetAt: authoritativeResetAt,
|
|
4914
|
-
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
4915
|
-
failurePayload,
|
|
4916
|
-
});
|
|
4917
|
-
if (armed.action === "waiting") {
|
|
4918
|
-
await publishDurableSessionEvents(
|
|
4919
|
-
bus,
|
|
5102
|
+
if (effectiveCodexCredentialId) {
|
|
5103
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
5104
|
+
await recordSessionActiveCodexCredential(
|
|
5105
|
+
db,
|
|
4920
5106
|
input.workspaceId,
|
|
4921
5107
|
input.sessionId,
|
|
4922
|
-
|
|
5108
|
+
effectiveCodexCredentialId,
|
|
4923
5109
|
);
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
status: "waiting_capacity",
|
|
4928
|
-
capacityWait: {
|
|
4929
|
-
waiterId: armed.waiter.id,
|
|
4930
|
-
generation: armed.waiter.generation,
|
|
4931
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
4932
|
-
wakeRevision: armed.waiter.wakeRevision,
|
|
4933
|
-
},
|
|
4934
|
-
});
|
|
4935
|
-
}
|
|
4936
|
-
if (
|
|
4937
|
-
!(await settle!({
|
|
4938
|
-
events: [
|
|
4939
|
-
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
4940
|
-
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
4941
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
4942
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
4943
|
-
// the proactive path while the reactive path spares it.
|
|
5110
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
5111
|
+
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
5112
|
+
await publish([
|
|
4944
5113
|
{
|
|
4945
|
-
type: "
|
|
5114
|
+
type: "codex.account.switched",
|
|
4946
5115
|
payload: {
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
rotated:
|
|
5116
|
+
fromAccountId: priorAccountId,
|
|
5117
|
+
toAccountId: effectiveCodexCredentialId,
|
|
5118
|
+
reason: rotated ? "rotation" : "manual",
|
|
4950
5119
|
},
|
|
4951
5120
|
},
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
}
|
|
4972
|
-
: { status: "idle" },
|
|
4973
|
-
);
|
|
4974
|
-
}
|
|
4975
|
-
if (effectiveCodexCredentialId) {
|
|
4976
|
-
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
4977
|
-
await recordSessionActiveCodexCredential(
|
|
4978
|
-
db,
|
|
4979
|
-
input.workspaceId,
|
|
4980
|
-
input.sessionId,
|
|
4981
|
-
effectiveCodexCredentialId,
|
|
4982
|
-
);
|
|
4983
|
-
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
4984
|
-
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
5121
|
+
]);
|
|
5122
|
+
}
|
|
5123
|
+
|
|
5124
|
+
const selectionReason = leased.reused
|
|
5125
|
+
? "lease_reused"
|
|
5126
|
+
: sessionPin === effectiveCodexCredentialId
|
|
5127
|
+
? "pin"
|
|
5128
|
+
: rotationDecision.kind === "active" && rotationDecision.moved
|
|
5129
|
+
? "rotation"
|
|
5130
|
+
: "active";
|
|
5131
|
+
observability.incrementCounter({
|
|
5132
|
+
name: "opengeni_codex_credential_selections_total",
|
|
5133
|
+
help: "Codex credential selections by strategy and reason.",
|
|
5134
|
+
labels: {
|
|
5135
|
+
workspace_key: codexWorkspaceKey,
|
|
5136
|
+
strategy: leased.rotationStrategy,
|
|
5137
|
+
reason: selectionReason,
|
|
5138
|
+
},
|
|
5139
|
+
});
|
|
4985
5140
|
await publish([
|
|
4986
5141
|
{
|
|
4987
|
-
type: "codex.
|
|
5142
|
+
type: "codex.credential.selected",
|
|
4988
5143
|
payload: {
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
reason:
|
|
5144
|
+
credentialId: effectiveCodexCredentialId,
|
|
5145
|
+
strategy: leased.rotationStrategy,
|
|
5146
|
+
reason: selectionReason,
|
|
5147
|
+
eligibleCount,
|
|
5148
|
+
connectedCount: leased.accounts.length,
|
|
5149
|
+
reused: leased.reused,
|
|
4992
5150
|
},
|
|
4993
5151
|
},
|
|
4994
5152
|
]);
|
|
4995
5153
|
}
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
help: "Codex credential selections by strategy and reason.",
|
|
5007
|
-
labels: {
|
|
5008
|
-
workspace_key: codexWorkspaceKey,
|
|
5009
|
-
strategy: leased.rotationStrategy,
|
|
5010
|
-
reason: selectionReason,
|
|
5011
|
-
},
|
|
5154
|
+
} catch (error) {
|
|
5155
|
+
credentialSelectionOutcome = "failed";
|
|
5156
|
+
throw error;
|
|
5157
|
+
} finally {
|
|
5158
|
+
recordTurnStartupPhase(observability, {
|
|
5159
|
+
phase: "credential_selection",
|
|
5160
|
+
provider: "codex-subscription",
|
|
5161
|
+
backend: turn.sandboxBackend,
|
|
5162
|
+
outcome: credentialSelectionOutcome,
|
|
5163
|
+
durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1_000,
|
|
5012
5164
|
});
|
|
5013
|
-
await publish([
|
|
5014
|
-
{
|
|
5015
|
-
type: "codex.credential.selected",
|
|
5016
|
-
payload: {
|
|
5017
|
-
credentialId: effectiveCodexCredentialId,
|
|
5018
|
-
strategy: leased.rotationStrategy,
|
|
5019
|
-
reason: selectionReason,
|
|
5020
|
-
eligibleCount,
|
|
5021
|
-
connectedCount: leased.accounts.length,
|
|
5022
|
-
reused: leased.reused,
|
|
5023
|
-
},
|
|
5024
|
-
},
|
|
5025
|
-
]);
|
|
5026
5165
|
}
|
|
5027
5166
|
}
|
|
5028
5167
|
|
|
@@ -5232,6 +5371,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5232
5371
|
});
|
|
5233
5372
|
}
|
|
5234
5373
|
|
|
5374
|
+
const runtimePreparationStartedAt = performance.now();
|
|
5375
|
+
|
|
5235
5376
|
// Pack-scoped runtime: enabled packs may declare the sandbox image this
|
|
5236
5377
|
// workspace's sessions run in and skills for the sandbox skill index.
|
|
5237
5378
|
// Resolved after turn.started so a composition conflict (two enabled
|
|
@@ -5369,9 +5510,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5369
5510
|
modelCanReceiveRetainedSessionImages = supportsImageInput;
|
|
5370
5511
|
const attachmentProjector = createModelHistoryAttachmentProjector(
|
|
5371
5512
|
db,
|
|
5372
|
-
|
|
5513
|
+
{
|
|
5514
|
+
accountId: input.accountId,
|
|
5515
|
+
workspaceId: input.workspaceId,
|
|
5516
|
+
subjectId: fileAuthoritySubjectId,
|
|
5517
|
+
},
|
|
5373
5518
|
modelInputPolicy,
|
|
5374
|
-
objectStorage
|
|
5519
|
+
objectStorage
|
|
5520
|
+
? async (file) => {
|
|
5521
|
+
await requireFileForSubject(db, {
|
|
5522
|
+
accountId: input.accountId,
|
|
5523
|
+
workspaceId: input.workspaceId,
|
|
5524
|
+
subjectId: fileAuthoritySubjectId,
|
|
5525
|
+
fileId: file.id,
|
|
5526
|
+
});
|
|
5527
|
+
return await objectStorage.getFileBytes(file);
|
|
5528
|
+
}
|
|
5529
|
+
: undefined,
|
|
5375
5530
|
);
|
|
5376
5531
|
const modelHistoryProjector = async (items: Array<Record<string, unknown>>) =>
|
|
5377
5532
|
projectModelInputForCapabilities(await attachmentProjector(items), modelInputPolicy);
|
|
@@ -5469,6 +5624,11 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5469
5624
|
// call on the same codex client) and the main run; otherwise the summarizer
|
|
5470
5625
|
// would hit the codex backend unauthenticated.
|
|
5471
5626
|
let codexModelRequestSequence = 0;
|
|
5627
|
+
let firstModelRequestPreparationStartedAt: number | null = null;
|
|
5628
|
+
let firstModelRequestPreparationRecorded = false;
|
|
5629
|
+
let firstModelRequestAuditRecorded = false;
|
|
5630
|
+
let firstModelRequestCheckpointAt: number | null = null;
|
|
5631
|
+
const firstModelRequestCheckpoints = new Set<string>();
|
|
5472
5632
|
const codexContext: CodexRequestContext | null =
|
|
5473
5633
|
resolvedModel?.provider.kind === "codex-subscription"
|
|
5474
5634
|
? ((): CodexRequestContext => {
|
|
@@ -5499,10 +5659,51 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5499
5659
|
onUsageHeaders: (snapshot) => {
|
|
5500
5660
|
latestCodexUsage = snapshot;
|
|
5501
5661
|
}, // latest wins; flushed once in finally
|
|
5662
|
+
onRequestPreparationDiagnostic: (phase) => {
|
|
5663
|
+
if (
|
|
5664
|
+
firstModelRequestCheckpointAt === null ||
|
|
5665
|
+
firstModelRequestCheckpoints.has(phase)
|
|
5666
|
+
) {
|
|
5667
|
+
return;
|
|
5668
|
+
}
|
|
5669
|
+
const now = performance.now();
|
|
5670
|
+
const metricPhase =
|
|
5671
|
+
phase === "transport_entry"
|
|
5672
|
+
? "model_sdk_serialization"
|
|
5673
|
+
: phase === "credential_ready"
|
|
5674
|
+
? "model_credential_resolution"
|
|
5675
|
+
: "model_wire_normalization";
|
|
5676
|
+
firstModelRequestCheckpoints.add(phase);
|
|
5677
|
+
recordTurnStartupPhase(observability, {
|
|
5678
|
+
phase: metricPhase,
|
|
5679
|
+
provider: turnExecutionPolicy.providerId,
|
|
5680
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5681
|
+
outcome: "completed",
|
|
5682
|
+
durationSeconds: (now - firstModelRequestCheckpointAt) / 1_000,
|
|
5683
|
+
count: turnTools.length,
|
|
5684
|
+
});
|
|
5685
|
+
firstModelRequestCheckpointAt = now;
|
|
5686
|
+
},
|
|
5502
5687
|
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
5503
5688
|
lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
5504
5689
|
},
|
|
5505
5690
|
onModelRequestDiagnostic: (event) => {
|
|
5691
|
+
if (
|
|
5692
|
+
event.phase === "started" &&
|
|
5693
|
+
firstModelRequestPreparationStartedAt !== null &&
|
|
5694
|
+
!firstModelRequestPreparationRecorded
|
|
5695
|
+
) {
|
|
5696
|
+
firstModelRequestPreparationRecorded = true;
|
|
5697
|
+
recordTurnStartupPhase(observability, {
|
|
5698
|
+
phase: "model_request_preparation",
|
|
5699
|
+
provider: turnExecutionPolicy.providerId,
|
|
5700
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5701
|
+
outcome: "completed",
|
|
5702
|
+
durationSeconds:
|
|
5703
|
+
(performance.now() - firstModelRequestPreparationStartedAt) / 1_000,
|
|
5704
|
+
count: turnTools.length,
|
|
5705
|
+
});
|
|
5706
|
+
}
|
|
5506
5707
|
const phase =
|
|
5507
5708
|
event.phase === "headers"
|
|
5508
5709
|
? "headers"
|
|
@@ -5532,19 +5733,42 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5532
5733
|
if (!publish || !turnId) {
|
|
5533
5734
|
throw new Error("Codex model request started before the turn event producer");
|
|
5534
5735
|
}
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5736
|
+
const shouldRecordStartedAudit =
|
|
5737
|
+
event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
5738
|
+
if (shouldRecordStartedAudit) {
|
|
5739
|
+
firstModelRequestAuditRecorded = true;
|
|
5740
|
+
}
|
|
5741
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
5742
|
+
let auditOutcome: "completed" | "failed" = "completed";
|
|
5743
|
+
try {
|
|
5744
|
+
await publish([
|
|
5745
|
+
{
|
|
5746
|
+
type: "agent.model.request",
|
|
5747
|
+
payload: {
|
|
5748
|
+
...event,
|
|
5749
|
+
provider: "codex-subscription",
|
|
5750
|
+
turnId,
|
|
5751
|
+
attemptId: input.attemptId,
|
|
5752
|
+
dispatchId,
|
|
5753
|
+
executionGeneration,
|
|
5754
|
+
},
|
|
5545
5755
|
},
|
|
5546
|
-
|
|
5547
|
-
|
|
5756
|
+
]);
|
|
5757
|
+
} catch (error) {
|
|
5758
|
+
auditOutcome = "failed";
|
|
5759
|
+
throw error;
|
|
5760
|
+
} finally {
|
|
5761
|
+
if (auditStartedAt !== null) {
|
|
5762
|
+
recordTurnStartupPhase(observability, {
|
|
5763
|
+
phase: "model_request_audit",
|
|
5764
|
+
provider: turnExecutionPolicy.providerId,
|
|
5765
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
5766
|
+
outcome: auditOutcome,
|
|
5767
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1_000,
|
|
5768
|
+
count: turnTools.length,
|
|
5769
|
+
});
|
|
5770
|
+
}
|
|
5771
|
+
}
|
|
5548
5772
|
},
|
|
5549
5773
|
};
|
|
5550
5774
|
})()
|
|
@@ -5903,7 +6127,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5903
6127
|
availableMcpServerIds: runSettings.mcpServers.map((server) => server.id),
|
|
5904
6128
|
defaultMcpServerIds: defaultSessionMcpServerIds(capabilitySettings.mcpServers),
|
|
5905
6129
|
});
|
|
5906
|
-
const
|
|
6130
|
+
const mcpAvailabilityNote = unavailableMcpOperationalContext({
|
|
5907
6131
|
droppedIds: resolvedToolPolicy.effectivePolicy.droppedIds,
|
|
5908
6132
|
droppedCount: resolvedToolPolicy.effectivePolicy.counts.dropped,
|
|
5909
6133
|
});
|
|
@@ -6085,17 +6309,27 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6085
6309
|
cancellationSignal,
|
|
6086
6310
|
undefined,
|
|
6087
6311
|
);
|
|
6088
|
-
const
|
|
6089
|
-
lazyProvisionEnabled(settings)
|
|
6090
|
-
|
|
6091
|
-
runSettings.sandboxBackend
|
|
6312
|
+
const establishDecision = sandboxEstablishPolicyDecision({
|
|
6313
|
+
lazyEnabled: lazyProvisionEnabled(settings),
|
|
6314
|
+
machinePrimary,
|
|
6315
|
+
sandboxBackend: runSettings.sandboxBackend,
|
|
6092
6316
|
// Resolved run credentials must be written to one exact leased sandbox
|
|
6093
6317
|
// before agent execution. A warm active pointer can bypass the lazy
|
|
6094
6318
|
// provisioner entirely, which previously skipped materialization.
|
|
6095
|
-
|
|
6096
|
-
requiredGeneratedVideoFiles.length
|
|
6097
|
-
|
|
6098
|
-
|
|
6319
|
+
hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
|
|
6320
|
+
generatedVideoFileCount: requiredGeneratedVideoFiles.length,
|
|
6321
|
+
hasSignedFileResources:
|
|
6322
|
+
requiresSignedFileResourceDownloads(
|
|
6323
|
+
runSettings,
|
|
6324
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
6325
|
+
) && turnResources.some((resource) => resource.kind === "file"),
|
|
6326
|
+
});
|
|
6327
|
+
const establishPolicy = establishDecision.policy;
|
|
6328
|
+
recordTurnSandboxEstablishPolicy(observability, {
|
|
6329
|
+
policy: establishDecision.policy,
|
|
6330
|
+
reason: establishDecision.reason,
|
|
6331
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
6332
|
+
});
|
|
6099
6333
|
// Computed exactly ONCE per turn and reused for BOTH the box manifest
|
|
6100
6334
|
// (resumeBoxForTurn -> establishSandboxSessionFromEnvelope, below) AND the
|
|
6101
6335
|
// agent (runtime.buildAgent, below). sandboxEnvironmentForRun mints a FRESH
|
|
@@ -6118,6 +6352,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6118
6352
|
if (activeSandboxBackend !== "selfhosted") {
|
|
6119
6353
|
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
6120
6354
|
}
|
|
6355
|
+
await assertFileResourcesRemainAuthorized(
|
|
6356
|
+
db,
|
|
6357
|
+
input.accountId,
|
|
6358
|
+
input.workspaceId,
|
|
6359
|
+
fileAuthoritySubjectId,
|
|
6360
|
+
turnResources,
|
|
6361
|
+
);
|
|
6121
6362
|
const authorizeGitHubTokenMint: GitHubTokenMintAuthorization = async (selection) => {
|
|
6122
6363
|
await assertGitHubTokenMintSelectionAuthorized(
|
|
6123
6364
|
db,
|
|
@@ -6568,6 +6809,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6568
6809
|
// applies the agent's manifest to this provided session and throws on ANY
|
|
6569
6810
|
// variableSet delta (validateNoEnvironmentDelta). Passing sandboxEnvironment
|
|
6570
6811
|
// here makes current==target so the delta is empty.
|
|
6812
|
+
recordTurnStartupPhase(observability, {
|
|
6813
|
+
phase: "runtime_preparation",
|
|
6814
|
+
provider: turnExecutionPolicy.providerId,
|
|
6815
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
6816
|
+
outcome: "completed",
|
|
6817
|
+
durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1_000,
|
|
6818
|
+
});
|
|
6571
6819
|
if (
|
|
6572
6820
|
shouldEstablishSandboxForTurn(
|
|
6573
6821
|
settings.sandboxOwnershipEnabled,
|
|
@@ -6575,242 +6823,277 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6575
6823
|
machinePrimary,
|
|
6576
6824
|
)
|
|
6577
6825
|
) {
|
|
6578
|
-
const
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
// STAGE D honest-label guard: a machine-home session carries
|
|
6586
|
-
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
6587
|
-
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
6588
|
-
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
6589
|
-
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6590
|
-
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6591
|
-
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6592
|
-
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6593
|
-
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6594
|
-
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6595
|
-
if (machinePrimary) {
|
|
6596
|
-
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6597
|
-
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6598
|
-
// the machine route's own epoch fence; the managed-home lease separately
|
|
6599
|
-
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6600
|
-
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6601
|
-
// healthy explicitly selected machine.
|
|
6602
|
-
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6603
|
-
// (refreshed on every connect). Read only when the server flag is on —
|
|
6604
|
-
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6605
|
-
const machineOpStream =
|
|
6606
|
-
settings.agentOpStreamEnabled === true
|
|
6607
|
-
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6608
|
-
?.opStream === true
|
|
6609
|
-
: false;
|
|
6610
|
-
const established = await establishSelfhostedTurnSession(
|
|
6611
|
-
{
|
|
6612
|
-
db,
|
|
6613
|
-
settings,
|
|
6614
|
-
bus,
|
|
6615
|
-
onOp: machineOpObserver.observer,
|
|
6616
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6617
|
-
opJournal,
|
|
6618
|
-
},
|
|
6619
|
-
{
|
|
6620
|
-
workspaceId: input.workspaceId,
|
|
6621
|
-
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6622
|
-
opStream: machineOpStream,
|
|
6623
|
-
epoch: activeSandboxPointer!.activeEpoch,
|
|
6624
|
-
environment: sandboxEnvironment,
|
|
6625
|
-
...(transientCodemodeEnvironment
|
|
6626
|
-
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6627
|
-
: {}),
|
|
6628
|
-
workingDir: activeSandboxPointer!.workingDir,
|
|
6629
|
-
},
|
|
6826
|
+
const sandboxEstablishStartedAt = performance.now();
|
|
6827
|
+
let sandboxEstablishOutcome: "completed" | "failed" = "completed";
|
|
6828
|
+
try {
|
|
6829
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
6830
|
+
machinePrimary,
|
|
6831
|
+
input.attemptId,
|
|
6832
|
+
session.sandboxGroupId,
|
|
6630
6833
|
);
|
|
6631
|
-
|
|
6632
|
-
|
|
6633
|
-
|
|
6634
|
-
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6834
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
6835
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
6836
|
+
// STAGE D honest-label guard: a machine-home session carries
|
|
6837
|
+
// turn.sandboxBackend "selfhosted", but a turn is only machine-PRIMARY
|
|
6838
|
+
// when a live machine pointer resolves (activeSandboxBackend==='selfhosted'
|
|
6839
|
+
// + enrollmentId). When it is NOT primary — the agent swapped back to the
|
|
6840
|
+
// group box (sandbox_swap 'session'/'default'/groupId clears the pointer) or
|
|
6841
|
+
// selfhosted routing is flag-OFF (the pointer is ignored) — the else-branch
|
|
6842
|
+
// must resume a REAL cloud group box, not a "selfhosted" one: the registry
|
|
6843
|
+
// SelfhostedSandboxClient has no bound agentId and throws. Fall the group-box
|
|
6844
|
+
// backend back to the deployment default cloud backend so swap-away / flag-off
|
|
6845
|
+
// degrade to a genuine cloud box exactly like today (home=modal did).
|
|
6846
|
+
if (machinePrimary) {
|
|
6847
|
+
// STAGE D D1-lite: the active sandbox is a connected machine, so DO NOT
|
|
6848
|
+
// establish OR lease the managed home box. The active-sandbox pointer is
|
|
6849
|
+
// the machine route's own epoch fence; the managed-home lease separately
|
|
6850
|
+
// owns cloud recovery, snapshots, billing, and reaping. Mixing those two
|
|
6851
|
+
// ownership domains lets an unrelated degraded cloud archive block a
|
|
6852
|
+
// healthy explicitly selected machine.
|
|
6853
|
+
// Whether the machine's latest Hello advertised the op-stream engine
|
|
6854
|
+
// (refreshed on every connect). Read only when the server flag is on —
|
|
6855
|
+
// one indexed lookup, and the flag off keeps this path byte-identical.
|
|
6856
|
+
const machineOpStream =
|
|
6857
|
+
settings.agentOpStreamEnabled === true
|
|
6858
|
+
? (await getEnrollment(db, input.workspaceId, activeSandboxRecord!.enrollmentId!))
|
|
6859
|
+
?.opStream === true
|
|
6860
|
+
: false;
|
|
6861
|
+
const established = await establishSelfhostedTurnSession(
|
|
6643
6862
|
{
|
|
6644
6863
|
db,
|
|
6645
6864
|
settings,
|
|
6646
6865
|
bus,
|
|
6647
|
-
opJournal,
|
|
6648
6866
|
onOp: machineOpObserver.observer,
|
|
6649
6867
|
onSandboxOperation: sandboxOperationObserver,
|
|
6650
|
-
|
|
6651
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6868
|
+
opJournal,
|
|
6652
6869
|
},
|
|
6653
6870
|
{
|
|
6654
6871
|
workspaceId: input.workspaceId,
|
|
6655
|
-
|
|
6872
|
+
agentId: activeSandboxRecord!.enrollmentId!,
|
|
6873
|
+
opStream: machineOpStream,
|
|
6874
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6656
6875
|
environment: sandboxEnvironment,
|
|
6657
6876
|
...(transientCodemodeEnvironment
|
|
6658
6877
|
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6659
6878
|
: {}),
|
|
6660
|
-
|
|
6661
|
-
|
|
6662
|
-
|
|
6663
|
-
|
|
6664
|
-
|
|
6879
|
+
workingDir: activeSandboxPointer!.workingDir,
|
|
6880
|
+
},
|
|
6881
|
+
);
|
|
6882
|
+
// The machine-primary establish narrows `session` to SelfhostedSession
|
|
6883
|
+
// (buildSelfhostedBackendSession); EstablishedSandboxSession widens it.
|
|
6884
|
+
machinePrimarySession =
|
|
6885
|
+
established.session as import("@opengeni/runtime").SelfhostedSession;
|
|
6886
|
+
setupBoxSession = established.session;
|
|
6887
|
+
resolvedSandbox = {
|
|
6888
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
6889
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
6890
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
6891
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
6892
|
+
// instance — no two-instance manifest divergence.
|
|
6893
|
+
established: wrapTurnBoxWithRouting(
|
|
6894
|
+
{
|
|
6895
|
+
db,
|
|
6896
|
+
settings,
|
|
6897
|
+
bus,
|
|
6898
|
+
opJournal,
|
|
6899
|
+
onOp: machineOpObserver.observer,
|
|
6900
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
6901
|
+
onHomeSandboxRebound,
|
|
6902
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6665
6903
|
},
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6904
|
+
{
|
|
6905
|
+
workspaceId: input.workspaceId,
|
|
6906
|
+
sessionId: input.sessionId,
|
|
6907
|
+
environment: sandboxEnvironment,
|
|
6908
|
+
...(transientCodemodeEnvironment
|
|
6909
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6910
|
+
: {}),
|
|
6911
|
+
workspaceMutationFence: {
|
|
6912
|
+
accountId: input.accountId,
|
|
6913
|
+
turnId: turn.id,
|
|
6914
|
+
executionGeneration,
|
|
6915
|
+
attemptId: input.attemptId,
|
|
6916
|
+
},
|
|
6917
|
+
pinnedSelfhosted: {
|
|
6918
|
+
sandboxId: activeSandboxPointer!.activeSandboxId!,
|
|
6919
|
+
epoch: activeSandboxPointer!.activeEpoch,
|
|
6920
|
+
},
|
|
6921
|
+
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
6922
|
+
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
6923
|
+
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
6924
|
+
// session merely PINNED to a machine this turn never established its group
|
|
6925
|
+
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
6926
|
+
// (`home_unavailable_this_turn`) rather than silently serving the machine;
|
|
6927
|
+
// the detach takes effect next turn. (Lazy home-box establishment on such a
|
|
6928
|
+
// clear is a deferred follow-up; issue #341.)
|
|
6929
|
+
defaultIsHome: session.sandboxBackend === "selfhosted",
|
|
6669
6930
|
},
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
//
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
} else {
|
|
6694
|
-
resolvedSandbox = await waitForTurnOperation(
|
|
6695
|
-
resumeBoxForTurn(
|
|
6696
|
-
{
|
|
6697
|
-
db,
|
|
6698
|
-
settings: runSettings,
|
|
6699
|
-
logicalFallbackSettings: logicalSandboxSettings,
|
|
6700
|
-
cancellationSignal: sandboxResumeSignal,
|
|
6701
|
-
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6702
|
-
onSandboxLost: publishSandboxLost,
|
|
6703
|
-
},
|
|
6704
|
-
{
|
|
6705
|
-
accountId: input.accountId,
|
|
6706
|
-
workspaceId: input.workspaceId,
|
|
6707
|
-
sandboxGroupId: session.sandboxGroupId,
|
|
6708
|
-
sessionId: input.sessionId,
|
|
6709
|
-
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
6710
|
-
// is not machine-primary resumes a real cloud group box (the
|
|
6711
|
-
// deployment default), never a "selfhosted" box (which would throw
|
|
6712
|
-
// for lack of a bound agentId).
|
|
6713
|
-
backend: groupBoxBackend,
|
|
6714
|
-
os: session.sandboxOs,
|
|
6715
|
-
environment: sandboxEnvironment,
|
|
6716
|
-
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
6717
|
-
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
6718
|
-
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6719
|
-
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6720
|
-
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6721
|
-
// (establishSelfhostedTurnSession) NEVER passes
|
|
6722
|
-
// an image — B3 lives only on this Modal else-branch.
|
|
6723
|
-
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6724
|
-
? {
|
|
6725
|
-
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6726
|
-
}
|
|
6727
|
-
: {}),
|
|
6728
|
-
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
6729
|
-
// conflicts on a live shared box set up under a different rig (solo
|
|
6730
|
-
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6731
|
-
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6732
|
-
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6733
|
-
},
|
|
6734
|
-
"turn",
|
|
6735
|
-
managedOwnership!.holderId,
|
|
6736
|
-
),
|
|
6737
|
-
sandboxResumeSignal,
|
|
6738
|
-
releaseLateSandbox,
|
|
6739
|
-
);
|
|
6740
|
-
setupBoxSession = resolvedSandbox.established.session;
|
|
6741
|
-
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6742
|
-
// record every box transition in session_events so the NEXT box loss is
|
|
6743
|
-
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6744
|
-
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6745
|
-
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6746
|
-
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6747
|
-
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6748
|
-
// into the run. The SDK binds to this ONE object once and calls its
|
|
6749
|
-
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
6750
|
-
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
6751
|
-
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
6752
|
-
// the flag off the established group box is injected unchanged (today's
|
|
6753
|
-
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
6754
|
-
// routing veneer, not an owner.
|
|
6755
|
-
resolvedSandbox = {
|
|
6756
|
-
...resolvedSandbox,
|
|
6757
|
-
established: wrapTurnBoxWithRouting(
|
|
6758
|
-
{
|
|
6759
|
-
db,
|
|
6760
|
-
settings,
|
|
6761
|
-
bus,
|
|
6762
|
-
opJournal,
|
|
6763
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
6764
|
-
onHomeSandboxLost: publishSandboxLost,
|
|
6765
|
-
onHomeSandboxRebound,
|
|
6766
|
-
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
6767
|
-
},
|
|
6768
|
-
// Thread the SAME declared environment the group box was created with
|
|
6769
|
-
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
6770
|
-
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
6771
|
-
// (no "cannot change manifest environment variables" throw).
|
|
6772
|
-
{
|
|
6773
|
-
workspaceId: input.workspaceId,
|
|
6774
|
-
sessionId: input.sessionId,
|
|
6775
|
-
environment: sandboxEnvironment,
|
|
6776
|
-
...(transientCodemodeEnvironment
|
|
6777
|
-
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
6778
|
-
: {}),
|
|
6779
|
-
workspaceMutationFence: {
|
|
6780
|
-
accountId: input.accountId,
|
|
6781
|
-
turnId: turn.id,
|
|
6782
|
-
executionGeneration,
|
|
6783
|
-
attemptId: input.attemptId,
|
|
6931
|
+
established,
|
|
6932
|
+
),
|
|
6933
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
6934
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
6935
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
6936
|
+
leaseEpoch: activeSandboxPointer!.activeEpoch,
|
|
6937
|
+
release: async () => undefined,
|
|
6938
|
+
};
|
|
6939
|
+
} else if (establishPolicy === "on-demand") {
|
|
6940
|
+
// Lazy sandbox provisioning: holder/group ids are fixed at turn start,
|
|
6941
|
+
// but the lease acquire + box establish + setup move behind the routing
|
|
6942
|
+
// proxy's first default-pointer op. A chat-only turn never calls it, so
|
|
6943
|
+
// no lease row, no provider box, no warm-meter interval.
|
|
6944
|
+
} else {
|
|
6945
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
6946
|
+
resumeBoxForTurn(
|
|
6947
|
+
{
|
|
6948
|
+
db,
|
|
6949
|
+
settings: runSettings,
|
|
6950
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
6951
|
+
cancellationSignal: sandboxResumeSignal,
|
|
6952
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
6953
|
+
onSandboxLost: publishSandboxLost,
|
|
6784
6954
|
},
|
|
6785
|
-
|
|
6955
|
+
{
|
|
6786
6956
|
accountId: input.accountId,
|
|
6957
|
+
workspaceId: input.workspaceId,
|
|
6787
6958
|
sandboxGroupId: session.sandboxGroupId,
|
|
6788
|
-
|
|
6789
|
-
|
|
6959
|
+
sessionId: input.sessionId,
|
|
6960
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
6961
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
6962
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
6963
|
+
// for lack of a bound agentId).
|
|
6790
6964
|
backend: groupBoxBackend,
|
|
6965
|
+
os: session.sandboxOs,
|
|
6966
|
+
environment: sandboxEnvironment,
|
|
6967
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
6968
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
6969
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
6970
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
6971
|
+
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
6972
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
6973
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
6974
|
+
...((runSettings.modalImageRef ?? runSettings.dockerImage)
|
|
6975
|
+
? {
|
|
6976
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
6977
|
+
}
|
|
6978
|
+
: {}),
|
|
6979
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
6980
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
6981
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
6982
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
6983
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
6791
6984
|
},
|
|
6792
|
-
|
|
6793
|
-
|
|
6794
|
-
|
|
6795
|
-
|
|
6796
|
-
|
|
6797
|
-
|
|
6798
|
-
|
|
6985
|
+
"turn",
|
|
6986
|
+
managedOwnership!.holderId,
|
|
6987
|
+
),
|
|
6988
|
+
sandboxResumeSignal,
|
|
6989
|
+
releaseLateSandbox,
|
|
6990
|
+
);
|
|
6991
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
6992
|
+
// Durable box-lifecycle events (sandbox-file-persistence observability):
|
|
6993
|
+
// record every box transition in session_events so the NEXT box loss is
|
|
6994
|
+
// attributable from the DB alone — worker logs rotate within hours, which
|
|
6995
|
+
// left both 2026-07-06 incidents without a durable trace. Best-effort.
|
|
6996
|
+
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
6997
|
+
// M7 hot-swap: when the selfhosted feature is on, wrap the established
|
|
6998
|
+
// group box in the STABLE routing proxy before it is injected NON-OWNED
|
|
6999
|
+
// into the run. The SDK binds to this ONE object once and calls its
|
|
7000
|
+
// methods per tool call; the proxy re-reads (active_sandbox_id,
|
|
7001
|
+
// active_epoch) per op and dispatches to the currently-active backend, so
|
|
7002
|
+
// a sandbox_swap mid-turn lands the NEXT tool call on the new box. With
|
|
7003
|
+
// the flag off the established group box is injected unchanged (today's
|
|
7004
|
+
// path). The lease still owns the group box lifecycle — the proxy is a
|
|
7005
|
+
// routing veneer, not an owner.
|
|
7006
|
+
resolvedSandbox = {
|
|
7007
|
+
...resolvedSandbox,
|
|
7008
|
+
established: wrapTurnBoxWithRouting(
|
|
7009
|
+
{
|
|
7010
|
+
db,
|
|
7011
|
+
settings,
|
|
7012
|
+
bus,
|
|
7013
|
+
opJournal,
|
|
7014
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
7015
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
7016
|
+
onHomeSandboxRebound,
|
|
7017
|
+
...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
|
|
7018
|
+
},
|
|
7019
|
+
// Thread the SAME declared environment the group box was created with
|
|
7020
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
7021
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
7022
|
+
// (no "cannot change manifest environment variables" throw).
|
|
7023
|
+
{
|
|
7024
|
+
workspaceId: input.workspaceId,
|
|
7025
|
+
sessionId: input.sessionId,
|
|
7026
|
+
environment: sandboxEnvironment,
|
|
7027
|
+
...(transientCodemodeEnvironment
|
|
7028
|
+
? { transientExecEnvironment: transientCodemodeEnvironment }
|
|
7029
|
+
: {}),
|
|
7030
|
+
workspaceMutationFence: {
|
|
7031
|
+
accountId: input.accountId,
|
|
7032
|
+
turnId: turn.id,
|
|
7033
|
+
executionGeneration,
|
|
7034
|
+
attemptId: input.attemptId,
|
|
7035
|
+
},
|
|
7036
|
+
homeLease: {
|
|
7037
|
+
accountId: input.accountId,
|
|
7038
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
7039
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
7040
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
7041
|
+
backend: groupBoxBackend,
|
|
7042
|
+
},
|
|
7043
|
+
},
|
|
7044
|
+
resolvedSandbox.established,
|
|
7045
|
+
),
|
|
7046
|
+
};
|
|
7047
|
+
}
|
|
7048
|
+
if (resolvedSandbox) {
|
|
7049
|
+
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
7050
|
+
}
|
|
7051
|
+
} catch (error) {
|
|
7052
|
+
sandboxEstablishOutcome = "failed";
|
|
7053
|
+
throw error;
|
|
7054
|
+
} finally {
|
|
7055
|
+
recordTurnStartupPhase(observability, {
|
|
7056
|
+
phase: "sandbox_establish",
|
|
7057
|
+
provider: turnExecutionPolicy.providerId,
|
|
7058
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
7059
|
+
outcome: sandboxEstablishOutcome,
|
|
7060
|
+
durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1_000,
|
|
7061
|
+
});
|
|
6799
7062
|
}
|
|
6800
7063
|
}
|
|
6801
7064
|
|
|
6802
|
-
const ordinaryFileResourceDownloads = await
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
7065
|
+
const ordinaryFileResourceDownloads = await (async () => {
|
|
7066
|
+
const fileResolutionStartedAt = performance.now();
|
|
7067
|
+
let fileResolutionOutcome: "completed" | "failed" = "completed";
|
|
7068
|
+
try {
|
|
7069
|
+
return await waitForTurnOperation(
|
|
7070
|
+
sandboxFileDownloadsForRun(
|
|
7071
|
+
runSettings,
|
|
7072
|
+
db,
|
|
7073
|
+
objectStorage,
|
|
7074
|
+
input.accountId,
|
|
7075
|
+
input.workspaceId,
|
|
7076
|
+
fileAuthoritySubjectId,
|
|
7077
|
+
turnResources,
|
|
7078
|
+
activeSandboxBackend ?? groupBoxBackend,
|
|
7079
|
+
),
|
|
7080
|
+
cancellationSignal,
|
|
7081
|
+
undefined,
|
|
7082
|
+
);
|
|
7083
|
+
} catch (error) {
|
|
7084
|
+
fileResolutionOutcome = "failed";
|
|
7085
|
+
throw error;
|
|
7086
|
+
} finally {
|
|
7087
|
+
recordTurnStartupPhase(observability, {
|
|
7088
|
+
phase: "file_resolution",
|
|
7089
|
+
provider: turnExecutionPolicy.providerId,
|
|
7090
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7091
|
+
outcome: fileResolutionOutcome,
|
|
7092
|
+
durationSeconds: (performance.now() - fileResolutionStartedAt) / 1_000,
|
|
7093
|
+
count: turnResources.length,
|
|
7094
|
+
});
|
|
7095
|
+
}
|
|
7096
|
+
})();
|
|
6814
7097
|
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
6815
7098
|
throw new Error("Generated video materialization requires object storage");
|
|
6816
7099
|
}
|
|
@@ -6840,6 +7123,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6840
7123
|
]),
|
|
6841
7124
|
).values(),
|
|
6842
7125
|
];
|
|
7126
|
+
const toolContextPreparationStartedAt = performance.now();
|
|
6843
7127
|
throwIfWorkerShuttingDown();
|
|
6844
7128
|
throwIfTurnCancelled();
|
|
6845
7129
|
const mcpCredentialRootSessionId =
|
|
@@ -6880,6 +7164,37 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6880
7164
|
}
|
|
6881
7165
|
return result;
|
|
6882
7166
|
};
|
|
7167
|
+
const connectorActionIdentity = {
|
|
7168
|
+
accountId: input.accountId,
|
|
7169
|
+
workspaceId: input.workspaceId,
|
|
7170
|
+
sessionId: input.sessionId,
|
|
7171
|
+
turnId: turn.id,
|
|
7172
|
+
attemptId: input.attemptId,
|
|
7173
|
+
executionGeneration,
|
|
7174
|
+
initiator: {
|
|
7175
|
+
kind: turn.initiator.kind,
|
|
7176
|
+
subjectId: turn.initiator.subjectId,
|
|
7177
|
+
},
|
|
7178
|
+
} as const;
|
|
7179
|
+
const googleDrivePublicationTarget = objectStorage
|
|
7180
|
+
? await resolveGoogleDrivePublicationTarget(
|
|
7181
|
+
db,
|
|
7182
|
+
input.workspaceId,
|
|
7183
|
+
personalConnectionDelegations,
|
|
7184
|
+
)
|
|
7185
|
+
: null;
|
|
7186
|
+
const googleDrivePublicationTool =
|
|
7187
|
+
objectStorage && googleDrivePublicationTarget
|
|
7188
|
+
? createGoogleDrivePublicationAttemptTool({
|
|
7189
|
+
db,
|
|
7190
|
+
objectStorage,
|
|
7191
|
+
identity: connectorActionIdentity,
|
|
7192
|
+
subjectId: googleDrivePublicationTarget.ownerSubjectId,
|
|
7193
|
+
target: googleDrivePublicationTarget,
|
|
7194
|
+
resolveCredential,
|
|
7195
|
+
...(runtimeCancellationSignal ? { signal: runtimeCancellationSignal } : {}),
|
|
7196
|
+
})
|
|
7197
|
+
: null;
|
|
6883
7198
|
const publishToolAuthNeeded = async (payload: ToolAuthNeededPayload): Promise<void> => {
|
|
6884
7199
|
if (!shouldPublishToolAuthNeededForTurn(payload, trigger, turn)) {
|
|
6885
7200
|
return;
|
|
@@ -6938,59 +7253,140 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
6938
7253
|
runSettings,
|
|
6939
7254
|
session.firstPartyMcpTools,
|
|
6940
7255
|
);
|
|
6941
|
-
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
|
|
6945
|
-
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
|
|
6949
|
-
|
|
6950
|
-
|
|
6951
|
-
|
|
6952
|
-
|
|
6953
|
-
|
|
6954
|
-
|
|
6955
|
-
|
|
6956
|
-
|
|
6957
|
-
|
|
6958
|
-
|
|
6959
|
-
|
|
6960
|
-
|
|
7256
|
+
const googleDrivePublicationAllowed =
|
|
7257
|
+
selectedFirstPartyMcpTools.includes("editable_artifact_export") &&
|
|
7258
|
+
selectedFirstPartyMcpTools.includes("editable_artifact_export_status") &&
|
|
7259
|
+
(!session.firstPartyMcpPermissions?.length ||
|
|
7260
|
+
(session.firstPartyMcpPermissions.includes("artifacts:read") &&
|
|
7261
|
+
session.firstPartyMcpPermissions.includes("artifacts:publish")));
|
|
7262
|
+
const googleDriveConnectorBindings: readonly AttemptConnectorActionBinding[] =
|
|
7263
|
+
googleDrivePublicationTool && googleDrivePublicationTarget && googleDrivePublicationAllowed
|
|
7264
|
+
? [
|
|
7265
|
+
{
|
|
7266
|
+
modelName: googleDrivePublicationTool.modelName,
|
|
7267
|
+
call: (approvalId, arguments_) =>
|
|
7268
|
+
googleDrivePublicationConnectorCall(
|
|
7269
|
+
googleDrivePublicationTarget,
|
|
7270
|
+
arguments_,
|
|
7271
|
+
approvalId,
|
|
7272
|
+
),
|
|
7273
|
+
},
|
|
7274
|
+
]
|
|
7275
|
+
: [];
|
|
7276
|
+
const attemptToolDefinitions = [
|
|
7277
|
+
...createFirstPartyInteractionAttemptToolDefinitions({
|
|
7278
|
+
settings: runSettings,
|
|
7279
|
+
scope: {
|
|
7280
|
+
accountId: input.accountId,
|
|
7281
|
+
workspaceId: input.workspaceId,
|
|
7282
|
+
sessionId: input.sessionId,
|
|
7283
|
+
turnId: turn.id,
|
|
7284
|
+
attemptId: input.attemptId,
|
|
7285
|
+
executionGeneration,
|
|
6961
7286
|
},
|
|
6962
|
-
// Manager-style sessions carry a creation-validated permission set
|
|
6963
|
-
// for their first-party MCP token; null keeps the fixed default.
|
|
6964
7287
|
...(session.firstPartyMcpPermissions?.length
|
|
6965
|
-
? {
|
|
7288
|
+
? { permissions: session.firstPartyMcpPermissions }
|
|
6966
7289
|
: {}),
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
6977
|
-
|
|
6978
|
-
|
|
7290
|
+
selectedTools: selectedFirstPartyMcpTools,
|
|
7291
|
+
subjectId: "worker:first-party-mcp",
|
|
7292
|
+
subjectLabel: "OpenGeni worker",
|
|
7293
|
+
...(interactionInterventionResume
|
|
7294
|
+
? { interventionResume: interactionInterventionResume }
|
|
7295
|
+
: {}),
|
|
7296
|
+
}),
|
|
7297
|
+
...(googleDrivePublicationTool && googleDrivePublicationAllowed
|
|
7298
|
+
? [googleDrivePublicationTool]
|
|
7299
|
+
: []),
|
|
7300
|
+
];
|
|
7301
|
+
recordTurnStartupPhase(observability, {
|
|
7302
|
+
phase: "tool_context_preparation",
|
|
7303
|
+
provider: turnExecutionPolicy.providerId,
|
|
7304
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7305
|
+
outcome: "completed",
|
|
7306
|
+
durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1_000,
|
|
7307
|
+
count: turnTools.length,
|
|
7308
|
+
});
|
|
7309
|
+
const toolPreparationStartedAt = performance.now();
|
|
7310
|
+
let toolPreparationOutcome: "completed" | "failed" = "completed";
|
|
7311
|
+
try {
|
|
7312
|
+
preparedTools = await waitForTurnOperation(
|
|
7313
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
7314
|
+
accountId: input.accountId,
|
|
7315
|
+
workspaceId: input.workspaceId,
|
|
7316
|
+
sessionId: input.sessionId,
|
|
7317
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
7318
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
7319
|
+
// live active pointer.
|
|
7320
|
+
...(turnId ? { turnId } : {}),
|
|
7321
|
+
attemptId: input.attemptId,
|
|
7322
|
+
executionGeneration,
|
|
7323
|
+
subjectId: "worker:first-party-mcp",
|
|
7324
|
+
subjectLabel: "OpenGeni worker",
|
|
7325
|
+
...(credentialSubjectId ? { credentialSubjectId } : {}),
|
|
7326
|
+
...(codexAppsAuth ? { codexAppsAuth } : {}),
|
|
7327
|
+
resolveCredential,
|
|
7328
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
7329
|
+
localMcpServers,
|
|
7330
|
+
onPreparationPhase: (measurement) => {
|
|
7331
|
+
recordTurnStartupPhase(observability, {
|
|
7332
|
+
phase: `tool_${measurement.phase}`,
|
|
7333
|
+
provider: turnExecutionPolicy.providerId,
|
|
7334
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7335
|
+
outcome: measurement.outcome,
|
|
7336
|
+
durationSeconds: measurement.durationSeconds,
|
|
7337
|
+
count: turnTools.length,
|
|
7338
|
+
});
|
|
7339
|
+
},
|
|
7340
|
+
onAttemptToolCatalog: async (catalog) => {
|
|
7341
|
+
await persistAttemptToolCatalog(db, catalog);
|
|
6979
7342
|
},
|
|
7343
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
7344
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
6980
7345
|
...(session.firstPartyMcpPermissions?.length
|
|
6981
|
-
? {
|
|
7346
|
+
? { firstPartyPermissions: session.firstPartyMcpPermissions }
|
|
6982
7347
|
: {}),
|
|
6983
|
-
|
|
6984
|
-
|
|
6985
|
-
|
|
6986
|
-
|
|
6987
|
-
|
|
7348
|
+
firstPartyTools: selectedFirstPartyMcpTools,
|
|
7349
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
7350
|
+
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
7351
|
+
attemptToolDefinitions,
|
|
7352
|
+
...(googleDrivePublicationTarget && googleDrivePublicationAllowed
|
|
7353
|
+
? {
|
|
7354
|
+
attemptToolAuthorize: async ({ call }) => {
|
|
7355
|
+
if (
|
|
7356
|
+
call.caller.kind !== "codemode" ||
|
|
7357
|
+
call.identity.serverId !== "google-drive-publishing" ||
|
|
7358
|
+
call.identity.toolName !== "google_drive_publish_file"
|
|
7359
|
+
) {
|
|
7360
|
+
return;
|
|
7361
|
+
}
|
|
7362
|
+
await authorizeGoogleDrivePublicationAttempt({
|
|
7363
|
+
db,
|
|
7364
|
+
identity: connectorActionIdentity,
|
|
7365
|
+
target: googleDrivePublicationTarget,
|
|
7366
|
+
approvalId: call.operationId,
|
|
7367
|
+
arguments: call.arguments,
|
|
7368
|
+
});
|
|
7369
|
+
},
|
|
7370
|
+
}
|
|
6988
7371
|
: {}),
|
|
6989
7372
|
}),
|
|
6990
|
-
|
|
6991
|
-
|
|
6992
|
-
|
|
6993
|
-
)
|
|
7373
|
+
cancellationSignal,
|
|
7374
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
7375
|
+
);
|
|
7376
|
+
} catch (error) {
|
|
7377
|
+
toolPreparationOutcome = "failed";
|
|
7378
|
+
throw error;
|
|
7379
|
+
} finally {
|
|
7380
|
+
recordTurnStartupPhase(observability, {
|
|
7381
|
+
phase: "tool_preparation",
|
|
7382
|
+
provider: turnExecutionPolicy.providerId,
|
|
7383
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7384
|
+
outcome: toolPreparationOutcome,
|
|
7385
|
+
durationSeconds: (performance.now() - toolPreparationStartedAt) / 1_000,
|
|
7386
|
+
count: turnTools.length,
|
|
7387
|
+
});
|
|
7388
|
+
}
|
|
7389
|
+
const postToolPreparationStartedAt = performance.now();
|
|
6994
7390
|
if (turnId && preparedTools.attemptToolEnvironment) {
|
|
6995
7391
|
codemodeDispatcher = new CodemodeAttemptDispatcher(
|
|
6996
7392
|
db,
|
|
@@ -7052,7 +7448,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7052
7448
|
await resolveImageGenerationReferences({
|
|
7053
7449
|
db,
|
|
7054
7450
|
objectStorage: objectStorage!,
|
|
7451
|
+
accountId: input.accountId,
|
|
7055
7452
|
workspaceId: input.workspaceId,
|
|
7453
|
+
subjectId: fileAuthoritySubjectId,
|
|
7056
7454
|
references,
|
|
7057
7455
|
readSandboxFile: async (path, maxBytes) => {
|
|
7058
7456
|
const imageReferenceSession = (setupBoxSession ??
|
|
@@ -7362,18 +7760,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7362
7760
|
turnExecutionPolicy.providerId,
|
|
7363
7761
|
turnExecutionPolicy.latencyMode,
|
|
7364
7762
|
);
|
|
7365
|
-
const connectorActionIdentity = {
|
|
7366
|
-
accountId: input.accountId,
|
|
7367
|
-
workspaceId: input.workspaceId,
|
|
7368
|
-
sessionId: input.sessionId,
|
|
7369
|
-
turnId,
|
|
7370
|
-
attemptId: input.attemptId,
|
|
7371
|
-
executionGeneration,
|
|
7372
|
-
initiator: {
|
|
7373
|
-
kind: turn.initiator.kind,
|
|
7374
|
-
subjectId: turn.initiator.subjectId,
|
|
7375
|
-
},
|
|
7376
|
-
} as const;
|
|
7377
7763
|
const connectorActionPolicy: ConnectorActionPolicyHooks = {
|
|
7378
7764
|
prepare: async (call) =>
|
|
7379
7765
|
await prepareConnectorActionApproval(db, connectorActionIdentity, call),
|
|
@@ -7391,190 +7777,207 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7391
7777
|
const runtimeSkillActivations = [
|
|
7392
7778
|
...installedSkillRuntime.activations,
|
|
7393
7779
|
...packRuntime.skillActivations,
|
|
7394
|
-
...session.skills.map((skill) => ({
|
|
7395
|
-
source: "session" as const,
|
|
7396
|
-
id: `session:${session.id}:${skill.name}`,
|
|
7397
|
-
artifact: {
|
|
7398
|
-
name: skill.name,
|
|
7399
|
-
description: skill.description ?? null,
|
|
7400
|
-
files: skill.files.map((file) => ({
|
|
7401
|
-
path: file.path,
|
|
7402
|
-
content: file.content,
|
|
7403
|
-
})),
|
|
7404
|
-
},
|
|
7405
|
-
reason: "attached to session",
|
|
7406
|
-
})),
|
|
7407
|
-
];
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
genesisTitleHint: isGenesisTurn,
|
|
7415
|
-
persistentSessionSettings: {
|
|
7416
|
-
titleIsSet: Boolean(session.title?.trim()),
|
|
7417
|
-
},
|
|
7418
|
-
sandboxEnvironment,
|
|
7419
|
-
...(preparedTools.attemptToolCatalog
|
|
7420
|
-
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
7421
|
-
: {}),
|
|
7422
|
-
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
7423
|
-
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
7424
|
-
onToolCancellationFence: (fence) => {
|
|
7425
|
-
toolCancellationFenceRef.current = fence;
|
|
7426
|
-
},
|
|
7427
|
-
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
7428
|
-
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
7429
|
-
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
7430
|
-
// AND the mint actually produced a token (repo resources present). The runtime
|
|
7431
|
-
// seeds it to the box's token file before the repository-clone runs; it never
|
|
7432
|
-
// touches the box/agent manifest env.
|
|
7433
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
7434
|
-
? { gitTokenSeeds: sandboxGitTokens }
|
|
7435
|
-
: {}),
|
|
7436
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
7437
|
-
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
7438
|
-
: {}),
|
|
7439
|
-
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
7440
|
-
? { gitTokenSeed: sandboxGitToken }
|
|
7441
|
-
: {}),
|
|
7442
|
-
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7443
|
-
// Managed boxes receive the bearer through their protected per-session
|
|
7444
|
-
// token file. Connected Machines use transient per-exec delivery above,
|
|
7445
|
-
// so they must not run the file-seeding lifecycle hook.
|
|
7446
|
-
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
7447
|
-
? {
|
|
7448
|
-
codemodeTokenSeed: sandboxCodemodeToken,
|
|
7449
|
-
codemodeTokenSessionId: input.sessionId,
|
|
7450
|
-
}
|
|
7451
|
-
: {}),
|
|
7452
|
-
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
7453
|
-
fileResourceDownloads,
|
|
7454
|
-
mcpServers: preparedTools.mcpServers,
|
|
7455
|
-
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
7456
|
-
connectorActionPolicy,
|
|
7457
|
-
// LIVE by-reference connector namespaces (fills during this turn's
|
|
7458
|
-
// codex_apps tools/list): the codex tool_search description reads it per
|
|
7459
|
-
// model call so the model sees the account's real connected sources.
|
|
7460
|
-
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
7461
|
-
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
7462
|
-
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
7463
|
-
// NOT a Model instance: an instance only survives the in-process
|
|
7464
|
-
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
7465
|
-
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
7466
|
-
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
7467
|
-
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
7468
|
-
// client. The gating still comes from the resolved provider: server-side
|
|
7469
|
-
// store/compaction follow the provider's compaction mode (registry
|
|
7470
|
-
// providers resolve to "client"); encrypted reasoning is only
|
|
7471
|
-
// round-tripped on the Responses wire API; hosted web search is attached
|
|
7472
|
-
// whenever the provider declares it runnable and is independent of the
|
|
7473
|
-
// session's MCP allow-list; the effective context window drives the
|
|
7474
|
-
// compaction threshold.
|
|
7475
|
-
hostedWebSearch,
|
|
7476
|
-
...imageGenerationOption,
|
|
7477
|
-
...videoGenerationOption,
|
|
7478
|
-
lazyToolTransport,
|
|
7479
|
-
supportsImageInput,
|
|
7480
|
-
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
7481
|
-
...(resolvedModel
|
|
7482
|
-
? {
|
|
7483
|
-
encryptedReasoning:
|
|
7484
|
-
resolvedModel.provider.api === "responses" &&
|
|
7485
|
-
runSettings.openaiReasoningEncryptedContent,
|
|
7486
|
-
contextWindowTokens:
|
|
7487
|
-
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
7488
|
-
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
7489
|
-
// tool. Gateway Responses routes likewise expose ordinary function
|
|
7490
|
-
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
7491
|
-
// function apply_patch and wrap successful view_image results as
|
|
7492
|
-
// typed input_image content. Chat wires have no proven typed image
|
|
7493
|
-
// result transport and therefore receive no view_image tool.
|
|
7494
|
-
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
7495
|
-
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
7496
|
-
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
7497
|
-
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
7498
|
-
// constructor name. See {@link computerToolModeForTurn}.
|
|
7499
|
-
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
7500
|
-
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
7501
|
-
}
|
|
7502
|
-
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
7503
|
-
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
7504
|
-
// client). That backend has real hosted support, so pin computerToolMode to
|
|
7505
|
-
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
7506
|
-
{
|
|
7507
|
-
computerToolMode: computerToolModeForTurn(null),
|
|
7508
|
-
promptCacheKey: input.sessionId,
|
|
7509
|
-
}),
|
|
7510
|
-
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
7511
|
-
// selects a computer tool, then this hook begins the optional proof
|
|
7512
|
-
// recording. Shell/filesystem turns never invoke either operation.
|
|
7513
|
-
onComputerUseReady: async () => {
|
|
7514
|
-
if (!resolvedSandbox) {
|
|
7515
|
-
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
7516
|
-
}
|
|
7517
|
-
// This callback is the authoritative execution boundary. Record the
|
|
7518
|
-
// action before async ffmpeg startup so transport-event ordering cannot
|
|
7519
|
-
// make settlement misclassify a real computer turn as unused.
|
|
7520
|
-
didComputerUse = true;
|
|
7521
|
-
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
7522
|
-
},
|
|
7523
|
-
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
7524
|
-
...(runtimeSkillActivations.length > 0
|
|
7525
|
-
? { skillActivations: runtimeSkillActivations }
|
|
7526
|
-
: {}),
|
|
7527
|
-
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
7528
|
-
? { instructionsTemplate: workspaceAgentInstructions }
|
|
7529
|
-
: {}),
|
|
7530
|
-
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
7531
|
-
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7532
|
-
goalSnapshot: turn.goalSnapshot,
|
|
7533
|
-
// Per-session persona tier (session > workspace > deployment default).
|
|
7534
|
-
// Composed system-level AFTER the workspace persona so it refines it for
|
|
7535
|
-
// this one session; absent ⇒ byte-identical to today's composition.
|
|
7536
|
-
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
7537
|
-
// Exact host context captured when this turn was accepted. It is
|
|
7538
|
-
// system-level and disappears with the turn rather than entering chat.
|
|
7539
|
-
...(turn.turnInstructions || mcpAvailabilityInstructions
|
|
7540
|
-
? {
|
|
7541
|
-
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions]
|
|
7542
|
-
.filter((value): value is string => Boolean(value))
|
|
7543
|
-
.join(" "),
|
|
7544
|
-
}
|
|
7545
|
-
: {}),
|
|
7546
|
-
...workspaceEnvironmentOption,
|
|
7547
|
-
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
7548
|
-
// the frozen version carries a non-empty script), and the rig credential
|
|
7549
|
-
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
7550
|
-
...(rigVersion && rigName
|
|
7551
|
-
? {
|
|
7552
|
-
rig: { name: rigName, version: rigVersion.version },
|
|
7553
|
-
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
7554
|
-
? {
|
|
7555
|
-
rigSetup: {
|
|
7556
|
-
rigId: session.rigId!,
|
|
7557
|
-
versionId: rigVersion.id,
|
|
7558
|
-
rigName,
|
|
7559
|
-
script: rigVersion.setupScript,
|
|
7560
|
-
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
7561
|
-
contentHash: rigProviderImageContentHash({
|
|
7562
|
-
backend: turn.sandboxBackend,
|
|
7563
|
-
sourceImage: rigProviderImageSourceImage(
|
|
7564
|
-
logicalSandboxSettings,
|
|
7565
|
-
turn.sandboxBackend,
|
|
7566
|
-
),
|
|
7567
|
-
definition: rigVersion,
|
|
7568
|
-
}),
|
|
7569
|
-
},
|
|
7570
|
-
}
|
|
7571
|
-
: {}),
|
|
7572
|
-
...(rigVersion.credentialHooks.length > 0
|
|
7573
|
-
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
7574
|
-
: {}),
|
|
7575
|
-
}
|
|
7576
|
-
: {}),
|
|
7780
|
+
...session.skills.map((skill) => ({
|
|
7781
|
+
source: "session" as const,
|
|
7782
|
+
id: `session:${session.id}:${skill.name}`,
|
|
7783
|
+
artifact: {
|
|
7784
|
+
name: skill.name,
|
|
7785
|
+
description: skill.description ?? null,
|
|
7786
|
+
files: skill.files.map((file) => ({
|
|
7787
|
+
path: file.path,
|
|
7788
|
+
content: file.content,
|
|
7789
|
+
})),
|
|
7790
|
+
},
|
|
7791
|
+
reason: "attached to session",
|
|
7792
|
+
})),
|
|
7793
|
+
];
|
|
7794
|
+
recordTurnStartupPhase(observability, {
|
|
7795
|
+
phase: "post_tool_preparation",
|
|
7796
|
+
provider: turnExecutionPolicy.providerId,
|
|
7797
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7798
|
+
outcome: "completed",
|
|
7799
|
+
durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1_000,
|
|
7577
7800
|
});
|
|
7801
|
+
const agent = (() => {
|
|
7802
|
+
const agentConstructionStartedAt = performance.now();
|
|
7803
|
+
let agentConstructionOutcome: "completed" | "failed" = "completed";
|
|
7804
|
+
try {
|
|
7805
|
+
return runtime.buildAgent(modelRunSettings, turnResources, {
|
|
7806
|
+
reasoningEffort: turn.reasoningEffort,
|
|
7807
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
7808
|
+
...(serviceTier ? { serviceTier } : {}),
|
|
7809
|
+
...(humanInputResume ? { humanInputResponse: humanInputResume } : {}),
|
|
7810
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
7811
|
+
genesisTitleHint: isGenesisTurn,
|
|
7812
|
+
persistentSessionSettings: {
|
|
7813
|
+
titleIsSet: Boolean(session.title?.trim()),
|
|
7814
|
+
},
|
|
7815
|
+
sandboxEnvironment,
|
|
7816
|
+
...(preparedTools.attemptToolCatalog
|
|
7817
|
+
? { attemptToolCatalog: preparedTools.attemptToolCatalog }
|
|
7818
|
+
: {}),
|
|
7819
|
+
...(sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {}),
|
|
7820
|
+
...(cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {}),
|
|
7821
|
+
onToolCancellationFence: (fence) => {
|
|
7822
|
+
toolCancellationFenceRef.current = fence;
|
|
7823
|
+
},
|
|
7824
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
7825
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
7826
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
7827
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
7828
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
7829
|
+
// touches the box/agent manifest env.
|
|
7830
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitTokens
|
|
7831
|
+
? { gitTokenSeeds: sandboxGitTokens }
|
|
7832
|
+
: {}),
|
|
7833
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings
|
|
7834
|
+
? { gitCredentialBindings: sandboxGitCredentialBindings }
|
|
7835
|
+
: {}),
|
|
7836
|
+
...(activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken
|
|
7837
|
+
? { gitTokenSeed: sandboxGitToken }
|
|
7838
|
+
: {}),
|
|
7839
|
+
...(sandboxCodemodeToken ? { codemodeAvailable: true } : {}),
|
|
7840
|
+
// Managed boxes receive the bearer through their protected per-session
|
|
7841
|
+
// token file. Connected Machines use transient per-exec delivery above,
|
|
7842
|
+
// so they must not run the file-seeding lifecycle hook.
|
|
7843
|
+
...(activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken
|
|
7844
|
+
? {
|
|
7845
|
+
codemodeTokenSeed: sandboxCodemodeToken,
|
|
7846
|
+
codemodeTokenSessionId: input.sessionId,
|
|
7847
|
+
}
|
|
7848
|
+
: {}),
|
|
7849
|
+
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
7850
|
+
fileResourceDownloads,
|
|
7851
|
+
mcpServers: preparedTools.mcpServers,
|
|
7852
|
+
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
7853
|
+
connectorActionPolicy,
|
|
7854
|
+
attemptConnectorActionBindings: googleDriveConnectorBindings,
|
|
7855
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
7856
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
7857
|
+
// model call so the model sees the account's real connected sources.
|
|
7858
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
7859
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
7860
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
7861
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
7862
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
7863
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
7864
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
7865
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
7866
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
7867
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
7868
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
7869
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
7870
|
+
// whenever the provider declares it runnable and is independent of the
|
|
7871
|
+
// session's MCP allow-list; the effective context window drives the
|
|
7872
|
+
// compaction threshold.
|
|
7873
|
+
hostedWebSearch,
|
|
7874
|
+
...imageGenerationOption,
|
|
7875
|
+
...videoGenerationOption,
|
|
7876
|
+
lazyToolTransport,
|
|
7877
|
+
supportsImageInput,
|
|
7878
|
+
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
7879
|
+
...(resolvedModel
|
|
7880
|
+
? {
|
|
7881
|
+
encryptedReasoning:
|
|
7882
|
+
resolvedModel.provider.api === "responses" &&
|
|
7883
|
+
runSettings.openaiReasoningEncryptedContent,
|
|
7884
|
+
contextWindowTokens:
|
|
7885
|
+
resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
7886
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
7887
|
+
// tool. Gateway Responses routes likewise expose ordinary function
|
|
7888
|
+
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
7889
|
+
// function apply_patch and wrap successful view_image results as
|
|
7890
|
+
// typed input_image content. Chat wires have no proven typed image
|
|
7891
|
+
// result transport and therefore receive no view_image tool.
|
|
7892
|
+
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
7893
|
+
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
7894
|
+
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
7895
|
+
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
7896
|
+
// constructor name. See {@link computerToolModeForTurn}.
|
|
7897
|
+
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
7898
|
+
...(promptCacheKey ? { promptCacheKey } : {}),
|
|
7899
|
+
}
|
|
7900
|
+
: // LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
7901
|
+
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
7902
|
+
// client). That backend has real hosted support, so pin computerToolMode to
|
|
7903
|
+
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
7904
|
+
{
|
|
7905
|
+
computerToolMode: computerToolModeForTurn(null),
|
|
7906
|
+
promptCacheKey: input.sessionId,
|
|
7907
|
+
}),
|
|
7908
|
+
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
7909
|
+
// selects a computer tool, then this hook begins the optional proof
|
|
7910
|
+
// recording. Shell/filesystem turns never invoke either operation.
|
|
7911
|
+
onComputerUseReady: async () => {
|
|
7912
|
+
if (!resolvedSandbox) {
|
|
7913
|
+
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
7914
|
+
}
|
|
7915
|
+
// This callback is the authoritative execution boundary. Record the
|
|
7916
|
+
// action before async ffmpeg startup so transport-event ordering cannot
|
|
7917
|
+
// make settlement misclassify a real computer turn as unused.
|
|
7918
|
+
didComputerUse = true;
|
|
7919
|
+
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
7920
|
+
},
|
|
7921
|
+
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
7922
|
+
...(runtimeSkillActivations.length > 0
|
|
7923
|
+
? { skillActivations: runtimeSkillActivations }
|
|
7924
|
+
: {}),
|
|
7925
|
+
...(!structuredWorkspacePolicyActive && workspaceAgentInstructions
|
|
7926
|
+
? { instructionsTemplate: workspaceAgentInstructions }
|
|
7927
|
+
: {}),
|
|
7928
|
+
...(workspaceGovernance ? { workspaceGovernance } : {}),
|
|
7929
|
+
...(workspaceMemory ? { workspaceMemory } : {}),
|
|
7930
|
+
goalSnapshot: turn.goalSnapshot,
|
|
7931
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
7932
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
7933
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
7934
|
+
...(session.instructions ? { sessionInstructions: session.instructions } : {}),
|
|
7935
|
+
...workspaceEnvironmentOption,
|
|
7936
|
+
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
7937
|
+
// the frozen version carries a non-empty script), and the rig credential
|
|
7938
|
+
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
7939
|
+
...(rigVersion && rigName
|
|
7940
|
+
? {
|
|
7941
|
+
rig: { name: rigName, version: rigVersion.version },
|
|
7942
|
+
...(rigVersion.setupScript && rigVersion.setupScript.trim().length > 0
|
|
7943
|
+
? {
|
|
7944
|
+
rigSetup: {
|
|
7945
|
+
rigId: session.rigId!,
|
|
7946
|
+
versionId: rigVersion.id,
|
|
7947
|
+
rigName,
|
|
7948
|
+
script: rigVersion.setupScript,
|
|
7949
|
+
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
7950
|
+
contentHash: rigProviderImageContentHash({
|
|
7951
|
+
backend: turn.sandboxBackend,
|
|
7952
|
+
sourceImage: rigProviderImageSourceImage(
|
|
7953
|
+
logicalSandboxSettings,
|
|
7954
|
+
turn.sandboxBackend,
|
|
7955
|
+
),
|
|
7956
|
+
definition: rigVersion,
|
|
7957
|
+
}),
|
|
7958
|
+
},
|
|
7959
|
+
}
|
|
7960
|
+
: {}),
|
|
7961
|
+
...(rigVersion.credentialHooks.length > 0
|
|
7962
|
+
? { rigCredentialHookIds: rigVersion.credentialHooks }
|
|
7963
|
+
: {}),
|
|
7964
|
+
}
|
|
7965
|
+
: {}),
|
|
7966
|
+
});
|
|
7967
|
+
} catch (error) {
|
|
7968
|
+
agentConstructionOutcome = "failed";
|
|
7969
|
+
throw error;
|
|
7970
|
+
} finally {
|
|
7971
|
+
recordTurnStartupPhase(observability, {
|
|
7972
|
+
phase: "agent_construction",
|
|
7973
|
+
provider: turnExecutionPolicy.providerId,
|
|
7974
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
7975
|
+
outcome: agentConstructionOutcome,
|
|
7976
|
+
durationSeconds: (performance.now() - agentConstructionStartedAt) / 1_000,
|
|
7977
|
+
});
|
|
7978
|
+
}
|
|
7979
|
+
})();
|
|
7980
|
+
const postAgentPreparationStartedAt = performance.now();
|
|
7578
7981
|
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
7579
7982
|
throw new Error(
|
|
7580
7983
|
"Sandbox agent construction did not install the mandatory turn tool cancellation fence",
|
|
@@ -7629,6 +8032,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
7629
8032
|
image: runSettings.modalImageRef ?? runSettings.dockerImage,
|
|
7630
8033
|
}
|
|
7631
8034
|
: {}),
|
|
8035
|
+
// The lazy acquire must enforce the same frozen rig authority
|
|
8036
|
+
// as the eager acquire; otherwise a warm box for another rig
|
|
8037
|
+
// can bypass the shared-state conflict/rotation fence.
|
|
8038
|
+
...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
|
|
7632
8039
|
},
|
|
7633
8040
|
"turn",
|
|
7634
8041
|
lazyHolderId,
|
|
@@ -8083,69 +8490,109 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8083
8490
|
return claimedResult({ status: "idle" });
|
|
8084
8491
|
}
|
|
8085
8492
|
}
|
|
8493
|
+
recordTurnStartupPhase(observability, {
|
|
8494
|
+
phase: "post_agent_preparation",
|
|
8495
|
+
provider: turnExecutionPolicy.providerId,
|
|
8496
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8497
|
+
outcome: "completed",
|
|
8498
|
+
durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1_000,
|
|
8499
|
+
});
|
|
8086
8500
|
let fileMaterializationFailures: SandboxFileDownloadFailure[] = [];
|
|
8087
8501
|
let fileDownloadsMaterializedForRun = false;
|
|
8088
8502
|
if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
|
|
8089
|
-
const
|
|
8090
|
-
|
|
8091
|
-
// memoize successful downloads. A connected machine is user-owned: files
|
|
8092
|
-
// can be changed or removed between turns, so verify/materialize every
|
|
8093
|
-
// attached file each turn instead of trusting the managed-box cache.
|
|
8503
|
+
const fileMaterializationStartedAt = performance.now();
|
|
8504
|
+
let fileMaterializationOutcome: "completed" | "failed" = "completed";
|
|
8094
8505
|
const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
|
|
8095
|
-
|
|
8096
|
-
?
|
|
8097
|
-
|
|
8098
|
-
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
|
|
8104
|
-
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
downloadsToMaterialize,
|
|
8117
|
-
{
|
|
8118
|
-
onRuntimeEvent: async (event) => {
|
|
8119
|
-
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8120
|
-
},
|
|
8121
|
-
...(runAs ? { runAs } : {}),
|
|
8122
|
-
...(toolCancellationFenceRef.current
|
|
8123
|
-
? {
|
|
8124
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8125
|
-
toolCancellationFenceRef.current,
|
|
8126
|
-
),
|
|
8127
|
-
}
|
|
8128
|
-
: {}),
|
|
8129
|
-
},
|
|
8130
|
-
),
|
|
8506
|
+
let fileMaterializationCache: "hit" | "miss" | "disabled" = cacheMaterialization
|
|
8507
|
+
? "miss"
|
|
8508
|
+
: "disabled";
|
|
8509
|
+
try {
|
|
8510
|
+
const boxInstanceId = resolvedSandbox.established.instanceId;
|
|
8511
|
+
// Managed boxes are immutable platform state, so their durable lease can
|
|
8512
|
+
// memoize successful downloads. A connected machine is user-owned: files
|
|
8513
|
+
// can be changed or removed between turns, so verify/materialize every
|
|
8514
|
+
// attached file each turn instead of trusting the managed-box cache.
|
|
8515
|
+
const alreadyMaterialized = cacheMaterialization
|
|
8516
|
+
? await getMaterializedSandboxFileResources(db, {
|
|
8517
|
+
accountId: input.accountId,
|
|
8518
|
+
workspaceId: input.workspaceId,
|
|
8519
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8520
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8521
|
+
instanceId: boxInstanceId,
|
|
8522
|
+
})
|
|
8523
|
+
: new Set<string>();
|
|
8524
|
+
const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
|
|
8525
|
+
fileResourceDownloads,
|
|
8526
|
+
alreadyMaterialized,
|
|
8131
8527
|
);
|
|
8132
|
-
|
|
8133
|
-
|
|
8134
|
-
|
|
8135
|
-
|
|
8136
|
-
|
|
8137
|
-
|
|
8138
|
-
|
|
8139
|
-
|
|
8140
|
-
|
|
8141
|
-
|
|
8142
|
-
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8528
|
+
if (cacheMaterialization && downloadsToMaterialize.length === 0) {
|
|
8529
|
+
fileMaterializationCache = "hit";
|
|
8530
|
+
}
|
|
8531
|
+
const runAs = sandboxRunAs(runSettings);
|
|
8532
|
+
if (downloadsToMaterialize.length > 0) {
|
|
8533
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
8534
|
+
resolvedSandbox,
|
|
8535
|
+
"fileMaterialization",
|
|
8536
|
+
async () =>
|
|
8537
|
+
await materializeSandboxFileDownloads(
|
|
8538
|
+
setupBoxSession as CodemodeTokenWriterSession,
|
|
8539
|
+
downloadsToMaterialize,
|
|
8540
|
+
{
|
|
8541
|
+
onRuntimeEvent: async (event) => {
|
|
8542
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8543
|
+
},
|
|
8544
|
+
...(runAs ? { runAs } : {}),
|
|
8545
|
+
...(toolCancellationFenceRef.current
|
|
8546
|
+
? {
|
|
8547
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8548
|
+
toolCancellationFenceRef.current,
|
|
8549
|
+
),
|
|
8550
|
+
}
|
|
8551
|
+
: {}),
|
|
8552
|
+
},
|
|
8553
|
+
),
|
|
8554
|
+
);
|
|
8555
|
+
fileMaterializationFailures = materialized.failures;
|
|
8556
|
+
fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
|
|
8557
|
+
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
8558
|
+
const succeededFileIds = downloadsToMaterialize
|
|
8559
|
+
.map((download) => download.fileId)
|
|
8560
|
+
.filter((fileId) => !failedFileIds.has(fileId));
|
|
8561
|
+
if (cacheMaterialization && succeededFileIds.length > 0) {
|
|
8562
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
8563
|
+
accountId: input.accountId,
|
|
8564
|
+
workspaceId: input.workspaceId,
|
|
8565
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
8566
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
8567
|
+
instanceId: boxInstanceId,
|
|
8568
|
+
fileIds: succeededFileIds,
|
|
8569
|
+
});
|
|
8570
|
+
}
|
|
8146
8571
|
}
|
|
8572
|
+
fileDownloadsMaterializedForRun = true;
|
|
8573
|
+
} catch (error) {
|
|
8574
|
+
fileMaterializationOutcome = "failed";
|
|
8575
|
+
throw error;
|
|
8576
|
+
} finally {
|
|
8577
|
+
recordTurnStartupPhase(observability, {
|
|
8578
|
+
phase: "file_materialization",
|
|
8579
|
+
provider: turnExecutionPolicy.providerId,
|
|
8580
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8581
|
+
outcome: fileMaterializationOutcome,
|
|
8582
|
+
durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1_000,
|
|
8583
|
+
count: fileResourceDownloads.length,
|
|
8584
|
+
cache: fileMaterializationCache,
|
|
8585
|
+
});
|
|
8147
8586
|
}
|
|
8148
|
-
|
|
8587
|
+
} else {
|
|
8588
|
+
recordTurnStartupPhase(observability, {
|
|
8589
|
+
phase: "file_materialization",
|
|
8590
|
+
provider: turnExecutionPolicy.providerId,
|
|
8591
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8592
|
+
outcome: "completed",
|
|
8593
|
+
durationSeconds: 0,
|
|
8594
|
+
count: 0,
|
|
8595
|
+
});
|
|
8149
8596
|
}
|
|
8150
8597
|
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
8151
8598
|
const failedRequiredVideo = fileMaterializationFailures.find((failure) =>
|
|
@@ -8190,52 +8637,115 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8190
8637
|
}
|
|
8191
8638
|
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
8192
8639
|
const prepareRunAttemptInput = async () => {
|
|
8193
|
-
const
|
|
8194
|
-
|
|
8195
|
-
|
|
8196
|
-
|
|
8197
|
-
|
|
8198
|
-
|
|
8199
|
-
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
|
|
8203
|
-
|
|
8204
|
-
|
|
8205
|
-
|
|
8640
|
+
const historyPreparationStartedAt = performance.now();
|
|
8641
|
+
let historyPreparationOutcome: "completed" | "failed" = "completed";
|
|
8642
|
+
let preparedHistoryCount: number | null = null;
|
|
8643
|
+
try {
|
|
8644
|
+
const prepared = await turnInput(db, runtime, agent, trigger, {
|
|
8645
|
+
turnId: activeTurnId,
|
|
8646
|
+
fileAuthority: {
|
|
8647
|
+
accountId: input.accountId,
|
|
8648
|
+
subjectId: fileAuthoritySubjectId,
|
|
8649
|
+
},
|
|
8650
|
+
recovering: turn.executionGeneration > 1,
|
|
8651
|
+
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
8652
|
+
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
8653
|
+
...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
|
|
8654
|
+
providerApi,
|
|
8655
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
8656
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
8657
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
8658
|
+
projectModelHistory: modelHistoryProjector,
|
|
8659
|
+
onPreparationPhase: (measurement) => {
|
|
8660
|
+
recordTurnStartupPhase(observability, {
|
|
8661
|
+
phase: `history_${measurement.phase}`,
|
|
8662
|
+
provider: turnExecutionPolicy.providerId,
|
|
8663
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8664
|
+
outcome: measurement.outcome,
|
|
8665
|
+
durationSeconds: measurement.durationSeconds,
|
|
8666
|
+
});
|
|
8667
|
+
},
|
|
8668
|
+
});
|
|
8669
|
+
const generatedImageMaterializationStartedAt = performance.now();
|
|
8670
|
+
let generatedImageMaterializationOutcome: "completed" | "failed" = "completed";
|
|
8671
|
+
try {
|
|
8672
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8673
|
+
await materializeGeneratedImage(receipt);
|
|
8674
|
+
}
|
|
8675
|
+
} catch (error) {
|
|
8676
|
+
generatedImageMaterializationOutcome = "failed";
|
|
8677
|
+
throw error;
|
|
8678
|
+
} finally {
|
|
8679
|
+
recordTurnStartupPhase(observability, {
|
|
8680
|
+
phase: "history_generated_image_materialization",
|
|
8681
|
+
provider: turnExecutionPolicy.providerId,
|
|
8682
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8683
|
+
outcome: generatedImageMaterializationOutcome,
|
|
8684
|
+
durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1_000,
|
|
8685
|
+
count: generatedImageReceiptsByArtifactId.size,
|
|
8686
|
+
});
|
|
8687
|
+
}
|
|
8688
|
+
runInput = prepared.input;
|
|
8689
|
+
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8690
|
+
// Slice index = the length of the model-facing (active) history this turn
|
|
8691
|
+
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
8692
|
+
// generated items) are the ones to persist. After a compaction this is the
|
|
8693
|
+
// short [summary, ...tail] active set, NOT the total row count. The
|
|
8694
|
+
// absolute write position is tracked separately (next whole number past
|
|
8695
|
+
// the max existing position) because the fractional summary row means
|
|
8696
|
+
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
8697
|
+
// the old total-count value, so the common path is unchanged.
|
|
8698
|
+
//
|
|
8699
|
+
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
8700
|
+
// count. `prepareRunInput` builds `state.history` from
|
|
8701
|
+
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
8702
|
+
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
8703
|
+
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
8704
|
+
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
8705
|
+
// the raw count (K too high) skips K genuinely-new items, and a
|
|
8706
|
+
// `function_call` left in that skipped region can later have its
|
|
8707
|
+
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
8708
|
+
// and bricks the session (issue-61). The repaired seed is already
|
|
8709
|
+
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
8710
|
+
// slice begins exactly at the first genuinely-new item.
|
|
8711
|
+
// prepareInput already sanitized the exact durable prefix represented
|
|
8712
|
+
// by state.history. Carry its count forward instead of loading and
|
|
8713
|
+
// retaining the full active transcript a second time beside runInput.
|
|
8714
|
+
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
8715
|
+
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
8716
|
+
const historyPositionStartedAt = performance.now();
|
|
8717
|
+
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
8718
|
+
try {
|
|
8719
|
+
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
8720
|
+
db,
|
|
8721
|
+
input.workspaceId,
|
|
8722
|
+
input.sessionId,
|
|
8723
|
+
);
|
|
8724
|
+
} catch (error) {
|
|
8725
|
+
historyPositionOutcome = "failed";
|
|
8726
|
+
throw error;
|
|
8727
|
+
} finally {
|
|
8728
|
+
recordTurnStartupPhase(observability, {
|
|
8729
|
+
phase: "history_position_load",
|
|
8730
|
+
provider: turnExecutionPolicy.providerId,
|
|
8731
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8732
|
+
outcome: historyPositionOutcome,
|
|
8733
|
+
durationSeconds: (performance.now() - historyPositionStartedAt) / 1_000,
|
|
8734
|
+
});
|
|
8735
|
+
}
|
|
8736
|
+
} catch (error) {
|
|
8737
|
+
historyPreparationOutcome = "failed";
|
|
8738
|
+
throw error;
|
|
8739
|
+
} finally {
|
|
8740
|
+
recordTurnStartupPhase(observability, {
|
|
8741
|
+
phase: "history_preparation",
|
|
8742
|
+
provider: turnExecutionPolicy.providerId,
|
|
8743
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8744
|
+
outcome: historyPreparationOutcome,
|
|
8745
|
+
durationSeconds: (performance.now() - historyPreparationStartedAt) / 1_000,
|
|
8746
|
+
count: preparedHistoryCount,
|
|
8747
|
+
});
|
|
8206
8748
|
}
|
|
8207
|
-
runInput = prepared.input;
|
|
8208
|
-
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
8209
|
-
// Slice index = the length of the model-facing (active) history this turn
|
|
8210
|
-
// is seeded from; new items beyond it (the trigger message + this turn's
|
|
8211
|
-
// generated items) are the ones to persist. After a compaction this is the
|
|
8212
|
-
// short [summary, ...tail] active set, NOT the total row count. The
|
|
8213
|
-
// absolute write position is tracked separately (next whole number past
|
|
8214
|
-
// the max existing position) because the fractional summary row means
|
|
8215
|
-
// total rows no longer equal max(position)+1. Pre-compaction both reduce to
|
|
8216
|
-
// the old total-count value, so the common path is unchanged.
|
|
8217
|
-
//
|
|
8218
|
-
// CRITICAL: seed from the structurally repaired active-row length, not the raw active
|
|
8219
|
-
// count. `prepareRunInput` builds `state.history` from
|
|
8220
|
-
// `sanitizeHistoryItemsForModel(activeRows)`, so when sanitization drops K
|
|
8221
|
-
// rows (a legacy orphan/dangling pair), the in-memory history this turn
|
|
8222
|
-
// starts from is K shorter than the raw row count. The reconcile slices the
|
|
8223
|
-
// repaired `state.history` off `persistedHistoryCount`; seeding it from
|
|
8224
|
-
// the raw count (K too high) skips K genuinely-new items, and a
|
|
8225
|
-
// `function_call` left in that skipped region can later have its
|
|
8226
|
-
// `function_call_result` persisted alone — the orphan that 400s on replay
|
|
8227
|
-
// and bricks the session (issue-61). The repaired seed is already
|
|
8228
|
-
// orphan-free, so it is a stable prefix of the re-sanitized history and the
|
|
8229
|
-
// slice begins exactly at the first genuinely-new item.
|
|
8230
|
-
// prepareInput already sanitized the exact durable prefix represented
|
|
8231
|
-
// by state.history. Carry its count forward instead of loading and
|
|
8232
|
-
// retaining the full active transcript a second time beside runInput.
|
|
8233
|
-
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
8234
|
-
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
8235
|
-
db,
|
|
8236
|
-
input.workspaceId,
|
|
8237
|
-
input.sessionId,
|
|
8238
|
-
);
|
|
8239
8749
|
};
|
|
8240
8750
|
|
|
8241
8751
|
const forceContextCompaction = async (
|
|
@@ -8287,6 +8797,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8287
8797
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
8288
8798
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
8289
8799
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
8800
|
+
let workerPreparationTotalRecorded = false;
|
|
8290
8801
|
const runStreamAttempt = async (): Promise<RunAgentTurnResult> => {
|
|
8291
8802
|
if (!runInput) {
|
|
8292
8803
|
throw new Error("Run input was not prepared");
|
|
@@ -8313,139 +8824,203 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8313
8824
|
// heartbeat capture. Lazy setup already runs under the same wrapper in
|
|
8314
8825
|
// its first-operation provisioner above.
|
|
8315
8826
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
8316
|
-
const
|
|
8317
|
-
|
|
8318
|
-
|
|
8319
|
-
|
|
8320
|
-
|
|
8321
|
-
|
|
8322
|
-
|
|
8323
|
-
|
|
8324
|
-
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8337
|
-
|
|
8338
|
-
|
|
8339
|
-
|
|
8340
|
-
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
|
|
8344
|
-
|
|
8345
|
-
|
|
8346
|
-
|
|
8347
|
-
|
|
8348
|
-
|
|
8349
|
-
|
|
8350
|
-
|
|
8351
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8352
|
-
toolCancellationFenceRef.current,
|
|
8353
|
-
),
|
|
8827
|
+
const ownedSandboxSetupStartedAt = performance.now();
|
|
8828
|
+
let ownedSandboxSetupOutcome: "completed" | "failed" = "completed";
|
|
8829
|
+
try {
|
|
8830
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
8831
|
+
// `deferredSetup: true` below tells the runtime that the worker owns
|
|
8832
|
+
// platform setup, so the runtime intentionally skips its credential
|
|
8833
|
+
// session callback. Materialize host-managed run credentials here
|
|
8834
|
+
// before any setup command (including provider login hooks), then
|
|
8835
|
+
// decorate setup commands so they source the active generation.
|
|
8836
|
+
await attachRunCredentialRenewal(
|
|
8837
|
+
eagerSetupSession as RunCredentialCommandSession,
|
|
8838
|
+
resolvedSandbox,
|
|
8839
|
+
);
|
|
8840
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial
|
|
8841
|
+
? withRunCredentialsSession(eagerSetupSession as object, input.sessionId)
|
|
8842
|
+
: eagerSetupSession;
|
|
8843
|
+
await runWorkspaceMutationForSandbox(
|
|
8844
|
+
resolvedSandbox,
|
|
8845
|
+
"eagerOwnedSandboxSetup",
|
|
8846
|
+
async () =>
|
|
8847
|
+
await runOwnedSandboxSetup(
|
|
8848
|
+
agent,
|
|
8849
|
+
ownedEstablished.session as never,
|
|
8850
|
+
eagerCredentialSetupSession as never,
|
|
8851
|
+
{
|
|
8852
|
+
settings: runSettings,
|
|
8853
|
+
environment: sandboxEnvironment,
|
|
8854
|
+
preparedInput: runInput!,
|
|
8855
|
+
...(fileDownloadsMaterializedForRun
|
|
8856
|
+
? { fileDownloadsMaterialized: true }
|
|
8857
|
+
: {}),
|
|
8858
|
+
onRuntimeEvent: async (event) => {
|
|
8859
|
+
await renewServingCredentialLease("runtime_event");
|
|
8860
|
+
if (servingCredentialLeaseLost()) {
|
|
8861
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
8354
8862
|
}
|
|
8355
|
-
|
|
8356
|
-
|
|
8357
|
-
|
|
8358
|
-
|
|
8359
|
-
|
|
8360
|
-
|
|
8361
|
-
|
|
8362
|
-
|
|
8863
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8864
|
+
},
|
|
8865
|
+
...(toolCancellationFenceRef.current
|
|
8866
|
+
? {
|
|
8867
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
8868
|
+
toolCancellationFenceRef.current,
|
|
8869
|
+
),
|
|
8870
|
+
}
|
|
8871
|
+
: {}),
|
|
8872
|
+
},
|
|
8873
|
+
),
|
|
8874
|
+
);
|
|
8875
|
+
await attachGitCredentialRenewal(
|
|
8876
|
+
eagerSetupSession as GitCredentialTokenWriterSession,
|
|
8877
|
+
initialGitCredentials,
|
|
8878
|
+
);
|
|
8879
|
+
} catch (error) {
|
|
8880
|
+
ownedSandboxSetupOutcome = "failed";
|
|
8881
|
+
throw error;
|
|
8882
|
+
} finally {
|
|
8883
|
+
recordTurnStartupPhase(observability, {
|
|
8884
|
+
phase: "owned_sandbox_setup",
|
|
8885
|
+
provider: turnExecutionPolicy.providerId,
|
|
8886
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8887
|
+
outcome: ownedSandboxSetupOutcome,
|
|
8888
|
+
durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1_000,
|
|
8889
|
+
count: fileResourceDownloads.length,
|
|
8890
|
+
});
|
|
8891
|
+
}
|
|
8363
8892
|
}
|
|
8364
8893
|
// Conservative request boundary: once control enters runStream, the
|
|
8365
8894
|
// provider may have accepted work even if no response/event follows.
|
|
8366
8895
|
// Escaped MCP setup timeouts are automatically recoverable only
|
|
8367
8896
|
// before this line.
|
|
8368
8897
|
recordCompanyBrainContributionReceiptOnce();
|
|
8369
|
-
|
|
8370
|
-
|
|
8371
|
-
|
|
8372
|
-
|
|
8373
|
-
|
|
8374
|
-
|
|
8375
|
-
|
|
8376
|
-
|
|
8377
|
-
|
|
8378
|
-
|
|
8379
|
-
|
|
8380
|
-
|
|
8381
|
-
|
|
8382
|
-
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
|
|
8898
|
+
const providerDispatchStartedAt = performance.now();
|
|
8899
|
+
let providerDispatchOutcome: "completed" | "failed" = "completed";
|
|
8900
|
+
try {
|
|
8901
|
+
// This histogram describes worker preparation until the first entry
|
|
8902
|
+
// into the runtime. Lazy SDK request preparation and the durable
|
|
8903
|
+
// model-request audit happen after this boundary and have their own
|
|
8904
|
+
// phase metrics. Context-compaction recovery re-enters
|
|
8905
|
+
// runStreamAttempt; recording again would fold the prior request and
|
|
8906
|
+
// compaction into a bogus second startup sample.
|
|
8907
|
+
if (!workerPreparationTotalRecorded) {
|
|
8908
|
+
workerPreparationTotalRecorded = true;
|
|
8909
|
+
recordTurnWorkerPreparationTotal(observability, {
|
|
8910
|
+
provider: turnExecutionPolicy.providerId,
|
|
8911
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
8912
|
+
outcome: "completed",
|
|
8913
|
+
durationSeconds: (performance.now() - workerPreparationStartedAt) / 1_000,
|
|
8914
|
+
});
|
|
8915
|
+
}
|
|
8916
|
+
modelRequestStarted = true;
|
|
8917
|
+
return await runtime.runStream(agent, runInput!, modelRunSettings, {
|
|
8918
|
+
signal: runtimeCancellationSignal,
|
|
8919
|
+
sandboxEnvironment,
|
|
8920
|
+
onRuntimeEvent: async (event) => {
|
|
8921
|
+
await renewServingCredentialLease("runtime_event");
|
|
8922
|
+
if (servingCredentialLeaseLost()) {
|
|
8923
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
8924
|
+
}
|
|
8925
|
+
await publish!([{ type: event.type, payload: event.payload }], true);
|
|
8926
|
+
},
|
|
8927
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
8928
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
8929
|
+
...(ownedEstablished
|
|
8930
|
+
? {
|
|
8931
|
+
ownedSandbox: {
|
|
8932
|
+
client: ownedEstablished.client,
|
|
8933
|
+
session: ownedEstablished.session,
|
|
8934
|
+
...(resolvedSandbox?.established.sessionState
|
|
8935
|
+
? {
|
|
8936
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
8937
|
+
}
|
|
8938
|
+
: {}),
|
|
8939
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
8940
|
+
// established box — never through the routing proxy, which would
|
|
8941
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
8942
|
+
...(setupBoxSession ? { setupSession: setupBoxSession } : {}),
|
|
8943
|
+
...(fileDownloadsMaterializedForRun
|
|
8944
|
+
? { fileDownloadsMaterialized: true }
|
|
8945
|
+
: {}),
|
|
8946
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
8947
|
+
// above under its exact admission, lazy in the provisioner.
|
|
8948
|
+
deferredSetup: true,
|
|
8949
|
+
},
|
|
8950
|
+
}
|
|
8951
|
+
: {}),
|
|
8952
|
+
...(activeSandboxBackend !== "selfhosted" &&
|
|
8953
|
+
sandboxCodemodeToken &&
|
|
8954
|
+
sandboxCodemodeTokenExpiresAt &&
|
|
8955
|
+
!lazyOwnedSandbox
|
|
8956
|
+
? {
|
|
8957
|
+
onCodemodeTokenSessionReady: async (
|
|
8958
|
+
tokenSession: CodemodeTokenWriterSession,
|
|
8959
|
+
) => {
|
|
8960
|
+
const renewalSession =
|
|
8961
|
+
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
8962
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
8963
|
+
},
|
|
8964
|
+
}
|
|
8965
|
+
: {}),
|
|
8966
|
+
...(runCredentialResolver
|
|
8967
|
+
? {
|
|
8968
|
+
runCredentialSessionId: input.sessionId,
|
|
8969
|
+
...(!lazyOwnedSandbox
|
|
8388
8970
|
? {
|
|
8389
|
-
|
|
8971
|
+
onRunCredentialSessionReady: async (
|
|
8972
|
+
credentialSession: RunCredentialCommandSession,
|
|
8973
|
+
) => {
|
|
8974
|
+
const pinnedCredentialSession = setupBoxSession
|
|
8975
|
+
? (setupBoxSession as RunCredentialCommandSession)
|
|
8976
|
+
: credentialSession;
|
|
8977
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
8978
|
+
},
|
|
8390
8979
|
}
|
|
8391
8980
|
: {}),
|
|
8392
|
-
|
|
8393
|
-
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
|
|
8398
|
-
|
|
8399
|
-
|
|
8400
|
-
},
|
|
8401
|
-
}
|
|
8402
|
-
: {}),
|
|
8403
|
-
...(activeSandboxBackend !== "selfhosted" &&
|
|
8404
|
-
sandboxCodemodeToken &&
|
|
8405
|
-
sandboxCodemodeTokenExpiresAt &&
|
|
8406
|
-
!lazyOwnedSandbox
|
|
8407
|
-
? {
|
|
8408
|
-
onCodemodeTokenSessionReady: async (tokenSession: CodemodeTokenWriterSession) => {
|
|
8409
|
-
const renewalSession =
|
|
8410
|
-
(setupBoxSession as CodemodeTokenWriterSession | null) ?? tokenSession;
|
|
8411
|
-
await attachCodemodeTokenRenewal(renewalSession);
|
|
8412
|
-
},
|
|
8413
|
-
}
|
|
8414
|
-
: {}),
|
|
8415
|
-
...(runCredentialResolver
|
|
8416
|
-
? {
|
|
8417
|
-
runCredentialSessionId: input.sessionId,
|
|
8418
|
-
...(!lazyOwnedSandbox
|
|
8419
|
-
? {
|
|
8420
|
-
onRunCredentialSessionReady: async (
|
|
8421
|
-
credentialSession: RunCredentialCommandSession,
|
|
8422
|
-
) => {
|
|
8423
|
-
const pinnedCredentialSession = setupBoxSession
|
|
8424
|
-
? (setupBoxSession as RunCredentialCommandSession)
|
|
8425
|
-
: credentialSession;
|
|
8426
|
-
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
8427
|
-
},
|
|
8981
|
+
}
|
|
8982
|
+
: {}),
|
|
8983
|
+
...(modelRunSettings.sandboxBackend !== "none"
|
|
8984
|
+
? {
|
|
8985
|
+
onSandboxSessionReady: async (sandboxSession: CodemodeTokenWriterSession) => {
|
|
8986
|
+
sdkOwnedSandboxSession = sandboxSession;
|
|
8987
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
8988
|
+
await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
8428
8989
|
}
|
|
8429
|
-
|
|
8430
|
-
|
|
8431
|
-
|
|
8432
|
-
|
|
8433
|
-
|
|
8434
|
-
|
|
8435
|
-
|
|
8436
|
-
|
|
8437
|
-
|
|
8438
|
-
|
|
8439
|
-
|
|
8440
|
-
|
|
8441
|
-
|
|
8442
|
-
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8448
|
-
|
|
8990
|
+
},
|
|
8991
|
+
}
|
|
8992
|
+
: {}),
|
|
8993
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
8994
|
+
contextCompactionRequested: () =>
|
|
8995
|
+
isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
8996
|
+
onModelPreparationPhase: (measurement) => {
|
|
8997
|
+
recordTurnStartupPhase(observability, {
|
|
8998
|
+
phase: `model_prepare_${measurement.phase}`,
|
|
8999
|
+
provider: turnExecutionPolicy.providerId,
|
|
9000
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
9001
|
+
outcome: measurement.outcome,
|
|
9002
|
+
durationSeconds: measurement.durationSeconds,
|
|
9003
|
+
...(measurement.count === undefined ? {} : { count: measurement.count }),
|
|
9004
|
+
});
|
|
9005
|
+
},
|
|
9006
|
+
...(toolCancellationFenceRef.current
|
|
9007
|
+
? {
|
|
9008
|
+
turnToolCancellationFence: toolCancellationFenceRef.current,
|
|
9009
|
+
}
|
|
9010
|
+
: {}),
|
|
9011
|
+
});
|
|
9012
|
+
} catch (error) {
|
|
9013
|
+
providerDispatchOutcome = "failed";
|
|
9014
|
+
throw error;
|
|
9015
|
+
} finally {
|
|
9016
|
+
recordTurnStartupPhase(observability, {
|
|
9017
|
+
phase: "provider_dispatch",
|
|
9018
|
+
provider: turnExecutionPolicy.providerId,
|
|
9019
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
9020
|
+
outcome: providerDispatchOutcome,
|
|
9021
|
+
durationSeconds: (performance.now() - providerDispatchStartedAt) / 1_000,
|
|
9022
|
+
});
|
|
9023
|
+
}
|
|
8449
9024
|
};
|
|
8450
9025
|
if (codexLeaseLost) {
|
|
8451
9026
|
throw new Error("Codex credential lease expired before the model run");
|
|
@@ -8471,11 +9046,30 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
8471
9046
|
},
|
|
8472
9047
|
);
|
|
8473
9048
|
|
|
9049
|
+
const streamBootstrapStartedAt = performance.now();
|
|
9050
|
+
let streamBootstrapRecorded = false;
|
|
9051
|
+
const recordStreamBootstrap = (): void => {
|
|
9052
|
+
if (streamBootstrapRecorded) return;
|
|
9053
|
+
streamBootstrapRecorded = true;
|
|
9054
|
+
recordTurnStartupPhase(observability, {
|
|
9055
|
+
phase: "stream_bootstrap",
|
|
9056
|
+
provider: turnExecutionPolicy.providerId,
|
|
9057
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
9058
|
+
outcome: "completed",
|
|
9059
|
+
durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1_000,
|
|
9060
|
+
count: turnTools.length,
|
|
9061
|
+
});
|
|
9062
|
+
};
|
|
9063
|
+
if (!firstModelRequestPreparationRecorded) {
|
|
9064
|
+
firstModelRequestPreparationStartedAt = performance.now();
|
|
9065
|
+
firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
|
|
9066
|
+
}
|
|
8474
9067
|
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
8475
9068
|
let streamDone = false;
|
|
8476
9069
|
try {
|
|
8477
9070
|
while (true) {
|
|
8478
9071
|
const next = await nextStreamEvent(iterator, activityContext);
|
|
9072
|
+
recordStreamBootstrap();
|
|
8479
9073
|
if (next.done) {
|
|
8480
9074
|
streamDone = true;
|
|
8481
9075
|
break;
|
|
@@ -10437,7 +11031,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
10437
11031
|
}
|
|
10438
11032
|
activityStatus = "failed";
|
|
10439
11033
|
activityError = error;
|
|
10440
|
-
if (!
|
|
11034
|
+
if (!turnId) {
|
|
11035
|
+
throw preClaimAdmissionFailure(error);
|
|
11036
|
+
}
|
|
11037
|
+
if (!publish || !turnStartedPublished) {
|
|
10441
11038
|
throw error;
|
|
10442
11039
|
}
|
|
10443
11040
|
// A partial/malformed stream may have emitted assistant/tool items (and
|
|
@@ -11081,6 +11678,28 @@ async function assertGitHubResourcesRemainAuthorized(
|
|
|
11081
11678
|
}
|
|
11082
11679
|
}
|
|
11083
11680
|
|
|
11681
|
+
async function assertFileResourcesRemainAuthorized(
|
|
11682
|
+
db: ActivityServices["db"],
|
|
11683
|
+
accountId: string,
|
|
11684
|
+
workspaceId: string,
|
|
11685
|
+
subjectId: string | null,
|
|
11686
|
+
resources: ResourceRef[],
|
|
11687
|
+
): Promise<void> {
|
|
11688
|
+
const fileIds = resources.flatMap((resource) =>
|
|
11689
|
+
resource.kind === "file" ? [resource.fileId] : [],
|
|
11690
|
+
);
|
|
11691
|
+
if (fileIds.length === 0) return;
|
|
11692
|
+
const authorized = await getFilesForSubject(db, {
|
|
11693
|
+
accountId,
|
|
11694
|
+
workspaceId,
|
|
11695
|
+
subjectId,
|
|
11696
|
+
fileIds,
|
|
11697
|
+
});
|
|
11698
|
+
if (authorized.length !== new Set(fileIds).size) {
|
|
11699
|
+
throw new Error("One or more file resources are unavailable or no longer authorized");
|
|
11700
|
+
}
|
|
11701
|
+
}
|
|
11702
|
+
|
|
11084
11703
|
async function assertGitHubTokenMintSelectionAuthorized(
|
|
11085
11704
|
db: Parameters<typeof areGitHubRepositoriesAllowedForWorkspace>[0],
|
|
11086
11705
|
workspaceId: string,
|
|
@@ -11994,7 +12613,9 @@ async function sandboxFileDownloadsForRun(
|
|
|
11994
12613
|
settings: Settings,
|
|
11995
12614
|
db: ActivityServices["db"],
|
|
11996
12615
|
objectStorage: ObjectStorage | null,
|
|
12616
|
+
accountId: string,
|
|
11997
12617
|
workspaceId: string,
|
|
12618
|
+
subjectId: string | null,
|
|
11998
12619
|
resources: ResourceRef[],
|
|
11999
12620
|
activeSandboxBackend: Settings["sandboxBackend"] = settings.sandboxBackend,
|
|
12000
12621
|
): Promise<SandboxFileDownload[]> {
|
|
@@ -12015,7 +12636,12 @@ async function sandboxFileDownloadsForRun(
|
|
|
12015
12636
|
const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
|
|
12016
12637
|
const downloads: SandboxFileDownload[] = [];
|
|
12017
12638
|
for (const resource of fileResources) {
|
|
12018
|
-
const file = await
|
|
12639
|
+
const file = await requireFileForSubject(db, {
|
|
12640
|
+
accountId,
|
|
12641
|
+
workspaceId,
|
|
12642
|
+
subjectId,
|
|
12643
|
+
fileId: resource.fileId,
|
|
12644
|
+
});
|
|
12019
12645
|
const url = await downloadStorage.createGetUrl({ key: file.objectKey });
|
|
12020
12646
|
downloads.push({
|
|
12021
12647
|
fileId: file.id,
|