@opengeni/worker-bundle 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +24 -1
- package/dist/activities/run-input.d.ts +8 -0
- package/dist/activities/session-state.d.ts +5 -3
- package/dist/activities/types.d.ts +27 -0
- package/dist/{activities-control-XURE4VKN.js → activities-control-SMKF2OBE.js} +54 -7
- package/dist/{activities-control-XURE4VKN.js.map → activities-control-SMKF2OBE.js.map} +1 -1
- package/dist/activities-control.d.ts +2 -2
- package/dist/{activities-turn-6EBXGGLS.js → activities-turn-IDS3WEHA.js} +1471 -975
- package/dist/activities-turn-IDS3WEHA.js.map +1 -0
- package/dist/activities.d.ts +3 -3
- package/dist/{chunk-TYNPCG4U.js → chunk-N5TJIFCE.js} +69 -1
- package/dist/chunk-N5TJIFCE.js.map +1 -0
- package/dist/{chunk-5GRJIP5H.js → chunk-P6Z2SLAA.js} +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/observability-metrics.d.ts +37 -0
- package/dist/workflow-bundle.js +174 -4
- package/dist/workflows/session.d.ts +17 -1
- package/package.json +11 -11
- package/src/activities/agent-turn.ts +1675 -1175
- package/src/activities/run-input.ts +137 -45
- package/src/activities/session-state.ts +69 -5
- package/src/activities/types.ts +27 -0
- package/src/activities.ts +2 -0
- package/src/observability-metrics.ts +184 -3
- package/src/workflows/session.ts +199 -8
- package/dist/activities-turn-6EBXGGLS.js.map +0 -1
- package/dist/chunk-TYNPCG4U.js.map +0 -1
- /package/dist/{chunk-5GRJIP5H.js.map → chunk-P6Z2SLAA.js.map} +0 -0
|
@@ -37,7 +37,7 @@ import {
|
|
|
37
37
|
withFirstPartyTools,
|
|
38
38
|
wrapLazyTurnBoxWithRouting,
|
|
39
39
|
wrapTurnBoxWithRouting
|
|
40
|
-
} from "./chunk-
|
|
40
|
+
} from "./chunk-P6Z2SLAA.js";
|
|
41
41
|
import {
|
|
42
42
|
StreamTimingMetrics,
|
|
43
43
|
makeMachineOpObserver,
|
|
@@ -51,9 +51,12 @@ import {
|
|
|
51
51
|
recordModelRequestPhase,
|
|
52
52
|
recordSessionEventAppendLatency,
|
|
53
53
|
recordSessionEventPublishLatency,
|
|
54
|
+
recordTurnSandboxEstablishPolicy,
|
|
55
|
+
recordTurnStartupPhase,
|
|
56
|
+
recordTurnWorkerPreparationTotal,
|
|
54
57
|
runtimeMetricsHooksForObservability,
|
|
55
58
|
turnLifecycleMetricsFor
|
|
56
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-N5TJIFCE.js";
|
|
57
60
|
|
|
58
61
|
// src/activities-turn.ts
|
|
59
62
|
import { createProductionAgentRuntime, summarizeForCompaction as summarizeForCompaction2 } from "@opengeni/runtime";
|
|
@@ -148,6 +151,7 @@ import {
|
|
|
148
151
|
SandboxImageConflictError,
|
|
149
152
|
ActiveSessionHistoryLimitExceededError,
|
|
150
153
|
ApprovalRunStateLimitExceededError,
|
|
154
|
+
isRetryableDatabaseTransportFailure,
|
|
151
155
|
isSessionEventPersistenceError,
|
|
152
156
|
getEnrollment,
|
|
153
157
|
abandonRecordingForTurnAttempt,
|
|
@@ -2938,6 +2942,25 @@ function resumeRunState(state) {
|
|
|
2938
2942
|
}
|
|
2939
2943
|
return projectRejectedProviderArtifactsFromSerializedRunState(state.serializedRunState);
|
|
2940
2944
|
}
|
|
2945
|
+
async function measureHistoryPreparationPhase(options, phase, operation) {
|
|
2946
|
+
const startedAt = performance.now();
|
|
2947
|
+
let outcome = "completed";
|
|
2948
|
+
try {
|
|
2949
|
+
return await operation();
|
|
2950
|
+
} catch (error) {
|
|
2951
|
+
outcome = "failed";
|
|
2952
|
+
throw error;
|
|
2953
|
+
} finally {
|
|
2954
|
+
try {
|
|
2955
|
+
options.onPreparationPhase?.({
|
|
2956
|
+
phase,
|
|
2957
|
+
outcome,
|
|
2958
|
+
durationSeconds: (performance.now() - startedAt) / 1e3
|
|
2959
|
+
});
|
|
2960
|
+
} catch {
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2941
2964
|
var MAX_INLINE_MODEL_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
|
2942
2965
|
var MODEL_IMAGE_CONTENT_TYPES = /* @__PURE__ */ new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]);
|
|
2943
2966
|
var MODEL_FILE_CONTENT_TYPES = /* @__PURE__ */ new Set([
|
|
@@ -3132,11 +3155,15 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
3132
3155
|
if (!trigger) {
|
|
3133
3156
|
throw new Error("Missing trigger event");
|
|
3134
3157
|
}
|
|
3135
|
-
const updates = await
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3158
|
+
const updates = await measureHistoryPreparationPhase(
|
|
3159
|
+
options,
|
|
3160
|
+
"system_update_load",
|
|
3161
|
+
async () => await listSessionSystemUpdatesForTurn(
|
|
3162
|
+
db,
|
|
3163
|
+
trigger.workspaceId,
|
|
3164
|
+
trigger.sessionId,
|
|
3165
|
+
options.turnId
|
|
3166
|
+
)
|
|
3140
3167
|
);
|
|
3141
3168
|
if (updates.length > 0) {
|
|
3142
3169
|
const historyItemIds = new Set(
|
|
@@ -3161,10 +3188,10 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
3161
3188
|
throw new Error("user.message payload is missing text and annotations");
|
|
3162
3189
|
}
|
|
3163
3190
|
const resources = Array.isArray(payload.resources) ? payload.resources : [];
|
|
3164
|
-
const fileAttachments = await
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
resources
|
|
3191
|
+
const fileAttachments = await measureHistoryPreparationPhase(
|
|
3192
|
+
options,
|
|
3193
|
+
"current_attachment_resolution",
|
|
3194
|
+
async () => await resolveUserMessageFileAttachments(db, trigger.workspaceId, resources)
|
|
3168
3195
|
);
|
|
3169
3196
|
const attachmentContext = userMessageAttachmentsContext(fileAttachments);
|
|
3170
3197
|
return await messageInput(
|
|
@@ -3179,7 +3206,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
3179
3206
|
options.projectCanonicalHistory,
|
|
3180
3207
|
options.materializeModelHistory,
|
|
3181
3208
|
options.projectModelHistory,
|
|
3182
|
-
options.loadActiveHistory
|
|
3209
|
+
options.loadActiveHistory,
|
|
3210
|
+
options
|
|
3183
3211
|
);
|
|
3184
3212
|
}
|
|
3185
3213
|
if (trigger.type === "system.update.delivered") {
|
|
@@ -3198,7 +3226,8 @@ async function turnInput(db, runtime, agent, trigger, options) {
|
|
|
3198
3226
|
options.projectCanonicalHistory,
|
|
3199
3227
|
options.materializeModelHistory,
|
|
3200
3228
|
options.projectModelHistory,
|
|
3201
|
-
options.loadActiveHistory
|
|
3229
|
+
options.loadActiveHistory,
|
|
3230
|
+
options
|
|
3202
3231
|
);
|
|
3203
3232
|
}
|
|
3204
3233
|
if (trigger.type === "user.approvalDecision") {
|
|
@@ -3258,33 +3287,76 @@ function joinInternalContext(...parts) {
|
|
|
3258
3287
|
const content = parts.map((part) => part?.trim()).filter((part) => Boolean(part));
|
|
3259
3288
|
return content.length > 0 ? content.join("\n\n") : void 0;
|
|
3260
3289
|
}
|
|
3261
|
-
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectCanonicalHistory, materializeModelHistory, projectModelHistory, loadActiveHistory = getActiveSessionHistoryItemsPaged) {
|
|
3262
|
-
const stored = await
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3290
|
+
async function messageInput(db, runtime, agent, trigger, text, internalContext, currentAttachmentRefs = [], providerApi = "responses", projectCanonicalHistory, materializeModelHistory, projectModelHistory, loadActiveHistory = getActiveSessionHistoryItemsPaged, preparationOptions = {}) {
|
|
3291
|
+
const [stored, envelope] = await Promise.all([
|
|
3292
|
+
measureHistoryPreparationPhase(
|
|
3293
|
+
preparationOptions,
|
|
3294
|
+
"durable_history_load",
|
|
3295
|
+
async () => loadActiveHistory(db, trigger.workspaceId, trigger.sessionId)
|
|
3296
|
+
),
|
|
3297
|
+
measureHistoryPreparationPhase(
|
|
3298
|
+
preparationOptions,
|
|
3299
|
+
"sandbox_envelope_load",
|
|
3300
|
+
async () => getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId)
|
|
3301
|
+
)
|
|
3302
|
+
]);
|
|
3303
|
+
const canonicalView = await measureHistoryPreparationPhase(
|
|
3304
|
+
preparationOptions,
|
|
3305
|
+
"canonical_projection",
|
|
3306
|
+
async () => {
|
|
3307
|
+
const active = projectRejectedProviderArtifacts(stored);
|
|
3308
|
+
return projectCanonicalHistory ? await projectCanonicalHistory(active) : active;
|
|
3309
|
+
}
|
|
3310
|
+
);
|
|
3311
|
+
const providerView = await measureHistoryPreparationPhase(
|
|
3312
|
+
preparationOptions,
|
|
3313
|
+
"provider_projection",
|
|
3314
|
+
() => projectHistoryForProvider(canonicalView, providerApi)
|
|
3315
|
+
);
|
|
3316
|
+
const referencedHistory = await measureHistoryPreparationPhase(
|
|
3317
|
+
preparationOptions,
|
|
3318
|
+
"attachment_ref_projection",
|
|
3319
|
+
() => withCurrentUserAttachmentRefs(providerView, currentAttachmentRefs)
|
|
3320
|
+
);
|
|
3321
|
+
const materializedHistory = await measureHistoryPreparationPhase(
|
|
3322
|
+
preparationOptions,
|
|
3323
|
+
"screenshot_materialization",
|
|
3324
|
+
async () => materializeModelHistory ? await materializeModelHistory(referencedHistory) : referencedHistory
|
|
3325
|
+
);
|
|
3326
|
+
const historyItems = await measureHistoryPreparationPhase(
|
|
3327
|
+
preparationOptions,
|
|
3328
|
+
"model_attachment_projection",
|
|
3329
|
+
async () => projectModelHistory ? await projectModelHistory(materializedHistory) : materializedHistory
|
|
3330
|
+
);
|
|
3331
|
+
const prepared = await measureHistoryPreparationPhase(
|
|
3332
|
+
preparationOptions,
|
|
3333
|
+
"runtime_input_assembly",
|
|
3334
|
+
async () => await runtime.prepareInput(agent, {
|
|
3335
|
+
kind: "message",
|
|
3336
|
+
...text ? { text } : {},
|
|
3337
|
+
...internalContext ? { internalContext } : {},
|
|
3338
|
+
historyItems,
|
|
3339
|
+
sandboxEnvelope: envelope,
|
|
3340
|
+
...projectModelHistory ? { modelInputAlreadyProjected: true } : {}
|
|
3341
|
+
})
|
|
3342
|
+
);
|
|
3343
|
+
const providerArtifactCandidates = await measureHistoryPreparationPhase(
|
|
3344
|
+
preparationOptions,
|
|
3345
|
+
"artifact_candidate_scan",
|
|
3346
|
+
() => {
|
|
3347
|
+
const preparedItems = Array.isArray(prepared.input) ? new Set(prepared.input) : /* @__PURE__ */ new Set();
|
|
3348
|
+
return {
|
|
3349
|
+
knownHistoryItemIds: stored.map((row) => row.id),
|
|
3350
|
+
historyItemIds: stored.filter(
|
|
3351
|
+
(row) => row.providerArtifactInvalidatedAt === null && hasOpaqueProviderArtifact(row.item) && preparedItems.has(row.item)
|
|
3352
|
+
).map((row) => row.id)
|
|
3353
|
+
};
|
|
3354
|
+
}
|
|
3355
|
+
);
|
|
3279
3356
|
return {
|
|
3280
3357
|
input: prepared,
|
|
3281
3358
|
persistedHistoryCount: prepared.persistedHistoryCount,
|
|
3282
|
-
providerArtifactCandidates
|
|
3283
|
-
knownHistoryItemIds: stored.map((row) => row.id),
|
|
3284
|
-
historyItemIds: stored.filter(
|
|
3285
|
-
(row) => row.providerArtifactInvalidatedAt === null && hasOpaqueProviderArtifact(row.item) && preparedItems.has(row.item)
|
|
3286
|
-
).map((row) => row.id)
|
|
3287
|
-
}
|
|
3359
|
+
providerArtifactCandidates
|
|
3288
3360
|
};
|
|
3289
3361
|
}
|
|
3290
3362
|
async function resolveUserMessageFileAttachments(db, workspaceId, resources) {
|
|
@@ -5241,6 +5313,8 @@ async function bindRunCredentialResolver(input) {
|
|
|
5241
5313
|
}
|
|
5242
5314
|
|
|
5243
5315
|
// src/activities/types.ts
|
|
5316
|
+
var PRE_CLAIM_FAILURE_TYPE = "OpenGeniPreClaimFailure";
|
|
5317
|
+
var PRE_CLAIM_FAILURE_MESSAGE = "Agent turn admission failed before attempt claim.";
|
|
5244
5318
|
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
|
|
5245
5319
|
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
|
|
5246
5320
|
|
|
@@ -6756,12 +6830,33 @@ function escapedMcpTimeoutRecoveryFailure(input) {
|
|
|
6756
6830
|
details: [input.detail]
|
|
6757
6831
|
});
|
|
6758
6832
|
}
|
|
6833
|
+
function preClaimAdmissionFailure(error) {
|
|
6834
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
6835
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
6836
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
6837
|
+
const retryable = Boolean(
|
|
6838
|
+
rawTransportFailure || persistenceFailure && (sqlState === null || sqlState.startsWith("08") || sqlState.startsWith("40") || sqlState.startsWith("53") || sqlState === "55P03" || sqlState === "57014" || sqlState === "57P01" || sqlState === "57P02" || sqlState === "57P03" || sqlState.startsWith("58"))
|
|
6839
|
+
);
|
|
6840
|
+
const detail = {
|
|
6841
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
6842
|
+
code: persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant")
|
|
6843
|
+
};
|
|
6844
|
+
return ApplicationFailure.create({
|
|
6845
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
6846
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
6847
|
+
nonRetryable: true,
|
|
6848
|
+
details: [detail]
|
|
6849
|
+
});
|
|
6850
|
+
}
|
|
6759
6851
|
function filterUnmaterializedSandboxFileDownloads(downloads, materializedFileIds) {
|
|
6760
6852
|
if (downloads.length === 0 || materializedFileIds.size === 0) {
|
|
6761
6853
|
return downloads;
|
|
6762
6854
|
}
|
|
6763
6855
|
return downloads.filter((download) => !materializedFileIds.has(download.fileId));
|
|
6764
6856
|
}
|
|
6857
|
+
function sandboxFileMaterializationOutcome(failures) {
|
|
6858
|
+
return failures.length === 0 ? "completed" : "failed";
|
|
6859
|
+
}
|
|
6765
6860
|
function codexWorkspaceMetricKey(workspaceId) {
|
|
6766
6861
|
return createHash8("sha256").update(workspaceId).digest("hex").slice(0, 12);
|
|
6767
6862
|
}
|
|
@@ -7432,6 +7527,21 @@ function managedSandboxOwnershipForTurn(machinePrimary, attemptId, sandboxGroupI
|
|
|
7432
7527
|
function shouldEstablishSandboxForTurn(sandboxOwnershipEnabled, homeBackend, machinePrimary) {
|
|
7433
7528
|
return sandboxOwnershipEnabled && (homeBackend !== "none" || machinePrimary);
|
|
7434
7529
|
}
|
|
7530
|
+
function sandboxEstablishPolicyDecision(input) {
|
|
7531
|
+
if (!input.lazyEnabled) return { policy: "eager", reason: "lazy_disabled" };
|
|
7532
|
+
if (input.machinePrimary) return { policy: "eager", reason: "machine_primary" };
|
|
7533
|
+
if (input.sandboxBackend === "none") return { policy: "eager", reason: "backend_none" };
|
|
7534
|
+
if (input.hasInitialRunCredentialMaterial) {
|
|
7535
|
+
return { policy: "eager", reason: "initial_run_credentials" };
|
|
7536
|
+
}
|
|
7537
|
+
if (input.generatedVideoFileCount > 0) {
|
|
7538
|
+
return { policy: "eager", reason: "generated_video_files" };
|
|
7539
|
+
}
|
|
7540
|
+
if (input.hasSignedFileResources) {
|
|
7541
|
+
return { policy: "eager", reason: "signed_file_resources" };
|
|
7542
|
+
}
|
|
7543
|
+
return { policy: "on-demand", reason: "eligible" };
|
|
7544
|
+
}
|
|
7435
7545
|
function pointerReconcileReason(record4) {
|
|
7436
7546
|
if (!record4) {
|
|
7437
7547
|
return "stale_pointer";
|
|
@@ -8760,7 +8870,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
8760
8870
|
let lastCodexRequestOpaqueArtifacts = [];
|
|
8761
8871
|
let triggerType = null;
|
|
8762
8872
|
try {
|
|
8763
|
-
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
8764
8873
|
const claim = await claimSessionWorkForAttempt(db, input.workspaceId, {
|
|
8765
8874
|
sessionId: input.sessionId,
|
|
8766
8875
|
workflowId: input.workflowId,
|
|
@@ -8775,6 +8884,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
8775
8884
|
}
|
|
8776
8885
|
const turn = claim.turn;
|
|
8777
8886
|
turnId = turn.id;
|
|
8887
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
8778
8888
|
executionGeneration = turn.executionGeneration;
|
|
8779
8889
|
providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
8780
8890
|
let installedApiIntegrations = [];
|
|
@@ -9031,6 +9141,14 @@ function createRunAgentTurnActivity(services) {
|
|
|
9031
9141
|
});
|
|
9032
9142
|
throwIfWorkerShuttingDown();
|
|
9033
9143
|
throwIfTurnCancelled();
|
|
9144
|
+
recordTurnStartupPhase(observability, {
|
|
9145
|
+
phase: "claim_and_policy",
|
|
9146
|
+
provider: turnExecutionPolicy.providerId,
|
|
9147
|
+
backend: turn.sandboxBackend,
|
|
9148
|
+
outcome: "completed",
|
|
9149
|
+
durationSeconds: (performance.now() - activityStarted) / 1e3
|
|
9150
|
+
});
|
|
9151
|
+
const turnStartSettlementStartedAt = performance.now();
|
|
9034
9152
|
if (!await settle({
|
|
9035
9153
|
events: [
|
|
9036
9154
|
{ type: "session.status.changed", payload: { status: "running" } },
|
|
@@ -9045,76 +9163,34 @@ function createRunAgentTurnActivity(services) {
|
|
|
9045
9163
|
})) {
|
|
9046
9164
|
return claimedResult({ status: "cancelled" });
|
|
9047
9165
|
}
|
|
9166
|
+
recordTurnStartupPhase(observability, {
|
|
9167
|
+
phase: "turn_start_settlement",
|
|
9168
|
+
provider: turnExecutionPolicy.providerId,
|
|
9169
|
+
backend: turn.sandboxBackend,
|
|
9170
|
+
outcome: "completed",
|
|
9171
|
+
durationSeconds: (performance.now() - turnStartSettlementStartedAt) / 1e3,
|
|
9172
|
+
count: 2
|
|
9173
|
+
});
|
|
9048
9174
|
turnStartedPublished = true;
|
|
9175
|
+
const workerPreparationStartedAt = performance.now();
|
|
9049
9176
|
if (isCodexTurn) {
|
|
9050
|
-
const
|
|
9051
|
-
|
|
9052
|
-
|
|
9053
|
-
|
|
9054
|
-
|
|
9055
|
-
|
|
9056
|
-
|
|
9057
|
-
|
|
9058
|
-
|
|
9059
|
-
|
|
9060
|
-
|
|
9061
|
-
|
|
9062
|
-
|
|
9063
|
-
|
|
9064
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
9065
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
9066
|
-
leased = await acquireCodexCredentialLease(
|
|
9067
|
-
db,
|
|
9068
|
-
{
|
|
9069
|
-
accountId: input.accountId,
|
|
9070
|
-
workspaceId: input.workspaceId,
|
|
9071
|
-
turnId,
|
|
9072
|
-
holderId: codexLeaseHolderId,
|
|
9073
|
-
advanceActivePointer: sessionPin === null
|
|
9074
|
-
},
|
|
9075
|
-
selectForTurn
|
|
9076
|
-
);
|
|
9077
|
-
} else {
|
|
9078
|
-
const [rotation, accounts] = await Promise.all([
|
|
9079
|
-
getCodexRotationSettings(db, input.workspaceId),
|
|
9080
|
-
listCodexAccountStatuses(db, input.workspaceId)
|
|
9081
|
-
]);
|
|
9082
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
9083
|
-
...account,
|
|
9084
|
-
activeLeaseCount: 0,
|
|
9085
|
-
selectionCount: 0,
|
|
9086
|
-
lastSelectedAt: null
|
|
9087
|
-
}));
|
|
9088
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
9089
|
-
const selected = selectForTurn({
|
|
9090
|
-
accounts: leaseAccounts,
|
|
9091
|
-
activeCredentialId,
|
|
9092
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9093
|
-
leaseRotationEnabled: false,
|
|
9094
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
9095
|
-
existingCredentialId: null,
|
|
9096
|
-
policyScope: null,
|
|
9097
|
-
unavailableDiagnostics: []
|
|
9098
|
-
});
|
|
9099
|
-
leased = {
|
|
9100
|
-
...selected,
|
|
9101
|
-
accounts: leaseAccounts,
|
|
9102
|
-
activeCredentialId,
|
|
9103
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9104
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
9105
|
-
reused: false,
|
|
9106
|
-
holderId: null,
|
|
9107
|
-
generation: null,
|
|
9108
|
-
leasedUntil: null,
|
|
9109
|
-
unavailableDiagnostics: [],
|
|
9110
|
-
advanceActivePointer: selected.advanceActivePointer !== false
|
|
9111
|
-
};
|
|
9112
|
-
}
|
|
9113
|
-
if (leased.decision.kind === "allCapped") {
|
|
9114
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
9115
|
-
signalCodexCapacityWorkflow,
|
|
9116
|
-
wakeSessionWorkflow
|
|
9177
|
+
const credentialSelectionStartedAt = performance.now();
|
|
9178
|
+
let credentialSelectionOutcome = "completed";
|
|
9179
|
+
try {
|
|
9180
|
+
const sessionCodex = await getSessionCodexState(db, input.workspaceId, input.sessionId);
|
|
9181
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
9182
|
+
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
9183
|
+
const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
|
|
9184
|
+
context,
|
|
9185
|
+
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
9186
|
+
sessionId: input.sessionId,
|
|
9187
|
+
sessionPinnedCredentialId: sessionPin,
|
|
9188
|
+
sessionPinSource,
|
|
9189
|
+
sessionLastCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
9190
|
+
now: /* @__PURE__ */ new Date()
|
|
9117
9191
|
});
|
|
9192
|
+
let leased;
|
|
9193
|
+
let leaseAcquisitionStartedAtMs = null;
|
|
9118
9194
|
if (settings.codexCredentialLeasingEnabled) {
|
|
9119
9195
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
9120
9196
|
leased = await acquireCodexCredentialLease(
|
|
@@ -9139,9 +9215,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
9139
9215
|
selectionCount: 0,
|
|
9140
9216
|
lastSelectedAt: null
|
|
9141
9217
|
}));
|
|
9218
|
+
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
9142
9219
|
const selected = selectForTurn({
|
|
9143
9220
|
accounts: leaseAccounts,
|
|
9144
|
-
activeCredentialId
|
|
9221
|
+
activeCredentialId,
|
|
9145
9222
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9146
9223
|
leaseRotationEnabled: false,
|
|
9147
9224
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
@@ -9152,7 +9229,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9152
9229
|
leased = {
|
|
9153
9230
|
...selected,
|
|
9154
9231
|
accounts: leaseAccounts,
|
|
9155
|
-
activeCredentialId
|
|
9232
|
+
activeCredentialId,
|
|
9156
9233
|
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9157
9234
|
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
9158
9235
|
reused: false,
|
|
@@ -9163,163 +9240,389 @@ function createRunAgentTurnActivity(services) {
|
|
|
9163
9240
|
advanceActivePointer: selected.advanceActivePointer !== false
|
|
9164
9241
|
};
|
|
9165
9242
|
}
|
|
9166
|
-
|
|
9167
|
-
|
|
9168
|
-
|
|
9169
|
-
|
|
9170
|
-
|
|
9171
|
-
|
|
9172
|
-
|
|
9173
|
-
|
|
9174
|
-
|
|
9175
|
-
|
|
9176
|
-
|
|
9177
|
-
|
|
9178
|
-
|
|
9179
|
-
|
|
9243
|
+
if (leased.decision.kind === "allCapped") {
|
|
9244
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
9245
|
+
signalCodexCapacityWorkflow,
|
|
9246
|
+
wakeSessionWorkflow
|
|
9247
|
+
});
|
|
9248
|
+
if (settings.codexCredentialLeasingEnabled) {
|
|
9249
|
+
leaseAcquisitionStartedAtMs = performance.now();
|
|
9250
|
+
leased = await acquireCodexCredentialLease(
|
|
9251
|
+
db,
|
|
9252
|
+
{
|
|
9253
|
+
accountId: input.accountId,
|
|
9254
|
+
workspaceId: input.workspaceId,
|
|
9255
|
+
turnId,
|
|
9256
|
+
holderId: codexLeaseHolderId,
|
|
9257
|
+
advanceActivePointer: sessionPin === null
|
|
9258
|
+
},
|
|
9259
|
+
selectForTurn
|
|
9260
|
+
);
|
|
9261
|
+
} else {
|
|
9262
|
+
const [rotation, accounts] = await Promise.all([
|
|
9263
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
9264
|
+
listCodexAccountStatuses(db, input.workspaceId)
|
|
9265
|
+
]);
|
|
9266
|
+
const leaseAccounts = accounts.map((account) => ({
|
|
9267
|
+
...account,
|
|
9268
|
+
activeLeaseCount: 0,
|
|
9269
|
+
selectionCount: 0,
|
|
9270
|
+
lastSelectedAt: null
|
|
9271
|
+
}));
|
|
9272
|
+
const selected = selectForTurn({
|
|
9273
|
+
accounts: leaseAccounts,
|
|
9274
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
9275
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9276
|
+
leaseRotationEnabled: false,
|
|
9277
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
9278
|
+
existingCredentialId: null,
|
|
9279
|
+
policyScope: null,
|
|
9280
|
+
unavailableDiagnostics: []
|
|
9281
|
+
});
|
|
9282
|
+
leased = {
|
|
9283
|
+
...selected,
|
|
9284
|
+
accounts: leaseAccounts,
|
|
9285
|
+
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
9286
|
+
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
9287
|
+
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
9288
|
+
reused: false,
|
|
9289
|
+
holderId: null,
|
|
9290
|
+
generation: null,
|
|
9291
|
+
leasedUntil: null,
|
|
9292
|
+
unavailableDiagnostics: [],
|
|
9293
|
+
advanceActivePointer: selected.advanceActivePointer !== false
|
|
9294
|
+
};
|
|
9295
|
+
}
|
|
9296
|
+
}
|
|
9297
|
+
const rotationDecision = leased.decision;
|
|
9298
|
+
const selectedPinDisposition = classifyCodexPin({
|
|
9299
|
+
pinnedCredentialId: sessionPin,
|
|
9300
|
+
pinSource: sessionPinSource,
|
|
9301
|
+
strategy: leased.rotationStrategy,
|
|
9302
|
+
rotationEnabled: leased.rotationEnabled
|
|
9303
|
+
});
|
|
9304
|
+
if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
|
|
9305
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
9306
|
+
db,
|
|
9307
|
+
{
|
|
9308
|
+
workspaceId: input.workspaceId,
|
|
9309
|
+
reason: "codex_policy_pin_changed"
|
|
9310
|
+
},
|
|
9311
|
+
async (tx) => {
|
|
9312
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
9313
|
+
tx,
|
|
9314
|
+
input.workspaceId,
|
|
9315
|
+
input.sessionId,
|
|
9316
|
+
leased.credentialId,
|
|
9317
|
+
"policy",
|
|
9318
|
+
{
|
|
9319
|
+
expected: {
|
|
9320
|
+
pinnedCredentialId: sessionPin,
|
|
9321
|
+
pinSource: sessionPinSource
|
|
9322
|
+
}
|
|
9323
|
+
}
|
|
9324
|
+
);
|
|
9325
|
+
return { result: changed, changed };
|
|
9326
|
+
}
|
|
9327
|
+
);
|
|
9328
|
+
await signalCodexCapacityWakeTargets(
|
|
9329
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
9330
|
+
pinMutation.wakeTargets
|
|
9331
|
+
);
|
|
9332
|
+
} else if (selectedPinDisposition === "clearStale") {
|
|
9333
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
9334
|
+
db,
|
|
9335
|
+
{
|
|
9336
|
+
workspaceId: input.workspaceId,
|
|
9337
|
+
reason: "codex_stale_policy_pin_cleared"
|
|
9338
|
+
},
|
|
9339
|
+
async (tx) => {
|
|
9340
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
9341
|
+
tx,
|
|
9342
|
+
input.workspaceId,
|
|
9343
|
+
input.sessionId,
|
|
9344
|
+
null,
|
|
9345
|
+
"policy",
|
|
9346
|
+
{
|
|
9347
|
+
expected: {
|
|
9348
|
+
pinnedCredentialId: sessionPin,
|
|
9349
|
+
pinSource: sessionPinSource
|
|
9350
|
+
}
|
|
9351
|
+
}
|
|
9352
|
+
);
|
|
9353
|
+
return { result: changed, changed };
|
|
9354
|
+
}
|
|
9355
|
+
);
|
|
9356
|
+
await signalCodexCapacityWakeTargets(
|
|
9357
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
9358
|
+
pinMutation.wakeTargets
|
|
9359
|
+
);
|
|
9360
|
+
}
|
|
9361
|
+
if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
|
|
9362
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
9363
|
+
}
|
|
9364
|
+
effectiveCodexCredentialId = leased.credentialId;
|
|
9365
|
+
codexLeaseGeneration = leased.generation;
|
|
9366
|
+
codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
|
|
9367
|
+
codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
|
|
9368
|
+
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
9369
|
+
const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
|
|
9370
|
+
const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
|
|
9371
|
+
const fencedInFlight = leased.reused;
|
|
9372
|
+
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
9373
|
+
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
9374
|
+
decision: {
|
|
9375
|
+
accounts: leased.accounts,
|
|
9376
|
+
actualCredentialId: effectiveCodexCredentialId,
|
|
9377
|
+
actualOutcome,
|
|
9378
|
+
actualReason,
|
|
9379
|
+
affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
|
|
9380
|
+
fencedInFlight,
|
|
9381
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
9382
|
+
now: /* @__PURE__ */ new Date(),
|
|
9383
|
+
aliasSeed: randomUUID4()
|
|
9180
9384
|
},
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9385
|
+
publish
|
|
9386
|
+
});
|
|
9387
|
+
if (shadowResult.outcome === "published") {
|
|
9388
|
+
const shadowPayload = shadowResult.payload;
|
|
9389
|
+
observability.incrementCounter({
|
|
9390
|
+
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
9391
|
+
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
9392
|
+
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
|
|
9393
|
+
});
|
|
9394
|
+
observability.info("Codex adaptive fleet shadow decision", {
|
|
9395
|
+
workspaceId: input.workspaceId,
|
|
9396
|
+
policyVersion: shadowPayload.replay.policyVersion,
|
|
9397
|
+
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
9398
|
+
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
9399
|
+
actualOutcome: shadowPayload.actual.outcome,
|
|
9400
|
+
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
9401
|
+
comparison: shadowPayload.comparison,
|
|
9402
|
+
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
9403
|
+
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
9404
|
+
payloadBytes: shadowResult.payloadBytes
|
|
9405
|
+
});
|
|
9406
|
+
} else if (shadowResult.outcome === "failed") {
|
|
9407
|
+
observability.incrementCounter({
|
|
9408
|
+
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
9409
|
+
help: "Shadow decision build/publication failures.",
|
|
9410
|
+
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
|
|
9411
|
+
});
|
|
9412
|
+
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
9413
|
+
stage: shadowResult.stage,
|
|
9414
|
+
reason: shadowResult.reason,
|
|
9415
|
+
errorClass: "CodexFleetShadowOperationError",
|
|
9416
|
+
errorCode: "codex_fleet_shadow_failed",
|
|
9417
|
+
origin: "worker",
|
|
9418
|
+
payloadBytes: shadowResult.payloadBytes
|
|
9419
|
+
});
|
|
9420
|
+
}
|
|
9421
|
+
const eligibleCount = leased.accounts.filter(
|
|
9422
|
+
(account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
|
|
9423
|
+
).length;
|
|
9424
|
+
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
9425
|
+
observability.incrementCounter({
|
|
9426
|
+
name: "opengeni_codex_pool_observations_total",
|
|
9427
|
+
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
9428
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
|
|
9429
|
+
});
|
|
9430
|
+
if (eligibleCount <= 1) {
|
|
9431
|
+
observability.incrementCounter({
|
|
9432
|
+
name: "opengeni_codex_pool_low_total",
|
|
9433
|
+
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
9434
|
+
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
|
|
9435
|
+
});
|
|
9436
|
+
observability.warn("Codex eligible credential pool is low", {
|
|
9437
|
+
workspaceId: input.workspaceId,
|
|
9438
|
+
eligibleCount,
|
|
9439
|
+
connectedCount: leased.accounts.length,
|
|
9440
|
+
depth: poolDepth
|
|
9441
|
+
});
|
|
9442
|
+
}
|
|
9443
|
+
if (effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
|
|
9444
|
+
if (turn.source === "compaction") {
|
|
9445
|
+
if (!await settle({
|
|
9446
|
+
events: [
|
|
9447
|
+
{
|
|
9448
|
+
type: "turn.cancelled",
|
|
9449
|
+
payload: {
|
|
9450
|
+
maintenance: "context_compaction",
|
|
9451
|
+
reason: "codex_allocator_disabled",
|
|
9452
|
+
requestPreserved: true
|
|
9453
|
+
}
|
|
9454
|
+
},
|
|
9455
|
+
{
|
|
9456
|
+
type: "session.status.changed",
|
|
9457
|
+
payload: { status: "idle" }
|
|
9458
|
+
}
|
|
9459
|
+
],
|
|
9460
|
+
turnStatus: "cancelled",
|
|
9461
|
+
sessionStatus: "idle",
|
|
9462
|
+
activeTurnId: null
|
|
9463
|
+
})) {
|
|
9464
|
+
return claimedResult({ status: "cancelled" });
|
|
9465
|
+
}
|
|
9466
|
+
turnMetricOutcome = "cancelled";
|
|
9467
|
+
activityStatus = "idle";
|
|
9468
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
9469
|
+
}
|
|
9470
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
9471
|
+
() => null
|
|
9472
|
+
);
|
|
9473
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
9474
|
+
const armed = await armCodexCapacityWait(db, {
|
|
9475
|
+
accountId: input.accountId,
|
|
9476
|
+
workspaceId: input.workspaceId,
|
|
9477
|
+
sessionId: input.sessionId,
|
|
9478
|
+
turnId,
|
|
9479
|
+
attemptId: input.attemptId,
|
|
9480
|
+
workflowId: input.workflowId,
|
|
9481
|
+
goalId: activeGoal?.id ?? null,
|
|
9482
|
+
goalVersion: activeGoal?.version ?? null,
|
|
9483
|
+
earliestResetAt: null,
|
|
9484
|
+
resetKind: "bounded_refresh",
|
|
9485
|
+
failurePayload: {
|
|
9486
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
9487
|
+
code: "codex_allocator_disabled",
|
|
9488
|
+
detail: "waiting for a credential to be re-enabled, reconnected, or added"
|
|
9489
|
+
}
|
|
9490
|
+
});
|
|
9491
|
+
if (armed.action === "waiting") {
|
|
9492
|
+
await publishDurableSessionEvents2(
|
|
9493
|
+
bus,
|
|
9184
9494
|
input.workspaceId,
|
|
9185
9495
|
input.sessionId,
|
|
9186
|
-
|
|
9187
|
-
|
|
9496
|
+
armed.events
|
|
9497
|
+
);
|
|
9498
|
+
turnMetricOutcome = "recovering";
|
|
9499
|
+
activityStatus = "waiting_capacity";
|
|
9500
|
+
return claimedResult({
|
|
9501
|
+
status: "waiting_capacity",
|
|
9502
|
+
capacityWait: {
|
|
9503
|
+
waiterId: armed.waiter.id,
|
|
9504
|
+
generation: armed.waiter.generation,
|
|
9505
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
9506
|
+
wakeRevision: armed.waiter.wakeRevision
|
|
9507
|
+
}
|
|
9508
|
+
});
|
|
9509
|
+
}
|
|
9510
|
+
if (!await settle({
|
|
9511
|
+
events: [
|
|
9188
9512
|
{
|
|
9189
|
-
|
|
9190
|
-
|
|
9191
|
-
|
|
9513
|
+
type: "turn.failed",
|
|
9514
|
+
payload: {
|
|
9515
|
+
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
9516
|
+
code: "codex_allocator_disabled",
|
|
9517
|
+
retryable: false,
|
|
9518
|
+
recovery: "user_message"
|
|
9192
9519
|
}
|
|
9520
|
+
},
|
|
9521
|
+
{
|
|
9522
|
+
type: "session.status.changed",
|
|
9523
|
+
payload: { status: "idle" }
|
|
9193
9524
|
}
|
|
9194
|
-
|
|
9195
|
-
|
|
9525
|
+
],
|
|
9526
|
+
turnStatus: "failed",
|
|
9527
|
+
sessionStatus: "idle",
|
|
9528
|
+
activeTurnId: null
|
|
9529
|
+
})) {
|
|
9530
|
+
return claimedResult({ status: "cancelled" });
|
|
9196
9531
|
}
|
|
9197
|
-
|
|
9198
|
-
|
|
9199
|
-
{
|
|
9200
|
-
|
|
9201
|
-
)
|
|
9202
|
-
|
|
9203
|
-
|
|
9204
|
-
|
|
9205
|
-
|
|
9532
|
+
turnMetricOutcome = "failed";
|
|
9533
|
+
activityStatus = "idle";
|
|
9534
|
+
return claimedResult({ status: "idle" });
|
|
9535
|
+
}
|
|
9536
|
+
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
9537
|
+
if (turn.source === "compaction") {
|
|
9538
|
+
if (!await settle({
|
|
9539
|
+
events: [
|
|
9540
|
+
{
|
|
9541
|
+
type: "turn.cancelled",
|
|
9542
|
+
payload: {
|
|
9543
|
+
maintenance: "context_compaction",
|
|
9544
|
+
reason: "codex_capacity_unavailable",
|
|
9545
|
+
requestPreserved: true
|
|
9546
|
+
}
|
|
9547
|
+
},
|
|
9548
|
+
{
|
|
9549
|
+
type: "session.status.changed",
|
|
9550
|
+
payload: { status: "idle" }
|
|
9551
|
+
}
|
|
9552
|
+
],
|
|
9553
|
+
turnStatus: "cancelled",
|
|
9554
|
+
sessionStatus: "idle",
|
|
9555
|
+
activeTurnId: null
|
|
9556
|
+
})) {
|
|
9557
|
+
return claimedResult({ status: "cancelled" });
|
|
9558
|
+
}
|
|
9559
|
+
turnMetricOutcome = "cancelled";
|
|
9560
|
+
activityStatus = "idle";
|
|
9561
|
+
return claimedResult({ status: "idle", deferredUntilWake: true });
|
|
9562
|
+
}
|
|
9563
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(
|
|
9564
|
+
() => null
|
|
9565
|
+
);
|
|
9566
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
9567
|
+
const resumeMs = computeIdleDelayMs(
|
|
9568
|
+
rotationDecision.earliestResetAt,
|
|
9569
|
+
/* @__PURE__ */ new Date(),
|
|
9570
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
9571
|
+
);
|
|
9572
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
9573
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1e3) },
|
|
9574
|
+
"all connected Codex subscriptions are rate-limited",
|
|
9575
|
+
{ allAccounts: true }
|
|
9576
|
+
);
|
|
9577
|
+
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
9578
|
+
leased.accounts,
|
|
9579
|
+
/* @__PURE__ */ new Date()
|
|
9580
|
+
);
|
|
9581
|
+
const armed = await armCodexCapacityWait(db, {
|
|
9582
|
+
accountId: input.accountId,
|
|
9206
9583
|
workspaceId: input.workspaceId,
|
|
9207
|
-
|
|
9208
|
-
|
|
9209
|
-
|
|
9210
|
-
|
|
9211
|
-
|
|
9584
|
+
sessionId: input.sessionId,
|
|
9585
|
+
turnId,
|
|
9586
|
+
attemptId: input.attemptId,
|
|
9587
|
+
workflowId: input.workflowId,
|
|
9588
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
9589
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
9590
|
+
earliestResetAt: authoritativeResetAt,
|
|
9591
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
9592
|
+
failurePayload
|
|
9593
|
+
});
|
|
9594
|
+
if (armed.action === "waiting") {
|
|
9595
|
+
await publishDurableSessionEvents2(
|
|
9596
|
+
bus,
|
|
9212
9597
|
input.workspaceId,
|
|
9213
9598
|
input.sessionId,
|
|
9214
|
-
|
|
9215
|
-
"policy",
|
|
9216
|
-
{
|
|
9217
|
-
expected: {
|
|
9218
|
-
pinnedCredentialId: sessionPin,
|
|
9219
|
-
pinSource: sessionPinSource
|
|
9220
|
-
}
|
|
9221
|
-
}
|
|
9599
|
+
armed.events
|
|
9222
9600
|
);
|
|
9223
|
-
|
|
9601
|
+
turnMetricOutcome = "recovering";
|
|
9602
|
+
activityStatus = "waiting_capacity";
|
|
9603
|
+
return claimedResult({
|
|
9604
|
+
status: "waiting_capacity",
|
|
9605
|
+
capacityWait: {
|
|
9606
|
+
waiterId: armed.waiter.id,
|
|
9607
|
+
generation: armed.waiter.generation,
|
|
9608
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
9609
|
+
wakeRevision: armed.waiter.wakeRevision
|
|
9610
|
+
}
|
|
9611
|
+
});
|
|
9224
9612
|
}
|
|
9225
|
-
);
|
|
9226
|
-
await signalCodexCapacityWakeTargets(
|
|
9227
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
9228
|
-
pinMutation.wakeTargets
|
|
9229
|
-
);
|
|
9230
|
-
}
|
|
9231
|
-
if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
|
|
9232
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
9233
|
-
}
|
|
9234
|
-
effectiveCodexCredentialId = leased.credentialId;
|
|
9235
|
-
codexLeaseGeneration = leased.generation;
|
|
9236
|
-
codexLeaseConfirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS : null;
|
|
9237
|
-
codexLeaseHeld = effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && codexLeaseConfirmedUntilMs !== null;
|
|
9238
|
-
if (codexLeaseHeld) startCodexLeaseHeartbeat();
|
|
9239
|
-
const actualOutcome = effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
|
|
9240
|
-
const actualReason = effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
|
|
9241
|
-
const fencedInFlight = leased.reused;
|
|
9242
|
-
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
9243
|
-
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
9244
|
-
decision: {
|
|
9245
|
-
accounts: leased.accounts,
|
|
9246
|
-
actualCredentialId: effectiveCodexCredentialId,
|
|
9247
|
-
actualOutcome,
|
|
9248
|
-
actualReason,
|
|
9249
|
-
affinityCredentialId: fencedInFlight ? effectiveCodexCredentialId : sessionPin ?? sessionCodex?.lastCredentialId ?? null,
|
|
9250
|
-
fencedInFlight,
|
|
9251
|
-
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
9252
|
-
now: /* @__PURE__ */ new Date(),
|
|
9253
|
-
aliasSeed: randomUUID4()
|
|
9254
|
-
},
|
|
9255
|
-
publish
|
|
9256
|
-
});
|
|
9257
|
-
if (shadowResult.outcome === "published") {
|
|
9258
|
-
const shadowPayload = shadowResult.payload;
|
|
9259
|
-
observability.incrementCounter({
|
|
9260
|
-
name: "opengeni_codex_fleet_shadow_decisions_total",
|
|
9261
|
-
help: "Shadow decisions by bounded actual/shadow outcome and comparison.",
|
|
9262
|
-
labels: codexFleetShadowDecisionMetricLabelsV1(shadowPayload)
|
|
9263
|
-
});
|
|
9264
|
-
observability.info("Codex adaptive fleet shadow decision", {
|
|
9265
|
-
workspaceId: input.workspaceId,
|
|
9266
|
-
policyVersion: shadowPayload.replay.policyVersion,
|
|
9267
|
-
inputFingerprint: shadowPayload.replay.inputFingerprint,
|
|
9268
|
-
decisionFingerprint: shadowPayload.replay.decisionFingerprint,
|
|
9269
|
-
actualOutcome: shadowPayload.actual.outcome,
|
|
9270
|
-
shadowOutcome: shadowPayload.replay.decision.outcome,
|
|
9271
|
-
comparison: shadowPayload.comparison,
|
|
9272
|
-
candidateCount: shadowPayload.replay.input.candidates.length,
|
|
9273
|
-
truncatedCandidateCount: shadowPayload.replay.truncatedCandidateCount,
|
|
9274
|
-
payloadBytes: shadowResult.payloadBytes
|
|
9275
|
-
});
|
|
9276
|
-
} else if (shadowResult.outcome === "failed") {
|
|
9277
|
-
observability.incrementCounter({
|
|
9278
|
-
name: "opengeni_codex_fleet_shadow_errors_total",
|
|
9279
|
-
help: "Shadow decision build/publication failures.",
|
|
9280
|
-
labels: codexFleetShadowErrorMetricLabelsV1(shadowResult)
|
|
9281
|
-
});
|
|
9282
|
-
observability.warn("Codex adaptive fleet shadow decision failed open", {
|
|
9283
|
-
stage: shadowResult.stage,
|
|
9284
|
-
reason: shadowResult.reason,
|
|
9285
|
-
errorClass: "CodexFleetShadowOperationError",
|
|
9286
|
-
errorCode: "codex_fleet_shadow_failed",
|
|
9287
|
-
origin: "worker",
|
|
9288
|
-
payloadBytes: shadowResult.payloadBytes
|
|
9289
|
-
});
|
|
9290
|
-
}
|
|
9291
|
-
const eligibleCount = leased.accounts.filter(
|
|
9292
|
-
(account) => isCodexCredentialEligible(account, /* @__PURE__ */ new Date())
|
|
9293
|
-
).length;
|
|
9294
|
-
const poolDepth = eligibleCount === 0 ? "zero" : eligibleCount === 1 ? "one" : "many";
|
|
9295
|
-
observability.incrementCounter({
|
|
9296
|
-
name: "opengeni_codex_pool_observations_total",
|
|
9297
|
-
help: "Observed eligible Codex pool depth buckets at turn selection.",
|
|
9298
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
|
|
9299
|
-
});
|
|
9300
|
-
if (eligibleCount <= 1) {
|
|
9301
|
-
observability.incrementCounter({
|
|
9302
|
-
name: "opengeni_codex_pool_low_total",
|
|
9303
|
-
help: "Alert signal emitted when the eligible Codex pool is zero or one.",
|
|
9304
|
-
labels: { workspace_key: codexWorkspaceKey, depth: poolDepth }
|
|
9305
|
-
});
|
|
9306
|
-
observability.warn("Codex eligible credential pool is low", {
|
|
9307
|
-
workspaceId: input.workspaceId,
|
|
9308
|
-
eligibleCount,
|
|
9309
|
-
connectedCount: leased.accounts.length,
|
|
9310
|
-
depth: poolDepth
|
|
9311
|
-
});
|
|
9312
|
-
}
|
|
9313
|
-
if (effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
|
|
9314
|
-
if (turn.source === "compaction") {
|
|
9315
9613
|
if (!await settle({
|
|
9316
9614
|
events: [
|
|
9615
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
9616
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
9617
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
9618
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
9619
|
+
// the proactive path while the reactive path spares it.
|
|
9317
9620
|
{
|
|
9318
|
-
type: "turn.
|
|
9621
|
+
type: "turn.failed",
|
|
9319
9622
|
payload: {
|
|
9320
|
-
|
|
9321
|
-
|
|
9322
|
-
|
|
9623
|
+
...failurePayload,
|
|
9624
|
+
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
9625
|
+
rotated: true
|
|
9323
9626
|
}
|
|
9324
9627
|
},
|
|
9325
9628
|
{
|
|
@@ -9327,233 +9630,78 @@ function createRunAgentTurnActivity(services) {
|
|
|
9327
9630
|
payload: { status: "idle" }
|
|
9328
9631
|
}
|
|
9329
9632
|
],
|
|
9330
|
-
turnStatus: "
|
|
9633
|
+
turnStatus: "failed",
|
|
9331
9634
|
sessionStatus: "idle",
|
|
9332
9635
|
activeTurnId: null
|
|
9333
9636
|
})) {
|
|
9334
9637
|
return claimedResult({ status: "cancelled" });
|
|
9335
9638
|
}
|
|
9336
|
-
turnMetricOutcome = "
|
|
9639
|
+
turnMetricOutcome = "failed";
|
|
9337
9640
|
activityStatus = "idle";
|
|
9338
|
-
return claimedResult(
|
|
9641
|
+
return claimedResult(
|
|
9642
|
+
goalActive ? {
|
|
9643
|
+
status: "idle",
|
|
9644
|
+
continueDelayMs: resumeMs,
|
|
9645
|
+
idleUntilReset: true
|
|
9646
|
+
} : { status: "idle" }
|
|
9647
|
+
);
|
|
9339
9648
|
}
|
|
9340
|
-
|
|
9341
|
-
|
|
9342
|
-
|
|
9343
|
-
|
|
9344
|
-
const armed = await armCodexCapacityWait(db, {
|
|
9345
|
-
accountId: input.accountId,
|
|
9346
|
-
workspaceId: input.workspaceId,
|
|
9347
|
-
sessionId: input.sessionId,
|
|
9348
|
-
turnId,
|
|
9349
|
-
attemptId: input.attemptId,
|
|
9350
|
-
workflowId: input.workflowId,
|
|
9351
|
-
goalId: activeGoal?.id ?? null,
|
|
9352
|
-
goalVersion: activeGoal?.version ?? null,
|
|
9353
|
-
earliestResetAt: null,
|
|
9354
|
-
resetKind: "bounded_refresh",
|
|
9355
|
-
failurePayload: {
|
|
9356
|
-
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
9357
|
-
code: "codex_allocator_disabled",
|
|
9358
|
-
detail: "waiting for a credential to be re-enabled, reconnected, or added"
|
|
9359
|
-
}
|
|
9360
|
-
});
|
|
9361
|
-
if (armed.action === "waiting") {
|
|
9362
|
-
await publishDurableSessionEvents2(
|
|
9363
|
-
bus,
|
|
9649
|
+
if (effectiveCodexCredentialId) {
|
|
9650
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
9651
|
+
await recordSessionActiveCodexCredential(
|
|
9652
|
+
db,
|
|
9364
9653
|
input.workspaceId,
|
|
9365
9654
|
input.sessionId,
|
|
9366
|
-
|
|
9655
|
+
effectiveCodexCredentialId
|
|
9367
9656
|
);
|
|
9368
|
-
|
|
9369
|
-
|
|
9370
|
-
|
|
9371
|
-
status: "waiting_capacity",
|
|
9372
|
-
capacityWait: {
|
|
9373
|
-
waiterId: armed.waiter.id,
|
|
9374
|
-
generation: armed.waiter.generation,
|
|
9375
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
9376
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
9377
|
-
}
|
|
9378
|
-
});
|
|
9379
|
-
}
|
|
9380
|
-
if (!await settle({
|
|
9381
|
-
events: [
|
|
9382
|
-
{
|
|
9383
|
-
type: "turn.failed",
|
|
9384
|
-
payload: {
|
|
9385
|
-
error: "All connected Codex subscriptions are disabled for new allocations.",
|
|
9386
|
-
code: "codex_allocator_disabled",
|
|
9387
|
-
retryable: false,
|
|
9388
|
-
recovery: "user_message"
|
|
9389
|
-
}
|
|
9390
|
-
},
|
|
9391
|
-
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
9392
|
-
],
|
|
9393
|
-
turnStatus: "failed",
|
|
9394
|
-
sessionStatus: "idle",
|
|
9395
|
-
activeTurnId: null
|
|
9396
|
-
})) {
|
|
9397
|
-
return claimedResult({ status: "cancelled" });
|
|
9398
|
-
}
|
|
9399
|
-
turnMetricOutcome = "failed";
|
|
9400
|
-
activityStatus = "idle";
|
|
9401
|
-
return claimedResult({ status: "idle" });
|
|
9402
|
-
}
|
|
9403
|
-
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
9404
|
-
if (turn.source === "compaction") {
|
|
9405
|
-
if (!await settle({
|
|
9406
|
-
events: [
|
|
9657
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
9658
|
+
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
9659
|
+
await publish([
|
|
9407
9660
|
{
|
|
9408
|
-
type: "
|
|
9661
|
+
type: "codex.account.switched",
|
|
9409
9662
|
payload: {
|
|
9410
|
-
|
|
9411
|
-
|
|
9412
|
-
|
|
9663
|
+
fromAccountId: priorAccountId,
|
|
9664
|
+
toAccountId: effectiveCodexCredentialId,
|
|
9665
|
+
reason: rotated ? "rotation" : "manual"
|
|
9413
9666
|
}
|
|
9414
|
-
},
|
|
9415
|
-
{
|
|
9416
|
-
type: "session.status.changed",
|
|
9417
|
-
payload: { status: "idle" }
|
|
9418
9667
|
}
|
|
9419
|
-
]
|
|
9420
|
-
turnStatus: "cancelled",
|
|
9421
|
-
sessionStatus: "idle",
|
|
9422
|
-
activeTurnId: null
|
|
9423
|
-
})) {
|
|
9424
|
-
return claimedResult({ status: "cancelled" });
|
|
9668
|
+
]);
|
|
9425
9669
|
}
|
|
9426
|
-
|
|
9427
|
-
|
|
9428
|
-
|
|
9429
|
-
|
|
9430
|
-
|
|
9431
|
-
|
|
9432
|
-
|
|
9433
|
-
|
|
9434
|
-
const resumeMs = computeIdleDelayMs(
|
|
9435
|
-
rotationDecision.earliestResetAt,
|
|
9436
|
-
/* @__PURE__ */ new Date(),
|
|
9437
|
-
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
9438
|
-
);
|
|
9439
|
-
const failurePayload = codexUsageLimitFailurePayload(
|
|
9440
|
-
{ resetsInSeconds: Math.ceil(resumeMs / 1e3) },
|
|
9441
|
-
"all connected Codex subscriptions are rate-limited",
|
|
9442
|
-
{ allAccounts: true }
|
|
9443
|
-
);
|
|
9444
|
-
const authoritativeResetAt = authoritativeCodexCapacityResetAt(
|
|
9445
|
-
leased.accounts,
|
|
9446
|
-
/* @__PURE__ */ new Date()
|
|
9447
|
-
);
|
|
9448
|
-
const armed = await armCodexCapacityWait(db, {
|
|
9449
|
-
accountId: input.accountId,
|
|
9450
|
-
workspaceId: input.workspaceId,
|
|
9451
|
-
sessionId: input.sessionId,
|
|
9452
|
-
turnId,
|
|
9453
|
-
attemptId: input.attemptId,
|
|
9454
|
-
workflowId: input.workflowId,
|
|
9455
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
9456
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
9457
|
-
earliestResetAt: authoritativeResetAt,
|
|
9458
|
-
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
9459
|
-
failurePayload
|
|
9460
|
-
});
|
|
9461
|
-
if (armed.action === "waiting") {
|
|
9462
|
-
await publishDurableSessionEvents2(
|
|
9463
|
-
bus,
|
|
9464
|
-
input.workspaceId,
|
|
9465
|
-
input.sessionId,
|
|
9466
|
-
armed.events
|
|
9467
|
-
);
|
|
9468
|
-
turnMetricOutcome = "recovering";
|
|
9469
|
-
activityStatus = "waiting_capacity";
|
|
9470
|
-
return claimedResult({
|
|
9471
|
-
status: "waiting_capacity",
|
|
9472
|
-
capacityWait: {
|
|
9473
|
-
waiterId: armed.waiter.id,
|
|
9474
|
-
generation: armed.waiter.generation,
|
|
9475
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
9476
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
9670
|
+
const selectionReason = leased.reused ? "lease_reused" : sessionPin === effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active";
|
|
9671
|
+
observability.incrementCounter({
|
|
9672
|
+
name: "opengeni_codex_credential_selections_total",
|
|
9673
|
+
help: "Codex credential selections by strategy and reason.",
|
|
9674
|
+
labels: {
|
|
9675
|
+
workspace_key: codexWorkspaceKey,
|
|
9676
|
+
strategy: leased.rotationStrategy,
|
|
9677
|
+
reason: selectionReason
|
|
9477
9678
|
}
|
|
9478
9679
|
});
|
|
9479
|
-
}
|
|
9480
|
-
if (!await settle({
|
|
9481
|
-
events: [
|
|
9482
|
-
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
9483
|
-
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
9484
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
9485
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
9486
|
-
// the proactive path while the reactive path spares it.
|
|
9487
|
-
{
|
|
9488
|
-
type: "turn.failed",
|
|
9489
|
-
payload: {
|
|
9490
|
-
...failurePayload,
|
|
9491
|
-
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
9492
|
-
rotated: true
|
|
9493
|
-
}
|
|
9494
|
-
},
|
|
9495
|
-
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
9496
|
-
],
|
|
9497
|
-
turnStatus: "failed",
|
|
9498
|
-
sessionStatus: "idle",
|
|
9499
|
-
activeTurnId: null
|
|
9500
|
-
})) {
|
|
9501
|
-
return claimedResult({ status: "cancelled" });
|
|
9502
|
-
}
|
|
9503
|
-
turnMetricOutcome = "failed";
|
|
9504
|
-
activityStatus = "idle";
|
|
9505
|
-
return claimedResult(
|
|
9506
|
-
goalActive ? {
|
|
9507
|
-
status: "idle",
|
|
9508
|
-
continueDelayMs: resumeMs,
|
|
9509
|
-
idleUntilReset: true
|
|
9510
|
-
} : { status: "idle" }
|
|
9511
|
-
);
|
|
9512
|
-
}
|
|
9513
|
-
if (effectiveCodexCredentialId) {
|
|
9514
|
-
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
9515
|
-
await recordSessionActiveCodexCredential(
|
|
9516
|
-
db,
|
|
9517
|
-
input.workspaceId,
|
|
9518
|
-
input.sessionId,
|
|
9519
|
-
effectiveCodexCredentialId
|
|
9520
|
-
);
|
|
9521
|
-
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
9522
|
-
const rotated = rotationDecision.kind === "active" && rotationDecision.moved;
|
|
9523
9680
|
await publish([
|
|
9524
9681
|
{
|
|
9525
|
-
type: "codex.
|
|
9682
|
+
type: "codex.credential.selected",
|
|
9526
9683
|
payload: {
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
reason:
|
|
9684
|
+
credentialId: effectiveCodexCredentialId,
|
|
9685
|
+
strategy: leased.rotationStrategy,
|
|
9686
|
+
reason: selectionReason,
|
|
9687
|
+
eligibleCount,
|
|
9688
|
+
connectedCount: leased.accounts.length,
|
|
9689
|
+
reused: leased.reused
|
|
9530
9690
|
}
|
|
9531
9691
|
}
|
|
9532
9692
|
]);
|
|
9533
9693
|
}
|
|
9534
|
-
|
|
9535
|
-
|
|
9536
|
-
|
|
9537
|
-
|
|
9538
|
-
|
|
9539
|
-
|
|
9540
|
-
|
|
9541
|
-
|
|
9542
|
-
|
|
9694
|
+
} catch (error) {
|
|
9695
|
+
credentialSelectionOutcome = "failed";
|
|
9696
|
+
throw error;
|
|
9697
|
+
} finally {
|
|
9698
|
+
recordTurnStartupPhase(observability, {
|
|
9699
|
+
phase: "credential_selection",
|
|
9700
|
+
provider: "codex-subscription",
|
|
9701
|
+
backend: turn.sandboxBackend,
|
|
9702
|
+
outcome: credentialSelectionOutcome,
|
|
9703
|
+
durationSeconds: (performance.now() - credentialSelectionStartedAt) / 1e3
|
|
9543
9704
|
});
|
|
9544
|
-
await publish([
|
|
9545
|
-
{
|
|
9546
|
-
type: "codex.credential.selected",
|
|
9547
|
-
payload: {
|
|
9548
|
-
credentialId: effectiveCodexCredentialId,
|
|
9549
|
-
strategy: leased.rotationStrategy,
|
|
9550
|
-
reason: selectionReason,
|
|
9551
|
-
eligibleCount,
|
|
9552
|
-
connectedCount: leased.accounts.length,
|
|
9553
|
-
reused: leased.reused
|
|
9554
|
-
}
|
|
9555
|
-
}
|
|
9556
|
-
]);
|
|
9557
9705
|
}
|
|
9558
9706
|
}
|
|
9559
9707
|
if (isXaiTurn) {
|
|
@@ -9733,6 +9881,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9733
9881
|
credentialId: effectiveXaiCredentialId
|
|
9734
9882
|
});
|
|
9735
9883
|
}
|
|
9884
|
+
const runtimePreparationStartedAt = performance.now();
|
|
9736
9885
|
const [packRuntime, installedSkillRuntime] = await Promise.all([
|
|
9737
9886
|
resolveWorkspacePackRuntime(db, input.workspaceId),
|
|
9738
9887
|
resolveWorkspaceInstalledSkillRuntime(db, input.workspaceId)
|
|
@@ -9880,6 +10029,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
9880
10029
|
});
|
|
9881
10030
|
}
|
|
9882
10031
|
let codexModelRequestSequence = 0;
|
|
10032
|
+
let firstModelRequestPreparationStartedAt = null;
|
|
10033
|
+
let firstModelRequestPreparationRecorded = false;
|
|
10034
|
+
let firstModelRequestAuditRecorded = false;
|
|
10035
|
+
let firstModelRequestCheckpointAt = null;
|
|
10036
|
+
const firstModelRequestCheckpoints = /* @__PURE__ */ new Set();
|
|
9883
10037
|
const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
|
|
9884
10038
|
const resolver = buildCodexTokenResolver(
|
|
9885
10039
|
db,
|
|
@@ -9906,10 +10060,38 @@ function createRunAgentTurnActivity(services) {
|
|
|
9906
10060
|
latestCodexUsage = snapshot;
|
|
9907
10061
|
},
|
|
9908
10062
|
// latest wins; flushed once in finally
|
|
10063
|
+
onRequestPreparationDiagnostic: (phase) => {
|
|
10064
|
+
if (firstModelRequestCheckpointAt === null || firstModelRequestCheckpoints.has(phase)) {
|
|
10065
|
+
return;
|
|
10066
|
+
}
|
|
10067
|
+
const now = performance.now();
|
|
10068
|
+
const metricPhase = phase === "transport_entry" ? "model_sdk_serialization" : phase === "credential_ready" ? "model_credential_resolution" : "model_wire_normalization";
|
|
10069
|
+
firstModelRequestCheckpoints.add(phase);
|
|
10070
|
+
recordTurnStartupPhase(observability, {
|
|
10071
|
+
phase: metricPhase,
|
|
10072
|
+
provider: turnExecutionPolicy.providerId,
|
|
10073
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
10074
|
+
outcome: "completed",
|
|
10075
|
+
durationSeconds: (now - firstModelRequestCheckpointAt) / 1e3,
|
|
10076
|
+
count: turnTools.length
|
|
10077
|
+
});
|
|
10078
|
+
firstModelRequestCheckpointAt = now;
|
|
10079
|
+
},
|
|
9909
10080
|
onRequestOpaqueArtifacts: ({ fingerprints }) => {
|
|
9910
10081
|
lastCodexRequestOpaqueArtifacts = fingerprints;
|
|
9911
10082
|
},
|
|
9912
10083
|
onModelRequestDiagnostic: (event) => {
|
|
10084
|
+
if (event.phase === "started" && firstModelRequestPreparationStartedAt !== null && !firstModelRequestPreparationRecorded) {
|
|
10085
|
+
firstModelRequestPreparationRecorded = true;
|
|
10086
|
+
recordTurnStartupPhase(observability, {
|
|
10087
|
+
phase: "model_request_preparation",
|
|
10088
|
+
provider: turnExecutionPolicy.providerId,
|
|
10089
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
10090
|
+
outcome: "completed",
|
|
10091
|
+
durationSeconds: (performance.now() - firstModelRequestPreparationStartedAt) / 1e3,
|
|
10092
|
+
count: turnTools.length
|
|
10093
|
+
});
|
|
10094
|
+
}
|
|
9913
10095
|
const phase = event.phase === "headers" ? "headers" : event.phase === "first_byte" ? "first_byte" : event.phase === "completed" || event.phase === "failed" || event.phase === "timed_out" ? "terminal" : null;
|
|
9914
10096
|
if (!phase) return;
|
|
9915
10097
|
recordModelRequestPhase(observability, {
|
|
@@ -9924,19 +10106,41 @@ function createRunAgentTurnActivity(services) {
|
|
|
9924
10106
|
if (!publish || !turnId) {
|
|
9925
10107
|
throw new Error("Codex model request started before the turn event producer");
|
|
9926
10108
|
}
|
|
9927
|
-
|
|
9928
|
-
|
|
9929
|
-
|
|
9930
|
-
|
|
9931
|
-
|
|
9932
|
-
|
|
9933
|
-
|
|
9934
|
-
|
|
9935
|
-
|
|
9936
|
-
|
|
10109
|
+
const shouldRecordStartedAudit = event.phase === "started" && !firstModelRequestAuditRecorded;
|
|
10110
|
+
if (shouldRecordStartedAudit) {
|
|
10111
|
+
firstModelRequestAuditRecorded = true;
|
|
10112
|
+
}
|
|
10113
|
+
const auditStartedAt = shouldRecordStartedAudit ? performance.now() : null;
|
|
10114
|
+
let auditOutcome = "completed";
|
|
10115
|
+
try {
|
|
10116
|
+
await publish([
|
|
10117
|
+
{
|
|
10118
|
+
type: "agent.model.request",
|
|
10119
|
+
payload: {
|
|
10120
|
+
...event,
|
|
10121
|
+
provider: "codex-subscription",
|
|
10122
|
+
turnId,
|
|
10123
|
+
attemptId: input.attemptId,
|
|
10124
|
+
dispatchId,
|
|
10125
|
+
executionGeneration
|
|
10126
|
+
}
|
|
9937
10127
|
}
|
|
10128
|
+
]);
|
|
10129
|
+
} catch (error) {
|
|
10130
|
+
auditOutcome = "failed";
|
|
10131
|
+
throw error;
|
|
10132
|
+
} finally {
|
|
10133
|
+
if (auditStartedAt !== null) {
|
|
10134
|
+
recordTurnStartupPhase(observability, {
|
|
10135
|
+
phase: "model_request_audit",
|
|
10136
|
+
provider: turnExecutionPolicy.providerId,
|
|
10137
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
10138
|
+
outcome: auditOutcome,
|
|
10139
|
+
durationSeconds: (performance.now() - auditStartedAt) / 1e3,
|
|
10140
|
+
count: turnTools.length
|
|
10141
|
+
});
|
|
9938
10142
|
}
|
|
9939
|
-
|
|
10143
|
+
}
|
|
9940
10144
|
}
|
|
9941
10145
|
};
|
|
9942
10146
|
})() : null;
|
|
@@ -10343,10 +10547,26 @@ function createRunAgentTurnActivity(services) {
|
|
|
10343
10547
|
cancellationSignal,
|
|
10344
10548
|
void 0
|
|
10345
10549
|
);
|
|
10346
|
-
const
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10550
|
+
const establishDecision = sandboxEstablishPolicyDecision({
|
|
10551
|
+
lazyEnabled: lazyProvisionEnabled(settings),
|
|
10552
|
+
machinePrimary,
|
|
10553
|
+
sandboxBackend: runSettings.sandboxBackend,
|
|
10554
|
+
// Resolved run credentials must be written to one exact leased sandbox
|
|
10555
|
+
// before agent execution. A warm active pointer can bypass the lazy
|
|
10556
|
+
// provisioner entirely, which previously skipped materialization.
|
|
10557
|
+
hasInitialRunCredentialMaterial: initialRunCredentialMaterial !== null,
|
|
10558
|
+
generatedVideoFileCount: requiredGeneratedVideoFiles.length,
|
|
10559
|
+
hasSignedFileResources: requiresSignedFileResourceDownloads(
|
|
10560
|
+
runSettings,
|
|
10561
|
+
activeSandboxBackend ?? groupBoxBackend
|
|
10562
|
+
) && turnResources.some((resource) => resource.kind === "file")
|
|
10563
|
+
});
|
|
10564
|
+
const establishPolicy = establishDecision.policy;
|
|
10565
|
+
recordTurnSandboxEstablishPolicy(observability, {
|
|
10566
|
+
policy: establishDecision.policy,
|
|
10567
|
+
reason: establishDecision.reason,
|
|
10568
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend
|
|
10569
|
+
});
|
|
10350
10570
|
if (activeSandboxBackend !== "selfhosted") {
|
|
10351
10571
|
await assertGitHubResourcesRemainAuthorized(db, input.workspaceId, turnResources);
|
|
10352
10572
|
}
|
|
@@ -10702,195 +10922,235 @@ function createRunAgentTurnActivity(services) {
|
|
|
10702
10922
|
}
|
|
10703
10923
|
runCredentialRenewal = controller;
|
|
10704
10924
|
};
|
|
10925
|
+
recordTurnStartupPhase(observability, {
|
|
10926
|
+
phase: "runtime_preparation",
|
|
10927
|
+
provider: turnExecutionPolicy.providerId,
|
|
10928
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
10929
|
+
outcome: "completed",
|
|
10930
|
+
durationSeconds: (performance.now() - runtimePreparationStartedAt) / 1e3
|
|
10931
|
+
});
|
|
10705
10932
|
if (shouldEstablishSandboxForTurn(
|
|
10706
10933
|
settings.sandboxOwnershipEnabled,
|
|
10707
10934
|
turn.sandboxBackend,
|
|
10708
10935
|
machinePrimary
|
|
10709
10936
|
)) {
|
|
10710
|
-
const
|
|
10711
|
-
|
|
10712
|
-
|
|
10713
|
-
|
|
10714
|
-
|
|
10715
|
-
|
|
10716
|
-
|
|
10717
|
-
if (machinePrimary) {
|
|
10718
|
-
const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
|
|
10719
|
-
const established = await establishSelfhostedTurnSession(
|
|
10720
|
-
{
|
|
10721
|
-
db,
|
|
10722
|
-
settings,
|
|
10723
|
-
bus,
|
|
10724
|
-
onOp: machineOpObserver.observer,
|
|
10725
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
10726
|
-
opJournal
|
|
10727
|
-
},
|
|
10728
|
-
{
|
|
10729
|
-
workspaceId: input.workspaceId,
|
|
10730
|
-
agentId: activeSandboxRecord.enrollmentId,
|
|
10731
|
-
opStream: machineOpStream,
|
|
10732
|
-
epoch: activeSandboxPointer.activeEpoch,
|
|
10733
|
-
environment: sandboxEnvironment,
|
|
10734
|
-
...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
|
|
10735
|
-
workingDir: activeSandboxPointer.workingDir
|
|
10736
|
-
}
|
|
10937
|
+
const sandboxEstablishStartedAt = performance.now();
|
|
10938
|
+
let sandboxEstablishOutcome = "completed";
|
|
10939
|
+
try {
|
|
10940
|
+
const managedOwnership = managedSandboxOwnershipForTurn(
|
|
10941
|
+
machinePrimary,
|
|
10942
|
+
input.attemptId,
|
|
10943
|
+
session.sandboxGroupId
|
|
10737
10944
|
);
|
|
10738
|
-
|
|
10739
|
-
|
|
10740
|
-
|
|
10741
|
-
|
|
10742
|
-
|
|
10743
|
-
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
10744
|
-
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
10745
|
-
// instance — no two-instance manifest divergence.
|
|
10746
|
-
established: wrapTurnBoxWithRouting(
|
|
10945
|
+
sandboxHolderId = managedOwnership?.holderId ?? null;
|
|
10946
|
+
sandboxGroupId = managedOwnership?.sandboxGroupId ?? null;
|
|
10947
|
+
if (machinePrimary) {
|
|
10948
|
+
const machineOpStream = settings.agentOpStreamEnabled === true ? (await getEnrollment(db, input.workspaceId, activeSandboxRecord.enrollmentId))?.opStream === true : false;
|
|
10949
|
+
const established = await establishSelfhostedTurnSession(
|
|
10747
10950
|
{
|
|
10748
10951
|
db,
|
|
10749
10952
|
settings,
|
|
10750
10953
|
bus,
|
|
10751
|
-
opJournal,
|
|
10752
10954
|
onOp: machineOpObserver.observer,
|
|
10753
10955
|
onSandboxOperation: sandboxOperationObserver,
|
|
10754
|
-
|
|
10755
|
-
...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
|
|
10956
|
+
opJournal
|
|
10756
10957
|
},
|
|
10757
10958
|
{
|
|
10758
10959
|
workspaceId: input.workspaceId,
|
|
10759
|
-
|
|
10960
|
+
agentId: activeSandboxRecord.enrollmentId,
|
|
10961
|
+
opStream: machineOpStream,
|
|
10962
|
+
epoch: activeSandboxPointer.activeEpoch,
|
|
10760
10963
|
environment: sandboxEnvironment,
|
|
10761
10964
|
...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10965
|
+
workingDir: activeSandboxPointer.workingDir
|
|
10966
|
+
}
|
|
10967
|
+
);
|
|
10968
|
+
machinePrimarySession = established.session;
|
|
10969
|
+
setupBoxSession = established.session;
|
|
10970
|
+
resolvedSandbox = {
|
|
10971
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
10972
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
10973
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
10974
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
10975
|
+
// instance — no two-instance manifest divergence.
|
|
10976
|
+
established: wrapTurnBoxWithRouting(
|
|
10977
|
+
{
|
|
10978
|
+
db,
|
|
10979
|
+
settings,
|
|
10980
|
+
bus,
|
|
10981
|
+
opJournal,
|
|
10982
|
+
onOp: machineOpObserver.observer,
|
|
10983
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
10984
|
+
onHomeSandboxRebound,
|
|
10985
|
+
...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
|
|
10767
10986
|
},
|
|
10768
|
-
|
|
10769
|
-
|
|
10770
|
-
|
|
10987
|
+
{
|
|
10988
|
+
workspaceId: input.workspaceId,
|
|
10989
|
+
sessionId: input.sessionId,
|
|
10990
|
+
environment: sandboxEnvironment,
|
|
10991
|
+
...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
|
|
10992
|
+
workspaceMutationFence: {
|
|
10993
|
+
accountId: input.accountId,
|
|
10994
|
+
turnId: turn.id,
|
|
10995
|
+
executionGeneration,
|
|
10996
|
+
attemptId: input.attemptId
|
|
10997
|
+
},
|
|
10998
|
+
pinnedSelfhosted: {
|
|
10999
|
+
sandboxId: activeSandboxPointer.activeSandboxId,
|
|
11000
|
+
epoch: activeSandboxPointer.activeEpoch
|
|
11001
|
+
},
|
|
11002
|
+
// HOME semantics for a mid-turn clear-to-null: only a genuine
|
|
11003
|
+
// machine-HOME session (its home IS this machine, session.sandboxBackend
|
|
11004
|
+
// === "selfhosted") resolves null back to the pinned machine. A Modal-HOME
|
|
11005
|
+
// session merely PINNED to a machine this turn never established its group
|
|
11006
|
+
// box, so defaultIsHome:false makes a clear-to-null fail typed
|
|
11007
|
+
// (`home_unavailable_this_turn`) rather than silently serving the machine;
|
|
11008
|
+
// the detach takes effect next turn. (Lazy home-box establishment on such a
|
|
11009
|
+
// clear is a deferred follow-up; issue #341.)
|
|
11010
|
+
defaultIsHome: session.sandboxBackend === "selfhosted"
|
|
10771
11011
|
},
|
|
10772
|
-
|
|
10773
|
-
|
|
10774
|
-
|
|
10775
|
-
|
|
10776
|
-
|
|
10777
|
-
|
|
10778
|
-
|
|
10779
|
-
|
|
10780
|
-
|
|
10781
|
-
|
|
10782
|
-
|
|
10783
|
-
|
|
10784
|
-
|
|
10785
|
-
|
|
10786
|
-
|
|
10787
|
-
|
|
10788
|
-
|
|
10789
|
-
|
|
10790
|
-
|
|
10791
|
-
} else {
|
|
10792
|
-
resolvedSandbox = await waitForTurnOperation(
|
|
10793
|
-
resumeBoxForTurn(
|
|
10794
|
-
{
|
|
10795
|
-
db,
|
|
10796
|
-
settings: runSettings,
|
|
10797
|
-
logicalFallbackSettings: logicalSandboxSettings,
|
|
10798
|
-
cancellationSignal: sandboxResumeSignal,
|
|
10799
|
-
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
10800
|
-
onSandboxLost: publishSandboxLost
|
|
10801
|
-
},
|
|
10802
|
-
{
|
|
10803
|
-
accountId: input.accountId,
|
|
10804
|
-
workspaceId: input.workspaceId,
|
|
10805
|
-
sandboxGroupId: session.sandboxGroupId,
|
|
10806
|
-
sessionId: input.sessionId,
|
|
10807
|
-
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
10808
|
-
// is not machine-primary resumes a real cloud group box (the
|
|
10809
|
-
// deployment default), never a "selfhosted" box (which would throw
|
|
10810
|
-
// for lack of a bound agentId).
|
|
10811
|
-
backend: groupBoxBackend,
|
|
10812
|
-
os: session.sandboxOs,
|
|
10813
|
-
environment: sandboxEnvironment,
|
|
10814
|
-
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
10815
|
-
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
10816
|
-
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
10817
|
-
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
10818
|
-
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
10819
|
-
// (establishSelfhostedTurnSession) NEVER passes
|
|
10820
|
-
// an image — B3 lives only on this Modal else-branch.
|
|
10821
|
-
...runSettings.modalImageRef ?? runSettings.dockerImage ? {
|
|
10822
|
-
image: runSettings.modalImageRef ?? runSettings.dockerImage
|
|
10823
|
-
} : {},
|
|
10824
|
-
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
10825
|
-
// conflicts on a live shared box set up under a different rig (solo
|
|
10826
|
-
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
10827
|
-
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
10828
|
-
...rigVersion ? { rigVersionId: rigVersion.id } : {}
|
|
10829
|
-
},
|
|
10830
|
-
"turn",
|
|
10831
|
-
managedOwnership.holderId
|
|
10832
|
-
),
|
|
10833
|
-
sandboxResumeSignal,
|
|
10834
|
-
releaseLateSandbox
|
|
10835
|
-
);
|
|
10836
|
-
setupBoxSession = resolvedSandbox.established.session;
|
|
10837
|
-
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
10838
|
-
resolvedSandbox = {
|
|
10839
|
-
...resolvedSandbox,
|
|
10840
|
-
established: wrapTurnBoxWithRouting(
|
|
10841
|
-
{
|
|
10842
|
-
db,
|
|
10843
|
-
settings,
|
|
10844
|
-
bus,
|
|
10845
|
-
opJournal,
|
|
10846
|
-
onSandboxOperation: sandboxOperationObserver,
|
|
10847
|
-
onHomeSandboxLost: publishSandboxLost,
|
|
10848
|
-
onHomeSandboxRebound,
|
|
10849
|
-
...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
|
|
10850
|
-
},
|
|
10851
|
-
// Thread the SAME declared environment the group box was created with
|
|
10852
|
-
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
10853
|
-
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
10854
|
-
// (no "cannot change manifest environment variables" throw).
|
|
10855
|
-
{
|
|
10856
|
-
workspaceId: input.workspaceId,
|
|
10857
|
-
sessionId: input.sessionId,
|
|
10858
|
-
environment: sandboxEnvironment,
|
|
10859
|
-
...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
|
|
10860
|
-
workspaceMutationFence: {
|
|
10861
|
-
accountId: input.accountId,
|
|
10862
|
-
turnId: turn.id,
|
|
10863
|
-
executionGeneration,
|
|
10864
|
-
attemptId: input.attemptId
|
|
11012
|
+
established
|
|
11013
|
+
),
|
|
11014
|
+
// For a machine route this is the active-pointer epoch, not a cloud
|
|
11015
|
+
// lease epoch. Cloud-only heartbeat/snapshot/capture paths require
|
|
11016
|
+
// sandboxGroupId and remain disabled for this branch.
|
|
11017
|
+
leaseEpoch: activeSandboxPointer.activeEpoch,
|
|
11018
|
+
release: async () => void 0
|
|
11019
|
+
};
|
|
11020
|
+
} else if (establishPolicy === "on-demand") {
|
|
11021
|
+
} else {
|
|
11022
|
+
resolvedSandbox = await waitForTurnOperation(
|
|
11023
|
+
resumeBoxForTurn(
|
|
11024
|
+
{
|
|
11025
|
+
db,
|
|
11026
|
+
settings: runSettings,
|
|
11027
|
+
logicalFallbackSettings: logicalSandboxSettings,
|
|
11028
|
+
cancellationSignal: sandboxResumeSignal,
|
|
11029
|
+
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
11030
|
+
onSandboxLost: publishSandboxLost
|
|
10865
11031
|
},
|
|
10866
|
-
|
|
11032
|
+
{
|
|
10867
11033
|
accountId: input.accountId,
|
|
11034
|
+
workspaceId: input.workspaceId,
|
|
10868
11035
|
sandboxGroupId: session.sandboxGroupId,
|
|
10869
|
-
|
|
10870
|
-
|
|
10871
|
-
|
|
10872
|
-
|
|
10873
|
-
|
|
10874
|
-
|
|
10875
|
-
|
|
10876
|
-
|
|
10877
|
-
|
|
10878
|
-
|
|
10879
|
-
|
|
11036
|
+
sessionId: input.sessionId,
|
|
11037
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
11038
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
11039
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
11040
|
+
// for lack of a bound agentId).
|
|
11041
|
+
backend: groupBoxBackend,
|
|
11042
|
+
os: session.sandboxOs,
|
|
11043
|
+
environment: sandboxEnvironment,
|
|
11044
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
11045
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
11046
|
+
// running a DIFFERENT image (solo → durable rotation; N-holders →
|
|
11047
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer
|
|
11048
|
+
// the explicit Modal image ref, else the docker image. The selfhosted branch
|
|
11049
|
+
// (establishSelfhostedTurnSession) NEVER passes
|
|
11050
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
11051
|
+
...runSettings.modalImageRef ?? runSettings.dockerImage ? {
|
|
11052
|
+
image: runSettings.modalImageRef ?? runSettings.dockerImage
|
|
11053
|
+
} : {},
|
|
11054
|
+
// RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
|
|
11055
|
+
// conflicts on a live shared box set up under a different rig (solo
|
|
11056
|
+
// durable rotation / N-holders SandboxRigConflictError). Omitted for a
|
|
11057
|
+
// rig-less turn -> never stamped or enforced (shares exactly as today).
|
|
11058
|
+
...rigVersion ? { rigVersionId: rigVersion.id } : {}
|
|
11059
|
+
},
|
|
11060
|
+
"turn",
|
|
11061
|
+
managedOwnership.holderId
|
|
11062
|
+
),
|
|
11063
|
+
sandboxResumeSignal,
|
|
11064
|
+
releaseLateSandbox
|
|
11065
|
+
);
|
|
11066
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
11067
|
+
await publishSandboxLifecycleEvents(resolvedSandbox);
|
|
11068
|
+
resolvedSandbox = {
|
|
11069
|
+
...resolvedSandbox,
|
|
11070
|
+
established: wrapTurnBoxWithRouting(
|
|
11071
|
+
{
|
|
11072
|
+
db,
|
|
11073
|
+
settings,
|
|
11074
|
+
bus,
|
|
11075
|
+
opJournal,
|
|
11076
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
11077
|
+
onHomeSandboxLost: publishSandboxLost,
|
|
11078
|
+
onHomeSandboxRebound,
|
|
11079
|
+
...runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}
|
|
11080
|
+
},
|
|
11081
|
+
// Thread the SAME declared environment the group box was created with
|
|
11082
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
11083
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
11084
|
+
// (no "cannot change manifest environment variables" throw).
|
|
11085
|
+
{
|
|
11086
|
+
workspaceId: input.workspaceId,
|
|
11087
|
+
sessionId: input.sessionId,
|
|
11088
|
+
environment: sandboxEnvironment,
|
|
11089
|
+
...transientCodemodeEnvironment ? { transientExecEnvironment: transientCodemodeEnvironment } : {},
|
|
11090
|
+
workspaceMutationFence: {
|
|
11091
|
+
accountId: input.accountId,
|
|
11092
|
+
turnId: turn.id,
|
|
11093
|
+
executionGeneration,
|
|
11094
|
+
attemptId: input.attemptId
|
|
11095
|
+
},
|
|
11096
|
+
homeLease: {
|
|
11097
|
+
accountId: input.accountId,
|
|
11098
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
11099
|
+
leaseEpoch: resolvedSandbox.leaseEpoch,
|
|
11100
|
+
instanceId: resolvedSandbox.established.instanceId,
|
|
11101
|
+
backend: groupBoxBackend
|
|
11102
|
+
}
|
|
11103
|
+
},
|
|
11104
|
+
resolvedSandbox.established
|
|
11105
|
+
)
|
|
11106
|
+
};
|
|
11107
|
+
}
|
|
11108
|
+
if (resolvedSandbox) {
|
|
11109
|
+
startLeaseHeartbeat(resolvedSandbox, activeSandboxBackend ?? groupBoxBackend);
|
|
11110
|
+
}
|
|
11111
|
+
} catch (error) {
|
|
11112
|
+
sandboxEstablishOutcome = "failed";
|
|
11113
|
+
throw error;
|
|
11114
|
+
} finally {
|
|
11115
|
+
recordTurnStartupPhase(observability, {
|
|
11116
|
+
phase: "sandbox_establish",
|
|
11117
|
+
provider: turnExecutionPolicy.providerId,
|
|
11118
|
+
backend: machinePrimary ? "selfhosted" : groupBoxBackend,
|
|
11119
|
+
outcome: sandboxEstablishOutcome,
|
|
11120
|
+
durationSeconds: (performance.now() - sandboxEstablishStartedAt) / 1e3
|
|
11121
|
+
});
|
|
10880
11122
|
}
|
|
10881
11123
|
}
|
|
10882
|
-
const ordinaryFileResourceDownloads = await
|
|
10883
|
-
|
|
10884
|
-
|
|
10885
|
-
|
|
10886
|
-
|
|
10887
|
-
|
|
10888
|
-
|
|
10889
|
-
|
|
10890
|
-
|
|
10891
|
-
|
|
10892
|
-
|
|
10893
|
-
|
|
11124
|
+
const ordinaryFileResourceDownloads = await (async () => {
|
|
11125
|
+
const fileResolutionStartedAt = performance.now();
|
|
11126
|
+
let fileResolutionOutcome = "completed";
|
|
11127
|
+
try {
|
|
11128
|
+
return await waitForTurnOperation(
|
|
11129
|
+
sandboxFileDownloadsForRun(
|
|
11130
|
+
runSettings,
|
|
11131
|
+
db,
|
|
11132
|
+
objectStorage,
|
|
11133
|
+
input.workspaceId,
|
|
11134
|
+
turnResources,
|
|
11135
|
+
activeSandboxBackend ?? groupBoxBackend
|
|
11136
|
+
),
|
|
11137
|
+
cancellationSignal,
|
|
11138
|
+
void 0
|
|
11139
|
+
);
|
|
11140
|
+
} catch (error) {
|
|
11141
|
+
fileResolutionOutcome = "failed";
|
|
11142
|
+
throw error;
|
|
11143
|
+
} finally {
|
|
11144
|
+
recordTurnStartupPhase(observability, {
|
|
11145
|
+
phase: "file_resolution",
|
|
11146
|
+
provider: turnExecutionPolicy.providerId,
|
|
11147
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11148
|
+
outcome: fileResolutionOutcome,
|
|
11149
|
+
durationSeconds: (performance.now() - fileResolutionStartedAt) / 1e3,
|
|
11150
|
+
count: turnResources.length
|
|
11151
|
+
});
|
|
11152
|
+
}
|
|
11153
|
+
})();
|
|
10894
11154
|
if (requiredGeneratedVideoFiles.length > 0 && !objectStorage) {
|
|
10895
11155
|
throw new Error("Generated video materialization requires object storage");
|
|
10896
11156
|
}
|
|
@@ -10920,6 +11180,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
10920
11180
|
])
|
|
10921
11181
|
).values()
|
|
10922
11182
|
];
|
|
11183
|
+
const toolContextPreparationStartedAt = performance.now();
|
|
10923
11184
|
throwIfWorkerShuttingDown();
|
|
10924
11185
|
throwIfTurnCancelled();
|
|
10925
11186
|
const mcpCredentialRootSessionId = connectionCredentials?.mcpCredentials && hostCredentialRootSessionId ? hostCredentialRootSessionId : input.sessionId;
|
|
@@ -11005,53 +11266,88 @@ function createRunAgentTurnActivity(services) {
|
|
|
11005
11266
|
runSettings,
|
|
11006
11267
|
session.firstPartyMcpTools
|
|
11007
11268
|
);
|
|
11008
|
-
|
|
11009
|
-
|
|
11010
|
-
|
|
11011
|
-
|
|
11012
|
-
|
|
11013
|
-
|
|
11014
|
-
|
|
11015
|
-
|
|
11016
|
-
|
|
11017
|
-
|
|
11018
|
-
|
|
11019
|
-
|
|
11020
|
-
|
|
11021
|
-
|
|
11022
|
-
|
|
11023
|
-
|
|
11024
|
-
|
|
11025
|
-
|
|
11026
|
-
|
|
11027
|
-
|
|
11028
|
-
|
|
11029
|
-
|
|
11030
|
-
// for their first-party MCP token; null keeps the fixed default.
|
|
11031
|
-
...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {},
|
|
11032
|
-
firstPartyTools: selectedFirstPartyMcpTools,
|
|
11033
|
-
nestedAgentDepth: session.nestedAgentDepth,
|
|
11034
|
-
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
11035
|
-
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
11036
|
-
settings: runSettings,
|
|
11037
|
-
scope: {
|
|
11038
|
-
accountId: input.accountId,
|
|
11039
|
-
workspaceId: input.workspaceId,
|
|
11040
|
-
sessionId: input.sessionId,
|
|
11041
|
-
turnId: turn.id,
|
|
11042
|
-
attemptId: input.attemptId,
|
|
11043
|
-
executionGeneration
|
|
11044
|
-
},
|
|
11045
|
-
...session.firstPartyMcpPermissions?.length ? { permissions: session.firstPartyMcpPermissions } : {},
|
|
11046
|
-
selectedTools: selectedFirstPartyMcpTools,
|
|
11269
|
+
recordTurnStartupPhase(observability, {
|
|
11270
|
+
phase: "tool_context_preparation",
|
|
11271
|
+
provider: turnExecutionPolicy.providerId,
|
|
11272
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11273
|
+
outcome: "completed",
|
|
11274
|
+
durationSeconds: (performance.now() - toolContextPreparationStartedAt) / 1e3,
|
|
11275
|
+
count: turnTools.length
|
|
11276
|
+
});
|
|
11277
|
+
const toolPreparationStartedAt = performance.now();
|
|
11278
|
+
let toolPreparationOutcome = "completed";
|
|
11279
|
+
try {
|
|
11280
|
+
preparedTools = await waitForTurnOperation(
|
|
11281
|
+
runtime.prepareTools(runSettings, turnTools, {
|
|
11282
|
+
accountId: input.accountId,
|
|
11283
|
+
workspaceId: input.workspaceId,
|
|
11284
|
+
sessionId: input.sessionId,
|
|
11285
|
+
// Sign the calling turn into the first-party token so tools classify
|
|
11286
|
+
// the caller by its own identity (sacred-pause guard), not the racy
|
|
11287
|
+
// live active pointer.
|
|
11288
|
+
...turnId ? { turnId } : {},
|
|
11289
|
+
attemptId: input.attemptId,
|
|
11290
|
+
executionGeneration,
|
|
11047
11291
|
subjectId: "worker:first-party-mcp",
|
|
11048
11292
|
subjectLabel: "OpenGeni worker",
|
|
11049
|
-
...
|
|
11050
|
-
|
|
11051
|
-
|
|
11052
|
-
|
|
11053
|
-
|
|
11054
|
-
|
|
11293
|
+
...credentialSubjectId ? { credentialSubjectId } : {},
|
|
11294
|
+
...codexAppsAuth ? { codexAppsAuth } : {},
|
|
11295
|
+
resolveCredential,
|
|
11296
|
+
onAuthNeeded: publishToolAuthNeeded,
|
|
11297
|
+
localMcpServers,
|
|
11298
|
+
onPreparationPhase: (measurement) => {
|
|
11299
|
+
recordTurnStartupPhase(observability, {
|
|
11300
|
+
phase: `tool_${measurement.phase}`,
|
|
11301
|
+
provider: turnExecutionPolicy.providerId,
|
|
11302
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11303
|
+
outcome: measurement.outcome,
|
|
11304
|
+
durationSeconds: measurement.durationSeconds,
|
|
11305
|
+
count: turnTools.length
|
|
11306
|
+
});
|
|
11307
|
+
},
|
|
11308
|
+
onAttemptToolCatalog: async (catalog) => {
|
|
11309
|
+
await persistAttemptToolCatalog(db, catalog);
|
|
11310
|
+
},
|
|
11311
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
11312
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
11313
|
+
...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {},
|
|
11314
|
+
firstPartyTools: selectedFirstPartyMcpTools,
|
|
11315
|
+
nestedAgentDepth: session.nestedAgentDepth,
|
|
11316
|
+
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
11317
|
+
attemptToolDefinitions: createFirstPartyInteractionAttemptToolDefinitions({
|
|
11318
|
+
settings: runSettings,
|
|
11319
|
+
scope: {
|
|
11320
|
+
accountId: input.accountId,
|
|
11321
|
+
workspaceId: input.workspaceId,
|
|
11322
|
+
sessionId: input.sessionId,
|
|
11323
|
+
turnId: turn.id,
|
|
11324
|
+
attemptId: input.attemptId,
|
|
11325
|
+
executionGeneration
|
|
11326
|
+
},
|
|
11327
|
+
...session.firstPartyMcpPermissions?.length ? { permissions: session.firstPartyMcpPermissions } : {},
|
|
11328
|
+
selectedTools: selectedFirstPartyMcpTools,
|
|
11329
|
+
subjectId: "worker:first-party-mcp",
|
|
11330
|
+
subjectLabel: "OpenGeni worker",
|
|
11331
|
+
...interactionInterventionResume ? { interventionResume: interactionInterventionResume } : {}
|
|
11332
|
+
})
|
|
11333
|
+
}),
|
|
11334
|
+
cancellationSignal,
|
|
11335
|
+
async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
|
|
11336
|
+
);
|
|
11337
|
+
} catch (error) {
|
|
11338
|
+
toolPreparationOutcome = "failed";
|
|
11339
|
+
throw error;
|
|
11340
|
+
} finally {
|
|
11341
|
+
recordTurnStartupPhase(observability, {
|
|
11342
|
+
phase: "tool_preparation",
|
|
11343
|
+
provider: turnExecutionPolicy.providerId,
|
|
11344
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11345
|
+
outcome: toolPreparationOutcome,
|
|
11346
|
+
durationSeconds: (performance.now() - toolPreparationStartedAt) / 1e3,
|
|
11347
|
+
count: turnTools.length
|
|
11348
|
+
});
|
|
11349
|
+
}
|
|
11350
|
+
const postToolPreparationStartedAt = performance.now();
|
|
11055
11351
|
if (turnId && preparedTools.attemptToolEnvironment) {
|
|
11056
11352
|
codemodeDispatcher = new CodemodeAttemptDispatcher(
|
|
11057
11353
|
db,
|
|
@@ -11399,146 +11695,171 @@ function createRunAgentTurnActivity(services) {
|
|
|
11399
11695
|
reason: "attached to session"
|
|
11400
11696
|
}))
|
|
11401
11697
|
];
|
|
11402
|
-
|
|
11403
|
-
|
|
11404
|
-
|
|
11405
|
-
|
|
11406
|
-
|
|
11407
|
-
|
|
11408
|
-
genesisTitleHint: isGenesisTurn,
|
|
11409
|
-
persistentSessionSettings: {
|
|
11410
|
-
titleIsSet: Boolean(session.title?.trim())
|
|
11411
|
-
},
|
|
11412
|
-
sandboxEnvironment,
|
|
11413
|
-
...preparedTools.attemptToolCatalog ? { attemptToolCatalog: preparedTools.attemptToolCatalog } : {},
|
|
11414
|
-
...sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {},
|
|
11415
|
-
...cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {},
|
|
11416
|
-
onToolCancellationFence: (fence) => {
|
|
11417
|
-
toolCancellationFenceRef.current = fence;
|
|
11418
|
-
},
|
|
11419
|
-
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
11420
|
-
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
11421
|
-
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
11422
|
-
// AND the mint actually produced a token (repo resources present). The runtime
|
|
11423
|
-
// seeds it to the box's token file before the repository-clone runs; it never
|
|
11424
|
-
// touches the box/agent manifest env.
|
|
11425
|
-
...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
|
|
11426
|
-
...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
|
|
11427
|
-
...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
11428
|
-
...sandboxCodemodeToken ? { codemodeAvailable: true } : {},
|
|
11429
|
-
// Managed boxes receive the bearer through their protected per-session
|
|
11430
|
-
// token file. Connected Machines use transient per-exec delivery above,
|
|
11431
|
-
// so they must not run the file-seeding lifecycle hook.
|
|
11432
|
-
...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken ? {
|
|
11433
|
-
codemodeTokenSeed: sandboxCodemodeToken,
|
|
11434
|
-
codemodeTokenSessionId: input.sessionId
|
|
11435
|
-
} : {},
|
|
11436
|
-
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
11437
|
-
fileResourceDownloads,
|
|
11438
|
-
mcpServers: preparedTools.mcpServers,
|
|
11439
|
-
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
11440
|
-
connectorActionPolicy,
|
|
11441
|
-
// LIVE by-reference connector namespaces (fills during this turn's
|
|
11442
|
-
// codex_apps tools/list): the codex tool_search description reads it per
|
|
11443
|
-
// model call so the model sees the account's real connected sources.
|
|
11444
|
-
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
11445
|
-
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
11446
|
-
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
11447
|
-
// NOT a Model instance: an instance only survives the in-process
|
|
11448
|
-
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
11449
|
-
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
11450
|
-
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
11451
|
-
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
11452
|
-
// client. The gating still comes from the resolved provider: server-side
|
|
11453
|
-
// store/compaction follow the provider's compaction mode (registry
|
|
11454
|
-
// providers resolve to "client"); encrypted reasoning is only
|
|
11455
|
-
// round-tripped on the Responses wire API; hosted web search is attached
|
|
11456
|
-
// whenever the provider declares it runnable and is independent of the
|
|
11457
|
-
// session's MCP allow-list; the effective context window drives the
|
|
11458
|
-
// compaction threshold.
|
|
11459
|
-
hostedWebSearch,
|
|
11460
|
-
...imageGenerationOption,
|
|
11461
|
-
...videoGenerationOption,
|
|
11462
|
-
lazyToolTransport,
|
|
11463
|
-
supportsImageInput,
|
|
11464
|
-
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
11465
|
-
...resolvedModel ? {
|
|
11466
|
-
encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
|
|
11467
|
-
contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
11468
|
-
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
11469
|
-
// tool. Gateway Responses routes likewise expose ordinary function
|
|
11470
|
-
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
11471
|
-
// function apply_patch and wrap successful view_image results as
|
|
11472
|
-
// typed input_image content. Chat wires have no proven typed image
|
|
11473
|
-
// result transport and therefore receive no view_image tool.
|
|
11474
|
-
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
11475
|
-
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
11476
|
-
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
11477
|
-
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
11478
|
-
// constructor name. See {@link computerToolModeForTurn}.
|
|
11479
|
-
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
11480
|
-
...promptCacheKey ? { promptCacheKey } : {}
|
|
11481
|
-
} : (
|
|
11482
|
-
// LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
11483
|
-
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
11484
|
-
// client). That backend has real hosted support, so pin computerToolMode to
|
|
11485
|
-
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
11486
|
-
{
|
|
11487
|
-
computerToolMode: computerToolModeForTurn(null),
|
|
11488
|
-
promptCacheKey: input.sessionId
|
|
11489
|
-
}
|
|
11490
|
-
),
|
|
11491
|
-
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
11492
|
-
// selects a computer tool, then this hook begins the optional proof
|
|
11493
|
-
// recording. Shell/filesystem turns never invoke either operation.
|
|
11494
|
-
onComputerUseReady: async () => {
|
|
11495
|
-
if (!resolvedSandbox) {
|
|
11496
|
-
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
11497
|
-
}
|
|
11498
|
-
didComputerUse = true;
|
|
11499
|
-
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
11500
|
-
},
|
|
11501
|
-
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
11502
|
-
...runtimeSkillActivations.length > 0 ? { skillActivations: runtimeSkillActivations } : {},
|
|
11503
|
-
...!structuredWorkspacePolicyActive && workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
11504
|
-
...workspaceGovernance ? { workspaceGovernance } : {},
|
|
11505
|
-
...workspaceMemory ? { workspaceMemory } : {},
|
|
11506
|
-
goalSnapshot: turn.goalSnapshot,
|
|
11507
|
-
// Per-session persona tier (session > workspace > deployment default).
|
|
11508
|
-
// Composed system-level AFTER the workspace persona so it refines it for
|
|
11509
|
-
// this one session; absent ⇒ byte-identical to today's composition.
|
|
11510
|
-
...session.instructions ? { sessionInstructions: session.instructions } : {},
|
|
11511
|
-
// Exact host context captured when this turn was accepted. It is
|
|
11512
|
-
// system-level and disappears with the turn rather than entering chat.
|
|
11513
|
-
...turn.turnInstructions || mcpAvailabilityInstructions ? {
|
|
11514
|
-
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions].filter((value) => Boolean(value)).join(" ")
|
|
11515
|
-
} : {},
|
|
11516
|
-
...workspaceEnvironmentOption,
|
|
11517
|
-
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
11518
|
-
// the frozen version carries a non-empty script), and the rig credential
|
|
11519
|
-
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
11520
|
-
...rigVersion && rigName ? {
|
|
11521
|
-
rig: { name: rigName, version: rigVersion.version },
|
|
11522
|
-
...rigVersion.setupScript && rigVersion.setupScript.trim().length > 0 ? {
|
|
11523
|
-
rigSetup: {
|
|
11524
|
-
rigId: session.rigId,
|
|
11525
|
-
versionId: rigVersion.id,
|
|
11526
|
-
rigName,
|
|
11527
|
-
script: rigVersion.setupScript,
|
|
11528
|
-
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
11529
|
-
contentHash: rigProviderImageContentHash({
|
|
11530
|
-
backend: turn.sandboxBackend,
|
|
11531
|
-
sourceImage: rigProviderImageSourceImage(
|
|
11532
|
-
logicalSandboxSettings,
|
|
11533
|
-
turn.sandboxBackend
|
|
11534
|
-
),
|
|
11535
|
-
definition: rigVersion
|
|
11536
|
-
})
|
|
11537
|
-
}
|
|
11538
|
-
} : {},
|
|
11539
|
-
...rigVersion.credentialHooks.length > 0 ? { rigCredentialHookIds: rigVersion.credentialHooks } : {}
|
|
11540
|
-
} : {}
|
|
11698
|
+
recordTurnStartupPhase(observability, {
|
|
11699
|
+
phase: "post_tool_preparation",
|
|
11700
|
+
provider: turnExecutionPolicy.providerId,
|
|
11701
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11702
|
+
outcome: "completed",
|
|
11703
|
+
durationSeconds: (performance.now() - postToolPreparationStartedAt) / 1e3
|
|
11541
11704
|
});
|
|
11705
|
+
const agent = (() => {
|
|
11706
|
+
const agentConstructionStartedAt = performance.now();
|
|
11707
|
+
let agentConstructionOutcome = "completed";
|
|
11708
|
+
try {
|
|
11709
|
+
return runtime.buildAgent(modelRunSettings, turnResources, {
|
|
11710
|
+
reasoningEffort: turn.reasoningEffort,
|
|
11711
|
+
latencyMode: turnExecutionPolicy.latencyMode,
|
|
11712
|
+
...serviceTier ? { serviceTier } : {},
|
|
11713
|
+
...humanInputResume ? { humanInputResponse: humanInputResume } : {},
|
|
11714
|
+
humanInputEnabled: agentHumanInputEnabled,
|
|
11715
|
+
genesisTitleHint: isGenesisTurn,
|
|
11716
|
+
persistentSessionSettings: {
|
|
11717
|
+
titleIsSet: Boolean(session.title?.trim())
|
|
11718
|
+
},
|
|
11719
|
+
sandboxEnvironment,
|
|
11720
|
+
...preparedTools.attemptToolCatalog ? { attemptToolCatalog: preparedTools.attemptToolCatalog } : {},
|
|
11721
|
+
...sandboxArtifactRuntime.available ? { artifactRuntimeAvailable: true } : {},
|
|
11722
|
+
...cancellationSignal ? { turnCancellationSignal: cancellationSignal } : {},
|
|
11723
|
+
onToolCancellationFence: (fence) => {
|
|
11724
|
+
toolCancellationFenceRef.current = fence;
|
|
11725
|
+
},
|
|
11726
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
11727
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
11728
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
11729
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
11730
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
11731
|
+
// touches the box/agent manifest env.
|
|
11732
|
+
...activeSandboxBackend !== "selfhosted" && sandboxGitTokens ? { gitTokenSeeds: sandboxGitTokens } : {},
|
|
11733
|
+
...activeSandboxBackend !== "selfhosted" && sandboxGitCredentialBindings ? { gitCredentialBindings: sandboxGitCredentialBindings } : {},
|
|
11734
|
+
...activeSandboxBackend !== "selfhosted" && !sandboxGitTokens && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
11735
|
+
...sandboxCodemodeToken ? { codemodeAvailable: true } : {},
|
|
11736
|
+
// Managed boxes receive the bearer through their protected per-session
|
|
11737
|
+
// token file. Connected Machines use transient per-exec delivery above,
|
|
11738
|
+
// so they must not run the file-seeding lifecycle hook.
|
|
11739
|
+
...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken ? {
|
|
11740
|
+
codemodeTokenSeed: sandboxCodemodeToken,
|
|
11741
|
+
codemodeTokenSessionId: input.sessionId
|
|
11742
|
+
} : {},
|
|
11743
|
+
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
11744
|
+
fileResourceDownloads,
|
|
11745
|
+
mcpServers: preparedTools.mcpServers,
|
|
11746
|
+
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
11747
|
+
connectorActionPolicy,
|
|
11748
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
11749
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
11750
|
+
// model call so the model sees the account's real connected sources.
|
|
11751
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
11752
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
11753
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
11754
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
11755
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
11756
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
11757
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
11758
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
11759
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
11760
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
11761
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
11762
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
11763
|
+
// whenever the provider declares it runnable and is independent of the
|
|
11764
|
+
// session's MCP allow-list; the effective context window drives the
|
|
11765
|
+
// compaction threshold.
|
|
11766
|
+
hostedWebSearch,
|
|
11767
|
+
...imageGenerationOption,
|
|
11768
|
+
...videoGenerationOption,
|
|
11769
|
+
lazyToolTransport,
|
|
11770
|
+
supportsImageInput,
|
|
11771
|
+
inputFileMediaTypes: modelInputPolicy.inputFileMediaTypes,
|
|
11772
|
+
...resolvedModel ? {
|
|
11773
|
+
encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
|
|
11774
|
+
contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
11775
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED apply_patch
|
|
11776
|
+
// tool. Gateway Responses routes likewise expose ordinary function
|
|
11777
|
+
// tools, not OpenAI-hosted sandbox tools. Tell buildAgent to use
|
|
11778
|
+
// function apply_patch and wrap successful view_image results as
|
|
11779
|
+
// typed input_image content. Chat wires have no proven typed image
|
|
11780
|
+
// result transport and therefore receive no view_image tool.
|
|
11781
|
+
structuredToolTransport: structuredToolTransportForTurn(resolvedModel),
|
|
11782
|
+
// EXPLICIT computer-use tool transport, derived from the resolved provider's
|
|
11783
|
+
// authoritative wire identity (codex → function-image, chat → disabled,
|
|
11784
|
+
// responses → hosted) so the runtime never string-sniffs the model instance's
|
|
11785
|
+
// constructor name. See {@link computerToolModeForTurn}.
|
|
11786
|
+
computerToolMode: computerToolModeForTurn(resolvedModel),
|
|
11787
|
+
...promptCacheKey ? { promptCacheKey } : {}
|
|
11788
|
+
} : (
|
|
11789
|
+
// LEGACY global-client fallback (resolveTurnModel returned null → the model
|
|
11790
|
+
// is not in the registry, served by the built-in OpenAI/Azure Responses
|
|
11791
|
+
// client). That backend has real hosted support, so pin computerToolMode to
|
|
11792
|
+
// "hosted" EXPLICITLY rather than leaving the runtime to sniff the instance.
|
|
11793
|
+
{
|
|
11794
|
+
computerToolMode: computerToolModeForTurn(null),
|
|
11795
|
+
promptCacheKey: input.sessionId
|
|
11796
|
+
}
|
|
11797
|
+
),
|
|
11798
|
+
// Lazy computer-use seam: runtime first brings up :0 only after the model
|
|
11799
|
+
// selects a computer tool, then this hook begins the optional proof
|
|
11800
|
+
// recording. Shell/filesystem turns never invoke either operation.
|
|
11801
|
+
onComputerUseReady: async () => {
|
|
11802
|
+
if (!resolvedSandbox) {
|
|
11803
|
+
throw new Error("Computer-use display became ready without a resolved sandbox");
|
|
11804
|
+
}
|
|
11805
|
+
didComputerUse = true;
|
|
11806
|
+
await maybeStartOnTurnRecording(resolvedSandbox, activeSandboxBackend);
|
|
11807
|
+
},
|
|
11808
|
+
onRetainableSessionImageOutput: retainSessionImageAtToolBoundary,
|
|
11809
|
+
...runtimeSkillActivations.length > 0 ? { skillActivations: runtimeSkillActivations } : {},
|
|
11810
|
+
...!structuredWorkspacePolicyActive && workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
11811
|
+
...workspaceGovernance ? { workspaceGovernance } : {},
|
|
11812
|
+
...workspaceMemory ? { workspaceMemory } : {},
|
|
11813
|
+
goalSnapshot: turn.goalSnapshot,
|
|
11814
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
11815
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
11816
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
11817
|
+
...session.instructions ? { sessionInstructions: session.instructions } : {},
|
|
11818
|
+
// Exact host context captured when this turn was accepted. It is
|
|
11819
|
+
// system-level and disappears with the turn rather than entering chat.
|
|
11820
|
+
...turn.turnInstructions || mcpAvailabilityInstructions ? {
|
|
11821
|
+
turnInstructions: [turn.turnInstructions, mcpAvailabilityInstructions].filter((value) => Boolean(value)).join(" ")
|
|
11822
|
+
} : {},
|
|
11823
|
+
...workspaceEnvironmentOption,
|
|
11824
|
+
// RIG RUNTIME (M3): the doctrine block, the setup-script hook (only when
|
|
11825
|
+
// the frozen version carries a non-empty script), and the rig credential
|
|
11826
|
+
// hooks. All absent for a rig-less turn (byte-for-byte today).
|
|
11827
|
+
...rigVersion && rigName ? {
|
|
11828
|
+
rig: { name: rigName, version: rigVersion.version },
|
|
11829
|
+
...rigVersion.setupScript && rigVersion.setupScript.trim().length > 0 ? {
|
|
11830
|
+
rigSetup: {
|
|
11831
|
+
rigId: session.rigId,
|
|
11832
|
+
versionId: rigVersion.id,
|
|
11833
|
+
rigName,
|
|
11834
|
+
script: rigVersion.setupScript,
|
|
11835
|
+
timeoutMs: runSettings.rigSetupTimeoutMs,
|
|
11836
|
+
contentHash: rigProviderImageContentHash({
|
|
11837
|
+
backend: turn.sandboxBackend,
|
|
11838
|
+
sourceImage: rigProviderImageSourceImage(
|
|
11839
|
+
logicalSandboxSettings,
|
|
11840
|
+
turn.sandboxBackend
|
|
11841
|
+
),
|
|
11842
|
+
definition: rigVersion
|
|
11843
|
+
})
|
|
11844
|
+
}
|
|
11845
|
+
} : {},
|
|
11846
|
+
...rigVersion.credentialHooks.length > 0 ? { rigCredentialHookIds: rigVersion.credentialHooks } : {}
|
|
11847
|
+
} : {}
|
|
11848
|
+
});
|
|
11849
|
+
} catch (error) {
|
|
11850
|
+
agentConstructionOutcome = "failed";
|
|
11851
|
+
throw error;
|
|
11852
|
+
} finally {
|
|
11853
|
+
recordTurnStartupPhase(observability, {
|
|
11854
|
+
phase: "agent_construction",
|
|
11855
|
+
provider: turnExecutionPolicy.providerId,
|
|
11856
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
11857
|
+
outcome: agentConstructionOutcome,
|
|
11858
|
+
durationSeconds: (performance.now() - agentConstructionStartedAt) / 1e3
|
|
11859
|
+
});
|
|
11860
|
+
}
|
|
11861
|
+
})();
|
|
11862
|
+
const postAgentPreparationStartedAt = performance.now();
|
|
11542
11863
|
if (modelRunSettings.sandboxBackend !== "none" && toolCancellationFenceRef.current === null) {
|
|
11543
11864
|
throw new Error(
|
|
11544
11865
|
"Sandbox agent construction did not install the mandatory turn tool cancellation fence"
|
|
@@ -11585,7 +11906,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
11585
11906
|
environment: sandboxEnvironment,
|
|
11586
11907
|
...runSettings.modalImageRef ?? runSettings.dockerImage ? {
|
|
11587
11908
|
image: runSettings.modalImageRef ?? runSettings.dockerImage
|
|
11588
|
-
} : {}
|
|
11909
|
+
} : {},
|
|
11910
|
+
// The lazy acquire must enforce the same frozen rig authority
|
|
11911
|
+
// as the eager acquire; otherwise a warm box for another rig
|
|
11912
|
+
// can bypass the shared-state conflict/rotation fence.
|
|
11913
|
+
...rigVersion ? { rigVersionId: rigVersion.id } : {}
|
|
11589
11914
|
},
|
|
11590
11915
|
"turn",
|
|
11591
11916
|
lazyHolderId
|
|
@@ -11983,58 +12308,96 @@ function createRunAgentTurnActivity(services) {
|
|
|
11983
12308
|
return claimedResult({ status: "idle" });
|
|
11984
12309
|
}
|
|
11985
12310
|
}
|
|
12311
|
+
recordTurnStartupPhase(observability, {
|
|
12312
|
+
phase: "post_agent_preparation",
|
|
12313
|
+
provider: turnExecutionPolicy.providerId,
|
|
12314
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12315
|
+
outcome: "completed",
|
|
12316
|
+
durationSeconds: (performance.now() - postAgentPreparationStartedAt) / 1e3
|
|
12317
|
+
});
|
|
11986
12318
|
let fileMaterializationFailures = [];
|
|
11987
12319
|
let fileDownloadsMaterializedForRun = false;
|
|
11988
12320
|
if (resolvedSandbox && setupBoxSession && fileResourceDownloads.length > 0) {
|
|
11989
|
-
const
|
|
12321
|
+
const fileMaterializationStartedAt = performance.now();
|
|
12322
|
+
let fileMaterializationOutcome = "completed";
|
|
11990
12323
|
const cacheMaterialization = resolvedSandbox.established.backendId !== "selfhosted";
|
|
11991
|
-
|
|
11992
|
-
|
|
11993
|
-
|
|
11994
|
-
|
|
11995
|
-
|
|
11996
|
-
|
|
11997
|
-
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
|
|
12003
|
-
|
|
12004
|
-
const materialized = await runWorkspaceMutationForSandbox(
|
|
12005
|
-
resolvedSandbox,
|
|
12006
|
-
"fileMaterialization",
|
|
12007
|
-
async () => await materializeSandboxFileDownloads(
|
|
12008
|
-
setupBoxSession,
|
|
12009
|
-
downloadsToMaterialize,
|
|
12010
|
-
{
|
|
12011
|
-
onRuntimeEvent: async (event) => {
|
|
12012
|
-
await publish([{ type: event.type, payload: event.payload }], true);
|
|
12013
|
-
},
|
|
12014
|
-
...runAs ? { runAs } : {},
|
|
12015
|
-
...toolCancellationFenceRef.current ? {
|
|
12016
|
-
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
12017
|
-
toolCancellationFenceRef.current
|
|
12018
|
-
)
|
|
12019
|
-
} : {}
|
|
12020
|
-
}
|
|
12021
|
-
)
|
|
12324
|
+
let fileMaterializationCache = cacheMaterialization ? "miss" : "disabled";
|
|
12325
|
+
try {
|
|
12326
|
+
const boxInstanceId = resolvedSandbox.established.instanceId;
|
|
12327
|
+
const alreadyMaterialized = cacheMaterialization ? await getMaterializedSandboxFileResources(db, {
|
|
12328
|
+
accountId: input.accountId,
|
|
12329
|
+
workspaceId: input.workspaceId,
|
|
12330
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
12331
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
12332
|
+
instanceId: boxInstanceId
|
|
12333
|
+
}) : /* @__PURE__ */ new Set();
|
|
12334
|
+
const downloadsToMaterialize = filterUnmaterializedSandboxFileDownloads(
|
|
12335
|
+
fileResourceDownloads,
|
|
12336
|
+
alreadyMaterialized
|
|
12022
12337
|
);
|
|
12023
|
-
|
|
12024
|
-
|
|
12025
|
-
|
|
12026
|
-
|
|
12027
|
-
|
|
12028
|
-
|
|
12029
|
-
|
|
12030
|
-
|
|
12031
|
-
|
|
12032
|
-
|
|
12033
|
-
|
|
12034
|
-
|
|
12338
|
+
if (cacheMaterialization && downloadsToMaterialize.length === 0) {
|
|
12339
|
+
fileMaterializationCache = "hit";
|
|
12340
|
+
}
|
|
12341
|
+
const runAs = sandboxRunAs(runSettings);
|
|
12342
|
+
if (downloadsToMaterialize.length > 0) {
|
|
12343
|
+
const materialized = await runWorkspaceMutationForSandbox(
|
|
12344
|
+
resolvedSandbox,
|
|
12345
|
+
"fileMaterialization",
|
|
12346
|
+
async () => await materializeSandboxFileDownloads(
|
|
12347
|
+
setupBoxSession,
|
|
12348
|
+
downloadsToMaterialize,
|
|
12349
|
+
{
|
|
12350
|
+
onRuntimeEvent: async (event) => {
|
|
12351
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
12352
|
+
},
|
|
12353
|
+
...runAs ? { runAs } : {},
|
|
12354
|
+
...toolCancellationFenceRef.current ? {
|
|
12355
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
12356
|
+
toolCancellationFenceRef.current
|
|
12357
|
+
)
|
|
12358
|
+
} : {}
|
|
12359
|
+
}
|
|
12360
|
+
)
|
|
12361
|
+
);
|
|
12362
|
+
fileMaterializationFailures = materialized.failures;
|
|
12363
|
+
fileMaterializationOutcome = sandboxFileMaterializationOutcome(materialized.failures);
|
|
12364
|
+
const failedFileIds = new Set(materialized.failures.map((failure) => failure.fileId));
|
|
12365
|
+
const succeededFileIds = downloadsToMaterialize.map((download) => download.fileId).filter((fileId) => !failedFileIds.has(fileId));
|
|
12366
|
+
if (cacheMaterialization && succeededFileIds.length > 0) {
|
|
12367
|
+
await markSandboxFileResourcesMaterialized(db, {
|
|
12368
|
+
accountId: input.accountId,
|
|
12369
|
+
workspaceId: input.workspaceId,
|
|
12370
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
12371
|
+
expectedEpoch: resolvedSandbox.leaseEpoch,
|
|
12372
|
+
instanceId: boxInstanceId,
|
|
12373
|
+
fileIds: succeededFileIds
|
|
12374
|
+
});
|
|
12375
|
+
}
|
|
12035
12376
|
}
|
|
12377
|
+
fileDownloadsMaterializedForRun = true;
|
|
12378
|
+
} catch (error) {
|
|
12379
|
+
fileMaterializationOutcome = "failed";
|
|
12380
|
+
throw error;
|
|
12381
|
+
} finally {
|
|
12382
|
+
recordTurnStartupPhase(observability, {
|
|
12383
|
+
phase: "file_materialization",
|
|
12384
|
+
provider: turnExecutionPolicy.providerId,
|
|
12385
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12386
|
+
outcome: fileMaterializationOutcome,
|
|
12387
|
+
durationSeconds: (performance.now() - fileMaterializationStartedAt) / 1e3,
|
|
12388
|
+
count: fileResourceDownloads.length,
|
|
12389
|
+
cache: fileMaterializationCache
|
|
12390
|
+
});
|
|
12036
12391
|
}
|
|
12037
|
-
|
|
12392
|
+
} else {
|
|
12393
|
+
recordTurnStartupPhase(observability, {
|
|
12394
|
+
phase: "file_materialization",
|
|
12395
|
+
provider: turnExecutionPolicy.providerId,
|
|
12396
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12397
|
+
outcome: "completed",
|
|
12398
|
+
durationSeconds: 0,
|
|
12399
|
+
count: 0
|
|
12400
|
+
});
|
|
12038
12401
|
}
|
|
12039
12402
|
const requiredVideoFileIds = new Set(requiredGeneratedVideoFiles.map((file) => file.fileId));
|
|
12040
12403
|
const failedRequiredVideo = fileMaterializationFailures.find(
|
|
@@ -12076,28 +12439,86 @@ function createRunAgentTurnActivity(services) {
|
|
|
12076
12439
|
}
|
|
12077
12440
|
let runInput = null;
|
|
12078
12441
|
const prepareRunAttemptInput = async () => {
|
|
12079
|
-
const
|
|
12080
|
-
|
|
12081
|
-
|
|
12082
|
-
|
|
12083
|
-
|
|
12084
|
-
|
|
12085
|
-
|
|
12086
|
-
|
|
12087
|
-
|
|
12088
|
-
|
|
12089
|
-
|
|
12090
|
-
|
|
12091
|
-
|
|
12442
|
+
const historyPreparationStartedAt = performance.now();
|
|
12443
|
+
let historyPreparationOutcome = "completed";
|
|
12444
|
+
let preparedHistoryCount = null;
|
|
12445
|
+
try {
|
|
12446
|
+
const prepared = await turnInput(db, runtime, agent, trigger, {
|
|
12447
|
+
turnId: activeTurnId,
|
|
12448
|
+
recovering: turn.executionGeneration > 1,
|
|
12449
|
+
...unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {},
|
|
12450
|
+
...runCredentialsNote ? { runCredentialsNote } : {},
|
|
12451
|
+
providerApi,
|
|
12452
|
+
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
12453
|
+
materializeModelHistory: materializeScreenshotHistory,
|
|
12454
|
+
materializeSerializedRunState: materializeScreenshotRunState,
|
|
12455
|
+
projectModelHistory: modelHistoryProjector,
|
|
12456
|
+
onPreparationPhase: (measurement) => {
|
|
12457
|
+
recordTurnStartupPhase(observability, {
|
|
12458
|
+
phase: `history_${measurement.phase}`,
|
|
12459
|
+
provider: turnExecutionPolicy.providerId,
|
|
12460
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12461
|
+
outcome: measurement.outcome,
|
|
12462
|
+
durationSeconds: measurement.durationSeconds
|
|
12463
|
+
});
|
|
12464
|
+
}
|
|
12465
|
+
});
|
|
12466
|
+
const generatedImageMaterializationStartedAt = performance.now();
|
|
12467
|
+
let generatedImageMaterializationOutcome = "completed";
|
|
12468
|
+
try {
|
|
12469
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
12470
|
+
await materializeGeneratedImage(receipt);
|
|
12471
|
+
}
|
|
12472
|
+
} catch (error) {
|
|
12473
|
+
generatedImageMaterializationOutcome = "failed";
|
|
12474
|
+
throw error;
|
|
12475
|
+
} finally {
|
|
12476
|
+
recordTurnStartupPhase(observability, {
|
|
12477
|
+
phase: "history_generated_image_materialization",
|
|
12478
|
+
provider: turnExecutionPolicy.providerId,
|
|
12479
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12480
|
+
outcome: generatedImageMaterializationOutcome,
|
|
12481
|
+
durationSeconds: (performance.now() - generatedImageMaterializationStartedAt) / 1e3,
|
|
12482
|
+
count: generatedImageReceiptsByArtifactId.size
|
|
12483
|
+
});
|
|
12484
|
+
}
|
|
12485
|
+
runInput = prepared.input;
|
|
12486
|
+
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
12487
|
+
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
12488
|
+
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
12489
|
+
const historyPositionStartedAt = performance.now();
|
|
12490
|
+
let historyPositionOutcome = "completed";
|
|
12491
|
+
try {
|
|
12492
|
+
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
12493
|
+
db,
|
|
12494
|
+
input.workspaceId,
|
|
12495
|
+
input.sessionId
|
|
12496
|
+
);
|
|
12497
|
+
} catch (error) {
|
|
12498
|
+
historyPositionOutcome = "failed";
|
|
12499
|
+
throw error;
|
|
12500
|
+
} finally {
|
|
12501
|
+
recordTurnStartupPhase(observability, {
|
|
12502
|
+
phase: "history_position_load",
|
|
12503
|
+
provider: turnExecutionPolicy.providerId,
|
|
12504
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12505
|
+
outcome: historyPositionOutcome,
|
|
12506
|
+
durationSeconds: (performance.now() - historyPositionStartedAt) / 1e3
|
|
12507
|
+
});
|
|
12508
|
+
}
|
|
12509
|
+
} catch (error) {
|
|
12510
|
+
historyPreparationOutcome = "failed";
|
|
12511
|
+
throw error;
|
|
12512
|
+
} finally {
|
|
12513
|
+
recordTurnStartupPhase(observability, {
|
|
12514
|
+
phase: "history_preparation",
|
|
12515
|
+
provider: turnExecutionPolicy.providerId,
|
|
12516
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12517
|
+
outcome: historyPreparationOutcome,
|
|
12518
|
+
durationSeconds: (performance.now() - historyPreparationStartedAt) / 1e3,
|
|
12519
|
+
count: preparedHistoryCount
|
|
12520
|
+
});
|
|
12092
12521
|
}
|
|
12093
|
-
runInput = prepared.input;
|
|
12094
|
-
providerArtifactCandidates = prepared.providerArtifactCandidates;
|
|
12095
|
-
persistedHistoryCount = prepared.persistedHistoryCount;
|
|
12096
|
-
nextHistoryPosition = await nextSessionHistoryPosition(
|
|
12097
|
-
db,
|
|
12098
|
-
input.workspaceId,
|
|
12099
|
-
input.sessionId
|
|
12100
|
-
);
|
|
12101
12522
|
};
|
|
12102
12523
|
const forceContextCompaction = async (triggerLabel, recoverySignalTokens) => {
|
|
12103
12524
|
const outcome = await waitForTurnOperation(
|
|
@@ -12140,6 +12561,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
12140
12561
|
return outcome;
|
|
12141
12562
|
};
|
|
12142
12563
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
12564
|
+
let workerPreparationTotalRecorded = false;
|
|
12143
12565
|
const runStreamAttempt = async () => {
|
|
12144
12566
|
if (!runInput) {
|
|
12145
12567
|
throw new Error("Run input was not prepared");
|
|
@@ -12154,102 +12576,154 @@ function createRunAgentTurnActivity(services) {
|
|
|
12154
12576
|
const ownedEstablished = resolvedSandbox?.established ?? lazyOwnedSandbox;
|
|
12155
12577
|
const runStreamOnce = async () => {
|
|
12156
12578
|
if (resolvedSandbox && !lazyOwnedSandbox && ownedEstablished) {
|
|
12157
|
-
const
|
|
12158
|
-
|
|
12159
|
-
|
|
12160
|
-
|
|
12161
|
-
|
|
12162
|
-
|
|
12163
|
-
|
|
12164
|
-
|
|
12165
|
-
|
|
12166
|
-
|
|
12167
|
-
|
|
12168
|
-
|
|
12169
|
-
|
|
12170
|
-
|
|
12171
|
-
|
|
12172
|
-
|
|
12173
|
-
|
|
12174
|
-
|
|
12175
|
-
|
|
12176
|
-
|
|
12177
|
-
|
|
12178
|
-
|
|
12179
|
-
|
|
12180
|
-
|
|
12181
|
-
|
|
12182
|
-
|
|
12183
|
-
|
|
12184
|
-
|
|
12185
|
-
|
|
12186
|
-
|
|
12187
|
-
|
|
12188
|
-
|
|
12189
|
-
|
|
12190
|
-
|
|
12191
|
-
|
|
12192
|
-
|
|
12193
|
-
|
|
12579
|
+
const ownedSandboxSetupStartedAt = performance.now();
|
|
12580
|
+
let ownedSandboxSetupOutcome = "completed";
|
|
12581
|
+
try {
|
|
12582
|
+
const eagerSetupSession = setupBoxSession ?? ownedEstablished.session;
|
|
12583
|
+
await attachRunCredentialRenewal(
|
|
12584
|
+
eagerSetupSession,
|
|
12585
|
+
resolvedSandbox
|
|
12586
|
+
);
|
|
12587
|
+
const eagerCredentialSetupSession = initialRunCredentialMaterial ? withRunCredentialsSession(eagerSetupSession, input.sessionId) : eagerSetupSession;
|
|
12588
|
+
await runWorkspaceMutationForSandbox(
|
|
12589
|
+
resolvedSandbox,
|
|
12590
|
+
"eagerOwnedSandboxSetup",
|
|
12591
|
+
async () => await runOwnedSandboxSetup(
|
|
12592
|
+
agent,
|
|
12593
|
+
ownedEstablished.session,
|
|
12594
|
+
eagerCredentialSetupSession,
|
|
12595
|
+
{
|
|
12596
|
+
settings: runSettings,
|
|
12597
|
+
environment: sandboxEnvironment,
|
|
12598
|
+
preparedInput: runInput,
|
|
12599
|
+
...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
|
|
12600
|
+
onRuntimeEvent: async (event) => {
|
|
12601
|
+
await renewServingCredentialLease("runtime_event");
|
|
12602
|
+
if (servingCredentialLeaseLost()) {
|
|
12603
|
+
throw new Error("Provider credential lease expired during sandbox setup");
|
|
12604
|
+
}
|
|
12605
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
12606
|
+
},
|
|
12607
|
+
...toolCancellationFenceRef.current ? {
|
|
12608
|
+
commandRunner: toolCancellationFenceRef.current.runSandboxCommand.bind(
|
|
12609
|
+
toolCancellationFenceRef.current
|
|
12610
|
+
)
|
|
12611
|
+
} : {}
|
|
12612
|
+
}
|
|
12613
|
+
)
|
|
12614
|
+
);
|
|
12615
|
+
await attachGitCredentialRenewal(
|
|
12616
|
+
eagerSetupSession,
|
|
12617
|
+
initialGitCredentials
|
|
12618
|
+
);
|
|
12619
|
+
} catch (error) {
|
|
12620
|
+
ownedSandboxSetupOutcome = "failed";
|
|
12621
|
+
throw error;
|
|
12622
|
+
} finally {
|
|
12623
|
+
recordTurnStartupPhase(observability, {
|
|
12624
|
+
phase: "owned_sandbox_setup",
|
|
12625
|
+
provider: turnExecutionPolicy.providerId,
|
|
12626
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12627
|
+
outcome: ownedSandboxSetupOutcome,
|
|
12628
|
+
durationSeconds: (performance.now() - ownedSandboxSetupStartedAt) / 1e3,
|
|
12629
|
+
count: fileResourceDownloads.length
|
|
12630
|
+
});
|
|
12631
|
+
}
|
|
12194
12632
|
}
|
|
12195
12633
|
recordCompanyBrainContributionReceiptOnce();
|
|
12196
|
-
|
|
12197
|
-
|
|
12198
|
-
|
|
12199
|
-
|
|
12200
|
-
|
|
12201
|
-
|
|
12202
|
-
|
|
12203
|
-
|
|
12204
|
-
|
|
12205
|
-
|
|
12206
|
-
|
|
12207
|
-
|
|
12208
|
-
|
|
12209
|
-
|
|
12210
|
-
|
|
12211
|
-
|
|
12212
|
-
|
|
12213
|
-
|
|
12214
|
-
|
|
12215
|
-
|
|
12216
|
-
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
12217
|
-
// established box — never through the routing proxy, which would
|
|
12218
|
-
// re-route those execs onto a machine swapped in mid-turn.
|
|
12219
|
-
...setupBoxSession ? { setupSession: setupBoxSession } : {},
|
|
12220
|
-
...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
|
|
12221
|
-
// Both owned paths execute setup outside runStream: eager just
|
|
12222
|
-
// above under its exact admission, lazy in the provisioner.
|
|
12223
|
-
deferredSetup: true
|
|
12224
|
-
}
|
|
12225
|
-
} : {},
|
|
12226
|
-
...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox ? {
|
|
12227
|
-
onCodemodeTokenSessionReady: async (tokenSession) => {
|
|
12228
|
-
const renewalSession = setupBoxSession ?? tokenSession;
|
|
12229
|
-
await attachCodemodeTokenRenewal(renewalSession);
|
|
12230
|
-
}
|
|
12231
|
-
} : {},
|
|
12232
|
-
...runCredentialResolver ? {
|
|
12233
|
-
runCredentialSessionId: input.sessionId,
|
|
12234
|
-
...!lazyOwnedSandbox ? {
|
|
12235
|
-
onRunCredentialSessionReady: async (credentialSession) => {
|
|
12236
|
-
const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
|
|
12237
|
-
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
12634
|
+
const providerDispatchStartedAt = performance.now();
|
|
12635
|
+
let providerDispatchOutcome = "completed";
|
|
12636
|
+
try {
|
|
12637
|
+
if (!workerPreparationTotalRecorded) {
|
|
12638
|
+
workerPreparationTotalRecorded = true;
|
|
12639
|
+
recordTurnWorkerPreparationTotal(observability, {
|
|
12640
|
+
provider: turnExecutionPolicy.providerId,
|
|
12641
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12642
|
+
outcome: "completed",
|
|
12643
|
+
durationSeconds: (performance.now() - workerPreparationStartedAt) / 1e3
|
|
12644
|
+
});
|
|
12645
|
+
}
|
|
12646
|
+
modelRequestStarted = true;
|
|
12647
|
+
return await runtime.runStream(agent, runInput, modelRunSettings, {
|
|
12648
|
+
signal: runtimeCancellationSignal,
|
|
12649
|
+
sandboxEnvironment,
|
|
12650
|
+
onRuntimeEvent: async (event) => {
|
|
12651
|
+
await renewServingCredentialLease("runtime_event");
|
|
12652
|
+
if (servingCredentialLeaseLost()) {
|
|
12653
|
+
throw new Error("Provider credential lease expired during the active turn");
|
|
12238
12654
|
}
|
|
12239
|
-
|
|
12240
|
-
|
|
12241
|
-
|
|
12242
|
-
|
|
12243
|
-
|
|
12244
|
-
|
|
12245
|
-
|
|
12655
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
12656
|
+
},
|
|
12657
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
12658
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
12659
|
+
...ownedEstablished ? {
|
|
12660
|
+
ownedSandbox: {
|
|
12661
|
+
client: ownedEstablished.client,
|
|
12662
|
+
session: ownedEstablished.session,
|
|
12663
|
+
...resolvedSandbox?.established.sessionState ? {
|
|
12664
|
+
sessionState: resolvedSandbox.established.sessionState
|
|
12665
|
+
} : {},
|
|
12666
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
12667
|
+
// established box — never through the routing proxy, which would
|
|
12668
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
12669
|
+
...setupBoxSession ? { setupSession: setupBoxSession } : {},
|
|
12670
|
+
...fileDownloadsMaterializedForRun ? { fileDownloadsMaterialized: true } : {},
|
|
12671
|
+
// Both owned paths execute setup outside runStream: eager just
|
|
12672
|
+
// above under its exact admission, lazy in the provisioner.
|
|
12673
|
+
deferredSetup: true
|
|
12246
12674
|
}
|
|
12247
|
-
}
|
|
12248
|
-
|
|
12249
|
-
|
|
12250
|
-
|
|
12251
|
-
|
|
12252
|
-
|
|
12675
|
+
} : {},
|
|
12676
|
+
...activeSandboxBackend !== "selfhosted" && sandboxCodemodeToken && sandboxCodemodeTokenExpiresAt && !lazyOwnedSandbox ? {
|
|
12677
|
+
onCodemodeTokenSessionReady: async (tokenSession) => {
|
|
12678
|
+
const renewalSession = setupBoxSession ?? tokenSession;
|
|
12679
|
+
await attachCodemodeTokenRenewal(renewalSession);
|
|
12680
|
+
}
|
|
12681
|
+
} : {},
|
|
12682
|
+
...runCredentialResolver ? {
|
|
12683
|
+
runCredentialSessionId: input.sessionId,
|
|
12684
|
+
...!lazyOwnedSandbox ? {
|
|
12685
|
+
onRunCredentialSessionReady: async (credentialSession) => {
|
|
12686
|
+
const pinnedCredentialSession = setupBoxSession ? setupBoxSession : credentialSession;
|
|
12687
|
+
await attachRunCredentialRenewal(pinnedCredentialSession);
|
|
12688
|
+
}
|
|
12689
|
+
} : {}
|
|
12690
|
+
} : {},
|
|
12691
|
+
...modelRunSettings.sandboxBackend !== "none" ? {
|
|
12692
|
+
onSandboxSessionReady: async (sandboxSession) => {
|
|
12693
|
+
sdkOwnedSandboxSession = sandboxSession;
|
|
12694
|
+
for (const receipt of generatedImageReceiptsByArtifactId.values()) {
|
|
12695
|
+
await materializeGeneratedImageInOwnedSdkSession(receipt, sandboxSession);
|
|
12696
|
+
}
|
|
12697
|
+
}
|
|
12698
|
+
} : {},
|
|
12699
|
+
contextCompactionSignal: () => modelResponseContextSignal(modelResponseState),
|
|
12700
|
+
contextCompactionRequested: () => isSessionCompactionRequested(db, input.workspaceId, input.sessionId),
|
|
12701
|
+
onModelPreparationPhase: (measurement) => {
|
|
12702
|
+
recordTurnStartupPhase(observability, {
|
|
12703
|
+
phase: `model_prepare_${measurement.phase}`,
|
|
12704
|
+
provider: turnExecutionPolicy.providerId,
|
|
12705
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12706
|
+
outcome: measurement.outcome,
|
|
12707
|
+
durationSeconds: measurement.durationSeconds,
|
|
12708
|
+
...measurement.count === void 0 ? {} : { count: measurement.count }
|
|
12709
|
+
});
|
|
12710
|
+
},
|
|
12711
|
+
...toolCancellationFenceRef.current ? {
|
|
12712
|
+
turnToolCancellationFence: toolCancellationFenceRef.current
|
|
12713
|
+
} : {}
|
|
12714
|
+
});
|
|
12715
|
+
} catch (error) {
|
|
12716
|
+
providerDispatchOutcome = "failed";
|
|
12717
|
+
throw error;
|
|
12718
|
+
} finally {
|
|
12719
|
+
recordTurnStartupPhase(observability, {
|
|
12720
|
+
phase: "provider_dispatch",
|
|
12721
|
+
provider: turnExecutionPolicy.providerId,
|
|
12722
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12723
|
+
outcome: providerDispatchOutcome,
|
|
12724
|
+
durationSeconds: (performance.now() - providerDispatchStartedAt) / 1e3
|
|
12725
|
+
});
|
|
12726
|
+
}
|
|
12253
12727
|
};
|
|
12254
12728
|
if (codexLeaseLost) {
|
|
12255
12729
|
throw new Error("Codex credential lease expired before the model run");
|
|
@@ -12270,11 +12744,30 @@ function createRunAgentTurnActivity(services) {
|
|
|
12270
12744
|
onFlush: ({ events, durationSeconds }) => recordBatchFlush(observability, { events, durationSeconds })
|
|
12271
12745
|
}
|
|
12272
12746
|
);
|
|
12747
|
+
const streamBootstrapStartedAt = performance.now();
|
|
12748
|
+
let streamBootstrapRecorded = false;
|
|
12749
|
+
const recordStreamBootstrap = () => {
|
|
12750
|
+
if (streamBootstrapRecorded) return;
|
|
12751
|
+
streamBootstrapRecorded = true;
|
|
12752
|
+
recordTurnStartupPhase(observability, {
|
|
12753
|
+
phase: "stream_bootstrap",
|
|
12754
|
+
provider: turnExecutionPolicy.providerId,
|
|
12755
|
+
backend: activeSandboxBackend ?? groupBoxBackend,
|
|
12756
|
+
outcome: "completed",
|
|
12757
|
+
durationSeconds: (performance.now() - streamBootstrapStartedAt) / 1e3,
|
|
12758
|
+
count: turnTools.length
|
|
12759
|
+
});
|
|
12760
|
+
};
|
|
12761
|
+
if (!firstModelRequestPreparationRecorded) {
|
|
12762
|
+
firstModelRequestPreparationStartedAt = performance.now();
|
|
12763
|
+
firstModelRequestCheckpointAt = firstModelRequestPreparationStartedAt;
|
|
12764
|
+
}
|
|
12273
12765
|
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
12274
12766
|
let streamDone = false;
|
|
12275
12767
|
try {
|
|
12276
12768
|
while (true) {
|
|
12277
12769
|
const next = await nextStreamEvent(iterator, activityContext);
|
|
12770
|
+
recordStreamBootstrap();
|
|
12278
12771
|
if (next.done) {
|
|
12279
12772
|
streamDone = true;
|
|
12280
12773
|
break;
|
|
@@ -13830,7 +14323,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
13830
14323
|
}
|
|
13831
14324
|
activityStatus = "failed";
|
|
13832
14325
|
activityError = error;
|
|
13833
|
-
if (!
|
|
14326
|
+
if (!turnId) {
|
|
14327
|
+
throw preClaimAdmissionFailure(error);
|
|
14328
|
+
}
|
|
14329
|
+
if (!publish || !turnStartedPublished) {
|
|
13834
14330
|
throw error;
|
|
13835
14331
|
}
|
|
13836
14332
|
await flushRuntimeBatcher();
|
|
@@ -16033,4 +16529,4 @@ export {
|
|
|
16033
16529
|
createTurnActivities,
|
|
16034
16530
|
createTurnActivitiesFromServices
|
|
16035
16531
|
};
|
|
16036
|
-
//# sourceMappingURL=activities-turn-
|
|
16532
|
+
//# sourceMappingURL=activities-turn-IDS3WEHA.js.map
|