@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/quiescence.d.ts +9 -6
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-CTEMLTHG.js} +45 -18
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-S5IU347A.js} +960 -815
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-TAUE3GYT.js} +117 -95
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/workflow-bundle.js +71 -8
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/quiescence.ts +21 -15
- package/src/activities/agent-turn/run.ts +18 -5
- package/src/activities/agent-turn/stream-attempt.ts +15 -1
- package/src/activities/agent-turn/tool-environment.ts +30 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/session-state.ts +17 -0
- package/src/activities/types.ts +16 -8
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -3,13 +3,13 @@ import {
|
|
|
3
3
|
PostCompactionContinuationEmptyError,
|
|
4
4
|
SandboxDeadlineRotationError,
|
|
5
5
|
agentRunFailurePayload,
|
|
6
|
+
armAndReconcileCodexCapacityWait,
|
|
6
7
|
armTurnQuiescenceWatchdog,
|
|
7
8
|
assertAgentStreamNotCancelled,
|
|
8
9
|
assertPhysicalToolQuiescenceForCancellation,
|
|
9
10
|
assertSessionAttemptQuiescenceRecoveryDurable,
|
|
10
11
|
assertSuccessfulAgentStreamCompletion,
|
|
11
12
|
authoritativeCodexCapacityResetAt,
|
|
12
|
-
chooseRotationActive,
|
|
13
13
|
classifyCodexCredentialFailure,
|
|
14
14
|
classifyCodexPin,
|
|
15
15
|
classifyContextWindowOverflowError,
|
|
@@ -22,7 +22,6 @@ import {
|
|
|
22
22
|
compactionFailureReason,
|
|
23
23
|
compactionFailureReasonFromError,
|
|
24
24
|
computeIdleDelayMs,
|
|
25
|
-
computeReactiveRotationResume,
|
|
26
25
|
createRuntimeBatcher,
|
|
27
26
|
createSharedActivityServices,
|
|
28
27
|
createTurnSandboxProvisioner,
|
|
@@ -30,7 +29,6 @@ import {
|
|
|
30
29
|
deliverChildRequiresActionToParent,
|
|
31
30
|
deliverFailedChildTurnToParent,
|
|
32
31
|
drainAttemptOwnedSandboxWriters,
|
|
33
|
-
earliestCodexReset,
|
|
34
32
|
escapedMcpTimeoutRecoveryFailure,
|
|
35
33
|
finalizeDurableTurnOpStreams,
|
|
36
34
|
isCodexCredentialEligible,
|
|
@@ -64,7 +62,6 @@ import {
|
|
|
64
62
|
sandboxRouteTransitionCode,
|
|
65
63
|
selectCodexCredentialLeaseForTurn,
|
|
66
64
|
settingsWithPackSandboxImage,
|
|
67
|
-
shardCredentialForSession,
|
|
68
65
|
shouldRecoverCompactionProviderFailure,
|
|
69
66
|
shouldRunTurnEndWorkspacePersistence,
|
|
70
67
|
shouldStartPeriodicWorkspaceSnapshot,
|
|
@@ -80,7 +77,7 @@ import {
|
|
|
80
77
|
waitForTurnOperation,
|
|
81
78
|
waitForWarmSnapshot,
|
|
82
79
|
withFirstPartyTools
|
|
83
|
-
} from "./chunk-
|
|
80
|
+
} from "./chunk-TAUE3GYT.js";
|
|
84
81
|
import {
|
|
85
82
|
StreamTimingMetrics,
|
|
86
83
|
makeMachineOpObserver,
|
|
@@ -106,7 +103,7 @@ import {
|
|
|
106
103
|
recordTurnWorkerPreparationTotal,
|
|
107
104
|
runtimeMetricsHooksForObservability,
|
|
108
105
|
turnLifecycleMetricsFor
|
|
109
|
-
} from "./chunk-
|
|
106
|
+
} from "./chunk-D4ZEFOYH.js";
|
|
110
107
|
|
|
111
108
|
// src/activities-turn.ts
|
|
112
109
|
import { createProductionAgentRuntime, summarizeForCompaction as summarizeForCompaction2 } from "@opengeni/runtime";
|
|
@@ -1259,12 +1256,21 @@ function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performan
|
|
|
1259
1256
|
}
|
|
1260
1257
|
|
|
1261
1258
|
// src/activities/agent-turn/credential-leases.ts
|
|
1259
|
+
var CodexCredentialLeaseLostError = class extends Error {
|
|
1260
|
+
constructor(reason) {
|
|
1261
|
+
super("Codex credential lease is not usable for provider dispatch");
|
|
1262
|
+
this.reason = reason;
|
|
1263
|
+
this.name = "CodexCredentialLeaseLostError";
|
|
1264
|
+
}
|
|
1265
|
+
code = "codex_credential_lease_lost";
|
|
1266
|
+
};
|
|
1262
1267
|
var CodexTurnLease = class {
|
|
1263
1268
|
constructor(deps) {
|
|
1264
1269
|
this.deps = deps;
|
|
1265
1270
|
}
|
|
1266
1271
|
held = false;
|
|
1267
1272
|
lost = false;
|
|
1273
|
+
lossReason = null;
|
|
1268
1274
|
holderId = null;
|
|
1269
1275
|
generation = null;
|
|
1270
1276
|
// Monotonic worker deadline, not a comparison between the Postgres and
|
|
@@ -1277,6 +1283,7 @@ var CodexTurnLease = class {
|
|
|
1277
1283
|
markLost = (reason) => {
|
|
1278
1284
|
if (this.lost) return;
|
|
1279
1285
|
this.lost = true;
|
|
1286
|
+
this.lossReason = reason;
|
|
1280
1287
|
this.deps.observability.incrementCounter({
|
|
1281
1288
|
name: "opengeni_codex_lease_renewals_total",
|
|
1282
1289
|
help: "Codex lease renewal checkpoints by outcome and reason.",
|
|
@@ -1288,6 +1295,16 @@ var CodexTurnLease = class {
|
|
|
1288
1295
|
reason
|
|
1289
1296
|
});
|
|
1290
1297
|
};
|
|
1298
|
+
/** Fail closed at the last worker-confirmed lease deadline before dispatch. */
|
|
1299
|
+
assertUsable = () => {
|
|
1300
|
+
if (this.lost) {
|
|
1301
|
+
throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
|
|
1302
|
+
}
|
|
1303
|
+
if (!this.held || this.holderId === null || this.generation === null || codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
|
|
1304
|
+
this.markLost("deadline");
|
|
1305
|
+
throw new CodexCredentialLeaseLostError("deadline");
|
|
1306
|
+
}
|
|
1307
|
+
};
|
|
1291
1308
|
renew = async (reason) => {
|
|
1292
1309
|
const turnId = this.deps.getTurnId();
|
|
1293
1310
|
if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
|
|
@@ -1299,6 +1316,7 @@ var CodexTurnLease = class {
|
|
|
1299
1316
|
}
|
|
1300
1317
|
if (this.heartbeatInFlight) return;
|
|
1301
1318
|
this.heartbeatInFlight = true;
|
|
1319
|
+
const priorConfirmedUntilMs = this.confirmedUntilMs;
|
|
1302
1320
|
const renewalStartedAtMs = performance.now();
|
|
1303
1321
|
try {
|
|
1304
1322
|
const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
|
|
@@ -1312,6 +1330,8 @@ var CodexTurnLease = class {
|
|
|
1312
1330
|
);
|
|
1313
1331
|
if (!renewedUntil) {
|
|
1314
1332
|
this.markLost("not_found");
|
|
1333
|
+
} else if (codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) || codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)) {
|
|
1334
|
+
this.markLost("deadline");
|
|
1315
1335
|
} else {
|
|
1316
1336
|
this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
|
|
1317
1337
|
this.deps.observability.incrementCounter({
|
|
@@ -3458,6 +3478,7 @@ function createTurnContext(input) {
|
|
|
3458
3478
|
},
|
|
3459
3479
|
attempt: {
|
|
3460
3480
|
turnId: void 0,
|
|
3481
|
+
dispatchId: "",
|
|
3461
3482
|
triggerEventId: void 0,
|
|
3462
3483
|
executionGeneration: 0,
|
|
3463
3484
|
providerRecoveryCount: 0,
|
|
@@ -3530,6 +3551,9 @@ function createTurnContext(input) {
|
|
|
3530
3551
|
},
|
|
3531
3552
|
providerTurn: {
|
|
3532
3553
|
effectiveCodexCredentialId: null,
|
|
3554
|
+
effectiveCodexCredentialVersion: null,
|
|
3555
|
+
codexCredentialFailoverLimit: 1,
|
|
3556
|
+
codexPolicySnapshot: null,
|
|
3533
3557
|
effectiveXaiCredentialId: null,
|
|
3534
3558
|
xaiRotationEnabled: false,
|
|
3535
3559
|
xaiAuthoritySnapshot: null,
|
|
@@ -4913,17 +4937,10 @@ async function finalizeTurnAttempt(deps) {
|
|
|
4913
4937
|
import {
|
|
4914
4938
|
requestSessionTurnRecovery,
|
|
4915
4939
|
getSessionGoal,
|
|
4916
|
-
armCodexCapacityWait,
|
|
4917
4940
|
armXaiCapacityWait,
|
|
4918
4941
|
reconcileXaiCapacityWait,
|
|
4919
|
-
getCodexRotationSettings,
|
|
4920
4942
|
listCodexAccountStatuses as listCodexAccountStatuses2,
|
|
4921
|
-
getSessionCodexState,
|
|
4922
|
-
setSessionCodexPinInTransaction,
|
|
4923
4943
|
quarantineCodexCredentialForLease,
|
|
4924
|
-
setCodexCredentialExhaustedWithWakeTargets,
|
|
4925
|
-
withSessionCodexCapacityMutation,
|
|
4926
|
-
countConsecutiveReactiveRotations,
|
|
4927
4944
|
recordUsageEvent as recordUsageEvent2,
|
|
4928
4945
|
getActiveSessionHistoryItemsPaged,
|
|
4929
4946
|
settleCodexCredentialLeaseLoss,
|
|
@@ -5633,6 +5650,59 @@ async function ensureRunAllowed(settings, db, accountId, workspaceId, isExternal
|
|
|
5633
5650
|
}
|
|
5634
5651
|
|
|
5635
5652
|
// src/activities/agent-turn/failure-settlement.ts
|
|
5653
|
+
function codexDefinitiveFailureDisposition(input) {
|
|
5654
|
+
const alternateAvailable = input.rotationEnabled && input.pinDisposition !== "manual" && input.decisionKind === "active" && input.decisionCredentialId !== null && input.decisionCredentialId !== input.servingCredentialId;
|
|
5655
|
+
if (alternateAvailable) return "failover";
|
|
5656
|
+
if (input.failureKind === "quota" || input.failureKind === "rate_limit" || input.decisionKind === "allCapped" || !input.rotationEnabled || input.pinDisposition === "manual") {
|
|
5657
|
+
return "wait";
|
|
5658
|
+
}
|
|
5659
|
+
return "terminal";
|
|
5660
|
+
}
|
|
5661
|
+
function codexCapacityWaitFailurePayload(input) {
|
|
5662
|
+
if (input.failureKind === "quota") {
|
|
5663
|
+
return codexUsageLimitFailurePayload(
|
|
5664
|
+
input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
|
|
5665
|
+
input.detail,
|
|
5666
|
+
input.allAccounts ? { allAccounts: true } : void 0
|
|
5667
|
+
);
|
|
5668
|
+
}
|
|
5669
|
+
if (input.failureKind === "rate_limit") {
|
|
5670
|
+
return {
|
|
5671
|
+
error: "The serving Codex subscription is temporarily rate limited.",
|
|
5672
|
+
code: "codex_account_rate_limited",
|
|
5673
|
+
detail: input.detail,
|
|
5674
|
+
retryable: false
|
|
5675
|
+
};
|
|
5676
|
+
}
|
|
5677
|
+
if (input.failureKind === "auth") {
|
|
5678
|
+
return {
|
|
5679
|
+
error: "The serving Codex account requires reconnection.",
|
|
5680
|
+
code: "codex_relogin_required",
|
|
5681
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
5682
|
+
retryable: false
|
|
5683
|
+
};
|
|
5684
|
+
}
|
|
5685
|
+
return {
|
|
5686
|
+
error: "The serving Codex account is not authorized for this request.",
|
|
5687
|
+
code: "codex_account_forbidden",
|
|
5688
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
5689
|
+
retryable: false
|
|
5690
|
+
};
|
|
5691
|
+
}
|
|
5692
|
+
function codexLeaseAccountsForSelection(accounts) {
|
|
5693
|
+
return accounts.map((account) => ({
|
|
5694
|
+
...account,
|
|
5695
|
+
activeLeaseCount: 0,
|
|
5696
|
+
selectionCount: 0,
|
|
5697
|
+
lastSelectedAt: null
|
|
5698
|
+
}));
|
|
5699
|
+
}
|
|
5700
|
+
function acceptedCodexPolicySnapshot(providerTurn) {
|
|
5701
|
+
if (!providerTurn.codexPolicySnapshot) {
|
|
5702
|
+
throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
|
|
5703
|
+
}
|
|
5704
|
+
return providerTurn.codexPolicySnapshot;
|
|
5705
|
+
}
|
|
5636
5706
|
async function settleTurnFailure(deps) {
|
|
5637
5707
|
const {
|
|
5638
5708
|
error,
|
|
@@ -5642,7 +5712,6 @@ async function settleTurnFailure(deps) {
|
|
|
5642
5712
|
bus,
|
|
5643
5713
|
observability,
|
|
5644
5714
|
wakeSessionWorkflow,
|
|
5645
|
-
signalCodexCapacityWorkflow,
|
|
5646
5715
|
cancellationSignal,
|
|
5647
5716
|
sandboxRotationController,
|
|
5648
5717
|
noteCancellationRequested,
|
|
@@ -5936,7 +6005,7 @@ async function settleTurnFailure(deps) {
|
|
|
5936
6005
|
control.turnMetricOutcome = "recovering";
|
|
5937
6006
|
return claimedResult({ status: "recovering" });
|
|
5938
6007
|
};
|
|
5939
|
-
if (leases.codex.lost
|
|
6008
|
+
if ((leases.codex.lost || error instanceof CodexCredentialLeaseLostError) && billingState.isCodexTurn && eventing.publish && attempt.turnId && eventing.turnStartedPublished && leases.codex.holderId && leases.codex.generation !== null) {
|
|
5940
6009
|
return await settleLostCodexAttempt(
|
|
5941
6010
|
attempt.turnId,
|
|
5942
6011
|
leases.codex.holderId,
|
|
@@ -5966,7 +6035,8 @@ async function settleTurnFailure(deps) {
|
|
|
5966
6035
|
control.turnMetricOutcome = "recovering";
|
|
5967
6036
|
return claimedResult({ status: "recovering" });
|
|
5968
6037
|
}
|
|
5969
|
-
const
|
|
6038
|
+
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
6039
|
+
const codexCredentialFailure = billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId ? classifyCodexCredentialFailure(error) : null;
|
|
5970
6040
|
if (codexCredentialFailure && providerTurn.effectiveCodexCredentialId && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
5971
6041
|
observability.incrementCounter({
|
|
5972
6042
|
name: "opengeni_codex_credential_failures_total",
|
|
@@ -6017,13 +6087,22 @@ async function settleTurnFailure(deps) {
|
|
|
6017
6087
|
} : {}
|
|
6018
6088
|
} : null;
|
|
6019
6089
|
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
6020
|
-
const
|
|
6090
|
+
const quarantineResult = leases.codex.holderId && leases.codex.generation !== null && providerTurn.effectiveCodexCredentialVersion !== null ? await quarantineCodexCredentialForLease(db, {
|
|
6021
6091
|
accountId: input.accountId,
|
|
6022
6092
|
workspaceId: input.workspaceId,
|
|
6093
|
+
sessionId: input.sessionId,
|
|
6023
6094
|
turnId: attempt.turnId,
|
|
6095
|
+
attemptId: input.attemptId,
|
|
6096
|
+
executionGeneration: attempt.executionGeneration,
|
|
6097
|
+
workflowId: input.workflowId,
|
|
6098
|
+
workflowRunId: input.workflowRunId,
|
|
6099
|
+
dispatchId: attempt.dispatchId,
|
|
6100
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6024
6101
|
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6102
|
+
credentialVersion: providerTurn.effectiveCodexCredentialVersion,
|
|
6025
6103
|
holderId: leases.codex.holderId,
|
|
6026
6104
|
generation: leases.codex.generation,
|
|
6105
|
+
maxFailovers: providerTurn.codexCredentialFailoverLimit,
|
|
6027
6106
|
quarantine: codexCredentialFailure.kind === "auth" ? {
|
|
6028
6107
|
kind: "status",
|
|
6029
6108
|
status: "needs_relogin",
|
|
@@ -6037,7 +6116,42 @@ async function settleTurnFailure(deps) {
|
|
|
6037
6116
|
until: cooldownUntil,
|
|
6038
6117
|
cooldownKind: codexCredentialFailure.kind
|
|
6039
6118
|
}
|
|
6040
|
-
}) :
|
|
6119
|
+
}) : null;
|
|
6120
|
+
if (quarantineResult?.action === "credential_changed" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6121
|
+
const recovery = await settleCodexCredentialLeaseLoss(db, {
|
|
6122
|
+
accountId: input.accountId,
|
|
6123
|
+
workspaceId: input.workspaceId,
|
|
6124
|
+
sessionId: input.sessionId,
|
|
6125
|
+
turnId: attempt.turnId,
|
|
6126
|
+
attemptId: input.attemptId,
|
|
6127
|
+
holderId: leases.codex.holderId,
|
|
6128
|
+
generation: leases.codex.generation,
|
|
6129
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6130
|
+
checkpointDurable: true,
|
|
6131
|
+
recoveryPayload: {
|
|
6132
|
+
triggerEventId: attempt.triggerEventId,
|
|
6133
|
+
reason: "codex_credential_version_changed"
|
|
6134
|
+
},
|
|
6135
|
+
failedPayload: {}
|
|
6136
|
+
});
|
|
6137
|
+
if (recovery.action === "recovering") {
|
|
6138
|
+
leases.codex.held = false;
|
|
6139
|
+
await publishDurableSessionEvents2(
|
|
6140
|
+
bus,
|
|
6141
|
+
input.workspaceId,
|
|
6142
|
+
input.sessionId,
|
|
6143
|
+
recovery.events
|
|
6144
|
+
);
|
|
6145
|
+
control.activityStatus = "recovering";
|
|
6146
|
+
control.turnMetricOutcome = "recovering";
|
|
6147
|
+
return claimedResult({ status: "recovering" });
|
|
6148
|
+
}
|
|
6149
|
+
acknowledgeLostAttemptOwnership();
|
|
6150
|
+
control.activityStatus = "cancelled";
|
|
6151
|
+
control.turnMetricOutcome = "cancelled";
|
|
6152
|
+
return claimedResult({ status: "cancelled" });
|
|
6153
|
+
}
|
|
6154
|
+
const statePersisted = quarantineResult?.action === "recorded";
|
|
6041
6155
|
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6042
6156
|
leases.codex.lost = true;
|
|
6043
6157
|
return await settleLostCodexAttempt(
|
|
@@ -6047,19 +6161,112 @@ async function settleTurnFailure(deps) {
|
|
|
6047
6161
|
true
|
|
6048
6162
|
);
|
|
6049
6163
|
}
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
|
|
6164
|
+
if (quarantineResult?.action === "recorded" && quarantineResult.exhausted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6165
|
+
const settlement = await settleCodexCredentialFailover(db, {
|
|
6166
|
+
accountId: input.accountId,
|
|
6167
|
+
workspaceId: input.workspaceId,
|
|
6168
|
+
sessionId: input.sessionId,
|
|
6169
|
+
turnId: attempt.turnId,
|
|
6170
|
+
attemptId: input.attemptId,
|
|
6171
|
+
holderId: leases.codex.holderId,
|
|
6172
|
+
generation: leases.codex.generation,
|
|
6173
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6174
|
+
maxFailovers: quarantineResult.maxFailovers,
|
|
6175
|
+
recoveryPayload: {
|
|
6176
|
+
triggerEventId: attempt.triggerEventId,
|
|
6177
|
+
reason: "codex_credential_failover",
|
|
6178
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6179
|
+
failureKind: codexCredentialFailure.kind
|
|
6180
|
+
},
|
|
6181
|
+
failedPayload: {
|
|
6182
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
6183
|
+
code: "codex_credential_failover_exhausted",
|
|
6184
|
+
retryable: false,
|
|
6185
|
+
recovery: "user_message",
|
|
6186
|
+
failoverCount: quarantineResult.failoverCount,
|
|
6187
|
+
maxFailovers: quarantineResult.maxFailovers
|
|
6188
|
+
}
|
|
6189
|
+
});
|
|
6190
|
+
if (settlement.action === "limit_exceeded") {
|
|
6191
|
+
leases.codex.held = false;
|
|
6192
|
+
await publishDurableSessionEvents2(
|
|
6193
|
+
bus,
|
|
6194
|
+
input.workspaceId,
|
|
6195
|
+
input.sessionId,
|
|
6196
|
+
settlement.events
|
|
6197
|
+
);
|
|
6198
|
+
control.activityError = error;
|
|
6199
|
+
control.activityStatus = "idle";
|
|
6200
|
+
control.turnMetricOutcome = "failed";
|
|
6201
|
+
await deliverFailedChildTurnToParent(
|
|
6202
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
6203
|
+
input.workspaceId,
|
|
6204
|
+
input.sessionId,
|
|
6205
|
+
attempt.turnId
|
|
6206
|
+
);
|
|
6207
|
+
return claimedResult({ status: "idle" });
|
|
6208
|
+
}
|
|
6209
|
+
if (settlement.action === "stale") {
|
|
6210
|
+
acknowledgeLostAttemptOwnership();
|
|
6211
|
+
control.activityStatus = "cancelled";
|
|
6212
|
+
control.turnMetricOutcome = "cancelled";
|
|
6213
|
+
return claimedResult({ status: "cancelled" });
|
|
6214
|
+
}
|
|
6215
|
+
throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
|
|
6216
|
+
}
|
|
6217
|
+
let accounts;
|
|
6218
|
+
try {
|
|
6219
|
+
accounts = await listCodexAccountStatuses2(db, input.workspaceId);
|
|
6220
|
+
} catch (metadataError) {
|
|
6221
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
6222
|
+
error: metadataError,
|
|
6223
|
+
turnId: attempt.turnId,
|
|
6224
|
+
triggerEventId: attempt.triggerEventId,
|
|
6225
|
+
executionGeneration: attempt.executionGeneration
|
|
6226
|
+
});
|
|
6227
|
+
if (recoveryFailure) {
|
|
6228
|
+
control.activityStatus = "recovering";
|
|
6229
|
+
control.turnMetricOutcome = "recovering";
|
|
6230
|
+
control.activityError = metadataError;
|
|
6231
|
+
throw recoveryFailure;
|
|
6232
|
+
}
|
|
6233
|
+
throw metadataError;
|
|
6234
|
+
}
|
|
6235
|
+
const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
|
|
6236
|
+
const selected = selectCodexCredentialLeaseForTurn({
|
|
6237
|
+
context: {
|
|
6238
|
+
accounts: codexLeaseAccountsForSelection(accounts),
|
|
6239
|
+
activeCredentialId: acceptedPolicy.activeCredentialId,
|
|
6240
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
6241
|
+
rotationStrategy: acceptedPolicy.rotationStrategy,
|
|
6242
|
+
existingCredentialId: null,
|
|
6243
|
+
failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
|
|
6244
|
+
policyScope: null,
|
|
6245
|
+
unavailableDiagnostics: []
|
|
6246
|
+
},
|
|
6247
|
+
sessionId: input.sessionId,
|
|
6248
|
+
sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
6249
|
+
sessionPinSource: acceptedPolicy.pinSource,
|
|
6250
|
+
sessionLastCredentialId: acceptedPolicy.lastCredentialId,
|
|
6251
|
+
now
|
|
6252
|
+
});
|
|
6253
|
+
const decisionKind = selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
|
|
6254
|
+
const pinDisposition = classifyCodexPin({
|
|
6255
|
+
pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
6256
|
+
pinSource: acceptedPolicy.pinSource,
|
|
6257
|
+
strategy: acceptedPolicy.rotationStrategy,
|
|
6258
|
+
rotationEnabled: acceptedPolicy.rotationEnabled
|
|
6259
|
+
});
|
|
6260
|
+
const failureDisposition = codexDefinitiveFailureDisposition({
|
|
6261
|
+
failureKind: codexCredentialFailure.kind,
|
|
6262
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
6263
|
+
pinDisposition,
|
|
6264
|
+
decisionKind,
|
|
6265
|
+
decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
|
|
6266
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId
|
|
6267
|
+
});
|
|
6268
|
+
const maxFailovers = providerTurn.codexCredentialFailoverLimit;
|
|
6269
|
+
if (statePersisted && failureDisposition === "failover" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6063
6270
|
const settlement = await settleCodexCredentialFailover(db, {
|
|
6064
6271
|
accountId: input.accountId,
|
|
6065
6272
|
workspaceId: input.workspaceId,
|
|
@@ -6069,13 +6276,21 @@ async function settleTurnFailure(deps) {
|
|
|
6069
6276
|
holderId: leases.codex.holderId,
|
|
6070
6277
|
generation: leases.codex.generation,
|
|
6071
6278
|
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
6072
|
-
maxFailovers
|
|
6279
|
+
maxFailovers,
|
|
6073
6280
|
recoveryPayload: {
|
|
6074
6281
|
triggerEventId: attempt.triggerEventId,
|
|
6075
6282
|
reason: "codex_credential_failover",
|
|
6076
6283
|
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
6077
6284
|
failureKind: codexCredentialFailure.kind,
|
|
6078
6285
|
...cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}
|
|
6286
|
+
},
|
|
6287
|
+
failedPayload: {
|
|
6288
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
6289
|
+
code: "codex_credential_failover_exhausted",
|
|
6290
|
+
retryable: false,
|
|
6291
|
+
recovery: "user_message",
|
|
6292
|
+
failoverCount: quarantineResult.failoverCount,
|
|
6293
|
+
maxFailovers: quarantineResult.maxFailovers
|
|
6079
6294
|
}
|
|
6080
6295
|
});
|
|
6081
6296
|
observability.incrementCounter({
|
|
@@ -6111,8 +6326,106 @@ async function settleTurnFailure(deps) {
|
|
|
6111
6326
|
acknowledgeLostAttemptOwnership();
|
|
6112
6327
|
control.activityStatus = "cancelled";
|
|
6113
6328
|
control.turnMetricOutcome = "cancelled";
|
|
6329
|
+
return claimedResult({ status: "cancelled" });
|
|
6330
|
+
}
|
|
6331
|
+
if (settlement.action === "limit_exceeded") {
|
|
6332
|
+
leases.codex.held = false;
|
|
6333
|
+
await publishDurableSessionEvents2(
|
|
6334
|
+
bus,
|
|
6335
|
+
input.workspaceId,
|
|
6336
|
+
input.sessionId,
|
|
6337
|
+
settlement.events
|
|
6338
|
+
);
|
|
6339
|
+
control.activityError = error;
|
|
6340
|
+
control.activityStatus = "idle";
|
|
6341
|
+
control.turnMetricOutcome = "failed";
|
|
6342
|
+
await deliverFailedChildTurnToParent(
|
|
6343
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
6344
|
+
input.workspaceId,
|
|
6345
|
+
input.sessionId,
|
|
6346
|
+
attempt.turnId
|
|
6347
|
+
);
|
|
6348
|
+
return claimedResult({ status: "idle" });
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
if (statePersisted && failureDisposition === "wait" && leases.codex.holderId && leases.codex.generation !== null) {
|
|
6352
|
+
let goal;
|
|
6353
|
+
try {
|
|
6354
|
+
goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
6355
|
+
} catch (metadataError) {
|
|
6356
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
6357
|
+
error: metadataError,
|
|
6358
|
+
turnId: attempt.turnId,
|
|
6359
|
+
triggerEventId: attempt.triggerEventId,
|
|
6360
|
+
executionGeneration: attempt.executionGeneration
|
|
6361
|
+
});
|
|
6362
|
+
if (recoveryFailure) {
|
|
6363
|
+
control.activityStatus = "recovering";
|
|
6364
|
+
control.turnMetricOutcome = "recovering";
|
|
6365
|
+
control.activityError = metadataError;
|
|
6366
|
+
throw recoveryFailure;
|
|
6367
|
+
}
|
|
6368
|
+
throw metadataError;
|
|
6369
|
+
}
|
|
6370
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
6371
|
+
const exactProviderReset = codexCredentialFailure.cooldownSeconds !== null && Number.isFinite(codexCredentialFailure.cooldownSeconds) && codexCredentialFailure.cooldownSeconds > 0;
|
|
6372
|
+
const policyCredentialId = pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId ? acceptedPolicy.pinnedCredentialId : !acceptedPolicy.rotationEnabled ? acceptedPolicy.activeCredentialId : null;
|
|
6373
|
+
const capacityAccounts = policyCredentialId ? accounts.filter((account) => account.id === policyCredentialId) : accounts;
|
|
6374
|
+
const authoritativeResetAt = exactProviderReset ? authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil : null;
|
|
6375
|
+
const allAccounts = acceptedPolicy.rotationEnabled && pinDisposition !== "manual" && decisionKind === "allCapped";
|
|
6376
|
+
const failurePayload = codexCapacityWaitFailurePayload({
|
|
6377
|
+
failureKind: codexCredentialFailure.kind,
|
|
6378
|
+
usageLimit,
|
|
6379
|
+
cooldownSeconds: codexCredentialFailure.cooldownSeconds,
|
|
6380
|
+
detail: codexCredentialFailure.kind === "quota" ? error instanceof Error ? error.message : String(error) : "the same accepted turn is waiting for eligible credential capacity",
|
|
6381
|
+
allAccounts
|
|
6382
|
+
});
|
|
6383
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
6384
|
+
{ db, bus },
|
|
6385
|
+
{
|
|
6386
|
+
accountId: input.accountId,
|
|
6387
|
+
workspaceId: input.workspaceId,
|
|
6388
|
+
sessionId: input.sessionId,
|
|
6389
|
+
turnId: attempt.turnId,
|
|
6390
|
+
attemptId: input.attemptId,
|
|
6391
|
+
workflowId: input.workflowId,
|
|
6392
|
+
goalId: activeGoal?.id ?? null,
|
|
6393
|
+
goalVersion: activeGoal?.version ?? null,
|
|
6394
|
+
earliestResetAt: authoritativeResetAt,
|
|
6395
|
+
resetKind: authoritativeResetAt ? "authoritative" : codexCredentialFailure.kind === "auth" || codexCredentialFailure.kind === "forbidden" ? "mutation_only" : "bounded_refresh",
|
|
6396
|
+
failurePayload,
|
|
6397
|
+
leaseFence: {
|
|
6398
|
+
holderId: leases.codex.holderId,
|
|
6399
|
+
generation: leases.codex.generation
|
|
6400
|
+
},
|
|
6401
|
+
expectedRedispatches: attempt.redispatchesAtDispatch
|
|
6402
|
+
},
|
|
6403
|
+
{ onArmed: () => leases.codex.held = false }
|
|
6404
|
+
);
|
|
6405
|
+
control.activityError = error;
|
|
6406
|
+
if (evaluated.action === "resumed") {
|
|
6407
|
+
control.activityStatus = "recovering";
|
|
6408
|
+
control.turnMetricOutcome = "recovering";
|
|
6114
6409
|
return claimedResult({ status: "recovering" });
|
|
6115
6410
|
}
|
|
6411
|
+
if (evaluated.action === "waiting") {
|
|
6412
|
+
control.activityError = error;
|
|
6413
|
+
control.activityStatus = "waiting_capacity";
|
|
6414
|
+
control.turnMetricOutcome = "recovering";
|
|
6415
|
+
return claimedResult({
|
|
6416
|
+
status: "waiting_capacity",
|
|
6417
|
+
capacityWait: {
|
|
6418
|
+
waiterId: evaluated.waiter.id,
|
|
6419
|
+
generation: evaluated.waiter.generation,
|
|
6420
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
6421
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
6422
|
+
}
|
|
6423
|
+
});
|
|
6424
|
+
}
|
|
6425
|
+
acknowledgeLostAttemptOwnership();
|
|
6426
|
+
control.activityStatus = "cancelled";
|
|
6427
|
+
control.turnMetricOutcome = "cancelled";
|
|
6428
|
+
return claimedResult({ status: "cancelled" });
|
|
6116
6429
|
}
|
|
6117
6430
|
}
|
|
6118
6431
|
}
|
|
@@ -6223,201 +6536,20 @@ async function settleTurnFailure(deps) {
|
|
|
6223
6536
|
control.activityError = error;
|
|
6224
6537
|
return claimedResult({ status: "recovering" });
|
|
6225
6538
|
}
|
|
6226
|
-
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
6227
6539
|
if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
6228
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
6229
|
-
const goalActive = Boolean(goal && goal.status === "active");
|
|
6230
6540
|
await flushRuntimeBatcher();
|
|
6231
6541
|
await historySink.reconcileConversationTruth();
|
|
6232
|
-
|
|
6233
|
-
let rotationResumeMs = null;
|
|
6234
|
-
let rotationResumeIdleUntilReset = false;
|
|
6235
|
-
let allCappedResetAt = null;
|
|
6236
|
-
let capacityAuthoritativeResetAt = null;
|
|
6237
|
-
let capacityNeedsBoundedRefresh = false;
|
|
6238
|
-
if (providerTurn.effectiveCodexCredentialId) {
|
|
6239
|
-
const [rotation, sessionCodex] = await Promise.all([
|
|
6240
|
-
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
6241
|
-
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null)
|
|
6242
|
-
]);
|
|
6243
|
-
const reactiveStrategy = rotation?.rotationStrategy ?? "most_remaining";
|
|
6244
|
-
const reactiveDisposition = classifyCodexPin({
|
|
6245
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
6246
|
-
pinSource: sessionCodex?.pinSource ?? null,
|
|
6247
|
-
strategy: reactiveStrategy,
|
|
6248
|
-
rotationEnabled: Boolean(rotation?.rotationEnabled)
|
|
6249
|
-
});
|
|
6250
|
-
const reactiveSharded = reactiveDisposition === "sharded";
|
|
6251
|
-
const rotating = Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) && reactiveDisposition !== "manual";
|
|
6252
|
-
if (rotating && rotation) {
|
|
6253
|
-
const accounts = await listCodexAccountStatuses2(db, input.workspaceId).catch(() => []);
|
|
6254
|
-
const serving = accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
|
|
6255
|
-
const until = codexCredentialCooldownUntil(
|
|
6256
|
-
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
6257
|
-
serving,
|
|
6258
|
-
/* @__PURE__ */ new Date()
|
|
6259
|
-
);
|
|
6260
|
-
const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
|
|
6261
|
-
db,
|
|
6262
|
-
input.workspaceId,
|
|
6263
|
-
providerTurn.effectiveCodexCredentialId,
|
|
6264
|
-
until,
|
|
6265
|
-
"quota"
|
|
6266
|
-
).catch(() => null);
|
|
6267
|
-
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
6268
|
-
if (cooldownMutation) {
|
|
6269
|
-
await signalCodexCapacityWakeTargets(
|
|
6270
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6271
|
-
cooldownMutation.wakeTargets
|
|
6272
|
-
);
|
|
6273
|
-
}
|
|
6274
|
-
const fresh = accounts.map(
|
|
6275
|
-
(a) => a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until, exhaustedKind: "quota" } : a
|
|
6276
|
-
);
|
|
6277
|
-
if (reactiveSharded) {
|
|
6278
|
-
const newHome = shardCredentialForSession({
|
|
6279
|
-
sessionId: input.sessionId,
|
|
6280
|
-
accounts: fresh,
|
|
6281
|
-
now: /* @__PURE__ */ new Date()
|
|
6282
|
-
});
|
|
6283
|
-
if (newHome) {
|
|
6284
|
-
rotated = true;
|
|
6285
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
6286
|
-
db,
|
|
6287
|
-
{
|
|
6288
|
-
workspaceId: input.workspaceId,
|
|
6289
|
-
reason: "codex_policy_pin_resharded"
|
|
6290
|
-
},
|
|
6291
|
-
async (tx) => {
|
|
6292
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
6293
|
-
tx,
|
|
6294
|
-
input.workspaceId,
|
|
6295
|
-
input.sessionId,
|
|
6296
|
-
newHome,
|
|
6297
|
-
"policy",
|
|
6298
|
-
{
|
|
6299
|
-
expected: {
|
|
6300
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
6301
|
-
pinSource: sessionCodex?.pinSource ?? null
|
|
6302
|
-
}
|
|
6303
|
-
}
|
|
6304
|
-
);
|
|
6305
|
-
return { result: changed, changed };
|
|
6306
|
-
}
|
|
6307
|
-
).catch(() => null);
|
|
6308
|
-
if (pinMutation) {
|
|
6309
|
-
await signalCodexCapacityWakeTargets(
|
|
6310
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
6311
|
-
pinMutation.wakeTargets
|
|
6312
|
-
);
|
|
6313
|
-
}
|
|
6314
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
6315
|
-
db,
|
|
6316
|
-
input.workspaceId,
|
|
6317
|
-
input.sessionId
|
|
6318
|
-
).catch(() => 0);
|
|
6319
|
-
const resume = computeReactiveRotationResume({
|
|
6320
|
-
cooldownPersisted,
|
|
6321
|
-
priorConsecutiveRotations,
|
|
6322
|
-
connectedAccountCount: accounts.length
|
|
6323
|
-
});
|
|
6324
|
-
rotationResumeMs = resume.continueDelayMs;
|
|
6325
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
6326
|
-
} else {
|
|
6327
|
-
rotated = true;
|
|
6328
|
-
allCappedResetAt = earliestCodexReset(fresh, /* @__PURE__ */ new Date());
|
|
6329
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
|
|
6330
|
-
capacityNeedsBoundedRefresh = true;
|
|
6331
|
-
}
|
|
6332
|
-
} else {
|
|
6333
|
-
const decision = chooseRotationActive({
|
|
6334
|
-
rotationStrategy: reactiveStrategy,
|
|
6335
|
-
activeCredentialId: rotation.activeCredentialId,
|
|
6336
|
-
priorCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
6337
|
-
accounts: fresh,
|
|
6338
|
-
now: /* @__PURE__ */ new Date()
|
|
6339
|
-
});
|
|
6340
|
-
if (decision.kind === "active") {
|
|
6341
|
-
rotated = true;
|
|
6342
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
6343
|
-
db,
|
|
6344
|
-
input.workspaceId,
|
|
6345
|
-
input.sessionId
|
|
6346
|
-
).catch(() => 0);
|
|
6347
|
-
const resume = computeReactiveRotationResume({
|
|
6348
|
-
cooldownPersisted,
|
|
6349
|
-
priorConsecutiveRotations,
|
|
6350
|
-
connectedAccountCount: accounts.length
|
|
6351
|
-
});
|
|
6352
|
-
rotationResumeMs = resume.continueDelayMs;
|
|
6353
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
6354
|
-
} else if (decision.kind === "allCapped") {
|
|
6355
|
-
rotated = true;
|
|
6356
|
-
allCappedResetAt = decision.earliestResetAt;
|
|
6357
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, /* @__PURE__ */ new Date());
|
|
6358
|
-
capacityNeedsBoundedRefresh = true;
|
|
6359
|
-
}
|
|
6360
|
-
}
|
|
6361
|
-
}
|
|
6362
|
-
}
|
|
6363
|
-
const failurePayload = allCappedResetAt ? codexUsageLimitFailurePayload(
|
|
6364
|
-
{
|
|
6365
|
-
resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1e3)
|
|
6366
|
-
},
|
|
6367
|
-
error instanceof Error ? error.message : String(error),
|
|
6368
|
-
{ allAccounts: true }
|
|
6369
|
-
) : codexUsageLimitFailurePayload(
|
|
6542
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
6370
6543
|
usageLimit,
|
|
6371
6544
|
error instanceof Error ? error.message : String(error)
|
|
6372
6545
|
);
|
|
6373
|
-
if (rotationResumeMs === null) {
|
|
6374
|
-
const providerResetAt = capacityAuthoritativeResetAt ?? (usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : null);
|
|
6375
|
-
const armed = await armCodexCapacityWait(db, {
|
|
6376
|
-
accountId: input.accountId,
|
|
6377
|
-
workspaceId: input.workspaceId,
|
|
6378
|
-
sessionId: input.sessionId,
|
|
6379
|
-
turnId: attempt.turnId,
|
|
6380
|
-
attemptId: input.attemptId,
|
|
6381
|
-
workflowId: input.workflowId,
|
|
6382
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
6383
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
6384
|
-
earliestResetAt: providerResetAt,
|
|
6385
|
-
resetKind: providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
|
|
6386
|
-
failurePayload,
|
|
6387
|
-
...leases.codex.holderId && leases.codex.generation !== null ? {
|
|
6388
|
-
leaseFence: {
|
|
6389
|
-
holderId: leases.codex.holderId,
|
|
6390
|
-
generation: leases.codex.generation
|
|
6391
|
-
},
|
|
6392
|
-
expectedRedispatches: attempt.redispatchesAtDispatch
|
|
6393
|
-
} : {}
|
|
6394
|
-
});
|
|
6395
|
-
if (armed.action === "waiting") {
|
|
6396
|
-
await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, armed.events);
|
|
6397
|
-
control.turnMetricOutcome = "recovering";
|
|
6398
|
-
control.activityStatus = "waiting_capacity";
|
|
6399
|
-
control.activityError = error;
|
|
6400
|
-
return claimedResult({
|
|
6401
|
-
status: "waiting_capacity",
|
|
6402
|
-
capacityWait: {
|
|
6403
|
-
waiterId: armed.waiter.id,
|
|
6404
|
-
generation: armed.waiter.generation,
|
|
6405
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
6406
|
-
wakeRevision: armed.waiter.wakeRevision
|
|
6407
|
-
}
|
|
6408
|
-
});
|
|
6409
|
-
}
|
|
6410
|
-
}
|
|
6411
6546
|
if (!await eventing.settle({
|
|
6412
6547
|
events: [
|
|
6413
|
-
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
6414
|
-
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
6415
6548
|
{
|
|
6416
6549
|
type: "turn.failed",
|
|
6417
6550
|
payload: {
|
|
6418
6551
|
...failurePayload,
|
|
6419
|
-
recovery:
|
|
6420
|
-
...rotated ? { rotated: true } : {}
|
|
6552
|
+
recovery: "user_message"
|
|
6421
6553
|
}
|
|
6422
6554
|
},
|
|
6423
6555
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
@@ -6431,21 +6563,6 @@ async function settleTurnFailure(deps) {
|
|
|
6431
6563
|
control.turnMetricOutcome = "failed";
|
|
6432
6564
|
control.activityStatus = "idle";
|
|
6433
6565
|
control.activityError = error;
|
|
6434
|
-
if (goalActive) {
|
|
6435
|
-
if (rotationResumeMs !== null) {
|
|
6436
|
-
return claimedResult({
|
|
6437
|
-
status: "idle",
|
|
6438
|
-
continueDelayMs: rotationResumeMs,
|
|
6439
|
-
...rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}
|
|
6440
|
-
});
|
|
6441
|
-
}
|
|
6442
|
-
const resumeMs = allCappedResetAt ? computeIdleDelayMs(allCappedResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS) : usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1e3, CODEX_USAGE_LIMIT_MAX_RESUME_MS) : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
|
|
6443
|
-
return claimedResult({
|
|
6444
|
-
status: "idle",
|
|
6445
|
-
continueDelayMs: resumeMs,
|
|
6446
|
-
...allCappedResetAt ? { idleUntilReset: true } : {}
|
|
6447
|
-
});
|
|
6448
|
-
}
|
|
6449
6566
|
return claimedResult({ status: "idle" });
|
|
6450
6567
|
}
|
|
6451
6568
|
if (error instanceof BudgetExhaustedError && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
@@ -6690,6 +6807,7 @@ import {
|
|
|
6690
6807
|
clearDurablePendingSessionToolCalls,
|
|
6691
6808
|
isSessionCompactionRequested,
|
|
6692
6809
|
nextSessionHistoryPosition as nextSessionHistoryPosition2,
|
|
6810
|
+
persistModelContextSnapshot,
|
|
6693
6811
|
updateSessionTitleWithEvent
|
|
6694
6812
|
} from "@opengeni/db";
|
|
6695
6813
|
import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
|
|
@@ -8520,6 +8638,17 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8520
8638
|
return await runtime.runStream(agent, runInput, eventing.modelRunSettings, {
|
|
8521
8639
|
signal: runtimeCancellationSignal,
|
|
8522
8640
|
sandboxEnvironment,
|
|
8641
|
+
onModelVisibleContext: async (snapshot) => {
|
|
8642
|
+
await persistModelContextSnapshot(db, {
|
|
8643
|
+
accountId: input.accountId,
|
|
8644
|
+
workspaceId: input.workspaceId,
|
|
8645
|
+
sessionId: input.sessionId,
|
|
8646
|
+
turnId: activeTurnId,
|
|
8647
|
+
attemptId: input.attemptId,
|
|
8648
|
+
executionGeneration: attempt.executionGeneration,
|
|
8649
|
+
snapshot
|
|
8650
|
+
});
|
|
8651
|
+
},
|
|
8523
8652
|
onRuntimeEvent: async (event) => {
|
|
8524
8653
|
await leases.renewServing("runtime_event");
|
|
8525
8654
|
if (leases.servingLost()) {
|
|
@@ -8605,7 +8734,9 @@ async function runTurnStreamAttempt(deps) {
|
|
|
8605
8734
|
});
|
|
8606
8735
|
}
|
|
8607
8736
|
};
|
|
8608
|
-
if (
|
|
8737
|
+
if (billingState.isCodexTurn) {
|
|
8738
|
+
leases.codex.assertUsable();
|
|
8739
|
+
} else if (leases.codex.lost) {
|
|
8609
8740
|
throw new Error("Codex credential lease expired before the model run");
|
|
8610
8741
|
}
|
|
8611
8742
|
if (leases.xai.lost) {
|
|
@@ -9664,6 +9795,13 @@ import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
|
9664
9795
|
function turnAttemptProducerId(input, turnId) {
|
|
9665
9796
|
return `${input.workflowId}:${turnId}:${input.attemptId}`;
|
|
9666
9797
|
}
|
|
9798
|
+
function codexCredentialLeaseHolderId(input, turnId) {
|
|
9799
|
+
const parts = [input.workflowId, turnId, input.attemptId];
|
|
9800
|
+
if (parts.some((part) => part.trim().length === 0)) {
|
|
9801
|
+
throw new Error("Codex credential lease holder requires a complete turn attempt identity");
|
|
9802
|
+
}
|
|
9803
|
+
return `codex-turn:${parts.map((part) => encodeURIComponent(part)).join(":")}`;
|
|
9804
|
+
}
|
|
9667
9805
|
async function claimTurnAttempt(deps) {
|
|
9668
9806
|
const {
|
|
9669
9807
|
input,
|
|
@@ -9712,9 +9850,11 @@ async function claimTurnAttempt(deps) {
|
|
|
9712
9850
|
}
|
|
9713
9851
|
const turn = claim.turn;
|
|
9714
9852
|
attempt.turnId = turn.id;
|
|
9853
|
+
attempt.dispatchId = dispatchId;
|
|
9715
9854
|
attempt.executionGeneration = turn.executionGeneration;
|
|
9716
9855
|
attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
|
|
9717
9856
|
attempt.triggerEventId = turn.triggerEventId;
|
|
9857
|
+
leases.codex.holderId = codexCredentialLeaseHolderId(input, turn.id);
|
|
9718
9858
|
attempt.redispatchesAtDispatch = Number(
|
|
9719
9859
|
turn.metadata?.workerDeathRedispatches ?? 0
|
|
9720
9860
|
);
|
|
@@ -9863,7 +10003,6 @@ async function claimTurnAttempt(deps) {
|
|
|
9863
10003
|
eventing.heartbeatTimer = startActivityHeartbeat(activityContext, heartbeatDetails);
|
|
9864
10004
|
let producerSeq = 0;
|
|
9865
10005
|
const producerId = turnAttemptProducerId(input, attempt.turnId);
|
|
9866
|
-
leases.codex.holderId = dispatchId;
|
|
9867
10006
|
const modelUsageDispatchId = input.attemptId;
|
|
9868
10007
|
const claimedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
9869
10008
|
const emittedModelUsageSourceKeys = /* @__PURE__ */ new Set();
|
|
@@ -9949,6 +10088,7 @@ async function claimTurnAttempt(deps) {
|
|
|
9949
10088
|
turnStatus: inputSettlement.turnStatus,
|
|
9950
10089
|
sessionStatus: inputSettlement.sessionStatus,
|
|
9951
10090
|
activeTurnId: inputSettlement.activeTurnId,
|
|
10091
|
+
...inputSettlement.suppressGoalContinuation !== void 0 ? { suppressGoalContinuation: inputSettlement.suppressGoalContinuation } : {},
|
|
9952
10092
|
events: inputs,
|
|
9953
10093
|
...runState ? { runState } : {},
|
|
9954
10094
|
...compactionRequestFailure ? { compactionRequestFailure } : {}
|
|
@@ -10042,16 +10182,15 @@ async function claimTurnAttempt(deps) {
|
|
|
10042
10182
|
import {
|
|
10043
10183
|
getSessionGoal as getSessionGoal2,
|
|
10044
10184
|
acquireCodexCredentialLease,
|
|
10045
|
-
|
|
10185
|
+
CodexCredentialLeaseAttemptFencedError,
|
|
10186
|
+
CodexCredentialFailoverExhaustedError,
|
|
10046
10187
|
CODEX_CREDENTIAL_LEASE_TTL_MS as CODEX_CREDENTIAL_LEASE_TTL_MS2,
|
|
10047
|
-
getCodexRotationSettings as getCodexRotationSettings2,
|
|
10048
|
-
listCodexAccountStatuses as listCodexAccountStatuses3,
|
|
10049
|
-
getSessionCodexState as getSessionCodexState2,
|
|
10050
10188
|
recordSessionActiveCodexCredential,
|
|
10051
|
-
setSessionCodexPinInTransaction
|
|
10052
|
-
|
|
10053
|
-
withSessionCodexCapacityMutation
|
|
10189
|
+
setSessionCodexPinInTransaction,
|
|
10190
|
+
settleCodexCredentialFailover as settleCodexCredentialFailover2,
|
|
10191
|
+
withSessionCodexCapacityMutation
|
|
10054
10192
|
} from "@opengeni/db";
|
|
10193
|
+
import { CodexReloginRequired } from "@opengeni/codex";
|
|
10055
10194
|
import { publishDurableSessionEvents as publishDurableSessionEvents5 } from "@opengeni/events";
|
|
10056
10195
|
|
|
10057
10196
|
// src/activities/codex-fleet-shadow.ts
|
|
@@ -10255,6 +10394,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10255
10394
|
observability,
|
|
10256
10395
|
wakeSessionWorkflow,
|
|
10257
10396
|
signalCodexCapacityWorkflow,
|
|
10397
|
+
dispatchId,
|
|
10258
10398
|
control,
|
|
10259
10399
|
attempt,
|
|
10260
10400
|
billingState,
|
|
@@ -10262,6 +10402,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10262
10402
|
providerTurn,
|
|
10263
10403
|
leases,
|
|
10264
10404
|
claimedResult,
|
|
10405
|
+
acknowledgeLostAttemptOwnership,
|
|
10265
10406
|
turn,
|
|
10266
10407
|
codexWorkspaceKey
|
|
10267
10408
|
} = deps;
|
|
@@ -10277,123 +10418,62 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10277
10418
|
const credentialSelectionStartedAt = performance.now();
|
|
10278
10419
|
let credentialSelectionOutcome = "completed";
|
|
10279
10420
|
try {
|
|
10280
|
-
const
|
|
10281
|
-
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
10282
|
-
const sessionPinSource = sessionCodex?.pinSource ?? null;
|
|
10283
|
-
const selectForTurn = (context) => selectCodexCredentialLeaseForTurn({
|
|
10421
|
+
const selectForTurn = (context, lockedSessionCodexState2) => selectCodexCredentialLeaseForTurn({
|
|
10284
10422
|
context,
|
|
10285
|
-
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
10286
10423
|
sessionId: input.sessionId,
|
|
10287
|
-
sessionPinnedCredentialId:
|
|
10288
|
-
sessionPinSource,
|
|
10289
|
-
sessionLastCredentialId:
|
|
10424
|
+
sessionPinnedCredentialId: lockedSessionCodexState2.pinnedCredentialId,
|
|
10425
|
+
sessionPinSource: lockedSessionCodexState2.pinSource,
|
|
10426
|
+
sessionLastCredentialId: lockedSessionCodexState2.lastCredentialId,
|
|
10290
10427
|
now: /* @__PURE__ */ new Date()
|
|
10291
10428
|
});
|
|
10292
|
-
let
|
|
10293
|
-
let
|
|
10294
|
-
|
|
10429
|
+
let leaseAcquisitionStartedAtMs = performance.now();
|
|
10430
|
+
let leased = await acquireCodexCredentialLease(
|
|
10431
|
+
db,
|
|
10432
|
+
{
|
|
10433
|
+
accountId: input.accountId,
|
|
10434
|
+
workspaceId: input.workspaceId,
|
|
10435
|
+
sessionId: input.sessionId,
|
|
10436
|
+
turnId,
|
|
10437
|
+
attemptId: input.attemptId,
|
|
10438
|
+
executionGeneration: attempt.executionGeneration,
|
|
10439
|
+
workflowId: input.workflowId,
|
|
10440
|
+
workflowRunId: input.workflowRunId,
|
|
10441
|
+
dispatchId,
|
|
10442
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10443
|
+
holderId,
|
|
10444
|
+
advanceActivePointer: true
|
|
10445
|
+
},
|
|
10446
|
+
selectForTurn
|
|
10447
|
+
);
|
|
10448
|
+
if (leased.decision.kind === "allCapped") {
|
|
10449
|
+
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
10450
|
+
signalCodexCapacityWorkflow,
|
|
10451
|
+
wakeSessionWorkflow
|
|
10452
|
+
});
|
|
10295
10453
|
leaseAcquisitionStartedAtMs = performance.now();
|
|
10296
10454
|
leased = await acquireCodexCredentialLease(
|
|
10297
10455
|
db,
|
|
10298
10456
|
{
|
|
10299
10457
|
accountId: input.accountId,
|
|
10300
10458
|
workspaceId: input.workspaceId,
|
|
10459
|
+
sessionId: input.sessionId,
|
|
10301
10460
|
turnId,
|
|
10461
|
+
attemptId: input.attemptId,
|
|
10462
|
+
executionGeneration: attempt.executionGeneration,
|
|
10463
|
+
workflowId: input.workflowId,
|
|
10464
|
+
workflowRunId: input.workflowRunId,
|
|
10465
|
+
dispatchId,
|
|
10466
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10302
10467
|
holderId,
|
|
10303
|
-
advanceActivePointer:
|
|
10468
|
+
advanceActivePointer: true
|
|
10304
10469
|
},
|
|
10305
10470
|
selectForTurn
|
|
10306
10471
|
);
|
|
10307
|
-
} else {
|
|
10308
|
-
const [rotation, accounts] = await Promise.all([
|
|
10309
|
-
getCodexRotationSettings2(db, input.workspaceId),
|
|
10310
|
-
listCodexAccountStatuses3(db, input.workspaceId)
|
|
10311
|
-
]);
|
|
10312
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
10313
|
-
...account,
|
|
10314
|
-
activeLeaseCount: 0,
|
|
10315
|
-
selectionCount: 0,
|
|
10316
|
-
lastSelectedAt: null
|
|
10317
|
-
}));
|
|
10318
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
10319
|
-
const selected = selectForTurn({
|
|
10320
|
-
accounts: leaseAccounts,
|
|
10321
|
-
activeCredentialId,
|
|
10322
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10323
|
-
leaseRotationEnabled: false,
|
|
10324
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10325
|
-
existingCredentialId: null,
|
|
10326
|
-
policyScope: null,
|
|
10327
|
-
unavailableDiagnostics: []
|
|
10328
|
-
});
|
|
10329
|
-
leased = {
|
|
10330
|
-
...selected,
|
|
10331
|
-
accounts: leaseAccounts,
|
|
10332
|
-
activeCredentialId,
|
|
10333
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10334
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10335
|
-
reused: false,
|
|
10336
|
-
holderId: null,
|
|
10337
|
-
generation: null,
|
|
10338
|
-
leasedUntil: null,
|
|
10339
|
-
unavailableDiagnostics: [],
|
|
10340
|
-
advanceActivePointer: selected.advanceActivePointer !== false
|
|
10341
|
-
};
|
|
10342
|
-
}
|
|
10343
|
-
if (leased.decision.kind === "allCapped") {
|
|
10344
|
-
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
10345
|
-
signalCodexCapacityWorkflow,
|
|
10346
|
-
wakeSessionWorkflow
|
|
10347
|
-
});
|
|
10348
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
10349
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
10350
|
-
leased = await acquireCodexCredentialLease(
|
|
10351
|
-
db,
|
|
10352
|
-
{
|
|
10353
|
-
accountId: input.accountId,
|
|
10354
|
-
workspaceId: input.workspaceId,
|
|
10355
|
-
turnId,
|
|
10356
|
-
holderId,
|
|
10357
|
-
advanceActivePointer: sessionPin === null
|
|
10358
|
-
},
|
|
10359
|
-
selectForTurn
|
|
10360
|
-
);
|
|
10361
|
-
} else {
|
|
10362
|
-
const [rotation, accounts] = await Promise.all([
|
|
10363
|
-
getCodexRotationSettings2(db, input.workspaceId),
|
|
10364
|
-
listCodexAccountStatuses3(db, input.workspaceId)
|
|
10365
|
-
]);
|
|
10366
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
10367
|
-
...account,
|
|
10368
|
-
activeLeaseCount: 0,
|
|
10369
|
-
selectionCount: 0,
|
|
10370
|
-
lastSelectedAt: null
|
|
10371
|
-
}));
|
|
10372
|
-
const selected = selectForTurn({
|
|
10373
|
-
accounts: leaseAccounts,
|
|
10374
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
10375
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10376
|
-
leaseRotationEnabled: false,
|
|
10377
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10378
|
-
existingCredentialId: null,
|
|
10379
|
-
policyScope: null,
|
|
10380
|
-
unavailableDiagnostics: []
|
|
10381
|
-
});
|
|
10382
|
-
leased = {
|
|
10383
|
-
...selected,
|
|
10384
|
-
accounts: leaseAccounts,
|
|
10385
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
10386
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
10387
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
10388
|
-
reused: false,
|
|
10389
|
-
holderId: null,
|
|
10390
|
-
generation: null,
|
|
10391
|
-
leasedUntil: null,
|
|
10392
|
-
unavailableDiagnostics: [],
|
|
10393
|
-
advanceActivePointer: selected.advanceActivePointer !== false
|
|
10394
|
-
};
|
|
10395
|
-
}
|
|
10396
10472
|
}
|
|
10473
|
+
const lockedSessionCodexState = leased.sessionCodexState;
|
|
10474
|
+
providerTurn.codexPolicySnapshot = leased.codexPolicySnapshot;
|
|
10475
|
+
const sessionPin = lockedSessionCodexState.pinnedCredentialId;
|
|
10476
|
+
const sessionPinSource = lockedSessionCodexState.pinSource;
|
|
10397
10477
|
const rotationDecision = leased.decision;
|
|
10398
10478
|
const selectedPinDisposition = classifyCodexPin({
|
|
10399
10479
|
pinnedCredentialId: sessionPin,
|
|
@@ -10401,15 +10481,15 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10401
10481
|
strategy: leased.rotationStrategy,
|
|
10402
10482
|
rotationEnabled: leased.rotationEnabled
|
|
10403
10483
|
});
|
|
10404
|
-
if (selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
|
|
10405
|
-
const pinMutation = await
|
|
10484
|
+
if (!leased.codexPolicySnapshotReused && selectedPinDisposition === "sharded" && leased.credentialId !== null && (sessionPinSource !== "policy" || sessionPin !== leased.credentialId)) {
|
|
10485
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
10406
10486
|
db,
|
|
10407
10487
|
{
|
|
10408
10488
|
workspaceId: input.workspaceId,
|
|
10409
10489
|
reason: "codex_policy_pin_changed"
|
|
10410
10490
|
},
|
|
10411
10491
|
async (tx) => {
|
|
10412
|
-
const changed = await
|
|
10492
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
10413
10493
|
tx,
|
|
10414
10494
|
input.workspaceId,
|
|
10415
10495
|
input.sessionId,
|
|
@@ -10430,14 +10510,14 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10430
10510
|
pinMutation.wakeTargets
|
|
10431
10511
|
);
|
|
10432
10512
|
} else if (selectedPinDisposition === "clearStale") {
|
|
10433
|
-
const pinMutation = await
|
|
10513
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
10434
10514
|
db,
|
|
10435
10515
|
{
|
|
10436
10516
|
workspaceId: input.workspaceId,
|
|
10437
10517
|
reason: "codex_stale_policy_pin_cleared"
|
|
10438
10518
|
},
|
|
10439
10519
|
async (tx) => {
|
|
10440
|
-
const changed = await
|
|
10520
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
10441
10521
|
tx,
|
|
10442
10522
|
input.workspaceId,
|
|
10443
10523
|
input.sessionId,
|
|
@@ -10458,16 +10538,14 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10458
10538
|
pinMutation.wakeTargets
|
|
10459
10539
|
);
|
|
10460
10540
|
}
|
|
10461
|
-
if (!settings.codexCredentialLeasingEnabled && leased.advanceActivePointer && sessionPin === null && rotationDecision.kind === "active" && rotationDecision.moved) {
|
|
10462
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
10463
|
-
}
|
|
10464
10541
|
providerTurn.effectiveCodexCredentialId = leased.credentialId;
|
|
10542
|
+
providerTurn.codexCredentialFailoverLimit = leased.failoverLimit;
|
|
10465
10543
|
leases.codex.generation = leased.generation;
|
|
10466
10544
|
leases.codex.confirmedUntilMs = leased.leasedUntil && leaseAcquisitionStartedAtMs !== null ? leaseAcquisitionStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS2 : null;
|
|
10467
10545
|
leases.codex.held = providerTurn.effectiveCodexCredentialId !== null && leased.holderId !== null && leased.generation !== null && leases.codex.confirmedUntilMs !== null;
|
|
10468
10546
|
if (leases.codex.held) leases.codex.startHeartbeat();
|
|
10469
|
-
const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" ? "waiting" : "none";
|
|
10470
|
-
const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : "none";
|
|
10547
|
+
const actualOutcome = providerTurn.effectiveCodexCredentialId ? "selected" : rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled" ? "waiting" : "none";
|
|
10548
|
+
const actualReason = providerTurn.effectiveCodexCredentialId ? leased.reused ? "lease_reused" : sessionPin === providerTurn.effectiveCodexCredentialId ? "pin" : rotationDecision.kind === "active" && rotationDecision.moved ? "rotation" : "active" : rotationDecision.kind === "allCapped" ? "all_capped" : rotationDecision.kind === "allocatorDisabled" ? "allocator_disabled" : "none";
|
|
10471
10549
|
const fencedInFlight = leased.reused;
|
|
10472
10550
|
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
10473
10551
|
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
@@ -10476,7 +10554,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10476
10554
|
actualCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
10477
10555
|
actualOutcome,
|
|
10478
10556
|
actualReason,
|
|
10479
|
-
affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ??
|
|
10557
|
+
affinityCredentialId: fencedInFlight ? providerTurn.effectiveCodexCredentialId : sessionPin ?? lockedSessionCodexState.lastCredentialId ?? null,
|
|
10480
10558
|
fencedInFlight,
|
|
10481
10559
|
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
10482
10560
|
now: /* @__PURE__ */ new Date(),
|
|
@@ -10540,7 +10618,7 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10540
10618
|
depth: poolDepth
|
|
10541
10619
|
});
|
|
10542
10620
|
}
|
|
10543
|
-
if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && leased.accounts.every((account) => !account.allocatorEnabled) && turnId) {
|
|
10621
|
+
if (providerTurn.effectiveCodexCredentialId === null && leased.accounts.length > 0 && (rotationDecision.kind === "allocatorDisabled" || leased.accounts.every((account) => !account.allocatorEnabled)) && turnId) {
|
|
10544
10622
|
if (turn.source === "compaction") {
|
|
10545
10623
|
if (!await eventing.settle({
|
|
10546
10624
|
events: [
|
|
@@ -10567,66 +10645,146 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10567
10645
|
control.activityStatus = "idle";
|
|
10568
10646
|
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10569
10647
|
}
|
|
10570
|
-
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId)
|
|
10648
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10571
10649
|
const activeGoal = goal?.status === "active" ? goal : null;
|
|
10572
|
-
const
|
|
10573
|
-
|
|
10574
|
-
|
|
10575
|
-
|
|
10576
|
-
|
|
10577
|
-
|
|
10578
|
-
|
|
10579
|
-
|
|
10580
|
-
|
|
10581
|
-
|
|
10582
|
-
|
|
10583
|
-
|
|
10584
|
-
|
|
10585
|
-
|
|
10586
|
-
|
|
10650
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10651
|
+
{ db, bus },
|
|
10652
|
+
{
|
|
10653
|
+
accountId: input.accountId,
|
|
10654
|
+
workspaceId: input.workspaceId,
|
|
10655
|
+
sessionId: input.sessionId,
|
|
10656
|
+
turnId,
|
|
10657
|
+
attemptId: input.attemptId,
|
|
10658
|
+
workflowId: input.workflowId,
|
|
10659
|
+
goalId: activeGoal?.id ?? null,
|
|
10660
|
+
goalVersion: activeGoal?.version ?? null,
|
|
10661
|
+
earliestResetAt: null,
|
|
10662
|
+
resetKind: "mutation_only",
|
|
10663
|
+
failurePayload: {
|
|
10664
|
+
error: rotationDecision.kind === "allocatorDisabled" ? "The policy-selected Codex subscription is disabled for new allocations." : "All connected Codex subscriptions are disabled for new allocations.",
|
|
10665
|
+
code: "codex_allocator_disabled",
|
|
10666
|
+
detail: rotationDecision.kind === "allocatorDisabled" ? "waiting for the selected credential to be re-enabled under the accepted source policy" : "waiting for a credential in the accepted source pool to be re-enabled, reconnected, or added"
|
|
10667
|
+
}
|
|
10587
10668
|
}
|
|
10588
|
-
|
|
10589
|
-
if (
|
|
10590
|
-
|
|
10669
|
+
);
|
|
10670
|
+
if (evaluated.action === "resumed") {
|
|
10671
|
+
control.turnMetricOutcome = "recovering";
|
|
10672
|
+
control.activityStatus = "recovering";
|
|
10673
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10674
|
+
}
|
|
10675
|
+
if (evaluated.action === "waiting") {
|
|
10591
10676
|
control.turnMetricOutcome = "recovering";
|
|
10592
10677
|
control.activityStatus = "waiting_capacity";
|
|
10593
10678
|
return {
|
|
10594
10679
|
exit: claimedResult({
|
|
10595
10680
|
status: "waiting_capacity",
|
|
10596
10681
|
capacityWait: {
|
|
10597
|
-
waiterId:
|
|
10598
|
-
generation:
|
|
10599
|
-
nextCheckAt:
|
|
10600
|
-
wakeRevision:
|
|
10682
|
+
waiterId: evaluated.waiter.id,
|
|
10683
|
+
generation: evaluated.waiter.generation,
|
|
10684
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10685
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10601
10686
|
}
|
|
10602
10687
|
})
|
|
10603
10688
|
};
|
|
10604
10689
|
}
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10613
|
-
|
|
10690
|
+
acknowledgeLostAttemptOwnership();
|
|
10691
|
+
control.turnMetricOutcome = "cancelled";
|
|
10692
|
+
control.activityStatus = "cancelled";
|
|
10693
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10694
|
+
}
|
|
10695
|
+
if (rotationDecision.kind === "none") {
|
|
10696
|
+
if (leased.poolAccountCount === 0 && !leased.codexPolicySnapshotReused && leased.codexPolicySnapshot?.source !== "disabled") {
|
|
10697
|
+
throw new CodexReloginRequired("No Codex subscription is connected for this workspace.");
|
|
10698
|
+
}
|
|
10699
|
+
const noneReason = sessionPinSource === "manual" && sessionPin !== null ? {
|
|
10700
|
+
code: "codex_manual_pin_unavailable",
|
|
10701
|
+
error: "The pinned Codex subscription is unavailable for this turn.",
|
|
10702
|
+
detail: "waiting for the pinned account to reconnect or become allocatable under the accepted source policy"
|
|
10703
|
+
} : !leased.rotationEnabled && leased.activeCredentialId === null ? {
|
|
10704
|
+
code: "codex_active_pointer_unavailable",
|
|
10705
|
+
error: "The Codex active subscription pointer is unavailable for this turn.",
|
|
10706
|
+
detail: "waiting for the accepted active account to be restored or become allocatable"
|
|
10707
|
+
} : leased.poolAccountCount > leased.accounts.length ? {
|
|
10708
|
+
code: "codex_policy_pool_unavailable",
|
|
10709
|
+
error: "No Codex subscription matches the accepted account-selection policy.",
|
|
10710
|
+
detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
|
|
10711
|
+
} : {
|
|
10712
|
+
code: "codex_credential_unavailable",
|
|
10713
|
+
error: "No Codex subscription is currently available under the accepted policy.",
|
|
10714
|
+
detail: "waiting for a matching credential to reconnect or become allocatable under the accepted source policy"
|
|
10715
|
+
};
|
|
10716
|
+
if (turn.source === "compaction") {
|
|
10717
|
+
if (!await eventing.settle({
|
|
10718
|
+
events: [
|
|
10719
|
+
{
|
|
10720
|
+
type: "turn.cancelled",
|
|
10721
|
+
payload: {
|
|
10722
|
+
maintenance: "context_compaction",
|
|
10723
|
+
reason: noneReason.code,
|
|
10724
|
+
requestPreserved: true
|
|
10725
|
+
}
|
|
10726
|
+
},
|
|
10727
|
+
{
|
|
10728
|
+
type: "session.status.changed",
|
|
10729
|
+
payload: { status: "idle" }
|
|
10614
10730
|
}
|
|
10615
|
-
|
|
10616
|
-
|
|
10617
|
-
|
|
10618
|
-
|
|
10731
|
+
],
|
|
10732
|
+
turnStatus: "cancelled",
|
|
10733
|
+
sessionStatus: "idle",
|
|
10734
|
+
activeTurnId: null
|
|
10735
|
+
})) {
|
|
10736
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10737
|
+
}
|
|
10738
|
+
control.turnMetricOutcome = "cancelled";
|
|
10739
|
+
control.activityStatus = "idle";
|
|
10740
|
+
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10741
|
+
}
|
|
10742
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10743
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
10744
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10745
|
+
{ db, bus },
|
|
10746
|
+
{
|
|
10747
|
+
accountId: input.accountId,
|
|
10748
|
+
workspaceId: input.workspaceId,
|
|
10749
|
+
sessionId: input.sessionId,
|
|
10750
|
+
turnId,
|
|
10751
|
+
attemptId: input.attemptId,
|
|
10752
|
+
workflowId: input.workflowId,
|
|
10753
|
+
goalId: activeGoal?.id ?? null,
|
|
10754
|
+
goalVersion: activeGoal?.version ?? null,
|
|
10755
|
+
earliestResetAt: null,
|
|
10756
|
+
resetKind: "mutation_only",
|
|
10757
|
+
failurePayload: {
|
|
10758
|
+
error: noneReason.error,
|
|
10759
|
+
code: noneReason.code,
|
|
10760
|
+
detail: noneReason.detail
|
|
10619
10761
|
}
|
|
10620
|
-
|
|
10621
|
-
|
|
10622
|
-
|
|
10623
|
-
|
|
10624
|
-
|
|
10625
|
-
return { exit: claimedResult({ status: "
|
|
10762
|
+
}
|
|
10763
|
+
);
|
|
10764
|
+
if (evaluated.action === "resumed") {
|
|
10765
|
+
control.turnMetricOutcome = "recovering";
|
|
10766
|
+
control.activityStatus = "recovering";
|
|
10767
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10626
10768
|
}
|
|
10627
|
-
|
|
10628
|
-
|
|
10629
|
-
|
|
10769
|
+
if (evaluated.action === "waiting") {
|
|
10770
|
+
control.turnMetricOutcome = "recovering";
|
|
10771
|
+
control.activityStatus = "waiting_capacity";
|
|
10772
|
+
return {
|
|
10773
|
+
exit: claimedResult({
|
|
10774
|
+
status: "waiting_capacity",
|
|
10775
|
+
capacityWait: {
|
|
10776
|
+
waiterId: evaluated.waiter.id,
|
|
10777
|
+
generation: evaluated.waiter.generation,
|
|
10778
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10779
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10780
|
+
}
|
|
10781
|
+
})
|
|
10782
|
+
};
|
|
10783
|
+
}
|
|
10784
|
+
acknowledgeLostAttemptOwnership();
|
|
10785
|
+
control.turnMetricOutcome = "cancelled";
|
|
10786
|
+
control.activityStatus = "cancelled";
|
|
10787
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10630
10788
|
}
|
|
10631
10789
|
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
10632
10790
|
if (turn.source === "compaction") {
|
|
@@ -10655,88 +10813,64 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10655
10813
|
control.activityStatus = "idle";
|
|
10656
10814
|
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
10657
10815
|
}
|
|
10658
|
-
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId)
|
|
10816
|
+
const goal = await getSessionGoal2(db, input.workspaceId, input.sessionId);
|
|
10659
10817
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
10660
10818
|
const resumeMs = computeIdleDelayMs(
|
|
10661
10819
|
rotationDecision.earliestResetAt,
|
|
10662
10820
|
/* @__PURE__ */ new Date(),
|
|
10663
10821
|
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
10664
10822
|
);
|
|
10823
|
+
const allConnectedAccountsWereConsidered = leased.rotationEnabled && selectedPinDisposition !== "manual";
|
|
10824
|
+
const capacityDetail = allConnectedAccountsWereConsidered ? "all connected Codex subscriptions are rate-limited" : selectedPinDisposition === "manual" ? "the pinned Codex subscription is rate-limited" : "the active Codex subscription is rate-limited";
|
|
10665
10825
|
const failurePayload = codexUsageLimitFailurePayload(
|
|
10666
10826
|
{ resetsInSeconds: Math.ceil(resumeMs / 1e3) },
|
|
10667
|
-
|
|
10668
|
-
{ allAccounts:
|
|
10827
|
+
capacityDetail,
|
|
10828
|
+
{ allAccounts: allConnectedAccountsWereConsidered }
|
|
10669
10829
|
);
|
|
10670
10830
|
const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, /* @__PURE__ */ new Date());
|
|
10671
|
-
const
|
|
10672
|
-
|
|
10673
|
-
|
|
10674
|
-
|
|
10675
|
-
|
|
10676
|
-
|
|
10677
|
-
|
|
10678
|
-
|
|
10679
|
-
|
|
10680
|
-
|
|
10681
|
-
|
|
10682
|
-
|
|
10683
|
-
|
|
10684
|
-
|
|
10685
|
-
|
|
10831
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
10832
|
+
{ db, bus },
|
|
10833
|
+
{
|
|
10834
|
+
accountId: input.accountId,
|
|
10835
|
+
workspaceId: input.workspaceId,
|
|
10836
|
+
sessionId: input.sessionId,
|
|
10837
|
+
turnId,
|
|
10838
|
+
attemptId: input.attemptId,
|
|
10839
|
+
workflowId: input.workflowId,
|
|
10840
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
10841
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
10842
|
+
earliestResetAt: authoritativeResetAt,
|
|
10843
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
10844
|
+
failurePayload
|
|
10845
|
+
}
|
|
10846
|
+
);
|
|
10847
|
+
if (evaluated.action === "resumed") {
|
|
10848
|
+
control.turnMetricOutcome = "recovering";
|
|
10849
|
+
control.activityStatus = "recovering";
|
|
10850
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
10851
|
+
}
|
|
10852
|
+
if (evaluated.action === "waiting") {
|
|
10686
10853
|
control.turnMetricOutcome = "recovering";
|
|
10687
10854
|
control.activityStatus = "waiting_capacity";
|
|
10688
10855
|
return {
|
|
10689
10856
|
exit: claimedResult({
|
|
10690
10857
|
status: "waiting_capacity",
|
|
10691
10858
|
capacityWait: {
|
|
10692
|
-
waiterId:
|
|
10693
|
-
generation:
|
|
10694
|
-
nextCheckAt:
|
|
10695
|
-
wakeRevision:
|
|
10859
|
+
waiterId: evaluated.waiter.id,
|
|
10860
|
+
generation: evaluated.waiter.generation,
|
|
10861
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
10862
|
+
wakeRevision: evaluated.waiter.wakeRevision
|
|
10696
10863
|
}
|
|
10697
10864
|
})
|
|
10698
10865
|
};
|
|
10699
10866
|
}
|
|
10700
|
-
|
|
10701
|
-
|
|
10702
|
-
|
|
10703
|
-
|
|
10704
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
10705
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
10706
|
-
// the proactive path while the reactive path spares it.
|
|
10707
|
-
{
|
|
10708
|
-
type: "turn.failed",
|
|
10709
|
-
payload: {
|
|
10710
|
-
...failurePayload,
|
|
10711
|
-
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
10712
|
-
rotated: true
|
|
10713
|
-
}
|
|
10714
|
-
},
|
|
10715
|
-
{
|
|
10716
|
-
type: "session.status.changed",
|
|
10717
|
-
payload: { status: "idle" }
|
|
10718
|
-
}
|
|
10719
|
-
],
|
|
10720
|
-
turnStatus: "failed",
|
|
10721
|
-
sessionStatus: "idle",
|
|
10722
|
-
activeTurnId: null
|
|
10723
|
-
})) {
|
|
10724
|
-
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10725
|
-
}
|
|
10726
|
-
control.turnMetricOutcome = "failed";
|
|
10727
|
-
control.activityStatus = "idle";
|
|
10728
|
-
return {
|
|
10729
|
-
exit: claimedResult(
|
|
10730
|
-
goalActive ? {
|
|
10731
|
-
status: "idle",
|
|
10732
|
-
continueDelayMs: resumeMs,
|
|
10733
|
-
idleUntilReset: true
|
|
10734
|
-
} : { status: "idle" }
|
|
10735
|
-
)
|
|
10736
|
-
};
|
|
10867
|
+
acknowledgeLostAttemptOwnership();
|
|
10868
|
+
control.turnMetricOutcome = "cancelled";
|
|
10869
|
+
control.activityStatus = "cancelled";
|
|
10870
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10737
10871
|
}
|
|
10738
10872
|
if (providerTurn.effectiveCodexCredentialId) {
|
|
10739
|
-
const priorAccountId =
|
|
10873
|
+
const priorAccountId = lockedSessionCodexState.lastCredentialId;
|
|
10740
10874
|
if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
|
|
10741
10875
|
await recordSessionActiveCodexCredential(
|
|
10742
10876
|
db,
|
|
@@ -10780,8 +10914,60 @@ async function selectCodexTurnCapacity(deps) {
|
|
|
10780
10914
|
}
|
|
10781
10915
|
]);
|
|
10782
10916
|
}
|
|
10783
|
-
} catch (error) {
|
|
10784
|
-
credentialSelectionOutcome = "failed";
|
|
10917
|
+
} catch (error) {
|
|
10918
|
+
credentialSelectionOutcome = "failed";
|
|
10919
|
+
if (error instanceof CodexCredentialLeaseAttemptFencedError) {
|
|
10920
|
+
acknowledgeLostAttemptOwnership();
|
|
10921
|
+
control.activityStatus = "cancelled";
|
|
10922
|
+
control.turnMetricOutcome = "cancelled";
|
|
10923
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10924
|
+
}
|
|
10925
|
+
if (error instanceof CodexCredentialFailoverExhaustedError) {
|
|
10926
|
+
const settlement = await settleCodexCredentialFailover2(db, {
|
|
10927
|
+
accountId: input.accountId,
|
|
10928
|
+
workspaceId: input.workspaceId,
|
|
10929
|
+
sessionId: input.sessionId,
|
|
10930
|
+
turnId,
|
|
10931
|
+
attemptId: input.attemptId,
|
|
10932
|
+
holderId: null,
|
|
10933
|
+
generation: null,
|
|
10934
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
10935
|
+
maxFailovers: error.maxFailovers,
|
|
10936
|
+
recoveryPayload: {
|
|
10937
|
+
triggerEventId: attempt.triggerEventId,
|
|
10938
|
+
reason: "codex_credential_failover_exhausted"
|
|
10939
|
+
},
|
|
10940
|
+
failedPayload: {
|
|
10941
|
+
error: "Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
10942
|
+
code: "codex_credential_failover_exhausted",
|
|
10943
|
+
retryable: false,
|
|
10944
|
+
recovery: "user_message",
|
|
10945
|
+
failoverCount: error.failoverCount,
|
|
10946
|
+
maxFailovers: error.maxFailovers
|
|
10947
|
+
}
|
|
10948
|
+
});
|
|
10949
|
+
if (settlement.action === "limit_exceeded") {
|
|
10950
|
+
await publishDurableSessionEvents5(
|
|
10951
|
+
bus,
|
|
10952
|
+
input.workspaceId,
|
|
10953
|
+
input.sessionId,
|
|
10954
|
+
settlement.events
|
|
10955
|
+
);
|
|
10956
|
+
control.activityStatus = "idle";
|
|
10957
|
+
control.turnMetricOutcome = "failed";
|
|
10958
|
+
await deliverFailedChildTurnToParent(
|
|
10959
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
10960
|
+
input.workspaceId,
|
|
10961
|
+
input.sessionId,
|
|
10962
|
+
turnId
|
|
10963
|
+
);
|
|
10964
|
+
return { exit: claimedResult({ status: "idle" }) };
|
|
10965
|
+
}
|
|
10966
|
+
acknowledgeLostAttemptOwnership();
|
|
10967
|
+
control.activityStatus = "cancelled";
|
|
10968
|
+
control.turnMetricOutcome = "cancelled";
|
|
10969
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
10970
|
+
}
|
|
10785
10971
|
throw error;
|
|
10786
10972
|
} finally {
|
|
10787
10973
|
recordTurnStartupPhase(observability, {
|
|
@@ -10841,7 +11027,8 @@ import {
|
|
|
10841
11027
|
getGeneratedImageArtifact,
|
|
10842
11028
|
markImageGenerationOperationOutcomeUnknown,
|
|
10843
11029
|
markImageGenerationOperationRetentionFailed,
|
|
10844
|
-
prepareImageGenerationOperation
|
|
11030
|
+
prepareImageGenerationOperation,
|
|
11031
|
+
resetImageGenerationOperationBeforeProviderDispatch
|
|
10845
11032
|
} from "@opengeni/db";
|
|
10846
11033
|
import { createHash as createHash8 } from "crypto";
|
|
10847
11034
|
var ImageGenerationOutcomeUnknownError = class extends Error {
|
|
@@ -10866,6 +11053,7 @@ var ImageGenerationRetentionFailedError = class extends Error {
|
|
|
10866
11053
|
var imageGenerationOperationPorts = {
|
|
10867
11054
|
prepare: prepareImageGenerationOperation,
|
|
10868
11055
|
begin: beginImageGenerationOperation,
|
|
11056
|
+
resetBeforeProviderDispatch: resetImageGenerationOperationBeforeProviderDispatch,
|
|
10869
11057
|
retain: retainGeneratedImage,
|
|
10870
11058
|
complete: completeImageGenerationOperation,
|
|
10871
11059
|
markOutcomeUnknown: markImageGenerationOperationOutcomeUnknown,
|
|
@@ -10912,7 +11100,9 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
|
|
|
10912
11100
|
accountId: input.accountId,
|
|
10913
11101
|
workspaceId: input.workspaceId,
|
|
10914
11102
|
operationId: identity.operationId,
|
|
10915
|
-
operationKey: identity.operationKey
|
|
11103
|
+
operationKey: identity.operationKey,
|
|
11104
|
+
providerBindingHash: input.providerBindingHash,
|
|
11105
|
+
expectedArtifactId: identity.artifactId
|
|
10916
11106
|
});
|
|
10917
11107
|
if (!begun.started || begun.operation.status !== "provider_started") {
|
|
10918
11108
|
const recovered = await ports.recover(input, {
|
|
@@ -10927,6 +11117,16 @@ async function executeImageGenerationOperation(input, ports = imageGenerationOpe
|
|
|
10927
11117
|
try {
|
|
10928
11118
|
output = await input.generate();
|
|
10929
11119
|
} catch (error) {
|
|
11120
|
+
if (input.isProviderDispatchRejected?.(error) === true) {
|
|
11121
|
+
await ports.resetBeforeProviderDispatch(input.db, {
|
|
11122
|
+
accountId: input.accountId,
|
|
11123
|
+
workspaceId: input.workspaceId,
|
|
11124
|
+
operationId: identity.operationId,
|
|
11125
|
+
operationKey: identity.operationKey,
|
|
11126
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11127
|
+
}).catch(() => void 0);
|
|
11128
|
+
throw error;
|
|
11129
|
+
}
|
|
10930
11130
|
await ports.markOutcomeUnknown(input.db, {
|
|
10931
11131
|
accountId: input.accountId,
|
|
10932
11132
|
workspaceId: input.workspaceId,
|
|
@@ -11058,13 +11258,9 @@ function uuidFromDigest3(digestValue) {
|
|
|
11058
11258
|
// src/activities/agent-turn/tool-policy.ts
|
|
11059
11259
|
function structuredToolTransportForTurn(resolvedModel) {
|
|
11060
11260
|
if (!resolvedModel) return true;
|
|
11061
|
-
|
|
11062
|
-
return
|
|
11063
|
-
|
|
11064
|
-
"xai-subscription",
|
|
11065
|
-
"vercel-gateway-managed",
|
|
11066
|
-
"vercel-gateway-workspace"
|
|
11067
|
-
].includes(resolvedModel.provider.kind);
|
|
11261
|
+
const provider = resolvedModel.provider;
|
|
11262
|
+
if (provider.api === "chat" || provider.kind === "codex-subscription") return false;
|
|
11263
|
+
return provider.wireProfile === "azure-openai" || provider.builtin && provider.id === "openai" && provider.baseUrl === void 0;
|
|
11068
11264
|
}
|
|
11069
11265
|
function lazyToolTransportForTurn(resolvedModel) {
|
|
11070
11266
|
if (!resolvedModel) return "openai_native";
|
|
@@ -12341,11 +12537,13 @@ import { managedSessionGroupBackend, managedSessionGroupOs } from "@opengeni/cor
|
|
|
12341
12537
|
import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
|
|
12342
12538
|
import {
|
|
12343
12539
|
adoptConnectedMachineSessionBackgroundCommand,
|
|
12540
|
+
adoptManagedSessionBackgroundCommand,
|
|
12344
12541
|
advanceWorkspaceGenerationForRetainedProcess,
|
|
12345
12542
|
advanceWorkspaceGeneration as advanceWorkspaceGeneration2,
|
|
12346
12543
|
getRetainedProcess,
|
|
12347
12544
|
retainWorkspaceMutationProcess,
|
|
12348
12545
|
retainedProcessSettlementIdentity,
|
|
12546
|
+
settleConnectedMachineSessionBackgroundCommand,
|
|
12349
12547
|
settleRetainedProcess,
|
|
12350
12548
|
verifyRetainedProcessMutationSettlement,
|
|
12351
12549
|
verifyWorkspaceMutationSettlement as verifyWorkspaceMutationSettlement2,
|
|
@@ -12357,10 +12555,6 @@ import {
|
|
|
12357
12555
|
resolvePersonalMachineConnectionForAttempt,
|
|
12358
12556
|
SandboxRetainedProcessPromotionFencedError
|
|
12359
12557
|
} from "@opengeni/db";
|
|
12360
|
-
import {
|
|
12361
|
-
settleConnectedMachineSessionBackgroundCommand,
|
|
12362
|
-
settleSessionBackgroundCommandForRetainedProcess
|
|
12363
|
-
} from "@opengeni/db/session-background-commands";
|
|
12364
12558
|
import {
|
|
12365
12559
|
buildSelfhostedBackendSession,
|
|
12366
12560
|
ActiveBackendUnresolvableError,
|
|
@@ -12558,10 +12752,6 @@ function afterPersistableHomeMutation(services, ids, home) {
|
|
|
12558
12752
|
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
12559
12753
|
operation: op,
|
|
12560
12754
|
providerBinding: boundAdmission.providerBinding ?? null,
|
|
12561
|
-
backgroundCommand: {
|
|
12562
|
-
commandId: retainedProcess.id,
|
|
12563
|
-
command: op
|
|
12564
|
-
},
|
|
12565
12755
|
owner: {
|
|
12566
12756
|
kind: "turn",
|
|
12567
12757
|
turnId: fence.turnId,
|
|
@@ -12668,14 +12858,33 @@ function settleRetainedProcessForTurn(services, ids) {
|
|
|
12668
12858
|
if (settlement.process.state === "active") {
|
|
12669
12859
|
throw new Error("Retained-process settlement returned an active durable process");
|
|
12670
12860
|
}
|
|
12671
|
-
|
|
12861
|
+
if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
|
|
12862
|
+
await services.bus.publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents).catch(() => void 0);
|
|
12863
|
+
}
|
|
12864
|
+
};
|
|
12865
|
+
}
|
|
12866
|
+
function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
|
|
12867
|
+
const fence = ids.workspaceMutationFence;
|
|
12868
|
+
if (!fence) return void 0;
|
|
12869
|
+
return async ({ backend, process: process2 }) => {
|
|
12870
|
+
const durable = await getRetainedProcess(services.db, {
|
|
12871
|
+
workspaceId: ids.workspaceId,
|
|
12872
|
+
sessionId: ids.sessionId,
|
|
12873
|
+
processId: process2.id
|
|
12874
|
+
});
|
|
12875
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
12876
|
+
throw new Error("Background command adoption lost its exact retained-process identity");
|
|
12877
|
+
}
|
|
12878
|
+
await adoptManagedSessionBackgroundCommand(services.db, {
|
|
12672
12879
|
accountId: fence.accountId,
|
|
12673
12880
|
workspaceId: ids.workspaceId,
|
|
12674
12881
|
sessionId: ids.sessionId,
|
|
12675
|
-
|
|
12676
|
-
|
|
12677
|
-
|
|
12678
|
-
|
|
12882
|
+
turnId: fence.turnId,
|
|
12883
|
+
executionGeneration: fence.executionGeneration,
|
|
12884
|
+
attemptId: fence.attemptId,
|
|
12885
|
+
processId: process2.id,
|
|
12886
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
12887
|
+
command: "execCommand"
|
|
12679
12888
|
});
|
|
12680
12889
|
};
|
|
12681
12890
|
}
|
|
@@ -12696,6 +12905,10 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
12696
12905
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
12697
12906
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
12698
12907
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
12908
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
12909
|
+
services,
|
|
12910
|
+
ids
|
|
12911
|
+
);
|
|
12699
12912
|
const resolver = makeActiveBackendResolver({
|
|
12700
12913
|
workspaceId: ids.workspaceId,
|
|
12701
12914
|
defaultBackend: established.session,
|
|
@@ -12749,7 +12962,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
12749
12962
|
return { commandId: adopted.id };
|
|
12750
12963
|
},
|
|
12751
12964
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
12752
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
12965
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
12753
12966
|
accountId: backgroundCommandAccountId,
|
|
12754
12967
|
workspaceId: ids.workspaceId,
|
|
12755
12968
|
sessionId: ids.sessionId,
|
|
@@ -12762,6 +12975,9 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
12762
12975
|
exitCode: command.exitCode,
|
|
12763
12976
|
reason: command.reason
|
|
12764
12977
|
});
|
|
12978
|
+
if (settlement && bus) {
|
|
12979
|
+
await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
|
|
12980
|
+
}
|
|
12765
12981
|
}
|
|
12766
12982
|
} : {},
|
|
12767
12983
|
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
@@ -12836,6 +13052,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
12836
13052
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
12837
13053
|
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
12838
13054
|
...settleProcess ? { settleProcess } : {},
|
|
13055
|
+
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
12839
13056
|
...ids.homeLease ? {
|
|
12840
13057
|
onDefaultBackendError: async ({
|
|
12841
13058
|
error,
|
|
@@ -12880,6 +13097,10 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
12880
13097
|
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
12881
13098
|
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
12882
13099
|
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
13100
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
13101
|
+
services,
|
|
13102
|
+
ids
|
|
13103
|
+
);
|
|
12883
13104
|
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
12884
13105
|
const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
|
|
12885
13106
|
const syntheticSession = {
|
|
@@ -12941,7 +13162,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
12941
13162
|
return { commandId: adopted.id };
|
|
12942
13163
|
},
|
|
12943
13164
|
selfhostedSettleBackgroundCommand: async (command) => {
|
|
12944
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
13165
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
12945
13166
|
accountId: backgroundCommandAccountId,
|
|
12946
13167
|
workspaceId: ids.workspaceId,
|
|
12947
13168
|
sessionId: ids.sessionId,
|
|
@@ -12954,6 +13175,9 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
12954
13175
|
exitCode: command.exitCode,
|
|
12955
13176
|
reason: command.reason
|
|
12956
13177
|
});
|
|
13178
|
+
if (settlement && bus) {
|
|
13179
|
+
await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
|
|
13180
|
+
}
|
|
12957
13181
|
}
|
|
12958
13182
|
} : {},
|
|
12959
13183
|
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
@@ -13004,6 +13228,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
13004
13228
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
13005
13229
|
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
13006
13230
|
...settleProcess ? { settleProcess } : {},
|
|
13231
|
+
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
13007
13232
|
...args.homeLeaseIdentity ? {
|
|
13008
13233
|
onDefaultBackendError: async ({
|
|
13009
13234
|
error,
|
|
@@ -13167,7 +13392,7 @@ async function establishSelfhostedTurnSession(services, args) {
|
|
|
13167
13392
|
}
|
|
13168
13393
|
} : {},
|
|
13169
13394
|
settleBackgroundCommand: async (command) => {
|
|
13170
|
-
await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
13395
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
13171
13396
|
accountId: args.accountId,
|
|
13172
13397
|
workspaceId: args.workspaceId,
|
|
13173
13398
|
sessionId: args.sessionId,
|
|
@@ -13180,6 +13405,9 @@ async function establishSelfhostedTurnSession(services, args) {
|
|
|
13180
13405
|
exitCode: command.exitCode,
|
|
13181
13406
|
reason: command.reason
|
|
13182
13407
|
});
|
|
13408
|
+
if (settlement && bus) {
|
|
13409
|
+
await bus.publish(args.workspaceId, args.sessionId, settlement.events).catch(() => void 0);
|
|
13410
|
+
}
|
|
13183
13411
|
},
|
|
13184
13412
|
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
13185
13413
|
...onOp !== void 0 ? { onOp } : {},
|
|
@@ -15354,6 +15582,8 @@ async function prepareRunCredentials(deps) {
|
|
|
15354
15582
|
|
|
15355
15583
|
// src/activities/agent-turn/tool-environment.ts
|
|
15356
15584
|
import {
|
|
15585
|
+
beginConnectorActionExecution,
|
|
15586
|
+
completeConnectorActionExecution,
|
|
15357
15587
|
getScheduledVariableSetExpectedGenerationForAttempt,
|
|
15358
15588
|
getWorkspaceModelPolicy as getWorkspaceModelPolicy2,
|
|
15359
15589
|
listWorkspaceGatewayCustomModels as listWorkspaceGatewayCustomModels2,
|
|
@@ -15361,6 +15591,8 @@ import {
|
|
|
15361
15591
|
listOrganizationModelProviderCustomModelsForWorkspace as listOrganizationModelProviderCustomModelsForWorkspace2,
|
|
15362
15592
|
organizationModelProviderConnectionActiveForWorkspace,
|
|
15363
15593
|
persistAttemptToolCatalog,
|
|
15594
|
+
prepareConnectorActionApproval,
|
|
15595
|
+
previewConnectorActionApproval,
|
|
15364
15596
|
namedSubjectHasLiveWorkspaceAuthority as namedSubjectHasLiveWorkspaceAuthority2,
|
|
15365
15597
|
updateSessionTitleWithEvent as updateSessionTitleWithEvent2,
|
|
15366
15598
|
withCodexAppsRequestAuthorization,
|
|
@@ -15376,7 +15608,6 @@ import {
|
|
|
15376
15608
|
|
|
15377
15609
|
// src/activities/google-drive-publication.ts
|
|
15378
15610
|
import { createHash as createHash9 } from "crypto";
|
|
15379
|
-
import { AttemptToolApprovalRequiredError } from "@opengeni/codemode";
|
|
15380
15611
|
import {
|
|
15381
15612
|
GOOGLE_DRIVE_FILE_SCOPE,
|
|
15382
15613
|
GOOGLE_DRIVE_FULL_SCOPE,
|
|
@@ -15390,10 +15621,7 @@ import {
|
|
|
15390
15621
|
googleDriveScopesAllowCapability
|
|
15391
15622
|
} from "@opengeni/contracts/google-drive";
|
|
15392
15623
|
import {
|
|
15393
|
-
beginConnectorActionExecution,
|
|
15394
|
-
completeConnectorActionExecution,
|
|
15395
15624
|
getConnectionMetadata,
|
|
15396
|
-
prepareConnectorActionApproval,
|
|
15397
15625
|
requireFile as requireFile2,
|
|
15398
15626
|
namedSubjectHasLiveWorkspaceAuthority
|
|
15399
15627
|
} from "@opengeni/db";
|
|
@@ -15401,6 +15629,7 @@ import {
|
|
|
15401
15629
|
readAuthorizedEditableArtifactMaterialization
|
|
15402
15630
|
} from "@opengeni/db/editable-artifact-durable-export";
|
|
15403
15631
|
import { readResponseJsonBounded, undiciFetch } from "@opengeni/network";
|
|
15632
|
+
import { ConnectorActionExecutionError } from "@opengeni/runtime";
|
|
15404
15633
|
import { retryWhileMissing as retryWhileMissing4 } from "@opengeni/storage";
|
|
15405
15634
|
var DRIVE_API = "https://www.googleapis.com/drive/v3";
|
|
15406
15635
|
var DRIVE_UPLOAD_API = "https://www.googleapis.com/upload/drive/v3";
|
|
@@ -15416,9 +15645,6 @@ var defaultPorts = {
|
|
|
15416
15645
|
getMembership: namedSubjectHasLiveWorkspaceAuthority,
|
|
15417
15646
|
readMaterialization: readAuthorizedEditableArtifactMaterialization,
|
|
15418
15647
|
requireFile: requireFile2,
|
|
15419
|
-
prepare: prepareConnectorActionApproval,
|
|
15420
|
-
begin: beginConnectorActionExecution,
|
|
15421
|
-
complete: completeConnectorActionExecution,
|
|
15422
15648
|
fetch: undiciFetch
|
|
15423
15649
|
};
|
|
15424
15650
|
async function resolveGoogleDrivePublicationTarget(db, workspace, delegations, ports = defaultPorts) {
|
|
@@ -15589,24 +15815,6 @@ function createGoogleDrivePublicationAttemptTool(input) {
|
|
|
15589
15815
|
approval: "policy",
|
|
15590
15816
|
execute: async (raw, context) => {
|
|
15591
15817
|
const request = GoogleDrivePublicationToolInput.parse(raw);
|
|
15592
|
-
const connectorCall = googleDrivePublicationConnectorCall(
|
|
15593
|
-
input.target,
|
|
15594
|
-
request,
|
|
15595
|
-
context.operationId
|
|
15596
|
-
);
|
|
15597
|
-
let requestId = null;
|
|
15598
|
-
if (context.caller.kind === "codemode") {
|
|
15599
|
-
const admission = await ports.begin(input.db, input.identity, connectorCall);
|
|
15600
|
-
if (!admission.allowed) {
|
|
15601
|
-
throw new Error(
|
|
15602
|
-
admission.reason === "uncertain_retry" || admission.reason === "already_executed" ? "Google Drive publication outcome is unknown: a previous attempt may have already sent this publication to Google Drive. Verify in Drive before retrying with a new idempotency key." : admission.reason === "not_executed" ? "Google Drive publication was not executed: the previous attempt failed before any request reached Google Drive. Retrying with a new call is safe." : `Google Drive publication was not executed: ${admission.reason}. No request reached Google Drive.`
|
|
15603
|
-
);
|
|
15604
|
-
}
|
|
15605
|
-
if (!admission.managed) {
|
|
15606
|
-
throw new Error("Google Drive publication has no explicit connector action policy");
|
|
15607
|
-
}
|
|
15608
|
-
requestId = admission.requestId;
|
|
15609
|
-
}
|
|
15610
15818
|
let providerRequestStarted = false;
|
|
15611
15819
|
try {
|
|
15612
15820
|
const receipt = await executeGoogleDrivePublication(
|
|
@@ -15625,53 +15833,21 @@ function createGoogleDrivePublicationAttemptTool(input) {
|
|
|
15625
15833
|
},
|
|
15626
15834
|
ports
|
|
15627
15835
|
);
|
|
15628
|
-
if (requestId) {
|
|
15629
|
-
await ports.complete(input.db, {
|
|
15630
|
-
accountId: input.identity.accountId,
|
|
15631
|
-
workspaceId: input.identity.workspaceId,
|
|
15632
|
-
requestId,
|
|
15633
|
-
attemptId: input.identity.attemptId,
|
|
15634
|
-
outcome: "completed"
|
|
15635
|
-
});
|
|
15636
|
-
}
|
|
15637
15836
|
return {
|
|
15638
15837
|
content: [{ type: "text", text: JSON.stringify(receipt) }],
|
|
15639
15838
|
structuredContent: receipt
|
|
15640
15839
|
};
|
|
15641
15840
|
} catch (error) {
|
|
15642
|
-
if (requestId) {
|
|
15643
|
-
await ports.complete(input.db, {
|
|
15644
|
-
accountId: input.identity.accountId,
|
|
15645
|
-
workspaceId: input.identity.workspaceId,
|
|
15646
|
-
requestId,
|
|
15647
|
-
attemptId: input.identity.attemptId,
|
|
15648
|
-
outcome: providerRequestStarted ? "uncertain" : "not_executed"
|
|
15649
|
-
});
|
|
15650
|
-
}
|
|
15651
15841
|
const detail = error instanceof Error ? error.message : String(error);
|
|
15652
|
-
throw new
|
|
15842
|
+
throw new ConnectorActionExecutionError(
|
|
15653
15843
|
providerRequestStarted ? `Google Drive publication outcome is unknown: a provider request may have been sent before the failure. Verify in Drive before retrying. Cause: ${detail}` : `Google Drive publication was not executed: no request reached Google Drive. Cause: ${detail}`,
|
|
15844
|
+
providerRequestStarted ? "uncertain" : "not_executed",
|
|
15654
15845
|
{ cause: error }
|
|
15655
15846
|
);
|
|
15656
15847
|
}
|
|
15657
15848
|
}
|
|
15658
15849
|
};
|
|
15659
15850
|
}
|
|
15660
|
-
async function authorizeGoogleDrivePublicationAttempt(input) {
|
|
15661
|
-
const request = GoogleDrivePublicationToolInput.parse(input.arguments);
|
|
15662
|
-
const preparation = await (input.ports ?? defaultPorts).prepare(
|
|
15663
|
-
input.db,
|
|
15664
|
-
input.identity,
|
|
15665
|
-
googleDrivePublicationConnectorCall(input.target, request, input.approvalId)
|
|
15666
|
-
);
|
|
15667
|
-
if (!preparation.managed) {
|
|
15668
|
-
throw new Error("Google Drive publication has no explicit connector action policy");
|
|
15669
|
-
}
|
|
15670
|
-
if (preparation.decision === "ask") throw new AttemptToolApprovalRequiredError();
|
|
15671
|
-
if (preparation.decision === "block") {
|
|
15672
|
-
throw new Error("Google Drive publication is blocked by connector action policy");
|
|
15673
|
-
}
|
|
15674
|
-
}
|
|
15675
15851
|
async function executeGoogleDrivePublication(input, ports = defaultPorts) {
|
|
15676
15852
|
const request = GoogleDrivePublicationToolInput.parse(input.request);
|
|
15677
15853
|
const connection = await ports.getConnection(
|
|
@@ -16202,100 +16378,6 @@ function isGitHubApiDestination(value) {
|
|
|
16202
16378
|
}
|
|
16203
16379
|
}
|
|
16204
16380
|
|
|
16205
|
-
// src/activities/api-integrations.ts
|
|
16206
|
-
import {
|
|
16207
|
-
createGraphqlMcpServer,
|
|
16208
|
-
createOpenApiMcpServer,
|
|
16209
|
-
createPinnedIntegrationTransport
|
|
16210
|
-
} from "@opengeni/capabilities";
|
|
16211
|
-
function buildApiIntegrationServersForTurn(input) {
|
|
16212
|
-
const transport = createPinnedIntegrationTransport({
|
|
16213
|
-
network: input.settings,
|
|
16214
|
-
...input.fetchImpl ? { fetchImpl: input.fetchImpl } : {}
|
|
16215
|
-
});
|
|
16216
|
-
return input.integrations.map((integration) => {
|
|
16217
|
-
const authority = {
|
|
16218
|
-
...input.authority,
|
|
16219
|
-
...integration.connectionRef?.connectionId ? { connectionRef: integration.connectionRef.connectionId } : {}
|
|
16220
|
-
};
|
|
16221
|
-
const credentialResolver = integration.connectionRef ? integrationCredentialResolver(input, integration) : void 0;
|
|
16222
|
-
const server = integration.revision.protocol === "openapi" ? createOpenApiMcpServer({
|
|
16223
|
-
revision: integration.revision,
|
|
16224
|
-
transport,
|
|
16225
|
-
authority,
|
|
16226
|
-
...credentialResolver ? { credentialResolver } : {}
|
|
16227
|
-
}) : createGraphqlMcpServer({
|
|
16228
|
-
revision: integration.revision,
|
|
16229
|
-
endpoint: integration.baseUrl,
|
|
16230
|
-
transport,
|
|
16231
|
-
authority,
|
|
16232
|
-
...credentialResolver ? { credentialResolver } : {}
|
|
16233
|
-
});
|
|
16234
|
-
return {
|
|
16235
|
-
id: integration.serverId,
|
|
16236
|
-
server,
|
|
16237
|
-
...integration.connectionRef?.connectionId ? { resolvedConnectionId: integration.connectionRef.connectionId } : {}
|
|
16238
|
-
};
|
|
16239
|
-
});
|
|
16240
|
-
}
|
|
16241
|
-
function integrationCredentialResolver(input, integration) {
|
|
16242
|
-
const connectionRef = integration.connectionRef;
|
|
16243
|
-
if (!connectionRef) throw new Error("Integration credential resolver requires a connection");
|
|
16244
|
-
return {
|
|
16245
|
-
resolve: async (request) => {
|
|
16246
|
-
const result = await input.resolveCredential({
|
|
16247
|
-
workspaceId: input.authority.workspaceId,
|
|
16248
|
-
serverId: integration.serverId,
|
|
16249
|
-
toolName: request.operationKey,
|
|
16250
|
-
connectionRef,
|
|
16251
|
-
destinationUrl: request.destinationUrl,
|
|
16252
|
-
credentialTarget: "http_api",
|
|
16253
|
-
forceRefresh: request.forceRefresh === true
|
|
16254
|
-
});
|
|
16255
|
-
if (result.status === "auth_needed") {
|
|
16256
|
-
await publishAuthNeeded(
|
|
16257
|
-
input,
|
|
16258
|
-
integration.serverId,
|
|
16259
|
-
request.operationKey,
|
|
16260
|
-
result,
|
|
16261
|
-
connectionRef
|
|
16262
|
-
);
|
|
16263
|
-
return null;
|
|
16264
|
-
}
|
|
16265
|
-
const destination = new URL(request.destinationUrl);
|
|
16266
|
-
return {
|
|
16267
|
-
audience: { origin: destination.origin, pathPrefix: "/" },
|
|
16268
|
-
placements: result.placements ?? Object.entries(result.headers).map(([name, value]) => ({
|
|
16269
|
-
carrier: "header",
|
|
16270
|
-
name,
|
|
16271
|
-
value
|
|
16272
|
-
})),
|
|
16273
|
-
...result.authorizeProviderRequest ? { authorizeProviderRequest: result.authorizeProviderRequest } : {},
|
|
16274
|
-
...result.expiresAt ? { expiresAt: result.expiresAt.toISOString() } : {},
|
|
16275
|
-
...connectionRef.scopes ? { scope: [...connectionRef.scopes] } : {}
|
|
16276
|
-
};
|
|
16277
|
-
}
|
|
16278
|
-
};
|
|
16279
|
-
}
|
|
16280
|
-
async function publishAuthNeeded(input, serverId, toolName, result, connectionRef) {
|
|
16281
|
-
try {
|
|
16282
|
-
await input.onAuthNeeded?.({
|
|
16283
|
-
serverId,
|
|
16284
|
-
toolName,
|
|
16285
|
-
providerDomain: result.providerDomain,
|
|
16286
|
-
...result.provider ? { provider: result.provider } : {},
|
|
16287
|
-
reason: result.reason,
|
|
16288
|
-
...result.connectionId ? { connectionId: result.connectionId } : {},
|
|
16289
|
-
...result.authoritySource === "host" || connectionRef.authoritySource === "host" ? { authoritySource: "host" } : {},
|
|
16290
|
-
...result.scopes ? { scopes: result.scopes } : {},
|
|
16291
|
-
...result.resource ? { resource: result.resource } : {},
|
|
16292
|
-
...result.selectedResources ? { selectedResources: result.selectedResources } : {},
|
|
16293
|
-
...result.authorizationUrl ? { authorizationUrl: result.authorizationUrl } : {}
|
|
16294
|
-
});
|
|
16295
|
-
} catch {
|
|
16296
|
-
}
|
|
16297
|
-
}
|
|
16298
|
-
|
|
16299
16381
|
// src/github-rest-mcp.ts
|
|
16300
16382
|
import {
|
|
16301
16383
|
mergeToolRefs
|
|
@@ -16306,18 +16388,12 @@ import {
|
|
|
16306
16388
|
} from "@opengeni/contracts/personal-github";
|
|
16307
16389
|
import { personalGitHubRepositoryResources as personalGitHubRepositoryResources2 } from "@opengeni/core";
|
|
16308
16390
|
import {
|
|
16309
|
-
beginConnectorActionExecution as beginConnectorActionExecution2,
|
|
16310
|
-
completeConnectorActionExecution as completeConnectorActionExecution2,
|
|
16311
16391
|
getConnectionMetadata as getConnectionMetadata2,
|
|
16312
16392
|
getPersonalGitHubRepositorySelectionState,
|
|
16313
16393
|
hasAuditableGitHubInstallationAuthority,
|
|
16314
16394
|
listGitHubInstallationAccessForWorkspace,
|
|
16315
|
-
prepareConnectorActionApproval as prepareConnectorActionApproval2,
|
|
16316
16395
|
resolveAcceptedConnectionUse as resolveAcceptedConnectionUse2
|
|
16317
16396
|
} from "@opengeni/db";
|
|
16318
|
-
import {
|
|
16319
|
-
AttemptToolApprovalRequiredError as AttemptToolApprovalRequiredError2
|
|
16320
|
-
} from "@opengeni/codemode";
|
|
16321
16397
|
import { createGitHubAppInstallationTokenWithExpiry as createGitHubAppInstallationTokenWithExpiry2, githubAppBotIdentity as githubAppBotIdentity2 } from "@opengeni/github";
|
|
16322
16398
|
import {
|
|
16323
16399
|
ConnectorActionBindingRejectedError,
|
|
@@ -16331,8 +16407,7 @@ import {
|
|
|
16331
16407
|
GITHUB_REST_TOOL_NAMES,
|
|
16332
16408
|
GitHubRestAuthorityError,
|
|
16333
16409
|
githubRestConnectorActionOutcome,
|
|
16334
|
-
GitHubRestMcpServer
|
|
16335
|
-
GitHubRestMutationNotExecutedError
|
|
16410
|
+
GitHubRestMcpServer
|
|
16336
16411
|
} from "@opengeni/runtime/github-rest-mcp";
|
|
16337
16412
|
var PERSONAL_GITHUB_DELEGATION_SERVER_ID = "github:personal";
|
|
16338
16413
|
var INTERNAL_GITHUB_MCP_TIMEOUT_MS = 2e4;
|
|
@@ -16342,8 +16417,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16342
16417
|
settings: input.settings,
|
|
16343
16418
|
tools: input.tools,
|
|
16344
16419
|
localMcpServers: [],
|
|
16345
|
-
connectorBindings: []
|
|
16346
|
-
authorizeCodemodeCall: null
|
|
16420
|
+
connectorBindings: []
|
|
16347
16421
|
};
|
|
16348
16422
|
}
|
|
16349
16423
|
const appRepositories = appRepositoryAuthority(input.resources);
|
|
@@ -16356,8 +16430,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16356
16430
|
settings: input.settings,
|
|
16357
16431
|
tools: input.tools,
|
|
16358
16432
|
localMcpServers: [],
|
|
16359
|
-
connectorBindings: []
|
|
16360
|
-
authorizeCodemodeCall: null
|
|
16433
|
+
connectorBindings: []
|
|
16361
16434
|
};
|
|
16362
16435
|
}
|
|
16363
16436
|
const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
|
|
@@ -16372,7 +16445,6 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16372
16445
|
const localMcpServers = [];
|
|
16373
16446
|
const toolRefs = [];
|
|
16374
16447
|
const connectorBindings = [];
|
|
16375
|
-
const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
|
|
16376
16448
|
if (appRepositories.length > 0) {
|
|
16377
16449
|
const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
|
|
16378
16450
|
localMcpServers.push({
|
|
@@ -16381,8 +16453,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16381
16453
|
serverId: GITHUB_REST_MCP_APP_SERVER_ID,
|
|
16382
16454
|
authorityKind: "workspace_app",
|
|
16383
16455
|
repositories: appRepositories,
|
|
16384
|
-
resolveAuthority: appResolver
|
|
16385
|
-
completeCodemodeAction: codemodeActions.complete
|
|
16456
|
+
resolveAuthority: appResolver
|
|
16386
16457
|
})
|
|
16387
16458
|
});
|
|
16388
16459
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub \u2014 OpenGeni bot"));
|
|
@@ -16398,8 +16469,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16398
16469
|
serverId: GITHUB_REST_MCP_PERSONAL_SERVER_ID,
|
|
16399
16470
|
authorityKind: "personal_oauth",
|
|
16400
16471
|
repositories: personalRepositories,
|
|
16401
|
-
resolveAuthority: personalAuthorityResolver(input)
|
|
16402
|
-
completeCodemodeAction: codemodeActions.complete
|
|
16472
|
+
resolveAuthority: personalAuthorityResolver(input)
|
|
16403
16473
|
})
|
|
16404
16474
|
});
|
|
16405
16475
|
configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub \u2014 My account"));
|
|
@@ -16415,59 +16485,7 @@ async function buildGitHubRestMcpForTurn(input) {
|
|
|
16415
16485
|
},
|
|
16416
16486
|
tools: mergeToolRefs(input.tools, toolRefs),
|
|
16417
16487
|
localMcpServers,
|
|
16418
|
-
connectorBindings
|
|
16419
|
-
authorizeCodemodeCall: codemodeActions.authorize
|
|
16420
|
-
};
|
|
16421
|
-
}
|
|
16422
|
-
function createCodemodeActionLifecycle(input, bindings) {
|
|
16423
|
-
const identity = {
|
|
16424
|
-
accountId: input.accountId,
|
|
16425
|
-
workspaceId: input.workspaceId,
|
|
16426
|
-
sessionId: input.sessionId,
|
|
16427
|
-
turnId: input.turn.id,
|
|
16428
|
-
attemptId: input.attemptId,
|
|
16429
|
-
executionGeneration: input.turn.executionGeneration,
|
|
16430
|
-
initiator: input.turn.initiator
|
|
16431
|
-
};
|
|
16432
|
-
const executing = /* @__PURE__ */ new Map();
|
|
16433
|
-
return {
|
|
16434
|
-
authorize: async ({ call, entry }) => {
|
|
16435
|
-
if (call.caller.kind !== "codemode") return;
|
|
16436
|
-
const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
|
|
16437
|
-
if (!binding) return;
|
|
16438
|
-
const invocation = binding.call(call.operationId, call.arguments);
|
|
16439
|
-
const policy = await prepareConnectorActionApproval2(input.db, identity, invocation);
|
|
16440
|
-
if (policy.managed && policy.decision === "ask") {
|
|
16441
|
-
throw new AttemptToolApprovalRequiredError2();
|
|
16442
|
-
}
|
|
16443
|
-
if (policy.managed && policy.decision === "block") {
|
|
16444
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
16445
|
-
"GitHub connector action is blocked by policy"
|
|
16446
|
-
);
|
|
16447
|
-
}
|
|
16448
|
-
const admission = await beginConnectorActionExecution2(input.db, identity, invocation);
|
|
16449
|
-
if (!admission.allowed) {
|
|
16450
|
-
if (admission.reason === "approval_required") {
|
|
16451
|
-
throw new AttemptToolApprovalRequiredError2();
|
|
16452
|
-
}
|
|
16453
|
-
throw new GitHubRestMutationNotExecutedError(
|
|
16454
|
-
`GitHub connector action was not executed: ${admission.reason}`
|
|
16455
|
-
);
|
|
16456
|
-
}
|
|
16457
|
-
if (admission.managed) executing.set(call.operationId, admission.requestId);
|
|
16458
|
-
},
|
|
16459
|
-
complete: async ({ operationId, outcome }) => {
|
|
16460
|
-
const requestId = executing.get(operationId);
|
|
16461
|
-
if (!requestId) return;
|
|
16462
|
-
executing.delete(operationId);
|
|
16463
|
-
await completeConnectorActionExecution2(input.db, {
|
|
16464
|
-
accountId: input.accountId,
|
|
16465
|
-
workspaceId: input.workspaceId,
|
|
16466
|
-
requestId,
|
|
16467
|
-
attemptId: input.attemptId,
|
|
16468
|
-
outcome
|
|
16469
|
-
});
|
|
16470
|
-
}
|
|
16488
|
+
connectorBindings
|
|
16471
16489
|
};
|
|
16472
16490
|
}
|
|
16473
16491
|
function internalMcpConfig(id, name) {
|
|
@@ -16943,14 +16961,16 @@ import { allowedFirstPartyMcpToolsForSession } from "@opengeni/config";
|
|
|
16943
16961
|
// src/activities/codemode-dispatcher.ts
|
|
16944
16962
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
16945
16963
|
import {
|
|
16946
|
-
AttemptToolApprovalRequiredError
|
|
16964
|
+
AttemptToolApprovalRequiredError,
|
|
16947
16965
|
AttemptToolCatalogStaleError,
|
|
16966
|
+
AttemptToolInputValidationError,
|
|
16948
16967
|
AttemptToolNotFoundError,
|
|
16949
16968
|
codemodeDispatchSubject,
|
|
16950
16969
|
decodeCodemodeDispatchRequest,
|
|
16951
16970
|
encodeCodemodeDispatchAck
|
|
16952
16971
|
} from "@opengeni/codemode";
|
|
16953
16972
|
import {
|
|
16973
|
+
CODEMODE_CLAIM_LEASE_MS,
|
|
16954
16974
|
CODEMODE_CLAIM_HEARTBEAT_MS,
|
|
16955
16975
|
CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT
|
|
16956
16976
|
} from "@opengeni/contracts";
|
|
@@ -16958,6 +16978,8 @@ import {
|
|
|
16958
16978
|
cancelQueuedCodemodeOperationsForAttempt,
|
|
16959
16979
|
claimCodemodeOperation,
|
|
16960
16980
|
failCodemodeOperation,
|
|
16981
|
+
getSessionEventByClientEventId,
|
|
16982
|
+
getSessionToolCallCreatedEventByOperationId,
|
|
16961
16983
|
markCodemodeOperationExecutionStarted,
|
|
16962
16984
|
renewCodemodeOperationClaim,
|
|
16963
16985
|
settleCodemodeOperationWithOutput
|
|
@@ -16967,7 +16989,7 @@ import {
|
|
|
16967
16989
|
publishDurableSessionEvents as publishDurableSessionEvents8
|
|
16968
16990
|
} from "@opengeni/events";
|
|
16969
16991
|
var CodemodeAttemptDispatcher = class {
|
|
16970
|
-
constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT) {
|
|
16992
|
+
constructor(db, bus, environment, scope, turnSignal, maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT, timings = {}) {
|
|
16971
16993
|
this.db = db;
|
|
16972
16994
|
this.bus = bus;
|
|
16973
16995
|
this.environment = environment;
|
|
@@ -16980,12 +17002,22 @@ var CodemodeAttemptDispatcher = class {
|
|
|
16980
17002
|
if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
|
|
16981
17003
|
throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
|
|
16982
17004
|
}
|
|
17005
|
+
this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
|
|
17006
|
+
this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
|
|
17007
|
+
if (!Number.isSafeInteger(this.claimLeaseMs) || this.claimLeaseMs < 1e3 || this.claimLeaseMs > 10 * 6e4) {
|
|
17008
|
+
throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
|
|
17009
|
+
}
|
|
17010
|
+
if (!Number.isSafeInteger(this.claimHeartbeatMs) || this.claimHeartbeatMs < 1 || this.claimHeartbeatMs >= this.claimLeaseMs) {
|
|
17011
|
+
throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
|
|
17012
|
+
}
|
|
16983
17013
|
}
|
|
16984
17014
|
stopController = new AbortController();
|
|
16985
17015
|
inFlight = /* @__PURE__ */ new Map();
|
|
16986
17016
|
pendingClaims = 0;
|
|
16987
17017
|
unsubscribe = null;
|
|
16988
17018
|
closing = false;
|
|
17019
|
+
claimLeaseMs;
|
|
17020
|
+
claimHeartbeatMs;
|
|
16989
17021
|
start() {
|
|
16990
17022
|
if (this.unsubscribe || this.closing)
|
|
16991
17023
|
throw new Error("Codemode dispatcher cannot be restarted");
|
|
@@ -17013,12 +17045,15 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17013
17045
|
...this.scope,
|
|
17014
17046
|
catalogDigest: request.catalogDigest,
|
|
17015
17047
|
operationId: request.operationId,
|
|
17016
|
-
claimId: randomUUID3()
|
|
17048
|
+
claimId: randomUUID3(),
|
|
17049
|
+
claimLeaseMs: this.claimLeaseMs
|
|
17017
17050
|
});
|
|
17018
17051
|
if (claim.status === "claimed") {
|
|
17019
|
-
const execution = this.execute(claim.operation, claim.claimId).finally(
|
|
17020
|
-
|
|
17021
|
-
|
|
17052
|
+
const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
|
|
17053
|
+
() => {
|
|
17054
|
+
this.inFlight.delete(request.operationId);
|
|
17055
|
+
}
|
|
17056
|
+
);
|
|
17022
17057
|
this.inFlight.set(request.operationId, execution);
|
|
17023
17058
|
return encodeCodemodeDispatchAck({
|
|
17024
17059
|
version: 1,
|
|
@@ -17026,6 +17061,18 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17026
17061
|
status: "accepted"
|
|
17027
17062
|
});
|
|
17028
17063
|
}
|
|
17064
|
+
if (claim.status === "execution_owner_lost") {
|
|
17065
|
+
await this.settleWithOutput(claim.operation, claim.claimId, {
|
|
17066
|
+
state: "outcome_unknown",
|
|
17067
|
+
errorCode: "worker_lost_during_execution",
|
|
17068
|
+
errorMessage: "The execution owner disappeared after the tool call began. Inspect actual state before retrying."
|
|
17069
|
+
});
|
|
17070
|
+
return encodeCodemodeDispatchAck({
|
|
17071
|
+
version: 1,
|
|
17072
|
+
operationId: request.operationId,
|
|
17073
|
+
status: "terminal"
|
|
17074
|
+
});
|
|
17075
|
+
}
|
|
17029
17076
|
return encodeCodemodeDispatchAck({
|
|
17030
17077
|
version: 1,
|
|
17031
17078
|
operationId: request.operationId,
|
|
@@ -17054,7 +17101,26 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17054
17101
|
});
|
|
17055
17102
|
await Promise.allSettled([...this.inFlight.values()]);
|
|
17056
17103
|
}
|
|
17057
|
-
async execute(operation, claimId) {
|
|
17104
|
+
async execute(operation, claimId, reclaimed) {
|
|
17105
|
+
const claimAuthority = {
|
|
17106
|
+
accountId: this.scope.accountId,
|
|
17107
|
+
workspaceId: this.scope.workspaceId,
|
|
17108
|
+
attemptId: this.scope.attemptId,
|
|
17109
|
+
operationId: operation.operationId,
|
|
17110
|
+
claimId,
|
|
17111
|
+
claimLeaseMs: this.claimLeaseMs
|
|
17112
|
+
};
|
|
17113
|
+
const heartbeat = setInterval(() => {
|
|
17114
|
+
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
|
|
17115
|
+
}, this.claimHeartbeatMs);
|
|
17116
|
+
heartbeat.unref?.();
|
|
17117
|
+
try {
|
|
17118
|
+
await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
|
|
17119
|
+
} finally {
|
|
17120
|
+
clearInterval(heartbeat);
|
|
17121
|
+
}
|
|
17122
|
+
}
|
|
17123
|
+
async executeClaimed(operation, claimId, reclaimed, claimAuthority) {
|
|
17058
17124
|
const entry = this.environment.catalog.entries.find(
|
|
17059
17125
|
(candidate) => candidate.identity.serverId === operation.identity.serverId && candidate.identity.toolName === operation.identity.toolName
|
|
17060
17126
|
);
|
|
@@ -17077,38 +17143,7 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17077
17143
|
);
|
|
17078
17144
|
return;
|
|
17079
17145
|
}
|
|
17080
|
-
|
|
17081
|
-
this.db,
|
|
17082
|
-
this.bus,
|
|
17083
|
-
this.scope.workspaceId,
|
|
17084
|
-
this.scope.sessionId,
|
|
17085
|
-
this.scope.turnId,
|
|
17086
|
-
this.scope.executionGeneration,
|
|
17087
|
-
this.scope.attemptId,
|
|
17088
|
-
[
|
|
17089
|
-
{
|
|
17090
|
-
type: "agent.toolCall.created",
|
|
17091
|
-
turnId: this.scope.turnId,
|
|
17092
|
-
turnGeneration: this.scope.executionGeneration,
|
|
17093
|
-
turnAttemptId: this.scope.attemptId,
|
|
17094
|
-
producerId: operation.caller.subjectId,
|
|
17095
|
-
payload: {
|
|
17096
|
-
id: operation.operationId,
|
|
17097
|
-
name: entry.modelName,
|
|
17098
|
-
arguments: operation.arguments,
|
|
17099
|
-
origin: "codemode",
|
|
17100
|
-
subjectId: operation.caller.subjectId,
|
|
17101
|
-
raw: {
|
|
17102
|
-
type: "codemode_call",
|
|
17103
|
-
serverId: operation.identity.serverId,
|
|
17104
|
-
toolName: operation.identity.toolName,
|
|
17105
|
-
catalogDigest: operation.catalogDigest
|
|
17106
|
-
}
|
|
17107
|
-
}
|
|
17108
|
-
}
|
|
17109
|
-
]
|
|
17110
|
-
).catch(() => ({ accepted: false }));
|
|
17111
|
-
if (!created.accepted) {
|
|
17146
|
+
if (!await this.ensureToolCallCreated(operation, entry, reclaimed)) {
|
|
17112
17147
|
await this.failBeforeExecution(
|
|
17113
17148
|
operation.operationId,
|
|
17114
17149
|
claimId,
|
|
@@ -17125,24 +17160,9 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17125
17160
|
});
|
|
17126
17161
|
return;
|
|
17127
17162
|
}
|
|
17128
|
-
|
|
17129
|
-
accountId: this.scope.accountId,
|
|
17130
|
-
workspaceId: this.scope.workspaceId,
|
|
17131
|
-
attemptId: this.scope.attemptId,
|
|
17132
|
-
operationId: operation.operationId,
|
|
17133
|
-
claimId
|
|
17134
|
-
};
|
|
17135
|
-
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
17136
|
-
this.db,
|
|
17137
|
-
claimAuthority
|
|
17138
|
-
);
|
|
17139
|
-
if (!crossedExecutionBoundary) return;
|
|
17140
|
-
const heartbeat = setInterval(() => {
|
|
17141
|
-
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => void 0);
|
|
17142
|
-
}, CODEMODE_CLAIM_HEARTBEAT_MS);
|
|
17143
|
-
heartbeat.unref?.();
|
|
17163
|
+
let preparedCall;
|
|
17144
17164
|
try {
|
|
17145
|
-
|
|
17165
|
+
preparedCall = await this.environment.prepareCall(
|
|
17146
17166
|
{
|
|
17147
17167
|
operationId: operation.operationId,
|
|
17148
17168
|
catalogDigest: operation.catalogDigest,
|
|
@@ -17152,16 +17172,111 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17152
17172
|
},
|
|
17153
17173
|
{ signal }
|
|
17154
17174
|
);
|
|
17155
|
-
await this.settleWithOutput(operation, claimId, { state: "completed", result });
|
|
17156
17175
|
} catch (error) {
|
|
17157
|
-
|
|
17176
|
+
await this.settleWithOutput(operation, claimId, {
|
|
17177
|
+
state: "failed",
|
|
17178
|
+
errorCode: errorCode(error, signal, false),
|
|
17179
|
+
errorMessage: safeErrorMessage(error)
|
|
17180
|
+
});
|
|
17181
|
+
return;
|
|
17182
|
+
}
|
|
17183
|
+
if (signal.aborted) {
|
|
17184
|
+
await this.settleWithOutput(operation, claimId, {
|
|
17185
|
+
state: "failed",
|
|
17186
|
+
errorCode: "attempt_cancelled",
|
|
17187
|
+
errorMessage: "Execution attempt ended before the Codemode call started"
|
|
17188
|
+
});
|
|
17189
|
+
return;
|
|
17190
|
+
}
|
|
17191
|
+
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
17192
|
+
this.db,
|
|
17193
|
+
claimAuthority
|
|
17194
|
+
);
|
|
17195
|
+
if (!crossedExecutionBoundary) return;
|
|
17196
|
+
try {
|
|
17197
|
+
const result = await preparedCall.execute();
|
|
17198
|
+
await this.settleWithOutput(operation, claimId, {
|
|
17199
|
+
state: "completed",
|
|
17200
|
+
result
|
|
17201
|
+
});
|
|
17202
|
+
} catch (error) {
|
|
17203
|
+
const knownPreExecution = error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolNotFoundError || connectorActionWasNotExecuted(error);
|
|
17158
17204
|
await this.settleWithOutput(operation, claimId, {
|
|
17159
17205
|
state: knownPreExecution ? "failed" : "outcome_unknown",
|
|
17160
|
-
errorCode: errorCode(error, signal),
|
|
17206
|
+
errorCode: errorCode(error, signal, true),
|
|
17161
17207
|
errorMessage: !knownPreExecution ? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying." : safeErrorMessage(error)
|
|
17162
17208
|
});
|
|
17163
|
-
}
|
|
17164
|
-
|
|
17209
|
+
}
|
|
17210
|
+
}
|
|
17211
|
+
async ensureToolCallCreated(operation, entry, reclaimed) {
|
|
17212
|
+
const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
|
|
17213
|
+
if (reclaimed) {
|
|
17214
|
+
const existing = await getSessionEventByClientEventId(
|
|
17215
|
+
this.db,
|
|
17216
|
+
this.scope.workspaceId,
|
|
17217
|
+
this.scope.sessionId,
|
|
17218
|
+
clientEventId
|
|
17219
|
+
).catch(() => null);
|
|
17220
|
+
if (existing) {
|
|
17221
|
+
assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
|
|
17222
|
+
return true;
|
|
17223
|
+
}
|
|
17224
|
+
const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
|
|
17225
|
+
workspaceId: this.scope.workspaceId,
|
|
17226
|
+
sessionId: this.scope.sessionId,
|
|
17227
|
+
turnId: this.scope.turnId,
|
|
17228
|
+
attemptId: this.scope.attemptId,
|
|
17229
|
+
operationId: operation.operationId
|
|
17230
|
+
}).catch(() => null);
|
|
17231
|
+
if (legacyCreated) {
|
|
17232
|
+
assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
|
|
17233
|
+
return true;
|
|
17234
|
+
}
|
|
17235
|
+
}
|
|
17236
|
+
try {
|
|
17237
|
+
const created = await appendAndPublishTurnEventsFenced3(
|
|
17238
|
+
this.db,
|
|
17239
|
+
this.bus,
|
|
17240
|
+
this.scope.workspaceId,
|
|
17241
|
+
this.scope.sessionId,
|
|
17242
|
+
this.scope.turnId,
|
|
17243
|
+
this.scope.executionGeneration,
|
|
17244
|
+
this.scope.attemptId,
|
|
17245
|
+
[
|
|
17246
|
+
{
|
|
17247
|
+
type: "agent.toolCall.created",
|
|
17248
|
+
clientEventId,
|
|
17249
|
+
turnId: this.scope.turnId,
|
|
17250
|
+
turnGeneration: this.scope.executionGeneration,
|
|
17251
|
+
turnAttemptId: this.scope.attemptId,
|
|
17252
|
+
producerId: operation.caller.subjectId,
|
|
17253
|
+
payload: {
|
|
17254
|
+
id: operation.operationId,
|
|
17255
|
+
name: entry.modelName,
|
|
17256
|
+
arguments: operation.arguments,
|
|
17257
|
+
origin: "codemode",
|
|
17258
|
+
subjectId: operation.caller.subjectId,
|
|
17259
|
+
raw: {
|
|
17260
|
+
type: "codemode_call",
|
|
17261
|
+
serverId: operation.identity.serverId,
|
|
17262
|
+
toolName: operation.identity.toolName,
|
|
17263
|
+
catalogDigest: operation.catalogDigest
|
|
17264
|
+
}
|
|
17265
|
+
}
|
|
17266
|
+
}
|
|
17267
|
+
]
|
|
17268
|
+
);
|
|
17269
|
+
return created.accepted;
|
|
17270
|
+
} catch {
|
|
17271
|
+
const raced = await getSessionEventByClientEventId(
|
|
17272
|
+
this.db,
|
|
17273
|
+
this.scope.workspaceId,
|
|
17274
|
+
this.scope.sessionId,
|
|
17275
|
+
clientEventId
|
|
17276
|
+
).catch(() => null);
|
|
17277
|
+
if (!raced) return false;
|
|
17278
|
+
assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
|
|
17279
|
+
return true;
|
|
17165
17280
|
}
|
|
17166
17281
|
}
|
|
17167
17282
|
async settleWithOutput(operation, claimId, settlement) {
|
|
@@ -17193,6 +17308,19 @@ var CodemodeAttemptDispatcher = class {
|
|
|
17193
17308
|
});
|
|
17194
17309
|
}
|
|
17195
17310
|
};
|
|
17311
|
+
function codemodeToolCallCreatedClientEventId(operationId) {
|
|
17312
|
+
return `opengeni:codemode-tool-call-created:${operationId}`;
|
|
17313
|
+
}
|
|
17314
|
+
function assertMatchingCodemodeToolCallCreated(event, operation, entry, scope) {
|
|
17315
|
+
const payload = recordValue(event.payload);
|
|
17316
|
+
const raw = recordValue(payload?.raw);
|
|
17317
|
+
if (event.type !== "agent.toolCall.created" || event.turnId !== scope.turnId || event.turnGeneration !== scope.executionGeneration || event.turnAttemptId !== scope.attemptId || event.turnAssociation !== "current" || payload?.id !== operation.operationId || payload?.name !== entry.modelName || payload?.origin !== "codemode" || payload?.subjectId !== operation.caller.subjectId || raw?.type !== "codemode_call" || raw?.serverId !== operation.identity.serverId || raw?.toolName !== operation.identity.toolName || raw?.catalogDigest !== operation.catalogDigest) {
|
|
17318
|
+
throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
|
|
17319
|
+
}
|
|
17320
|
+
}
|
|
17321
|
+
function recordValue(value) {
|
|
17322
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
17323
|
+
}
|
|
17196
17324
|
function connectorActionWasNotExecuted(error) {
|
|
17197
17325
|
return Boolean(
|
|
17198
17326
|
error && typeof error === "object" && "connectorActionOutcome" in error && error.connectorActionOutcome === "not_executed"
|
|
@@ -17201,8 +17329,10 @@ function connectorActionWasNotExecuted(error) {
|
|
|
17201
17329
|
function combinedSignal(primary, secondary) {
|
|
17202
17330
|
return secondary ? AbortSignal.any([primary, secondary]) : primary;
|
|
17203
17331
|
}
|
|
17204
|
-
function errorCode(error, signal) {
|
|
17205
|
-
if (signal.aborted)
|
|
17332
|
+
function errorCode(error, signal, executionStarted) {
|
|
17333
|
+
if (signal.aborted) {
|
|
17334
|
+
return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
|
|
17335
|
+
}
|
|
17206
17336
|
if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
|
|
17207
17337
|
if (typeof error === "object" && error !== null && "code" in error) {
|
|
17208
17338
|
const code = error.code;
|
|
@@ -17211,7 +17341,7 @@ function errorCode(error, signal) {
|
|
|
17211
17341
|
return "tool_outcome_unknown";
|
|
17212
17342
|
}
|
|
17213
17343
|
function safeErrorMessage(error) {
|
|
17214
|
-
if (error instanceof
|
|
17344
|
+
if (error instanceof AttemptToolApprovalRequiredError || error instanceof AttemptToolCatalogStaleError || error instanceof AttemptToolInputValidationError || error instanceof AttemptToolNotFoundError) {
|
|
17215
17345
|
return error.message;
|
|
17216
17346
|
}
|
|
17217
17347
|
return "Codemode call failed before tool execution";
|
|
@@ -17223,6 +17353,7 @@ import {
|
|
|
17223
17353
|
defaultSessionMcpServerIds,
|
|
17224
17354
|
loadRigDefaultVariableSetEnvironment,
|
|
17225
17355
|
mergeRigDefaultVariableSetEnvironment,
|
|
17356
|
+
buildApiIntegrationMcpServers,
|
|
17226
17357
|
resolveCatalogSettings as resolveCatalogSettings2,
|
|
17227
17358
|
resolveWorkspaceModelSelection,
|
|
17228
17359
|
withFrozenPersonalConnectionDelegations,
|
|
@@ -17555,7 +17686,7 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17555
17686
|
);
|
|
17556
17687
|
};
|
|
17557
17688
|
const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
|
|
17558
|
-
const apiIntegrationMcpServers =
|
|
17689
|
+
const apiIntegrationMcpServers = buildApiIntegrationMcpServers({
|
|
17559
17690
|
settings: runSettings,
|
|
17560
17691
|
integrations: installedApiIntegrations.filter(
|
|
17561
17692
|
(integration) => selectedApiIntegrationServerIds.has(integration.serverId)
|
|
@@ -17633,6 +17764,22 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17633
17764
|
)
|
|
17634
17765
|
}
|
|
17635
17766
|
] : [];
|
|
17767
|
+
const attemptConnectorActionBindings = [
|
|
17768
|
+
...googleDriveConnectorBindings,
|
|
17769
|
+
...githubRestMcp.connectorBindings
|
|
17770
|
+
];
|
|
17771
|
+
const connectorActionPolicy = {
|
|
17772
|
+
preview: async (call) => await previewConnectorActionApproval(db, connectorActionIdentity, call),
|
|
17773
|
+
prepare: async (call) => await prepareConnectorActionApproval(db, connectorActionIdentity, call),
|
|
17774
|
+
begin: async (call) => await beginConnectorActionExecution(db, connectorActionIdentity, call),
|
|
17775
|
+
complete: async ({ requestId, outcome }) => await completeConnectorActionExecution(db, {
|
|
17776
|
+
accountId: input.accountId,
|
|
17777
|
+
workspaceId: input.workspaceId,
|
|
17778
|
+
requestId,
|
|
17779
|
+
attemptId: input.attemptId,
|
|
17780
|
+
outcome
|
|
17781
|
+
})
|
|
17782
|
+
};
|
|
17636
17783
|
const attemptToolDefinitions = [
|
|
17637
17784
|
createListModelsAttemptToolDefinition({
|
|
17638
17785
|
currentModelId: turnExecutionPolicy.productModelId,
|
|
@@ -17844,21 +17991,8 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17844
17991
|
nestedAgentDepth: session.nestedAgentDepth,
|
|
17845
17992
|
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
17846
17993
|
attemptToolDefinitions,
|
|
17847
|
-
|
|
17848
|
-
|
|
17849
|
-
const { call } = authorization;
|
|
17850
|
-
if (googleDrivePublicationTarget && googleDrivePublicationAllowed && call.caller.kind === "codemode" && call.identity.serverId === "google-drive-publishing" && call.identity.toolName === "google_drive_publish_file") {
|
|
17851
|
-
await authorizeGoogleDrivePublicationAttempt({
|
|
17852
|
-
db,
|
|
17853
|
-
identity: connectorActionIdentity,
|
|
17854
|
-
target: googleDrivePublicationTarget,
|
|
17855
|
-
approvalId: call.operationId,
|
|
17856
|
-
arguments: call.arguments
|
|
17857
|
-
});
|
|
17858
|
-
}
|
|
17859
|
-
await githubRestMcp.authorizeCodemodeCall?.(authorization);
|
|
17860
|
-
}
|
|
17861
|
-
} : {}
|
|
17994
|
+
connectorActionPolicy,
|
|
17995
|
+
attemptConnectorActionBindings
|
|
17862
17996
|
}),
|
|
17863
17997
|
cancellationSignal,
|
|
17864
17998
|
async (latePreparedTools) => await latePreparedTools.close().catch(() => void 0)
|
|
@@ -17917,11 +18051,8 @@ async function prepareTurnToolRuntime(deps) {
|
|
|
17917
18051
|
activatePreparedToolEnvironment(eventing.preparedTools);
|
|
17918
18052
|
}
|
|
17919
18053
|
return {
|
|
17920
|
-
attemptConnectorActionBindings
|
|
17921
|
-
|
|
17922
|
-
...githubRestMcp.connectorBindings
|
|
17923
|
-
],
|
|
17924
|
-
connectorActionIdentity,
|
|
18054
|
+
attemptConnectorActionBindings,
|
|
18055
|
+
connectorActionPolicy,
|
|
17925
18056
|
generateSessionTitleInParallel: titleToolPlan.generateTitleInParallel,
|
|
17926
18057
|
postToolPreparationStartedAt,
|
|
17927
18058
|
preparationIndependentToolNames: titleToolPlan.preparationIndependentToolNames
|
|
@@ -18119,10 +18250,7 @@ import {
|
|
|
18119
18250
|
getXaiSessionAccountPin as getXaiSessionAccountPin2,
|
|
18120
18251
|
setXaiSessionAccountPin as setXaiSessionAccountPin2,
|
|
18121
18252
|
getWorkspaceVideoGenerationPolicy,
|
|
18122
|
-
loadWorkspaceVercelAiGatewayCredentialLease
|
|
18123
|
-
beginConnectorActionExecution as beginConnectorActionExecution3,
|
|
18124
|
-
completeConnectorActionExecution as completeConnectorActionExecution3,
|
|
18125
|
-
prepareConnectorActionApproval as prepareConnectorActionApproval3
|
|
18253
|
+
loadWorkspaceVercelAiGatewayCredentialLease
|
|
18126
18254
|
} from "@opengeni/db";
|
|
18127
18255
|
import {
|
|
18128
18256
|
serviceTierForLatencyMode as serviceTierForLatencyMode2,
|
|
@@ -19059,18 +19187,27 @@ import {
|
|
|
19059
19187
|
var CODEX_IMAGE_MODEL = "gpt-image-2";
|
|
19060
19188
|
async function executeCodexImageGeneration(input) {
|
|
19061
19189
|
const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID2, input.credentialId);
|
|
19190
|
+
let providerDispatchAdmitted = false;
|
|
19191
|
+
const codexContext = {
|
|
19192
|
+
...input.codexContext,
|
|
19193
|
+
beforeProviderDispatch: async () => {
|
|
19194
|
+
await input.codexContext.beforeProviderDispatch?.();
|
|
19195
|
+
providerDispatchAdmitted = true;
|
|
19196
|
+
}
|
|
19197
|
+
};
|
|
19062
19198
|
return await executeImageGenerationOperation({
|
|
19063
19199
|
...input,
|
|
19064
19200
|
providerId: CODEX_PROVIDER_ID2,
|
|
19065
19201
|
providerBindingHash,
|
|
19066
19202
|
modelId: CODEX_IMAGE_MODEL,
|
|
19067
19203
|
...input.references ? { referenceDigests: input.references } : {},
|
|
19204
|
+
isProviderDispatchRejected: (error) => !providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
|
|
19068
19205
|
generate: async () => {
|
|
19069
19206
|
const generated = await generateCodexSubscriptionImage({
|
|
19070
19207
|
prompt: input.prompt,
|
|
19071
19208
|
...input.references ? { references: input.references } : {},
|
|
19072
19209
|
turnId: input.turnId,
|
|
19073
|
-
context:
|
|
19210
|
+
context: codexContext,
|
|
19074
19211
|
...input.abortSignal ? { abortSignal: input.abortSignal } : {}
|
|
19075
19212
|
});
|
|
19076
19213
|
return {
|
|
@@ -19326,7 +19463,8 @@ async function buildTurnAgent(deps) {
|
|
|
19326
19463
|
sandboxCodemodeToken,
|
|
19327
19464
|
fileResourceDownloads,
|
|
19328
19465
|
attemptConnectorActionBindings,
|
|
19329
|
-
|
|
19466
|
+
connectorActionPolicy,
|
|
19467
|
+
trigger,
|
|
19330
19468
|
preparationIndependentToolNames,
|
|
19331
19469
|
videoGenerationAcceptancesByCallId,
|
|
19332
19470
|
activeSandboxBackend,
|
|
@@ -19645,17 +19783,7 @@ async function buildTurnAgent(deps) {
|
|
|
19645
19783
|
turnExecutionPolicy.providerId,
|
|
19646
19784
|
turnExecutionPolicy.latencyMode
|
|
19647
19785
|
);
|
|
19648
|
-
const
|
|
19649
|
-
prepare: async (call) => await prepareConnectorActionApproval3(db, connectorActionIdentity, call),
|
|
19650
|
-
begin: async (call) => await beginConnectorActionExecution3(db, connectorActionIdentity, call),
|
|
19651
|
-
complete: async ({ requestId, outcome }) => await completeConnectorActionExecution3(db, {
|
|
19652
|
-
accountId: input.accountId,
|
|
19653
|
-
workspaceId: input.workspaceId,
|
|
19654
|
-
requestId,
|
|
19655
|
-
attemptId: input.attemptId,
|
|
19656
|
-
outcome
|
|
19657
|
-
})
|
|
19658
|
-
};
|
|
19786
|
+
const approvedToolCallId = approvedConnectorActionCallId(trigger);
|
|
19659
19787
|
const runtimeSkillActivations = [
|
|
19660
19788
|
...installedSkillRuntime.activations,
|
|
19661
19789
|
...packRuntime.skillActivations,
|
|
@@ -19732,6 +19860,7 @@ async function buildTurnAgent(deps) {
|
|
|
19732
19860
|
resolvedMcpConnectionIds: preparedTools.resolvedMcpConnectionIds,
|
|
19733
19861
|
connectorActionPolicy,
|
|
19734
19862
|
attemptConnectorActionBindings,
|
|
19863
|
+
...approvedToolCallId ? { approvedToolCallId } : {},
|
|
19735
19864
|
// LIVE by-reference connector namespaces (fills during this turn's
|
|
19736
19865
|
// codex_apps tools/list): the codex tool_search description reads it per
|
|
19737
19866
|
// model call so the model sees the account's real connected sources.
|
|
@@ -19838,6 +19967,13 @@ async function buildTurnAgent(deps) {
|
|
|
19838
19967
|
postAgentPreparationStartedAt
|
|
19839
19968
|
};
|
|
19840
19969
|
}
|
|
19970
|
+
function approvedConnectorActionCallId(trigger) {
|
|
19971
|
+
if (trigger.type !== "user.approvalDecision") {
|
|
19972
|
+
return null;
|
|
19973
|
+
}
|
|
19974
|
+
const payload = trigger.payload;
|
|
19975
|
+
return payload.decision === "approve" && typeof payload.approvalId === "string" ? payload.approvalId : null;
|
|
19976
|
+
}
|
|
19841
19977
|
|
|
19842
19978
|
// src/activities/agent-turn/run.ts
|
|
19843
19979
|
function sessionTitleCodexRequestContext(context, nextRequestId) {
|
|
@@ -19848,6 +19984,7 @@ function sessionTitleCodexRequestContext(context, nextRequestId) {
|
|
|
19848
19984
|
refresh: context.refresh,
|
|
19849
19985
|
resolveModel: context.resolveModel,
|
|
19850
19986
|
...context.onUsageHeaders ? { onUsageHeaders: context.onUsageHeaders } : {},
|
|
19987
|
+
...context.beforeProviderDispatch ? { beforeProviderDispatch: context.beforeProviderDispatch } : {},
|
|
19851
19988
|
...context.responseTimeoutPolicy ? { responseTimeoutPolicy: context.responseTimeoutPolicy } : {},
|
|
19852
19989
|
nextRequestId,
|
|
19853
19990
|
turnMetadata: { request_kind: "session_title" }
|
|
@@ -20177,6 +20314,11 @@ function createRunAgentTurnActivity(services) {
|
|
|
20177
20314
|
input.workspaceId,
|
|
20178
20315
|
providerTurn.effectiveCodexCredentialId ?? ""
|
|
20179
20316
|
);
|
|
20317
|
+
const resolveTrackedToken = async (resolve) => {
|
|
20318
|
+
const token = await resolve();
|
|
20319
|
+
providerTurn.effectiveCodexCredentialVersion = token.credentialVersion;
|
|
20320
|
+
return token;
|
|
20321
|
+
};
|
|
20180
20322
|
return {
|
|
20181
20323
|
clientVersion: CODEX_CLIENT_VERSION2,
|
|
20182
20324
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -20186,8 +20328,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
20186
20328
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
20187
20329
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
20188
20330
|
sessionId: input.sessionId,
|
|
20189
|
-
getToken: () => resolver.getToken
|
|
20190
|
-
refresh: () => resolver.refresh
|
|
20331
|
+
getToken: () => resolveTrackedToken(resolver.getToken),
|
|
20332
|
+
refresh: () => resolveTrackedToken(resolver.refresh),
|
|
20191
20333
|
resolveModel: buildModelResolver(
|
|
20192
20334
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
20193
20335
|
CODEX_FALLBACK_MODEL_SLUGS[0]
|
|
@@ -20196,6 +20338,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
20196
20338
|
providerTurn.latestCodexUsage = snapshot;
|
|
20197
20339
|
},
|
|
20198
20340
|
// latest wins; flushed once in finally
|
|
20341
|
+
beforeProviderDispatch: () => {
|
|
20342
|
+
leases.codex.assertUsable();
|
|
20343
|
+
},
|
|
20199
20344
|
onRequestPreparationDiagnostic: (phase) => {
|
|
20200
20345
|
if (eventing.firstModelRequestCheckpointAt === null || firstModelRequestCheckpoints.has(phase)) {
|
|
20201
20346
|
return;
|
|
@@ -20748,7 +20893,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
20748
20893
|
});
|
|
20749
20894
|
const {
|
|
20750
20895
|
attemptConnectorActionBindings,
|
|
20751
|
-
|
|
20896
|
+
connectorActionPolicy,
|
|
20752
20897
|
generateSessionTitleInParallel,
|
|
20753
20898
|
postToolPreparationStartedAt,
|
|
20754
20899
|
preparationIndependentToolNames
|
|
@@ -20802,7 +20947,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
20802
20947
|
sandboxCodemodeToken,
|
|
20803
20948
|
fileResourceDownloads,
|
|
20804
20949
|
attemptConnectorActionBindings,
|
|
20805
|
-
|
|
20950
|
+
connectorActionPolicy,
|
|
20951
|
+
trigger,
|
|
20806
20952
|
preparationIndependentToolNames,
|
|
20807
20953
|
videoGenerationAcceptancesByCallId,
|
|
20808
20954
|
activeSandboxBackend,
|
|
@@ -21119,7 +21265,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
21119
21265
|
bus,
|
|
21120
21266
|
observability,
|
|
21121
21267
|
wakeSessionWorkflow,
|
|
21122
|
-
signalCodexCapacityWorkflow,
|
|
21123
21268
|
cancellationSignal,
|
|
21124
21269
|
sandboxRotationController,
|
|
21125
21270
|
noteCancellationRequested,
|
|
@@ -22262,4 +22407,4 @@ export {
|
|
|
22262
22407
|
createTurnActivities,
|
|
22263
22408
|
createTurnActivitiesFromServices
|
|
22264
22409
|
};
|
|
22265
|
-
//# sourceMappingURL=activities-turn-
|
|
22410
|
+
//# sourceMappingURL=activities-turn-S5IU347A.js.map
|