@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -1,17 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
requestSessionTurnRecovery,
|
|
3
3
|
getSessionGoal,
|
|
4
|
-
armCodexCapacityWait,
|
|
5
4
|
armXaiCapacityWait,
|
|
6
5
|
reconcileXaiCapacityWait,
|
|
7
|
-
getCodexRotationSettings,
|
|
8
6
|
listCodexAccountStatuses,
|
|
9
|
-
getSessionCodexState,
|
|
10
|
-
setSessionCodexPinInTransaction,
|
|
11
7
|
quarantineCodexCredentialForLease,
|
|
12
|
-
setCodexCredentialExhaustedWithWakeTargets,
|
|
13
|
-
withSessionCodexCapacityMutation,
|
|
14
|
-
countConsecutiveReactiveRotations,
|
|
15
8
|
recordUsageEvent,
|
|
16
9
|
getActiveSessionHistoryItemsPaged,
|
|
17
10
|
settleCodexCredentialLeaseLoss,
|
|
@@ -19,21 +12,17 @@ import {
|
|
|
19
12
|
readLease,
|
|
20
13
|
SandboxLeaseSupersededError,
|
|
21
14
|
isSessionEventPersistenceError,
|
|
15
|
+
type CodexLeaseAccountStatus,
|
|
22
16
|
} from "@opengeni/db";
|
|
23
17
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
24
18
|
import { maxTurnsExceededRunState } from "@opengeni/runtime";
|
|
25
19
|
import { CancelledFailure } from "@temporalio/activity";
|
|
26
20
|
import {
|
|
27
21
|
authoritativeCodexCapacityResetAt,
|
|
28
|
-
chooseRotationActive,
|
|
29
22
|
classifyCodexPin,
|
|
30
|
-
|
|
31
|
-
computeReactiveRotationResume,
|
|
32
|
-
shardCredentialForSession,
|
|
33
|
-
earliestCodexReset,
|
|
23
|
+
selectCodexCredentialLeaseForTurn,
|
|
34
24
|
type CodexRotationStrategy,
|
|
35
25
|
} from "../codex-rotation";
|
|
36
|
-
import { signalCodexCapacityWakeTargets } from "../codex-capacity";
|
|
37
26
|
import type { Settings } from "@opengeni/config";
|
|
38
27
|
import {
|
|
39
28
|
classifyCodexEncryptedArtifactRejection,
|
|
@@ -48,7 +37,7 @@ import type {
|
|
|
48
37
|
RunAgentTurnInput,
|
|
49
38
|
RunAgentTurnResult,
|
|
50
39
|
} from "../types";
|
|
51
|
-
import { createTurnCredentialLeases } from "./credential-leases";
|
|
40
|
+
import { CodexCredentialLeaseLostError, createTurnCredentialLeases } from "./credential-leases";
|
|
52
41
|
import { createTurnHistorySink } from "./history-sink";
|
|
53
42
|
|
|
54
43
|
import { BudgetExhaustedError } from "./admission";
|
|
@@ -68,7 +57,6 @@ import {
|
|
|
68
57
|
codexCredentialCooldownUntil,
|
|
69
58
|
classifyCodexCredentialFailure,
|
|
70
59
|
codexUsageLimitFailurePayload,
|
|
71
|
-
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
72
60
|
} from "./errors";
|
|
73
61
|
import { selectRejectedProviderArtifactHistoryIds } from "./history";
|
|
74
62
|
import { waitForTurnFinalizerStep, turnFinalizerCancellationSignal } from "./quiescence";
|
|
@@ -84,6 +72,8 @@ import type {
|
|
|
84
72
|
ProviderTurnState,
|
|
85
73
|
TurnControlState,
|
|
86
74
|
} from "./turn-context";
|
|
75
|
+
import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
|
|
76
|
+
import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
|
|
87
77
|
|
|
88
78
|
export type TurnFailureDeps = {
|
|
89
79
|
error: unknown;
|
|
@@ -93,7 +83,6 @@ export type TurnFailureDeps = {
|
|
|
93
83
|
bus: ActivityServices["bus"];
|
|
94
84
|
observability: ActivityServices["observability"];
|
|
95
85
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
96
|
-
signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
|
|
97
86
|
cancellationSignal: AbortSignal | undefined;
|
|
98
87
|
sandboxRotationController: AbortController;
|
|
99
88
|
noteCancellationRequested: () => void;
|
|
@@ -116,6 +105,125 @@ export type TurnFailureDeps = {
|
|
|
116
105
|
acknowledgeRecoveryQuiescence: () => void;
|
|
117
106
|
};
|
|
118
107
|
|
|
108
|
+
export type CodexDefinitiveFailureDisposition = "failover" | "wait" | "terminal";
|
|
109
|
+
|
|
110
|
+
type CodexCapacityWaitFailurePayload = {
|
|
111
|
+
error: string;
|
|
112
|
+
code: string;
|
|
113
|
+
detail?: string;
|
|
114
|
+
retryable: false;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Pure policy for a definitive serving-credential refusal. A policy-constrained
|
|
119
|
+
* account and an all-unavailable pool wait for the same selected capacity to
|
|
120
|
+
* recover; only a truly empty/non-allocatable pool makes an auth/forbidden
|
|
121
|
+
* failure terminal. A different eligible account under rotation-on policy may
|
|
122
|
+
* recover the same durable turn immediately.
|
|
123
|
+
*/
|
|
124
|
+
export function codexDefinitiveFailureDisposition(input: {
|
|
125
|
+
failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
|
|
126
|
+
rotationEnabled: boolean;
|
|
127
|
+
pinDisposition: "manual" | "sharded" | "clearStale" | "unpinned";
|
|
128
|
+
decisionKind: "active" | "allCapped" | "none";
|
|
129
|
+
decisionCredentialId: string | null;
|
|
130
|
+
servingCredentialId: string;
|
|
131
|
+
}): CodexDefinitiveFailureDisposition {
|
|
132
|
+
const alternateAvailable =
|
|
133
|
+
input.rotationEnabled &&
|
|
134
|
+
input.pinDisposition !== "manual" &&
|
|
135
|
+
input.decisionKind === "active" &&
|
|
136
|
+
input.decisionCredentialId !== null &&
|
|
137
|
+
input.decisionCredentialId !== input.servingCredentialId;
|
|
138
|
+
if (alternateAvailable) return "failover";
|
|
139
|
+
if (
|
|
140
|
+
input.failureKind === "quota" ||
|
|
141
|
+
input.failureKind === "rate_limit" ||
|
|
142
|
+
input.decisionKind === "allCapped" ||
|
|
143
|
+
!input.rotationEnabled ||
|
|
144
|
+
input.pinDisposition === "manual"
|
|
145
|
+
) {
|
|
146
|
+
return "wait";
|
|
147
|
+
}
|
|
148
|
+
return "terminal";
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Bound one turn to the alternate credentials that policy actually permits.
|
|
153
|
+
* The effective account list is already workspace/organization scoped;
|
|
154
|
+
* allocator-disabled rows must not enlarge the retry budget. The database
|
|
155
|
+
* requires a positive bound even though one-account paths never fail over.
|
|
156
|
+
*/
|
|
157
|
+
export function codexCredentialFailoverLimit(
|
|
158
|
+
accounts: ReadonlyArray<{ id: string; allocatorEnabled: boolean }>,
|
|
159
|
+
servingCredentialId: string,
|
|
160
|
+
): number {
|
|
161
|
+
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled).length;
|
|
162
|
+
const servingIsAllocatable = accounts.some(
|
|
163
|
+
(account) => account.id === servingCredentialId && account.allocatorEnabled,
|
|
164
|
+
);
|
|
165
|
+
return Math.max(1, allocatableAccounts - (servingIsAllocatable ? 1 : 0));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Build the durable waiter payload without collapsing quota refusals into 403. */
|
|
169
|
+
export function codexCapacityWaitFailurePayload(input: {
|
|
170
|
+
failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
|
|
171
|
+
usageLimit: { resetsInSeconds: number | null } | null;
|
|
172
|
+
cooldownSeconds: number | null;
|
|
173
|
+
detail: string;
|
|
174
|
+
allAccounts: boolean;
|
|
175
|
+
}): CodexCapacityWaitFailurePayload {
|
|
176
|
+
if (input.failureKind === "quota") {
|
|
177
|
+
return codexUsageLimitFailurePayload(
|
|
178
|
+
input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
|
|
179
|
+
input.detail,
|
|
180
|
+
input.allAccounts ? { allAccounts: true } : undefined,
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
if (input.failureKind === "rate_limit") {
|
|
184
|
+
return {
|
|
185
|
+
error: "The serving Codex subscription is temporarily rate limited.",
|
|
186
|
+
code: "codex_account_rate_limited",
|
|
187
|
+
detail: input.detail,
|
|
188
|
+
retryable: false,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
if (input.failureKind === "auth") {
|
|
192
|
+
return {
|
|
193
|
+
error: "The serving Codex account requires reconnection.",
|
|
194
|
+
code: "codex_relogin_required",
|
|
195
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
196
|
+
retryable: false,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
error: "The serving Codex account is not authorized for this request.",
|
|
201
|
+
code: "codex_account_forbidden",
|
|
202
|
+
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
203
|
+
retryable: false,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function codexLeaseAccountsForSelection(
|
|
208
|
+
accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>,
|
|
209
|
+
): CodexLeaseAccountStatus[] {
|
|
210
|
+
return accounts.map((account) => ({
|
|
211
|
+
...account,
|
|
212
|
+
activeLeaseCount: 0,
|
|
213
|
+
selectionCount: 0,
|
|
214
|
+
lastSelectedAt: null,
|
|
215
|
+
}));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function acceptedCodexPolicySnapshot(
|
|
219
|
+
providerTurn: ProviderTurnState,
|
|
220
|
+
): CodexCredentialPolicySnapshotV1 {
|
|
221
|
+
if (!providerTurn.codexPolicySnapshot) {
|
|
222
|
+
throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
|
|
223
|
+
}
|
|
224
|
+
return providerTurn.codexPolicySnapshot;
|
|
225
|
+
}
|
|
226
|
+
|
|
119
227
|
export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
|
|
120
228
|
const {
|
|
121
229
|
error,
|
|
@@ -125,7 +233,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
125
233
|
bus,
|
|
126
234
|
observability,
|
|
127
235
|
wakeSessionWorkflow,
|
|
128
|
-
signalCodexCapacityWorkflow,
|
|
129
236
|
cancellationSignal,
|
|
130
237
|
sandboxRotationController,
|
|
131
238
|
noteCancellationRequested,
|
|
@@ -156,9 +263,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
156
263
|
// finished yet.
|
|
157
264
|
const recoveryTurnId = attempt.turnId;
|
|
158
265
|
// A true epoch supersession and a provider lifecycle transition are both
|
|
159
|
-
// recoverable control-plane states, never session failures.
|
|
160
|
-
//
|
|
161
|
-
//
|
|
266
|
+
// recoverable control-plane states, never session failures. A rotation
|
|
267
|
+
// persists an exact group/epoch wait marker so the workflow parks before
|
|
268
|
+
// another turn-worker dispatch; shorter non-rotation transitions retain
|
|
269
|
+
// their existing paced retry.
|
|
162
270
|
const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
|
|
163
271
|
const leaseControlError =
|
|
164
272
|
error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
|
|
@@ -175,6 +283,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
175
283
|
const transitionPending = lifecycleTransition !== null || rotationPending;
|
|
176
284
|
const deadlineRotationPending =
|
|
177
285
|
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
286
|
+
const sandboxLifecycleWait = rotationPending
|
|
287
|
+
? {
|
|
288
|
+
version: 1 as const,
|
|
289
|
+
sandboxGroupId: leaseControlError.sandboxGroupId,
|
|
290
|
+
leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
|
|
291
|
+
reason: "rotation_in_progress" as const,
|
|
292
|
+
}
|
|
293
|
+
: undefined;
|
|
178
294
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
179
295
|
sessionId: input.sessionId,
|
|
180
296
|
turnId: recoveryTurnId,
|
|
@@ -199,6 +315,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
199
315
|
},
|
|
200
316
|
}
|
|
201
317
|
: {}),
|
|
318
|
+
...(sandboxLifecycleWait ? { sandboxLifecycleWait } : {}),
|
|
202
319
|
});
|
|
203
320
|
if (recovery.action === "stale") {
|
|
204
321
|
acknowledgeLostAttemptOwnership();
|
|
@@ -212,7 +329,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
212
329
|
control.turnMetricOutcome = "recovering";
|
|
213
330
|
return claimedResult({
|
|
214
331
|
status: "recovering",
|
|
215
|
-
...(transitionPending
|
|
332
|
+
...(transitionPending && !sandboxLifecycleWait
|
|
216
333
|
? {
|
|
217
334
|
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
218
335
|
}
|
|
@@ -286,6 +403,12 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
286
403
|
sandboxGroupId: rotation.sandboxGroupId,
|
|
287
404
|
leaseEpoch: rotation.leaseEpoch,
|
|
288
405
|
},
|
|
406
|
+
sandboxLifecycleWait: {
|
|
407
|
+
version: 1,
|
|
408
|
+
sandboxGroupId: rotation.sandboxGroupId,
|
|
409
|
+
leaseEpoch: rotation.leaseEpoch,
|
|
410
|
+
reason: "rotation_in_progress",
|
|
411
|
+
},
|
|
289
412
|
});
|
|
290
413
|
if (recovery.action === "stale") {
|
|
291
414
|
acknowledgeLostAttemptOwnership();
|
|
@@ -297,10 +420,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
297
420
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
298
421
|
control.activityStatus = "recovering";
|
|
299
422
|
control.turnMetricOutcome = "recovering";
|
|
300
|
-
return claimedResult({
|
|
301
|
-
status: "recovering",
|
|
302
|
-
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
303
|
-
});
|
|
423
|
+
return claimedResult({ status: "recovering" });
|
|
304
424
|
} catch (recoveryError) {
|
|
305
425
|
console.error(
|
|
306
426
|
"sandbox deadline rotation recovery failed",
|
|
@@ -508,8 +628,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
508
628
|
// recoverable, but a successor attempt or worker recovery makes this activity
|
|
509
629
|
// stale and unable to clobber the shared turn/session.
|
|
510
630
|
if (
|
|
511
|
-
leases.codex.lost &&
|
|
512
|
-
settings.codexCredentialLeasingEnabled &&
|
|
631
|
+
(leases.codex.lost || error instanceof CodexCredentialLeaseLostError) &&
|
|
513
632
|
billingState.isCodexTurn &&
|
|
514
633
|
eventing.publish &&
|
|
515
634
|
attempt.turnId &&
|
|
@@ -558,10 +677,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
558
677
|
// malformed/partial 200 stream, invalid content, prompt 4xx, or provider
|
|
559
678
|
// 5xx does not classify here and therefore cannot consume another
|
|
560
679
|
// subscription or duplicate a side effect.
|
|
680
|
+
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
561
681
|
const codexCredentialFailure =
|
|
562
|
-
|
|
563
|
-
billingState.isCodexTurn &&
|
|
564
|
-
providerTurn.effectiveCodexCredentialId
|
|
682
|
+
billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
|
|
565
683
|
? classifyCodexCredentialFailure(error)
|
|
566
684
|
: null;
|
|
567
685
|
if (
|
|
@@ -625,15 +743,26 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
625
743
|
}
|
|
626
744
|
: null;
|
|
627
745
|
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
628
|
-
const
|
|
629
|
-
leases.codex.holderId &&
|
|
746
|
+
const quarantineResult =
|
|
747
|
+
leases.codex.holderId &&
|
|
748
|
+
leases.codex.generation !== null &&
|
|
749
|
+
providerTurn.effectiveCodexCredentialVersion !== null
|
|
630
750
|
? await quarantineCodexCredentialForLease(db, {
|
|
631
751
|
accountId: input.accountId,
|
|
632
752
|
workspaceId: input.workspaceId,
|
|
753
|
+
sessionId: input.sessionId,
|
|
633
754
|
turnId: attempt.turnId,
|
|
755
|
+
attemptId: input.attemptId,
|
|
756
|
+
executionGeneration: attempt.executionGeneration,
|
|
757
|
+
workflowId: input.workflowId,
|
|
758
|
+
workflowRunId: input.workflowRunId,
|
|
759
|
+
dispatchId: attempt.dispatchId,
|
|
760
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
634
761
|
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
762
|
+
credentialVersion: providerTurn.effectiveCodexCredentialVersion,
|
|
635
763
|
holderId: leases.codex.holderId,
|
|
636
764
|
generation: leases.codex.generation,
|
|
765
|
+
maxFailovers: providerTurn.codexCredentialFailoverLimit,
|
|
637
766
|
quarantine:
|
|
638
767
|
codexCredentialFailure.kind === "auth"
|
|
639
768
|
? {
|
|
@@ -653,7 +782,46 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
653
782
|
cooldownKind: codexCredentialFailure.kind,
|
|
654
783
|
},
|
|
655
784
|
})
|
|
656
|
-
:
|
|
785
|
+
: null;
|
|
786
|
+
if (
|
|
787
|
+
quarantineResult?.action === "credential_changed" &&
|
|
788
|
+
leases.codex.holderId &&
|
|
789
|
+
leases.codex.generation !== null
|
|
790
|
+
) {
|
|
791
|
+
const recovery = await settleCodexCredentialLeaseLoss(db, {
|
|
792
|
+
accountId: input.accountId,
|
|
793
|
+
workspaceId: input.workspaceId,
|
|
794
|
+
sessionId: input.sessionId,
|
|
795
|
+
turnId: attempt.turnId,
|
|
796
|
+
attemptId: input.attemptId,
|
|
797
|
+
holderId: leases.codex.holderId,
|
|
798
|
+
generation: leases.codex.generation,
|
|
799
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
800
|
+
checkpointDurable: true,
|
|
801
|
+
recoveryPayload: {
|
|
802
|
+
triggerEventId: attempt.triggerEventId!,
|
|
803
|
+
reason: "codex_credential_version_changed",
|
|
804
|
+
},
|
|
805
|
+
failedPayload: {},
|
|
806
|
+
});
|
|
807
|
+
if (recovery.action === "recovering") {
|
|
808
|
+
leases.codex.held = false;
|
|
809
|
+
await publishDurableSessionEvents(
|
|
810
|
+
bus,
|
|
811
|
+
input.workspaceId,
|
|
812
|
+
input.sessionId,
|
|
813
|
+
recovery.events,
|
|
814
|
+
);
|
|
815
|
+
control.activityStatus = "recovering";
|
|
816
|
+
control.turnMetricOutcome = "recovering";
|
|
817
|
+
return claimedResult({ status: "recovering" });
|
|
818
|
+
}
|
|
819
|
+
acknowledgeLostAttemptOwnership();
|
|
820
|
+
control.activityStatus = "cancelled";
|
|
821
|
+
control.turnMetricOutcome = "cancelled";
|
|
822
|
+
return claimedResult({ status: "cancelled" });
|
|
823
|
+
}
|
|
824
|
+
const statePersisted = quarantineResult?.action === "recorded";
|
|
657
825
|
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
658
826
|
leases.codex.lost = true;
|
|
659
827
|
return await settleLostCodexAttempt(
|
|
@@ -663,26 +831,129 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
663
831
|
true,
|
|
664
832
|
);
|
|
665
833
|
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
834
|
+
if (
|
|
835
|
+
quarantineResult?.action === "recorded" &&
|
|
836
|
+
quarantineResult.exhausted &&
|
|
837
|
+
leases.codex.holderId &&
|
|
838
|
+
leases.codex.generation !== null
|
|
839
|
+
) {
|
|
840
|
+
const settlement = await settleCodexCredentialFailover(db, {
|
|
841
|
+
accountId: input.accountId,
|
|
842
|
+
workspaceId: input.workspaceId,
|
|
843
|
+
sessionId: input.sessionId,
|
|
844
|
+
turnId: attempt.turnId,
|
|
845
|
+
attemptId: input.attemptId,
|
|
846
|
+
holderId: leases.codex.holderId,
|
|
847
|
+
generation: leases.codex.generation,
|
|
848
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
849
|
+
maxFailovers: quarantineResult.maxFailovers,
|
|
850
|
+
recoveryPayload: {
|
|
851
|
+
triggerEventId: attempt.triggerEventId!,
|
|
852
|
+
reason: "codex_credential_failover",
|
|
853
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
854
|
+
failureKind: codexCredentialFailure.kind,
|
|
855
|
+
},
|
|
856
|
+
failedPayload: {
|
|
857
|
+
error:
|
|
858
|
+
"Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
859
|
+
code: "codex_credential_failover_exhausted",
|
|
860
|
+
retryable: false,
|
|
861
|
+
recovery: "user_message",
|
|
862
|
+
failoverCount: quarantineResult.failoverCount,
|
|
863
|
+
maxFailovers: quarantineResult.maxFailovers,
|
|
864
|
+
},
|
|
865
|
+
});
|
|
866
|
+
if (settlement.action === "limit_exceeded") {
|
|
867
|
+
leases.codex.held = false;
|
|
868
|
+
await publishDurableSessionEvents(
|
|
869
|
+
bus,
|
|
870
|
+
input.workspaceId,
|
|
871
|
+
input.sessionId,
|
|
872
|
+
settlement.events,
|
|
873
|
+
);
|
|
874
|
+
control.activityError = error;
|
|
875
|
+
control.activityStatus = "idle";
|
|
876
|
+
control.turnMetricOutcome = "failed";
|
|
877
|
+
await deliverFailedChildTurnToParent(
|
|
878
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
879
|
+
input.workspaceId,
|
|
880
|
+
input.sessionId,
|
|
881
|
+
attempt.turnId,
|
|
882
|
+
);
|
|
883
|
+
return claimedResult({ status: "idle" });
|
|
884
|
+
}
|
|
885
|
+
if (settlement.action === "stale") {
|
|
886
|
+
acknowledgeLostAttemptOwnership();
|
|
887
|
+
control.activityStatus = "cancelled";
|
|
888
|
+
control.turnMetricOutcome = "cancelled";
|
|
889
|
+
return claimedResult({ status: "cancelled" });
|
|
890
|
+
}
|
|
891
|
+
throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
|
|
892
|
+
}
|
|
893
|
+
let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
|
|
894
|
+
try {
|
|
895
|
+
accounts = await listCodexAccountStatuses(db, input.workspaceId);
|
|
896
|
+
} catch (metadataError) {
|
|
897
|
+
// Current account health/cooldown metadata is still required after
|
|
898
|
+
// quarantine. Operational database failures re-enter the existing
|
|
899
|
+
// exact-attempt recovery lane; no policy choice is made from partial
|
|
900
|
+
// account state.
|
|
901
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
902
|
+
error: metadataError,
|
|
903
|
+
turnId: attempt.turnId,
|
|
904
|
+
triggerEventId: attempt.triggerEventId!,
|
|
905
|
+
executionGeneration: attempt.executionGeneration,
|
|
906
|
+
});
|
|
907
|
+
if (recoveryFailure) {
|
|
908
|
+
control.activityStatus = "recovering";
|
|
909
|
+
control.turnMetricOutcome = "recovering";
|
|
910
|
+
control.activityError = metadataError;
|
|
911
|
+
throw recoveryFailure;
|
|
912
|
+
}
|
|
913
|
+
throw metadataError;
|
|
914
|
+
}
|
|
915
|
+
const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
|
|
916
|
+
const selected = selectCodexCredentialLeaseForTurn({
|
|
917
|
+
context: {
|
|
918
|
+
accounts: codexLeaseAccountsForSelection(accounts),
|
|
919
|
+
activeCredentialId: acceptedPolicy.activeCredentialId,
|
|
920
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
921
|
+
rotationStrategy: acceptedPolicy.rotationStrategy,
|
|
922
|
+
existingCredentialId: null,
|
|
923
|
+
failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
|
|
924
|
+
policyScope: null,
|
|
925
|
+
unavailableDiagnostics: [],
|
|
926
|
+
},
|
|
927
|
+
sessionId: input.sessionId,
|
|
928
|
+
sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
929
|
+
sessionPinSource: acceptedPolicy.pinSource,
|
|
930
|
+
sessionLastCredentialId: acceptedPolicy.lastCredentialId,
|
|
931
|
+
now,
|
|
932
|
+
});
|
|
933
|
+
const decisionKind =
|
|
934
|
+
selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
|
|
935
|
+
const pinDisposition = classifyCodexPin({
|
|
936
|
+
pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
937
|
+
pinSource: acceptedPolicy.pinSource,
|
|
938
|
+
strategy: acceptedPolicy.rotationStrategy as CodexRotationStrategy,
|
|
939
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
940
|
+
});
|
|
941
|
+
const failureDisposition = codexDefinitiveFailureDisposition({
|
|
942
|
+
failureKind: codexCredentialFailure.kind,
|
|
943
|
+
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
944
|
+
pinDisposition,
|
|
945
|
+
decisionKind,
|
|
946
|
+
decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
|
|
947
|
+
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
948
|
+
});
|
|
949
|
+
const maxFailovers = providerTurn.codexCredentialFailoverLimit;
|
|
684
950
|
|
|
685
|
-
if (
|
|
951
|
+
if (
|
|
952
|
+
statePersisted &&
|
|
953
|
+
failureDisposition === "failover" &&
|
|
954
|
+
leases.codex.holderId &&
|
|
955
|
+
leases.codex.generation !== null
|
|
956
|
+
) {
|
|
686
957
|
const settlement = await settleCodexCredentialFailover(db, {
|
|
687
958
|
accountId: input.accountId,
|
|
688
959
|
workspaceId: input.workspaceId,
|
|
@@ -692,7 +963,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
692
963
|
holderId: leases.codex.holderId,
|
|
693
964
|
generation: leases.codex.generation,
|
|
694
965
|
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
695
|
-
maxFailovers
|
|
966
|
+
maxFailovers,
|
|
696
967
|
recoveryPayload: {
|
|
697
968
|
triggerEventId: attempt.triggerEventId!,
|
|
698
969
|
reason: "codex_credential_failover",
|
|
@@ -700,6 +971,15 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
700
971
|
failureKind: codexCredentialFailure.kind,
|
|
701
972
|
...(cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}),
|
|
702
973
|
},
|
|
974
|
+
failedPayload: {
|
|
975
|
+
error:
|
|
976
|
+
"Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
977
|
+
code: "codex_credential_failover_exhausted",
|
|
978
|
+
retryable: false,
|
|
979
|
+
recovery: "user_message",
|
|
980
|
+
failoverCount: quarantineResult.failoverCount,
|
|
981
|
+
maxFailovers: quarantineResult.maxFailovers,
|
|
982
|
+
},
|
|
703
983
|
});
|
|
704
984
|
observability.incrementCounter({
|
|
705
985
|
name: "opengeni_codex_failover_settlements_total",
|
|
@@ -738,8 +1018,137 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
738
1018
|
acknowledgeLostAttemptOwnership();
|
|
739
1019
|
control.activityStatus = "cancelled";
|
|
740
1020
|
control.turnMetricOutcome = "cancelled";
|
|
1021
|
+
return claimedResult({ status: "cancelled" });
|
|
1022
|
+
}
|
|
1023
|
+
if (settlement.action === "limit_exceeded") {
|
|
1024
|
+
leases.codex.held = false;
|
|
1025
|
+
await publishDurableSessionEvents(
|
|
1026
|
+
bus,
|
|
1027
|
+
input.workspaceId,
|
|
1028
|
+
input.sessionId,
|
|
1029
|
+
settlement.events,
|
|
1030
|
+
);
|
|
1031
|
+
control.activityError = error;
|
|
1032
|
+
control.activityStatus = "idle";
|
|
1033
|
+
control.turnMetricOutcome = "failed";
|
|
1034
|
+
await deliverFailedChildTurnToParent(
|
|
1035
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
1036
|
+
input.workspaceId,
|
|
1037
|
+
input.sessionId,
|
|
1038
|
+
attempt.turnId,
|
|
1039
|
+
);
|
|
1040
|
+
return claimedResult({ status: "idle" });
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
if (
|
|
1045
|
+
statePersisted &&
|
|
1046
|
+
failureDisposition === "wait" &&
|
|
1047
|
+
leases.codex.holderId &&
|
|
1048
|
+
leases.codex.generation !== null
|
|
1049
|
+
) {
|
|
1050
|
+
let goal: Awaited<ReturnType<typeof getSessionGoal>>;
|
|
1051
|
+
try {
|
|
1052
|
+
goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
1053
|
+
} catch (metadataError) {
|
|
1054
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
1055
|
+
error: metadataError,
|
|
1056
|
+
turnId: attempt.turnId,
|
|
1057
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1058
|
+
executionGeneration: attempt.executionGeneration,
|
|
1059
|
+
});
|
|
1060
|
+
if (recoveryFailure) {
|
|
1061
|
+
control.activityStatus = "recovering";
|
|
1062
|
+
control.turnMetricOutcome = "recovering";
|
|
1063
|
+
control.activityError = metadataError;
|
|
1064
|
+
throw recoveryFailure;
|
|
1065
|
+
}
|
|
1066
|
+
throw metadataError;
|
|
1067
|
+
}
|
|
1068
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
1069
|
+
const exactProviderReset =
|
|
1070
|
+
codexCredentialFailure.cooldownSeconds !== null &&
|
|
1071
|
+
Number.isFinite(codexCredentialFailure.cooldownSeconds) &&
|
|
1072
|
+
codexCredentialFailure.cooldownSeconds > 0;
|
|
1073
|
+
const policyCredentialId =
|
|
1074
|
+
pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId
|
|
1075
|
+
? acceptedPolicy.pinnedCredentialId
|
|
1076
|
+
: !acceptedPolicy.rotationEnabled
|
|
1077
|
+
? acceptedPolicy.activeCredentialId
|
|
1078
|
+
: null;
|
|
1079
|
+
const capacityAccounts = policyCredentialId
|
|
1080
|
+
? accounts.filter((account) => account.id === policyCredentialId)
|
|
1081
|
+
: accounts;
|
|
1082
|
+
const authoritativeResetAt = exactProviderReset
|
|
1083
|
+
? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
|
|
1084
|
+
: null;
|
|
1085
|
+
const allAccounts =
|
|
1086
|
+
acceptedPolicy.rotationEnabled &&
|
|
1087
|
+
pinDisposition !== "manual" &&
|
|
1088
|
+
decisionKind === "allCapped";
|
|
1089
|
+
const failurePayload = codexCapacityWaitFailurePayload({
|
|
1090
|
+
failureKind: codexCredentialFailure.kind,
|
|
1091
|
+
usageLimit,
|
|
1092
|
+
cooldownSeconds: codexCredentialFailure.cooldownSeconds,
|
|
1093
|
+
detail:
|
|
1094
|
+
codexCredentialFailure.kind === "quota"
|
|
1095
|
+
? error instanceof Error
|
|
1096
|
+
? error.message
|
|
1097
|
+
: String(error)
|
|
1098
|
+
: "the same accepted turn is waiting for eligible credential capacity",
|
|
1099
|
+
allAccounts,
|
|
1100
|
+
});
|
|
1101
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
1102
|
+
{ db, bus },
|
|
1103
|
+
{
|
|
1104
|
+
accountId: input.accountId,
|
|
1105
|
+
workspaceId: input.workspaceId,
|
|
1106
|
+
sessionId: input.sessionId,
|
|
1107
|
+
turnId: attempt.turnId,
|
|
1108
|
+
attemptId: input.attemptId,
|
|
1109
|
+
workflowId: input.workflowId,
|
|
1110
|
+
goalId: activeGoal?.id ?? null,
|
|
1111
|
+
goalVersion: activeGoal?.version ?? null,
|
|
1112
|
+
earliestResetAt: authoritativeResetAt,
|
|
1113
|
+
resetKind: authoritativeResetAt
|
|
1114
|
+
? "authoritative"
|
|
1115
|
+
: codexCredentialFailure.kind === "auth" ||
|
|
1116
|
+
codexCredentialFailure.kind === "forbidden"
|
|
1117
|
+
? "mutation_only"
|
|
1118
|
+
: "bounded_refresh",
|
|
1119
|
+
failurePayload,
|
|
1120
|
+
leaseFence: {
|
|
1121
|
+
holderId: leases.codex.holderId,
|
|
1122
|
+
generation: leases.codex.generation,
|
|
1123
|
+
},
|
|
1124
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
1125
|
+
},
|
|
1126
|
+
{ onArmed: () => (leases.codex.held = false) },
|
|
1127
|
+
);
|
|
1128
|
+
control.activityError = error;
|
|
1129
|
+
if (evaluated.action === "resumed") {
|
|
1130
|
+
control.activityStatus = "recovering";
|
|
1131
|
+
control.turnMetricOutcome = "recovering";
|
|
741
1132
|
return claimedResult({ status: "recovering" });
|
|
742
1133
|
}
|
|
1134
|
+
if (evaluated.action === "waiting") {
|
|
1135
|
+
control.activityError = error;
|
|
1136
|
+
control.activityStatus = "waiting_capacity";
|
|
1137
|
+
control.turnMetricOutcome = "recovering";
|
|
1138
|
+
return claimedResult({
|
|
1139
|
+
status: "waiting_capacity",
|
|
1140
|
+
capacityWait: {
|
|
1141
|
+
waiterId: evaluated.waiter.id,
|
|
1142
|
+
generation: evaluated.waiter.generation,
|
|
1143
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
1144
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
1145
|
+
},
|
|
1146
|
+
});
|
|
1147
|
+
}
|
|
1148
|
+
acknowledgeLostAttemptOwnership();
|
|
1149
|
+
control.activityStatus = "cancelled";
|
|
1150
|
+
control.turnMetricOutcome = "cancelled";
|
|
1151
|
+
return claimedResult({ status: "cancelled" });
|
|
743
1152
|
}
|
|
744
1153
|
}
|
|
745
1154
|
}
|
|
@@ -882,262 +1291,26 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
882
1291
|
control.activityError = error;
|
|
883
1292
|
return claimedResult({ status: "recovering" });
|
|
884
1293
|
}
|
|
885
|
-
//
|
|
886
|
-
//
|
|
887
|
-
//
|
|
888
|
-
//
|
|
889
|
-
//
|
|
890
|
-
// does not loop. For an active goal we hold the continuation for the reported
|
|
891
|
-
// reset window (capped) so it resumes itself when access returns, instead of
|
|
892
|
-
// hammering the capped backend.
|
|
893
|
-
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
1294
|
+
// The leased credential path above normally quarantines quota state and
|
|
1295
|
+
// either recovers the same turn or arms a durable capacity wait. This narrow
|
|
1296
|
+
// fallback covers failures before a credential lease existed, or a failed
|
|
1297
|
+
// durable checkpoint where replay would be unsafe. Keep the session usable,
|
|
1298
|
+
// but never synthesize another turn or walk an unfenced legacy pointer.
|
|
894
1299
|
if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
895
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
896
|
-
const goalActive = Boolean(goal && goal.status === "active");
|
|
897
1300
|
await flushRuntimeBatcher();
|
|
898
1301
|
await historySink.reconcileConversationTruth();
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
// catch deliberately does NOT move the active pointer — the re-dispatched turn's
|
|
904
|
-
// proactive seam (turn-start) is the single authoritative pointer-move + strip site.
|
|
905
|
-
let rotated = false;
|
|
906
|
-
let rotationResumeMs: number | null = null; // 0 ⇒ a candidate is available; re-dispatch now
|
|
907
|
-
let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
|
|
908
|
-
let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
|
|
909
|
-
let capacityAuthoritativeResetAt: Date | null = null;
|
|
910
|
-
let capacityNeedsBoundedRefresh = false;
|
|
911
|
-
if (providerTurn.effectiveCodexCredentialId) {
|
|
912
|
-
const [rotation, sessionCodex] = await Promise.all([
|
|
913
|
-
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
914
|
-
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null),
|
|
915
|
-
]);
|
|
916
|
-
const reactiveStrategy = (rotation?.rotationStrategy ??
|
|
917
|
-
"most_remaining") as CodexRotationStrategy;
|
|
918
|
-
const reactiveDisposition = classifyCodexPin({
|
|
919
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
920
|
-
pinSource: sessionCodex?.pinSource ?? null,
|
|
921
|
-
strategy: reactiveStrategy,
|
|
922
|
-
rotationEnabled: Boolean(rotation?.rotationEnabled),
|
|
923
|
-
});
|
|
924
|
-
const reactiveSharded = reactiveDisposition === "sharded";
|
|
925
|
-
const rotating =
|
|
926
|
-
Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) &&
|
|
927
|
-
reactiveDisposition !== "manual";
|
|
928
|
-
if (rotating && rotation) {
|
|
929
|
-
const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
|
|
930
|
-
const serving =
|
|
931
|
-
accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
|
|
932
|
-
// Both provider allowance windows bind. Use the same canonical
|
|
933
|
-
// quarantine calculation as the fenced failover path so a short
|
|
934
|
-
// five-hour reset can never overwrite a later weekly reset.
|
|
935
|
-
const until = codexCredentialCooldownUntil(
|
|
936
|
-
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
937
|
-
serving,
|
|
938
|
-
new Date(),
|
|
939
|
-
)!;
|
|
940
|
-
// Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
|
|
941
|
-
// write whose failure went unnoticed is exactly what lets the next proactive
|
|
942
|
-
// rank re-pick this just-capped account (stale-low cached usedPercent, not
|
|
943
|
-
// cooling) — so capture whether it PERSISTED and feed it into the resume floor.
|
|
944
|
-
const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
|
|
945
|
-
db,
|
|
946
|
-
input.workspaceId,
|
|
947
|
-
providerTurn.effectiveCodexCredentialId,
|
|
948
|
-
until,
|
|
949
|
-
"quota",
|
|
950
|
-
).catch(() => null);
|
|
951
|
-
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
952
|
-
if (cooldownMutation) {
|
|
953
|
-
await signalCodexCapacityWakeTargets(
|
|
954
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
955
|
-
cooldownMutation.wakeTargets,
|
|
956
|
-
);
|
|
957
|
-
}
|
|
958
|
-
// Re-rank over the fresh accounts; the in-memory list predates the cooldown
|
|
959
|
-
// write, so stamp the just-cooled account so the engine excludes it now. The
|
|
960
|
-
// serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
|
|
961
|
-
const fresh = accounts.map((a) =>
|
|
962
|
-
a.id === providerTurn.effectiveCodexCredentialId
|
|
963
|
-
? { ...a, exhaustedUntil: until, exhaustedKind: "quota" as const }
|
|
964
|
-
: a,
|
|
965
|
-
);
|
|
966
|
-
if (reactiveSharded) {
|
|
967
|
-
// AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
|
|
968
|
-
// marked cooling in `fresh` → excluded) so sessions sharing a capped account
|
|
969
|
-
// spread across the pool rather than re-concentrating on one first-eligible
|
|
970
|
-
// failover. AM-3: DURABLY REWRITE the session's POLICY pin to the new home —
|
|
971
|
-
// selectCodexCredentialForTurn returns a cooling pinned account with NO
|
|
972
|
-
// exhaustion check, so a pointer-only move would leave the re-dispatched turn on
|
|
973
|
-
// the capped pin. Like the classic path we do NOT touch the workspace active
|
|
974
|
-
// pointer; the session pin is the sharded home.
|
|
975
|
-
const newHome = shardCredentialForSession({
|
|
976
|
-
sessionId: input.sessionId,
|
|
977
|
-
accounts: fresh,
|
|
978
|
-
now: new Date(),
|
|
979
|
-
});
|
|
980
|
-
if (newHome) {
|
|
981
|
-
rotated = true;
|
|
982
|
-
const pinMutation = await withSessionCodexCapacityMutation(
|
|
983
|
-
db,
|
|
984
|
-
{
|
|
985
|
-
workspaceId: input.workspaceId,
|
|
986
|
-
reason: "codex_policy_pin_resharded",
|
|
987
|
-
},
|
|
988
|
-
async (tx) => {
|
|
989
|
-
const changed = await setSessionCodexPinInTransaction(
|
|
990
|
-
tx,
|
|
991
|
-
input.workspaceId,
|
|
992
|
-
input.sessionId,
|
|
993
|
-
newHome,
|
|
994
|
-
"policy",
|
|
995
|
-
{
|
|
996
|
-
expected: {
|
|
997
|
-
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
998
|
-
pinSource: sessionCodex?.pinSource ?? null,
|
|
999
|
-
},
|
|
1000
|
-
},
|
|
1001
|
-
);
|
|
1002
|
-
return { result: changed, changed };
|
|
1003
|
-
},
|
|
1004
|
-
).catch(() => null);
|
|
1005
|
-
if (pinMutation) {
|
|
1006
|
-
await signalCodexCapacityWakeTargets(
|
|
1007
|
-
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
1008
|
-
pinMutation.wakeTargets,
|
|
1009
|
-
);
|
|
1010
|
-
}
|
|
1011
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
1012
|
-
db,
|
|
1013
|
-
input.workspaceId,
|
|
1014
|
-
input.sessionId,
|
|
1015
|
-
).catch(() => 0);
|
|
1016
|
-
const resume = computeReactiveRotationResume({
|
|
1017
|
-
cooldownPersisted,
|
|
1018
|
-
priorConsecutiveRotations,
|
|
1019
|
-
connectedAccountCount: accounts.length,
|
|
1020
|
-
});
|
|
1021
|
-
rotationResumeMs = resume.continueDelayMs;
|
|
1022
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
1023
|
-
} else {
|
|
1024
|
-
// Every account capped/cooling → idle until the earliest reset across all.
|
|
1025
|
-
rotated = true;
|
|
1026
|
-
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
1027
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1028
|
-
capacityNeedsBoundedRefresh = true;
|
|
1029
|
-
}
|
|
1030
|
-
} else {
|
|
1031
|
-
const decision = chooseRotationActive({
|
|
1032
|
-
rotationStrategy: reactiveStrategy,
|
|
1033
|
-
activeCredentialId: rotation.activeCredentialId,
|
|
1034
|
-
priorCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
1035
|
-
accounts: fresh,
|
|
1036
|
-
now: new Date(),
|
|
1037
|
-
});
|
|
1038
|
-
if (decision.kind === "active") {
|
|
1039
|
-
rotated = true;
|
|
1040
|
-
// Finding 1: a live candidate normally re-dispatches NOW (0). Two second-order
|
|
1041
|
-
// faults would turn that 0 into a hot loop, so bound it. Count the consecutive
|
|
1042
|
-
// reactive failovers since the last successful turn (this one is not yet
|
|
1043
|
-
// published) and combine with the cooldown-persistence result.
|
|
1044
|
-
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
1045
|
-
db,
|
|
1046
|
-
input.workspaceId,
|
|
1047
|
-
input.sessionId,
|
|
1048
|
-
).catch(() => 0);
|
|
1049
|
-
const resume = computeReactiveRotationResume({
|
|
1050
|
-
cooldownPersisted,
|
|
1051
|
-
priorConsecutiveRotations,
|
|
1052
|
-
connectedAccountCount: accounts.length,
|
|
1053
|
-
});
|
|
1054
|
-
rotationResumeMs = resume.continueDelayMs; // 0 (happy path), a slow-retry floor, or the circuit-breaker idle
|
|
1055
|
-
rotationResumeIdleUntilReset = resume.idleUntilReset; // true only on the circuit-breaker fall (MANDATORY hold)
|
|
1056
|
-
} else if (decision.kind === "allCapped") {
|
|
1057
|
-
rotated = true;
|
|
1058
|
-
allCappedResetAt = decision.earliestResetAt;
|
|
1059
|
-
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1060
|
-
capacityNeedsBoundedRefresh = true;
|
|
1061
|
-
}
|
|
1062
|
-
// kind:"none" → fall through to today's single-account idle.
|
|
1063
|
-
}
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
const failurePayload = allCappedResetAt
|
|
1068
|
-
? codexUsageLimitFailurePayload(
|
|
1069
|
-
{
|
|
1070
|
-
resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1000),
|
|
1071
|
-
},
|
|
1072
|
-
error instanceof Error ? error.message : String(error),
|
|
1073
|
-
{ allAccounts: true },
|
|
1074
|
-
)
|
|
1075
|
-
: codexUsageLimitFailurePayload(
|
|
1076
|
-
usageLimit,
|
|
1077
|
-
error instanceof Error ? error.message : String(error),
|
|
1078
|
-
);
|
|
1079
|
-
// A live alternate is still handled by the existing immediate,
|
|
1080
|
-
// same-policy continuation path. When no alternate exists (all capped,
|
|
1081
|
-
// or a single non-rotating account), persist the native capacity wait
|
|
1082
|
-
// instead of an in-memory delay/user-message recovery.
|
|
1083
|
-
if (rotationResumeMs === null) {
|
|
1084
|
-
const providerResetAt =
|
|
1085
|
-
capacityAuthoritativeResetAt ??
|
|
1086
|
-
(usageLimit.resetsInSeconds !== null &&
|
|
1087
|
-
Number.isFinite(usageLimit.resetsInSeconds) &&
|
|
1088
|
-
usageLimit.resetsInSeconds > 0
|
|
1089
|
-
? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1000)
|
|
1090
|
-
: null);
|
|
1091
|
-
const armed = await armCodexCapacityWait(db, {
|
|
1092
|
-
accountId: input.accountId,
|
|
1093
|
-
workspaceId: input.workspaceId,
|
|
1094
|
-
sessionId: input.sessionId,
|
|
1095
|
-
turnId: attempt.turnId,
|
|
1096
|
-
attemptId: input.attemptId,
|
|
1097
|
-
workflowId: input.workflowId,
|
|
1098
|
-
goalId: goalActive && goal ? goal.id : null,
|
|
1099
|
-
goalVersion: goalActive && goal ? goal.version : null,
|
|
1100
|
-
earliestResetAt: providerResetAt,
|
|
1101
|
-
resetKind:
|
|
1102
|
-
providerResetAt && !capacityNeedsBoundedRefresh ? "authoritative" : "bounded_refresh",
|
|
1103
|
-
failurePayload,
|
|
1104
|
-
...(leases.codex.holderId && leases.codex.generation !== null
|
|
1105
|
-
? {
|
|
1106
|
-
leaseFence: {
|
|
1107
|
-
holderId: leases.codex.holderId,
|
|
1108
|
-
generation: leases.codex.generation,
|
|
1109
|
-
},
|
|
1110
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
1111
|
-
}
|
|
1112
|
-
: {}),
|
|
1113
|
-
});
|
|
1114
|
-
if (armed.action === "waiting") {
|
|
1115
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
1116
|
-
control.turnMetricOutcome = "recovering";
|
|
1117
|
-
control.activityStatus = "waiting_capacity";
|
|
1118
|
-
control.activityError = error;
|
|
1119
|
-
return claimedResult({
|
|
1120
|
-
status: "waiting_capacity",
|
|
1121
|
-
capacityWait: {
|
|
1122
|
-
waiterId: armed.waiter.id,
|
|
1123
|
-
generation: armed.waiter.generation,
|
|
1124
|
-
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
1125
|
-
wakeRevision: armed.waiter.wakeRevision,
|
|
1126
|
-
},
|
|
1127
|
-
});
|
|
1128
|
-
}
|
|
1129
|
-
}
|
|
1302
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
1303
|
+
usageLimit,
|
|
1304
|
+
error instanceof Error ? error.message : String(error),
|
|
1305
|
+
);
|
|
1130
1306
|
if (
|
|
1131
1307
|
!(await eventing.settle!({
|
|
1132
1308
|
events: [
|
|
1133
|
-
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
1134
|
-
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
1135
1309
|
{
|
|
1136
1310
|
type: "turn.failed",
|
|
1137
1311
|
payload: {
|
|
1138
1312
|
...failurePayload,
|
|
1139
|
-
recovery:
|
|
1140
|
-
...(rotated ? { rotated: true } : {}),
|
|
1313
|
+
recovery: "user_message",
|
|
1141
1314
|
},
|
|
1142
1315
|
},
|
|
1143
1316
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
@@ -1152,38 +1325,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1152
1325
|
control.turnMetricOutcome = "failed";
|
|
1153
1326
|
control.activityStatus = "idle";
|
|
1154
1327
|
control.activityError = error;
|
|
1155
|
-
if (goalActive) {
|
|
1156
|
-
// Rotation: a candidate is available → continue NOW (0). All-capped → idle until the
|
|
1157
|
-
// earliest reset across all accounts (capped at 1h). Else the unchanged single-account idle.
|
|
1158
|
-
if (rotationResumeMs !== null) {
|
|
1159
|
-
// A candidate IS available. Normally the just-failed account is now cooling so
|
|
1160
|
-
// the ranker cannot re-pick it → 0 (re-dispatch NOW, the legitimate skip-the-hold
|
|
1161
|
-
// case). Finding 1 bounds the two exceptions: a persistence fault yields a positive
|
|
1162
|
-
// slow-retry floor, and once consecutive failovers exceed the account count + margin
|
|
1163
|
-
// the circuit breaker returns a fixed MANDATORY idle (idleUntilReset) — never a 0-delay
|
|
1164
|
-
// hot loop against a capped backend + DB.
|
|
1165
|
-
return claimedResult({
|
|
1166
|
-
status: "idle",
|
|
1167
|
-
continueDelayMs: rotationResumeMs,
|
|
1168
|
-
...(rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}),
|
|
1169
|
-
});
|
|
1170
|
-
}
|
|
1171
|
-
// All-capped: clamp to [MIN_IDLE_MS, max] — a POSITIVE, BOUNDED hold (never 0,
|
|
1172
|
-
// so session.ts can never tight-loop). The post-idle continuation re-dispatch
|
|
1173
|
-
// hits the proactive seam, which refreshes usage and self-heals.
|
|
1174
|
-
const resumeMs = allCappedResetAt
|
|
1175
|
-
? computeIdleDelayMs(allCappedResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1176
|
-
: usageLimit.resetsInSeconds !== null &&
|
|
1177
|
-
Number.isFinite(usageLimit.resetsInSeconds) &&
|
|
1178
|
-
usageLimit.resetsInSeconds > 0
|
|
1179
|
-
? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1000, CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1180
|
-
: CODEX_USAGE_LIMIT_MAX_RESUME_MS;
|
|
1181
|
-
return claimedResult({
|
|
1182
|
-
status: "idle",
|
|
1183
|
-
continueDelayMs: resumeMs,
|
|
1184
|
-
...(allCappedResetAt ? { idleUntilReset: true } : {}),
|
|
1185
|
-
});
|
|
1186
|
-
}
|
|
1187
1328
|
return claimedResult({ status: "idle" });
|
|
1188
1329
|
}
|
|
1189
1330
|
// Budget/limit exhaustion between model calls is account state, not an
|