@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getSessionGoal,
|
|
3
3
|
acquireCodexCredentialLease,
|
|
4
|
-
|
|
4
|
+
CodexCredentialLeaseAttemptFencedError,
|
|
5
|
+
CodexCredentialFailoverExhaustedError,
|
|
5
6
|
CODEX_CREDENTIAL_LEASE_TTL_MS,
|
|
6
|
-
getCodexRotationSettings,
|
|
7
|
-
listCodexAccountStatuses,
|
|
8
|
-
getSessionCodexState,
|
|
9
7
|
recordSessionActiveCodexCredential,
|
|
10
8
|
setSessionCodexPinInTransaction,
|
|
11
|
-
|
|
9
|
+
settleCodexCredentialFailover,
|
|
12
10
|
withSessionCodexCapacityMutation,
|
|
13
11
|
type CodexCredentialLeaseResult,
|
|
12
|
+
type CodexCredentialLeaseSessionState,
|
|
14
13
|
type CodexCredentialLeaseSelectionContext,
|
|
15
14
|
} from "@opengeni/db";
|
|
16
|
-
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
17
15
|
import { type Settings } from "@opengeni/config";
|
|
16
|
+
import { CodexReloginRequired } from "@opengeni/codex";
|
|
17
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
18
18
|
import {
|
|
19
19
|
authoritativeCodexCapacityResetAt,
|
|
20
20
|
classifyCodexPin,
|
|
@@ -22,14 +22,17 @@ import {
|
|
|
22
22
|
isCodexCredentialEligible,
|
|
23
23
|
selectCodexCredentialLeaseForTurn,
|
|
24
24
|
type CodexRotationStrategy,
|
|
25
|
-
type
|
|
25
|
+
type CodexTurnLeaseDecision,
|
|
26
26
|
} from "../codex-rotation";
|
|
27
27
|
import {
|
|
28
28
|
codexFleetShadowDecisionMetricLabelsV1,
|
|
29
29
|
codexFleetShadowErrorMetricLabelsV1,
|
|
30
30
|
publishCodexFleetShadowDecisionV1,
|
|
31
31
|
} from "../codex-fleet-shadow";
|
|
32
|
-
import {
|
|
32
|
+
import {
|
|
33
|
+
armAndReconcileCodexCapacityWait,
|
|
34
|
+
signalCodexCapacityWakeTargets,
|
|
35
|
+
} from "../codex-capacity";
|
|
33
36
|
import type {
|
|
34
37
|
TurnActivityServices as ActivityServices,
|
|
35
38
|
RunAgentTurnInput,
|
|
@@ -37,6 +40,7 @@ import type {
|
|
|
37
40
|
} from "../types";
|
|
38
41
|
import { recordTurnStartupPhase } from "../../observability-metrics";
|
|
39
42
|
import { createTurnCredentialLeases } from "./credential-leases";
|
|
43
|
+
import { deliverFailedChildTurnToParent } from "../parent-wake";
|
|
40
44
|
import { randomUUID } from "node:crypto";
|
|
41
45
|
|
|
42
46
|
import { refreshCappedCodexUsageRows } from "./codex";
|
|
@@ -94,6 +98,7 @@ export async function selectCodexTurnCapacity(
|
|
|
94
98
|
observability,
|
|
95
99
|
wakeSessionWorkflow,
|
|
96
100
|
signalCodexCapacityWorkflow,
|
|
101
|
+
dispatchId,
|
|
97
102
|
control,
|
|
98
103
|
attempt,
|
|
99
104
|
billingState,
|
|
@@ -101,6 +106,7 @@ export async function selectCodexTurnCapacity(
|
|
|
101
106
|
providerTurn,
|
|
102
107
|
leases,
|
|
103
108
|
claimedResult,
|
|
109
|
+
acknowledgeLostAttemptOwnership,
|
|
104
110
|
turn,
|
|
105
111
|
codexWorkspaceKey,
|
|
106
112
|
} = deps;
|
|
@@ -117,129 +123,69 @@ export async function selectCodexTurnCapacity(
|
|
|
117
123
|
const credentialSelectionStartedAt = performance.now();
|
|
118
124
|
let credentialSelectionOutcome: "completed" | "failed" = "completed";
|
|
119
125
|
try {
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
126
|
+
const selectForTurn = (
|
|
127
|
+
context: CodexCredentialLeaseSelectionContext,
|
|
128
|
+
lockedSessionCodexState: CodexCredentialLeaseSessionState,
|
|
129
|
+
) =>
|
|
124
130
|
selectCodexCredentialLeaseForTurn({
|
|
125
131
|
context,
|
|
126
|
-
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
127
132
|
sessionId: input.sessionId,
|
|
128
|
-
sessionPinnedCredentialId:
|
|
129
|
-
sessionPinSource,
|
|
130
|
-
sessionLastCredentialId:
|
|
133
|
+
sessionPinnedCredentialId: lockedSessionCodexState.pinnedCredentialId,
|
|
134
|
+
sessionPinSource: lockedSessionCodexState.pinSource,
|
|
135
|
+
sessionLastCredentialId: lockedSessionCodexState.lastCredentialId,
|
|
131
136
|
now: new Date(),
|
|
132
137
|
});
|
|
133
138
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
let leased: CodexCredentialLeaseResult<RotationDecision>;
|
|
138
|
-
let leaseAcquisitionStartedAtMs: number | null = null;
|
|
139
|
-
if (settings.codexCredentialLeasingEnabled) {
|
|
140
|
-
leaseAcquisitionStartedAtMs = performance.now();
|
|
141
|
-
leased = await acquireCodexCredentialLease(
|
|
139
|
+
let leaseAcquisitionStartedAtMs = performance.now();
|
|
140
|
+
let leased: CodexCredentialLeaseResult<CodexTurnLeaseDecision> =
|
|
141
|
+
await acquireCodexCredentialLease(
|
|
142
142
|
db,
|
|
143
143
|
{
|
|
144
144
|
accountId: input.accountId,
|
|
145
145
|
workspaceId: input.workspaceId,
|
|
146
|
+
sessionId: input.sessionId,
|
|
146
147
|
turnId,
|
|
148
|
+
attemptId: input.attemptId,
|
|
149
|
+
executionGeneration: attempt.executionGeneration,
|
|
150
|
+
workflowId: input.workflowId,
|
|
151
|
+
workflowRunId: input.workflowRunId,
|
|
152
|
+
dispatchId,
|
|
153
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
147
154
|
holderId,
|
|
148
|
-
advanceActivePointer:
|
|
155
|
+
advanceActivePointer: true,
|
|
149
156
|
},
|
|
150
157
|
selectForTurn,
|
|
151
158
|
);
|
|
152
|
-
} else {
|
|
153
|
-
const [rotation, accounts] = await Promise.all([
|
|
154
|
-
getCodexRotationSettings(db, input.workspaceId),
|
|
155
|
-
listCodexAccountStatuses(db, input.workspaceId),
|
|
156
|
-
]);
|
|
157
|
-
const leaseAccounts = accounts.map((account) => ({
|
|
158
|
-
...account,
|
|
159
|
-
activeLeaseCount: 0,
|
|
160
|
-
selectionCount: 0,
|
|
161
|
-
lastSelectedAt: null,
|
|
162
|
-
}));
|
|
163
|
-
const activeCredentialId = rotation?.activeCredentialId ?? null;
|
|
164
|
-
const selected = selectForTurn({
|
|
165
|
-
accounts: leaseAccounts,
|
|
166
|
-
activeCredentialId,
|
|
167
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
168
|
-
leaseRotationEnabled: false,
|
|
169
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
170
|
-
existingCredentialId: null,
|
|
171
|
-
policyScope: null,
|
|
172
|
-
unavailableDiagnostics: [],
|
|
173
|
-
});
|
|
174
|
-
leased = {
|
|
175
|
-
...selected,
|
|
176
|
-
accounts: leaseAccounts,
|
|
177
|
-
activeCredentialId,
|
|
178
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
179
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
180
|
-
reused: false,
|
|
181
|
-
holderId: null,
|
|
182
|
-
generation: null,
|
|
183
|
-
leasedUntil: null,
|
|
184
|
-
unavailableDiagnostics: [],
|
|
185
|
-
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
186
|
-
};
|
|
187
|
-
}
|
|
188
159
|
if (leased.decision.kind === "allCapped") {
|
|
189
160
|
// Bounded self-heal of stale usage cache, then ONE new atomic selection.
|
|
190
161
|
await refreshCappedCodexUsageRows(db, settings, input.workspaceId, leased.accounts, {
|
|
191
162
|
signalCodexCapacityWorkflow,
|
|
192
163
|
wakeSessionWorkflow,
|
|
193
164
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
...account,
|
|
214
|
-
activeLeaseCount: 0,
|
|
215
|
-
selectionCount: 0,
|
|
216
|
-
lastSelectedAt: null,
|
|
217
|
-
}));
|
|
218
|
-
const selected = selectForTurn({
|
|
219
|
-
accounts: leaseAccounts,
|
|
220
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
221
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
222
|
-
leaseRotationEnabled: false,
|
|
223
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
224
|
-
existingCredentialId: null,
|
|
225
|
-
policyScope: null,
|
|
226
|
-
unavailableDiagnostics: [],
|
|
227
|
-
});
|
|
228
|
-
leased = {
|
|
229
|
-
...selected,
|
|
230
|
-
accounts: leaseAccounts,
|
|
231
|
-
activeCredentialId: rotation?.activeCredentialId ?? null,
|
|
232
|
-
rotationEnabled: rotation?.rotationEnabled ?? false,
|
|
233
|
-
rotationStrategy: rotation?.rotationStrategy ?? "most_remaining",
|
|
234
|
-
reused: false,
|
|
235
|
-
holderId: null,
|
|
236
|
-
generation: null,
|
|
237
|
-
leasedUntil: null,
|
|
238
|
-
unavailableDiagnostics: [],
|
|
239
|
-
advanceActivePointer: selected.advanceActivePointer !== false,
|
|
240
|
-
};
|
|
241
|
-
}
|
|
165
|
+
leaseAcquisitionStartedAtMs = performance.now();
|
|
166
|
+
leased = await acquireCodexCredentialLease(
|
|
167
|
+
db,
|
|
168
|
+
{
|
|
169
|
+
accountId: input.accountId,
|
|
170
|
+
workspaceId: input.workspaceId,
|
|
171
|
+
sessionId: input.sessionId,
|
|
172
|
+
turnId,
|
|
173
|
+
attemptId: input.attemptId,
|
|
174
|
+
executionGeneration: attempt.executionGeneration,
|
|
175
|
+
workflowId: input.workflowId,
|
|
176
|
+
workflowRunId: input.workflowRunId,
|
|
177
|
+
dispatchId,
|
|
178
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
179
|
+
holderId,
|
|
180
|
+
advanceActivePointer: true,
|
|
181
|
+
},
|
|
182
|
+
selectForTurn,
|
|
183
|
+
);
|
|
242
184
|
}
|
|
185
|
+
const lockedSessionCodexState = leased.sessionCodexState;
|
|
186
|
+
providerTurn.codexPolicySnapshot = leased.codexPolicySnapshot;
|
|
187
|
+
const sessionPin = lockedSessionCodexState.pinnedCredentialId;
|
|
188
|
+
const sessionPinSource = lockedSessionCodexState.pinSource;
|
|
243
189
|
const rotationDecision = leased.decision;
|
|
244
190
|
const selectedPinDisposition = classifyCodexPin({
|
|
245
191
|
pinnedCredentialId: sessionPin,
|
|
@@ -252,6 +198,7 @@ export async function selectCodexTurnCapacity(
|
|
|
252
198
|
// movement for manual/policy homes; this write only records the NEXT turn's
|
|
253
199
|
// policy home (or clears a policy pin whose strategy is no longer active).
|
|
254
200
|
if (
|
|
201
|
+
!leased.codexPolicySnapshotReused &&
|
|
255
202
|
selectedPinDisposition === "sharded" &&
|
|
256
203
|
leased.credentialId !== null &&
|
|
257
204
|
(sessionPinSource !== "policy" || sessionPin !== leased.credentialId)
|
|
@@ -312,16 +259,8 @@ export async function selectCodexTurnCapacity(
|
|
|
312
259
|
pinMutation.wakeTargets,
|
|
313
260
|
);
|
|
314
261
|
}
|
|
315
|
-
if (
|
|
316
|
-
!settings.codexCredentialLeasingEnabled &&
|
|
317
|
-
leased.advanceActivePointer &&
|
|
318
|
-
sessionPin === null &&
|
|
319
|
-
rotationDecision.kind === "active" &&
|
|
320
|
-
rotationDecision.moved
|
|
321
|
-
) {
|
|
322
|
-
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
323
|
-
}
|
|
324
262
|
providerTurn.effectiveCodexCredentialId = leased.credentialId;
|
|
263
|
+
providerTurn.codexCredentialFailoverLimit = leased.failoverLimit;
|
|
325
264
|
leases.codex.generation = leased.generation;
|
|
326
265
|
leases.codex.confirmedUntilMs =
|
|
327
266
|
leased.leasedUntil && leaseAcquisitionStartedAtMs !== null
|
|
@@ -336,7 +275,7 @@ export async function selectCodexTurnCapacity(
|
|
|
336
275
|
|
|
337
276
|
const actualOutcome = providerTurn.effectiveCodexCredentialId
|
|
338
277
|
? "selected"
|
|
339
|
-
: rotationDecision.kind === "allCapped"
|
|
278
|
+
: rotationDecision.kind === "allCapped" || rotationDecision.kind === "allocatorDisabled"
|
|
340
279
|
? "waiting"
|
|
341
280
|
: "none";
|
|
342
281
|
const actualReason = providerTurn.effectiveCodexCredentialId
|
|
@@ -349,7 +288,9 @@ export async function selectCodexTurnCapacity(
|
|
|
349
288
|
: "active"
|
|
350
289
|
: rotationDecision.kind === "allCapped"
|
|
351
290
|
? "all_capped"
|
|
352
|
-
: "
|
|
291
|
+
: rotationDecision.kind === "allocatorDisabled"
|
|
292
|
+
? "allocator_disabled"
|
|
293
|
+
: "none";
|
|
353
294
|
const fencedInFlight = leased.reused;
|
|
354
295
|
const shadowResult = await publishCodexFleetShadowDecisionV1({
|
|
355
296
|
enabled: settings.codexFleetPolicyShadowEnabled,
|
|
@@ -360,7 +301,7 @@ export async function selectCodexTurnCapacity(
|
|
|
360
301
|
actualReason,
|
|
361
302
|
affinityCredentialId: fencedInFlight
|
|
362
303
|
? providerTurn.effectiveCodexCredentialId
|
|
363
|
-
: (sessionPin ??
|
|
304
|
+
: (sessionPin ?? lockedSessionCodexState.lastCredentialId ?? null),
|
|
364
305
|
fencedInFlight,
|
|
365
306
|
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
366
307
|
now: new Date(),
|
|
@@ -432,7 +373,8 @@ export async function selectCodexTurnCapacity(
|
|
|
432
373
|
if (
|
|
433
374
|
providerTurn.effectiveCodexCredentialId === null &&
|
|
434
375
|
leased.accounts.length > 0 &&
|
|
435
|
-
|
|
376
|
+
(rotationDecision.kind === "allocatorDisabled" ||
|
|
377
|
+
leased.accounts.every((account) => !account.allocatorEnabled)) &&
|
|
436
378
|
turnId
|
|
437
379
|
) {
|
|
438
380
|
if (turn.source === "compaction") {
|
|
@@ -463,68 +405,175 @@ export async function selectCodexTurnCapacity(
|
|
|
463
405
|
control.activityStatus = "idle";
|
|
464
406
|
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
465
407
|
}
|
|
466
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId)
|
|
408
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
467
409
|
const activeGoal = goal?.status === "active" ? goal : null;
|
|
468
|
-
const
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
410
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
411
|
+
{ db, bus },
|
|
412
|
+
{
|
|
413
|
+
accountId: input.accountId,
|
|
414
|
+
workspaceId: input.workspaceId,
|
|
415
|
+
sessionId: input.sessionId,
|
|
416
|
+
turnId,
|
|
417
|
+
attemptId: input.attemptId,
|
|
418
|
+
workflowId: input.workflowId,
|
|
419
|
+
goalId: activeGoal?.id ?? null,
|
|
420
|
+
goalVersion: activeGoal?.version ?? null,
|
|
421
|
+
earliestResetAt: null,
|
|
422
|
+
resetKind: "mutation_only",
|
|
423
|
+
failurePayload: {
|
|
424
|
+
error:
|
|
425
|
+
rotationDecision.kind === "allocatorDisabled"
|
|
426
|
+
? "The policy-selected Codex subscription is disabled for new allocations."
|
|
427
|
+
: "All connected Codex subscriptions are disabled for new allocations.",
|
|
428
|
+
code: "codex_allocator_disabled",
|
|
429
|
+
detail:
|
|
430
|
+
rotationDecision.kind === "allocatorDisabled"
|
|
431
|
+
? "waiting for the selected credential to be re-enabled under the accepted source policy"
|
|
432
|
+
: "waiting for a credential in the accepted source pool to be re-enabled, reconnected, or added",
|
|
433
|
+
},
|
|
483
434
|
},
|
|
484
|
-
|
|
485
|
-
if (
|
|
486
|
-
|
|
435
|
+
);
|
|
436
|
+
if (evaluated.action === "resumed") {
|
|
437
|
+
control.turnMetricOutcome = "recovering";
|
|
438
|
+
control.activityStatus = "recovering";
|
|
439
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
440
|
+
}
|
|
441
|
+
if (evaluated.action === "waiting") {
|
|
487
442
|
control.turnMetricOutcome = "recovering";
|
|
488
443
|
control.activityStatus = "waiting_capacity";
|
|
489
444
|
return {
|
|
490
445
|
exit: claimedResult({
|
|
491
446
|
status: "waiting_capacity",
|
|
492
447
|
capacityWait: {
|
|
493
|
-
waiterId:
|
|
494
|
-
generation:
|
|
495
|
-
nextCheckAt:
|
|
496
|
-
wakeRevision:
|
|
448
|
+
waiterId: evaluated.waiter.id,
|
|
449
|
+
generation: evaluated.waiter.generation,
|
|
450
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
451
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
497
452
|
},
|
|
498
453
|
}),
|
|
499
454
|
};
|
|
500
455
|
}
|
|
456
|
+
acknowledgeLostAttemptOwnership();
|
|
457
|
+
control.turnMetricOutcome = "cancelled";
|
|
458
|
+
control.activityStatus = "cancelled";
|
|
459
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
if (rotationDecision.kind === "none") {
|
|
501
463
|
if (
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
464
|
+
leased.poolAccountCount === 0 &&
|
|
465
|
+
!leased.codexPolicySnapshotReused &&
|
|
466
|
+
leased.codexPolicySnapshot?.source !== "disabled"
|
|
467
|
+
) {
|
|
468
|
+
// Preserve the established no-account behavior. The empty-string
|
|
469
|
+
// token resolver used by the later model path raises this same typed
|
|
470
|
+
// reconnect outcome, but capacity must not report success and defer
|
|
471
|
+
// the failure to an unguarded provider setup.
|
|
472
|
+
throw new CodexReloginRequired("No Codex subscription is connected for this workspace.");
|
|
473
|
+
}
|
|
474
|
+
const noneReason =
|
|
475
|
+
sessionPinSource === "manual" && sessionPin !== null
|
|
476
|
+
? {
|
|
477
|
+
code: "codex_manual_pin_unavailable",
|
|
478
|
+
error: "The pinned Codex subscription is unavailable for this turn.",
|
|
479
|
+
detail:
|
|
480
|
+
"waiting for the pinned account to reconnect or become allocatable under the accepted source policy",
|
|
481
|
+
}
|
|
482
|
+
: !leased.rotationEnabled && leased.activeCredentialId === null
|
|
483
|
+
? {
|
|
484
|
+
code: "codex_active_pointer_unavailable",
|
|
485
|
+
error: "The Codex active subscription pointer is unavailable for this turn.",
|
|
486
|
+
detail:
|
|
487
|
+
"waiting for the accepted active account to be restored or become allocatable",
|
|
488
|
+
}
|
|
489
|
+
: leased.poolAccountCount > leased.accounts.length
|
|
490
|
+
? {
|
|
491
|
+
code: "codex_policy_pool_unavailable",
|
|
492
|
+
error: "No Codex subscription matches the accepted account-selection policy.",
|
|
493
|
+
detail:
|
|
494
|
+
"waiting for a matching credential to reconnect or become allocatable under the accepted source policy",
|
|
495
|
+
}
|
|
496
|
+
: {
|
|
497
|
+
code: "codex_credential_unavailable",
|
|
498
|
+
error:
|
|
499
|
+
"No Codex subscription is currently available under the accepted policy.",
|
|
500
|
+
detail:
|
|
501
|
+
"waiting for a matching credential to reconnect or become allocatable under the accepted source policy",
|
|
502
|
+
};
|
|
503
|
+
if (turn.source === "compaction") {
|
|
504
|
+
if (
|
|
505
|
+
!(await eventing.settle!({
|
|
506
|
+
events: [
|
|
507
|
+
{
|
|
508
|
+
type: "turn.cancelled",
|
|
509
|
+
payload: {
|
|
510
|
+
maintenance: "context_compaction",
|
|
511
|
+
reason: noneReason.code,
|
|
512
|
+
requestPreserved: true,
|
|
513
|
+
},
|
|
511
514
|
},
|
|
515
|
+
{
|
|
516
|
+
type: "session.status.changed",
|
|
517
|
+
payload: { status: "idle" },
|
|
518
|
+
},
|
|
519
|
+
],
|
|
520
|
+
turnStatus: "cancelled",
|
|
521
|
+
sessionStatus: "idle",
|
|
522
|
+
activeTurnId: null,
|
|
523
|
+
}))
|
|
524
|
+
) {
|
|
525
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
526
|
+
}
|
|
527
|
+
control.turnMetricOutcome = "cancelled";
|
|
528
|
+
control.activityStatus = "idle";
|
|
529
|
+
return { exit: claimedResult({ status: "idle", deferredUntilWake: true }) };
|
|
530
|
+
}
|
|
531
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
532
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
533
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
534
|
+
{ db, bus },
|
|
535
|
+
{
|
|
536
|
+
accountId: input.accountId,
|
|
537
|
+
workspaceId: input.workspaceId,
|
|
538
|
+
sessionId: input.sessionId,
|
|
539
|
+
turnId,
|
|
540
|
+
attemptId: input.attemptId,
|
|
541
|
+
workflowId: input.workflowId,
|
|
542
|
+
goalId: activeGoal?.id ?? null,
|
|
543
|
+
goalVersion: activeGoal?.version ?? null,
|
|
544
|
+
earliestResetAt: null,
|
|
545
|
+
resetKind: "mutation_only",
|
|
546
|
+
failurePayload: {
|
|
547
|
+
error: noneReason.error,
|
|
548
|
+
code: noneReason.code,
|
|
549
|
+
detail: noneReason.detail,
|
|
550
|
+
},
|
|
551
|
+
},
|
|
552
|
+
);
|
|
553
|
+
if (evaluated.action === "resumed") {
|
|
554
|
+
control.turnMetricOutcome = "recovering";
|
|
555
|
+
control.activityStatus = "recovering";
|
|
556
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
557
|
+
}
|
|
558
|
+
if (evaluated.action === "waiting") {
|
|
559
|
+
control.turnMetricOutcome = "recovering";
|
|
560
|
+
control.activityStatus = "waiting_capacity";
|
|
561
|
+
return {
|
|
562
|
+
exit: claimedResult({
|
|
563
|
+
status: "waiting_capacity",
|
|
564
|
+
capacityWait: {
|
|
565
|
+
waiterId: evaluated.waiter.id,
|
|
566
|
+
generation: evaluated.waiter.generation,
|
|
567
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
568
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
512
569
|
},
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
payload: { status: "idle" },
|
|
516
|
-
},
|
|
517
|
-
],
|
|
518
|
-
turnStatus: "failed",
|
|
519
|
-
sessionStatus: "idle",
|
|
520
|
-
activeTurnId: null,
|
|
521
|
-
}))
|
|
522
|
-
) {
|
|
523
|
-
return { exit: claimedResult({ status: "cancelled" }) };
|
|
570
|
+
}),
|
|
571
|
+
};
|
|
524
572
|
}
|
|
525
|
-
|
|
526
|
-
control.
|
|
527
|
-
|
|
573
|
+
acknowledgeLostAttemptOwnership();
|
|
574
|
+
control.turnMetricOutcome = "cancelled";
|
|
575
|
+
control.activityStatus = "cancelled";
|
|
576
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
528
577
|
}
|
|
529
578
|
|
|
530
579
|
if (rotationDecision.kind === "allCapped" && turnId) {
|
|
@@ -561,7 +610,7 @@ export async function selectCodexTurnCapacity(
|
|
|
561
610
|
// build) until the EARLIEST reset across all accounts — the multi-account
|
|
562
611
|
// generalization of #143's single-account idle-until-reset. No saveRunState:
|
|
563
612
|
// no model ran, nothing to freeze.
|
|
564
|
-
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId)
|
|
613
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
565
614
|
const goalActive = Boolean(goal && goal.status === "active");
|
|
566
615
|
// BOUNDED + POSITIVE: clamp to [MIN_IDLE_MS, max] so a null/elapsed/unknown
|
|
567
616
|
// reset can never yield a 0 (which session.ts would treat as "continue now",
|
|
@@ -571,87 +620,62 @@ export async function selectCodexTurnCapacity(
|
|
|
571
620
|
new Date(),
|
|
572
621
|
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
573
622
|
);
|
|
623
|
+
const allConnectedAccountsWereConsidered =
|
|
624
|
+
leased.rotationEnabled && selectedPinDisposition !== "manual";
|
|
625
|
+
const capacityDetail = allConnectedAccountsWereConsidered
|
|
626
|
+
? "all connected Codex subscriptions are rate-limited"
|
|
627
|
+
: selectedPinDisposition === "manual"
|
|
628
|
+
? "the pinned Codex subscription is rate-limited"
|
|
629
|
+
: "the active Codex subscription is rate-limited";
|
|
574
630
|
const failurePayload = codexUsageLimitFailurePayload(
|
|
575
631
|
{ resetsInSeconds: Math.ceil(resumeMs / 1000) },
|
|
576
|
-
|
|
577
|
-
{ allAccounts:
|
|
632
|
+
capacityDetail,
|
|
633
|
+
{ allAccounts: allConnectedAccountsWereConsidered },
|
|
578
634
|
);
|
|
579
635
|
const authoritativeResetAt = authoritativeCodexCapacityResetAt(leased.accounts, new Date());
|
|
580
|
-
const
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
636
|
+
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
637
|
+
{ db, bus },
|
|
638
|
+
{
|
|
639
|
+
accountId: input.accountId,
|
|
640
|
+
workspaceId: input.workspaceId,
|
|
641
|
+
sessionId: input.sessionId,
|
|
642
|
+
turnId,
|
|
643
|
+
attemptId: input.attemptId,
|
|
644
|
+
workflowId: input.workflowId,
|
|
645
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
646
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
647
|
+
earliestResetAt: authoritativeResetAt,
|
|
648
|
+
resetKind: authoritativeResetAt ? "authoritative" : "bounded_refresh",
|
|
649
|
+
failurePayload,
|
|
650
|
+
},
|
|
651
|
+
);
|
|
652
|
+
if (evaluated.action === "resumed") {
|
|
653
|
+
control.turnMetricOutcome = "recovering";
|
|
654
|
+
control.activityStatus = "recovering";
|
|
655
|
+
return { exit: claimedResult({ status: "recovering" }) };
|
|
656
|
+
}
|
|
657
|
+
if (evaluated.action === "waiting") {
|
|
595
658
|
control.turnMetricOutcome = "recovering";
|
|
596
659
|
control.activityStatus = "waiting_capacity";
|
|
597
660
|
return {
|
|
598
661
|
exit: claimedResult({
|
|
599
662
|
status: "waiting_capacity",
|
|
600
663
|
capacityWait: {
|
|
601
|
-
waiterId:
|
|
602
|
-
generation:
|
|
603
|
-
nextCheckAt:
|
|
604
|
-
wakeRevision:
|
|
664
|
+
waiterId: evaluated.waiter.id,
|
|
665
|
+
generation: evaluated.waiter.generation,
|
|
666
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
667
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
605
668
|
},
|
|
606
669
|
}),
|
|
607
670
|
};
|
|
608
671
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
614
|
-
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
615
|
-
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
616
|
-
// the proactive path while the reactive path spares it.
|
|
617
|
-
{
|
|
618
|
-
type: "turn.failed",
|
|
619
|
-
payload: {
|
|
620
|
-
...failurePayload,
|
|
621
|
-
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
622
|
-
rotated: true,
|
|
623
|
-
},
|
|
624
|
-
},
|
|
625
|
-
{
|
|
626
|
-
type: "session.status.changed",
|
|
627
|
-
payload: { status: "idle" },
|
|
628
|
-
},
|
|
629
|
-
],
|
|
630
|
-
turnStatus: "failed",
|
|
631
|
-
sessionStatus: "idle",
|
|
632
|
-
activeTurnId: null,
|
|
633
|
-
}))
|
|
634
|
-
) {
|
|
635
|
-
return { exit: claimedResult({ status: "cancelled" }) };
|
|
636
|
-
}
|
|
637
|
-
control.turnMetricOutcome = "failed";
|
|
638
|
-
control.activityStatus = "idle";
|
|
639
|
-
// idleUntilReset marks this a MANDATORY hold: session.ts must wait the full
|
|
640
|
-
// resumeMs even if a future change made it 0 — never a tight re-dispatch.
|
|
641
|
-
return {
|
|
642
|
-
exit: claimedResult(
|
|
643
|
-
goalActive
|
|
644
|
-
? {
|
|
645
|
-
status: "idle",
|
|
646
|
-
continueDelayMs: resumeMs,
|
|
647
|
-
idleUntilReset: true,
|
|
648
|
-
}
|
|
649
|
-
: { status: "idle" },
|
|
650
|
-
),
|
|
651
|
-
};
|
|
672
|
+
acknowledgeLostAttemptOwnership();
|
|
673
|
+
control.turnMetricOutcome = "cancelled";
|
|
674
|
+
control.activityStatus = "cancelled";
|
|
675
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
652
676
|
}
|
|
653
677
|
if (providerTurn.effectiveCodexCredentialId) {
|
|
654
|
-
const priorAccountId =
|
|
678
|
+
const priorAccountId = lockedSessionCodexState.lastCredentialId;
|
|
655
679
|
if (priorAccountId !== providerTurn.effectiveCodexCredentialId) {
|
|
656
680
|
await recordSessionActiveCodexCredential(
|
|
657
681
|
db,
|
|
@@ -704,6 +728,59 @@ export async function selectCodexTurnCapacity(
|
|
|
704
728
|
}
|
|
705
729
|
} catch (error) {
|
|
706
730
|
credentialSelectionOutcome = "failed";
|
|
731
|
+
if (error instanceof CodexCredentialLeaseAttemptFencedError) {
|
|
732
|
+
acknowledgeLostAttemptOwnership();
|
|
733
|
+
control.activityStatus = "cancelled";
|
|
734
|
+
control.turnMetricOutcome = "cancelled";
|
|
735
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
736
|
+
}
|
|
737
|
+
if (error instanceof CodexCredentialFailoverExhaustedError) {
|
|
738
|
+
const settlement = await settleCodexCredentialFailover(db, {
|
|
739
|
+
accountId: input.accountId,
|
|
740
|
+
workspaceId: input.workspaceId,
|
|
741
|
+
sessionId: input.sessionId,
|
|
742
|
+
turnId,
|
|
743
|
+
attemptId: input.attemptId,
|
|
744
|
+
holderId: null,
|
|
745
|
+
generation: null,
|
|
746
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
747
|
+
maxFailovers: error.maxFailovers,
|
|
748
|
+
recoveryPayload: {
|
|
749
|
+
triggerEventId: attempt.triggerEventId!,
|
|
750
|
+
reason: "codex_credential_failover_exhausted",
|
|
751
|
+
},
|
|
752
|
+
failedPayload: {
|
|
753
|
+
error:
|
|
754
|
+
"Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
755
|
+
code: "codex_credential_failover_exhausted",
|
|
756
|
+
retryable: false,
|
|
757
|
+
recovery: "user_message",
|
|
758
|
+
failoverCount: error.failoverCount,
|
|
759
|
+
maxFailovers: error.maxFailovers,
|
|
760
|
+
},
|
|
761
|
+
});
|
|
762
|
+
if (settlement.action === "limit_exceeded") {
|
|
763
|
+
await publishDurableSessionEvents(
|
|
764
|
+
bus,
|
|
765
|
+
input.workspaceId,
|
|
766
|
+
input.sessionId,
|
|
767
|
+
settlement.events,
|
|
768
|
+
);
|
|
769
|
+
control.activityStatus = "idle";
|
|
770
|
+
control.turnMetricOutcome = "failed";
|
|
771
|
+
await deliverFailedChildTurnToParent(
|
|
772
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
773
|
+
input.workspaceId,
|
|
774
|
+
input.sessionId,
|
|
775
|
+
turnId,
|
|
776
|
+
);
|
|
777
|
+
return { exit: claimedResult({ status: "idle" }) };
|
|
778
|
+
}
|
|
779
|
+
acknowledgeLostAttemptOwnership();
|
|
780
|
+
control.activityStatus = "cancelled";
|
|
781
|
+
control.turnMetricOutcome = "cancelled";
|
|
782
|
+
return { exit: claimedResult({ status: "cancelled" }) };
|
|
783
|
+
}
|
|
707
784
|
throw error;
|
|
708
785
|
} finally {
|
|
709
786
|
recordTurnStartupPhase(observability, {
|