@opengeni/worker-bundle 0.20.3 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,1436 @@
|
|
|
1
|
+
import {
|
|
2
|
+
requestSessionTurnRecovery,
|
|
3
|
+
getSessionGoal,
|
|
4
|
+
armCodexCapacityWait,
|
|
5
|
+
armXaiCapacityWait,
|
|
6
|
+
reconcileXaiCapacityWait,
|
|
7
|
+
getCodexRotationSettings,
|
|
8
|
+
listCodexAccountStatuses,
|
|
9
|
+
getSessionCodexState,
|
|
10
|
+
setSessionCodexPinInTransaction,
|
|
11
|
+
quarantineCodexCredentialForLease,
|
|
12
|
+
setCodexCredentialExhaustedWithWakeTargets,
|
|
13
|
+
withSessionCodexCapacityMutation,
|
|
14
|
+
countConsecutiveReactiveRotations,
|
|
15
|
+
recordUsageEvent,
|
|
16
|
+
getActiveSessionHistoryItemsPaged,
|
|
17
|
+
settleCodexCredentialLeaseLoss,
|
|
18
|
+
settleCodexCredentialFailover,
|
|
19
|
+
readLease,
|
|
20
|
+
SandboxLeaseSupersededError,
|
|
21
|
+
SandboxLeaseTransitionError,
|
|
22
|
+
isSessionEventPersistenceError,
|
|
23
|
+
} from "@opengeni/db";
|
|
24
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
25
|
+
import { maxTurnsExceededRunState } from "@opengeni/runtime";
|
|
26
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
27
|
+
import {
|
|
28
|
+
authoritativeCodexCapacityResetAt,
|
|
29
|
+
chooseRotationActive,
|
|
30
|
+
classifyCodexPin,
|
|
31
|
+
computeIdleDelayMs,
|
|
32
|
+
computeReactiveRotationResume,
|
|
33
|
+
shardCredentialForSession,
|
|
34
|
+
earliestCodexReset,
|
|
35
|
+
type CodexRotationStrategy,
|
|
36
|
+
} from "../codex-rotation";
|
|
37
|
+
import { signalCodexCapacityWakeTargets } from "../codex-capacity";
|
|
38
|
+
import type { Settings } from "@opengeni/config";
|
|
39
|
+
import {
|
|
40
|
+
classifyCodexEncryptedArtifactRejection,
|
|
41
|
+
classifyCodexUsageLimitError,
|
|
42
|
+
isCodexTransportError,
|
|
43
|
+
type CodexUsageHeaderSnapshot,
|
|
44
|
+
} from "@opengeni/codex";
|
|
45
|
+
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
46
|
+
import { deliverFailedChildTurnToParent } from "../parent-wake";
|
|
47
|
+
import type {
|
|
48
|
+
TurnActivityServices as ActivityServices,
|
|
49
|
+
RunAgentTurnInput,
|
|
50
|
+
RunAgentTurnResult,
|
|
51
|
+
} from "../types";
|
|
52
|
+
import { createTurnCredentialLeases } from "./credential-leases";
|
|
53
|
+
import { createTurnHistorySink } from "./history-sink";
|
|
54
|
+
|
|
55
|
+
import { BudgetExhaustedError } from "./admission";
|
|
56
|
+
import {
|
|
57
|
+
providerRecoveryExhaustedFailure,
|
|
58
|
+
providerRecoveryResult,
|
|
59
|
+
providerRetryAfterMs,
|
|
60
|
+
escapedMcpTimeoutRecoveryFailure,
|
|
61
|
+
preClaimAdmissionFailure,
|
|
62
|
+
isWorkerShutdownCancellation,
|
|
63
|
+
sandboxRouteTransitionCode,
|
|
64
|
+
safeErrorDiagnostic,
|
|
65
|
+
classifyXaiCredentialFailure,
|
|
66
|
+
agentRunFailurePayload,
|
|
67
|
+
codexCredentialCooldownUntil,
|
|
68
|
+
classifyCodexCredentialFailure,
|
|
69
|
+
codexUsageLimitFailurePayload,
|
|
70
|
+
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
71
|
+
} from "./errors";
|
|
72
|
+
import { selectRejectedProviderArtifactHistoryIds } from "./history";
|
|
73
|
+
import { waitForTurnFinalizerStep, turnFinalizerCancellationSignal } from "./quiescence";
|
|
74
|
+
import {
|
|
75
|
+
SandboxDeadlineRotationError,
|
|
76
|
+
turnOperationCancellationFailure,
|
|
77
|
+
sandboxDeadlineRotationRecoveryDelayMs,
|
|
78
|
+
} from "./sandbox-provision";
|
|
79
|
+
import type {
|
|
80
|
+
AttemptIdentityState,
|
|
81
|
+
BillingState,
|
|
82
|
+
EventingState,
|
|
83
|
+
ProviderTurnState,
|
|
84
|
+
TurnControlState,
|
|
85
|
+
} from "./turn-context";
|
|
86
|
+
|
|
87
|
+
export type TurnFailureDeps = {
|
|
88
|
+
error: unknown;
|
|
89
|
+
input: RunAgentTurnInput;
|
|
90
|
+
settings: Settings;
|
|
91
|
+
db: ActivityServices["db"];
|
|
92
|
+
bus: ActivityServices["bus"];
|
|
93
|
+
observability: ActivityServices["observability"];
|
|
94
|
+
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
95
|
+
signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
|
|
96
|
+
cancellationSignal: AbortSignal | undefined;
|
|
97
|
+
sandboxRotationController: AbortController;
|
|
98
|
+
noteCancellationRequested: () => void;
|
|
99
|
+
codexWorkspaceKey: string;
|
|
100
|
+
control: TurnControlState;
|
|
101
|
+
attempt: AttemptIdentityState;
|
|
102
|
+
billingState: BillingState;
|
|
103
|
+
eventing: EventingState;
|
|
104
|
+
providerTurn: ProviderTurnState;
|
|
105
|
+
leases: ReturnType<typeof createTurnCredentialLeases>;
|
|
106
|
+
historySink: ReturnType<typeof createTurnHistorySink>;
|
|
107
|
+
claimedResult: (
|
|
108
|
+
result: Omit<
|
|
109
|
+
Extract<RunAgentTurnResult, { status: Exclude<RunAgentTurnResult["status"], "unclaimed"> }>,
|
|
110
|
+
"turnId" | "attemptId"
|
|
111
|
+
>,
|
|
112
|
+
) => RunAgentTurnResult;
|
|
113
|
+
flushRuntimeBatcher: () => Promise<void>;
|
|
114
|
+
acknowledgeLostAttemptOwnership: () => void;
|
|
115
|
+
acknowledgeRecoveryQuiescence: () => void;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
|
|
119
|
+
const {
|
|
120
|
+
error,
|
|
121
|
+
input,
|
|
122
|
+
settings,
|
|
123
|
+
db,
|
|
124
|
+
bus,
|
|
125
|
+
observability,
|
|
126
|
+
wakeSessionWorkflow,
|
|
127
|
+
signalCodexCapacityWorkflow,
|
|
128
|
+
cancellationSignal,
|
|
129
|
+
sandboxRotationController,
|
|
130
|
+
noteCancellationRequested,
|
|
131
|
+
codexWorkspaceKey,
|
|
132
|
+
control,
|
|
133
|
+
attempt,
|
|
134
|
+
billingState,
|
|
135
|
+
eventing,
|
|
136
|
+
providerTurn,
|
|
137
|
+
leases,
|
|
138
|
+
historySink,
|
|
139
|
+
claimedResult,
|
|
140
|
+
flushRuntimeBatcher,
|
|
141
|
+
acknowledgeLostAttemptOwnership,
|
|
142
|
+
acknowledgeRecoveryQuiescence,
|
|
143
|
+
} = deps;
|
|
144
|
+
// Graceful worker shutdown (deploy / rollout restart): checkpoint the
|
|
145
|
+
// same current inference for a new fenced attempt instead of failing the
|
|
146
|
+
// session. Conversation truth is already persisted per model response;
|
|
147
|
+
// the final reconcile bounds loss to the one in-flight model step.
|
|
148
|
+
//
|
|
149
|
+
// The branch deliberately does NOT require turn.started to have been
|
|
150
|
+
// published: a shutdown landing during setup (claim/billing, before the
|
|
151
|
+
// turn visibly started) must also recover, not fail the session. In that
|
|
152
|
+
// early case nothing ran, so the new attempt uses the original trigger.
|
|
153
|
+
// The turn id falls
|
|
154
|
+
// back to the workflow-claimed turn when the local lookup had not
|
|
155
|
+
// finished yet.
|
|
156
|
+
const recoveryTurnId = attempt.turnId;
|
|
157
|
+
// A true epoch supersession and a provider lifecycle transition are both
|
|
158
|
+
// recoverable control-plane states, never session failures. The latter is
|
|
159
|
+
// paced briefly; the next acquire waits on the durable claim and resumes
|
|
160
|
+
// immediately when it clears.
|
|
161
|
+
if (
|
|
162
|
+
(error instanceof SandboxLeaseSupersededError ||
|
|
163
|
+
error instanceof SandboxLeaseTransitionError) &&
|
|
164
|
+
recoveryTurnId
|
|
165
|
+
) {
|
|
166
|
+
try {
|
|
167
|
+
const fencedLease = await readLease(db, input.workspaceId, error.sandboxGroupId).catch(
|
|
168
|
+
() => null,
|
|
169
|
+
);
|
|
170
|
+
const lifecycleTransition = error instanceof SandboxLeaseTransitionError;
|
|
171
|
+
const rotationPending =
|
|
172
|
+
fencedLease?.rotationRequestedAt != null ||
|
|
173
|
+
(lifecycleTransition && error.reason === "rotation_in_progress");
|
|
174
|
+
const transitionPending = lifecycleTransition || rotationPending;
|
|
175
|
+
const deadlineRotationPending =
|
|
176
|
+
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
177
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
178
|
+
sessionId: input.sessionId,
|
|
179
|
+
turnId: recoveryTurnId,
|
|
180
|
+
triggerEventId: attempt.triggerEventId!,
|
|
181
|
+
attemptId: input.attemptId,
|
|
182
|
+
reason: deadlineRotationPending
|
|
183
|
+
? "sandbox_deadline_rotation"
|
|
184
|
+
: lifecycleTransition
|
|
185
|
+
? "sandbox_lifecycle_transition"
|
|
186
|
+
: "sandbox_lease_superseded",
|
|
187
|
+
...(transitionPending
|
|
188
|
+
? {
|
|
189
|
+
detail: {
|
|
190
|
+
sandboxGroupId: error.sandboxGroupId,
|
|
191
|
+
leaseEpoch: error.leaseEpoch,
|
|
192
|
+
...(rotationPending
|
|
193
|
+
? {
|
|
194
|
+
rotationReason: fencedLease?.rotationReason ?? "operator",
|
|
195
|
+
}
|
|
196
|
+
: {}),
|
|
197
|
+
...(lifecycleTransition ? { transitionReason: error.reason } : {}),
|
|
198
|
+
},
|
|
199
|
+
}
|
|
200
|
+
: {}),
|
|
201
|
+
});
|
|
202
|
+
if (recovery.action === "stale") {
|
|
203
|
+
acknowledgeLostAttemptOwnership();
|
|
204
|
+
control.activityStatus = "cancelled";
|
|
205
|
+
control.turnMetricOutcome = "cancelled";
|
|
206
|
+
return claimedResult({ status: "cancelled" });
|
|
207
|
+
}
|
|
208
|
+
acknowledgeRecoveryQuiescence();
|
|
209
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
210
|
+
control.activityStatus = "recovering";
|
|
211
|
+
control.turnMetricOutcome = "recovering";
|
|
212
|
+
return claimedResult({
|
|
213
|
+
status: "recovering",
|
|
214
|
+
...(transitionPending
|
|
215
|
+
? {
|
|
216
|
+
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
217
|
+
}
|
|
218
|
+
: {}),
|
|
219
|
+
});
|
|
220
|
+
} catch (recoveryError) {
|
|
221
|
+
console.error("sandbox lifecycle recovery failed", safeErrorDiagnostic(recoveryError));
|
|
222
|
+
throw recoveryError;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// A managed-home session can start directly on a Connected Machine without
|
|
226
|
+
// creating or leasing its cloud home. If sandbox_attach/sandbox_swap then
|
|
227
|
+
// clears the durable pointer to home, the commit is valid but this exact
|
|
228
|
+
// attempt cannot serve a later home operation. Preserve the completed attach
|
|
229
|
+
// and every preceding model/tool receipt, close only the unresolved suffix,
|
|
230
|
+
// and continue the SAME logical turn in a fresh attempt. That next attempt
|
|
231
|
+
// starts from the now-null pointer and establishes home normally.
|
|
232
|
+
const routeTransitionCode = sandboxRouteTransitionCode(error);
|
|
233
|
+
if (routeTransitionCode && recoveryTurnId && eventing.publish && eventing.turnStartedPublished) {
|
|
234
|
+
try {
|
|
235
|
+
await flushRuntimeBatcher();
|
|
236
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
237
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
238
|
+
sessionId: input.sessionId,
|
|
239
|
+
turnId: recoveryTurnId,
|
|
240
|
+
triggerEventId: attempt.triggerEventId!,
|
|
241
|
+
attemptId: input.attemptId,
|
|
242
|
+
reason: "sandbox_route_transition",
|
|
243
|
+
detail: {
|
|
244
|
+
code: routeTransitionCode,
|
|
245
|
+
effectiveBoundary: "next_attempt",
|
|
246
|
+
},
|
|
247
|
+
});
|
|
248
|
+
if (recovery.action === "stale") {
|
|
249
|
+
acknowledgeLostAttemptOwnership();
|
|
250
|
+
control.activityStatus = "cancelled";
|
|
251
|
+
control.turnMetricOutcome = "cancelled";
|
|
252
|
+
return claimedResult({ status: "cancelled" });
|
|
253
|
+
}
|
|
254
|
+
if (recovery.action !== "recovering") {
|
|
255
|
+
throw new Error("Home sandbox route transition could not recover the current turn");
|
|
256
|
+
}
|
|
257
|
+
acknowledgeRecoveryQuiescence();
|
|
258
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
259
|
+
control.activityStatus = "recovering";
|
|
260
|
+
control.turnMetricOutcome = "recovering";
|
|
261
|
+
control.activityError = error;
|
|
262
|
+
return claimedResult({ status: "recovering" });
|
|
263
|
+
} catch (recoveryError) {
|
|
264
|
+
console.error("sandbox route-transition recovery failed", safeErrorDiagnostic(recoveryError));
|
|
265
|
+
throw recoveryError;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
if (
|
|
269
|
+
sandboxRotationController.signal.aborted &&
|
|
270
|
+
sandboxRotationController.signal.reason instanceof SandboxDeadlineRotationError &&
|
|
271
|
+
!cancellationSignal?.aborted &&
|
|
272
|
+
recoveryTurnId
|
|
273
|
+
) {
|
|
274
|
+
try {
|
|
275
|
+
await flushRuntimeBatcher();
|
|
276
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
277
|
+
const rotation = sandboxRotationController.signal.reason;
|
|
278
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
279
|
+
sessionId: input.sessionId,
|
|
280
|
+
turnId: recoveryTurnId,
|
|
281
|
+
triggerEventId: attempt.triggerEventId!,
|
|
282
|
+
attemptId: input.attemptId,
|
|
283
|
+
reason: "sandbox_deadline_rotation",
|
|
284
|
+
detail: {
|
|
285
|
+
sandboxGroupId: rotation.sandboxGroupId,
|
|
286
|
+
leaseEpoch: rotation.leaseEpoch,
|
|
287
|
+
},
|
|
288
|
+
});
|
|
289
|
+
if (recovery.action === "stale") {
|
|
290
|
+
acknowledgeLostAttemptOwnership();
|
|
291
|
+
control.activityStatus = "cancelled";
|
|
292
|
+
control.turnMetricOutcome = "cancelled";
|
|
293
|
+
return claimedResult({ status: "cancelled" });
|
|
294
|
+
}
|
|
295
|
+
acknowledgeRecoveryQuiescence();
|
|
296
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
297
|
+
control.activityStatus = "recovering";
|
|
298
|
+
control.turnMetricOutcome = "recovering";
|
|
299
|
+
return claimedResult({
|
|
300
|
+
status: "recovering",
|
|
301
|
+
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
302
|
+
});
|
|
303
|
+
} catch (recoveryError) {
|
|
304
|
+
console.error(
|
|
305
|
+
"sandbox deadline rotation recovery failed",
|
|
306
|
+
safeErrorDiagnostic(recoveryError),
|
|
307
|
+
);
|
|
308
|
+
throw recoveryError;
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const cancellationFailure = turnOperationCancellationFailure(error);
|
|
312
|
+
if (cancellationFailure && isWorkerShutdownCancellation(cancellationFailure) && recoveryTurnId) {
|
|
313
|
+
try {
|
|
314
|
+
await flushRuntimeBatcher();
|
|
315
|
+
await historySink.reconcileConversationTruth();
|
|
316
|
+
// An approval-decision rerun always replays its original trigger. The
|
|
317
|
+
// decision is applied through the exact durable open-suffix receipt and
|
|
318
|
+
// its paired history, so swapping the trigger for a resume notice could
|
|
319
|
+
// drop the user's decision. Re-applying an already-consumed approval
|
|
320
|
+
// re-enters at most the single approved step. Every approval-gated MCP
|
|
321
|
+
// action crosses the durable execution-admission fence before provider
|
|
322
|
+
// invocation, so a consumed step resumes as already-executed or
|
|
323
|
+
// outcome-unknown rather than calling MCP again.
|
|
324
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
325
|
+
sessionId: input.sessionId,
|
|
326
|
+
turnId: recoveryTurnId,
|
|
327
|
+
triggerEventId: attempt.triggerEventId!,
|
|
328
|
+
attemptId: input.attemptId,
|
|
329
|
+
reason: "worker_shutdown",
|
|
330
|
+
});
|
|
331
|
+
if (recovery.action === "stale") {
|
|
332
|
+
acknowledgeLostAttemptOwnership();
|
|
333
|
+
control.activityStatus = "cancelled";
|
|
334
|
+
control.turnMetricOutcome = "cancelled";
|
|
335
|
+
return claimedResult({ status: "cancelled" });
|
|
336
|
+
}
|
|
337
|
+
acknowledgeRecoveryQuiescence();
|
|
338
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
339
|
+
control.activityStatus = "recovering";
|
|
340
|
+
control.turnMetricOutcome = "recovering";
|
|
341
|
+
return claimedResult({ status: "recovering" });
|
|
342
|
+
} catch (recoveryError) {
|
|
343
|
+
// The database transition is atomic. If it could not commit, surface
|
|
344
|
+
// the failure so Temporal can retry on a healthy worker; never mutate
|
|
345
|
+
// the turn through a second cancellation path.
|
|
346
|
+
console.error(
|
|
347
|
+
"worker-shutdown recovery checkpoint failed",
|
|
348
|
+
safeErrorDiagnostic(recoveryError),
|
|
349
|
+
);
|
|
350
|
+
throw recoveryError;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
if (error instanceof TurnAttemptFencedError) {
|
|
354
|
+
control.activityStatus = "cancelled";
|
|
355
|
+
control.activityError = error;
|
|
356
|
+
control.acknowledgeQuiescence = true;
|
|
357
|
+
noteCancellationRequested();
|
|
358
|
+
await waitForTurnFinalizerStep(
|
|
359
|
+
flushRuntimeBatcher(),
|
|
360
|
+
turnFinalizerCancellationSignal(cancellationSignal, control.activityStatus),
|
|
361
|
+
);
|
|
362
|
+
// Ownership already moved to a newer attempt or an authoritative
|
|
363
|
+
// control transaction. Surface the exact transport cancellation rather
|
|
364
|
+
// than a normal result. Temporal terminalization remains diagnostic
|
|
365
|
+
// only; replacement admission waits for the activity-owned durable
|
|
366
|
+
// quiescence receipt written from the hard tool fence below.
|
|
367
|
+
control.turnMetricOutcome = "cancelled";
|
|
368
|
+
throw new CancelledFailure("TURN_ATTEMPT_FENCED", [], error);
|
|
369
|
+
}
|
|
370
|
+
if (cancellationFailure) {
|
|
371
|
+
control.activityStatus = "cancelled";
|
|
372
|
+
control.activityError = error;
|
|
373
|
+
control.acknowledgeQuiescence = true;
|
|
374
|
+
noteCancellationRequested();
|
|
375
|
+
await waitForTurnFinalizerStep(
|
|
376
|
+
flushRuntimeBatcher(),
|
|
377
|
+
turnFinalizerCancellationSignal(cancellationSignal, control.activityStatus),
|
|
378
|
+
);
|
|
379
|
+
// The workflow owns cancellation settlement: Pause/Steer controls use
|
|
380
|
+
// settleSessionControl, and heartbeat timeouts use worker-death
|
|
381
|
+
// recovery. A dying activity must never append a
|
|
382
|
+
// competing cancellation or mutate the turn/session on its own.
|
|
383
|
+
control.turnMetricOutcome = "cancelled";
|
|
384
|
+
throw cancellationFailure;
|
|
385
|
+
}
|
|
386
|
+
// The SDK's per-segment turn cap is a pacing valve, not a failure: end
|
|
387
|
+
// the turn gracefully and idle the session so an active goal continues
|
|
388
|
+
// via a synthesized continuation turn (or a user message resumes work).
|
|
389
|
+
// The run state captured at the cap keeps full conversation context for
|
|
390
|
+
// that resumption.
|
|
391
|
+
const maxTurns = maxTurnsExceededRunState(error);
|
|
392
|
+
if (maxTurns && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
393
|
+
await flushRuntimeBatcher();
|
|
394
|
+
// The SDK attaches the run state at the throw site; persisting it lets
|
|
395
|
+
// the continuation resume with this segment's full context. If capture
|
|
396
|
+
// ever fails, the continuation falls back to the previous snapshot --
|
|
397
|
+
// degraded context, flagged on the event, but still strictly better
|
|
398
|
+
// than a terminal failed session: the sandbox filesystem state
|
|
399
|
+
// persists independently and the agent re-derives from it.
|
|
400
|
+
await historySink.reconcileConversationTruth();
|
|
401
|
+
if (
|
|
402
|
+
!(await eventing.settle!({
|
|
403
|
+
events: [
|
|
404
|
+
{
|
|
405
|
+
type: "turn.completed",
|
|
406
|
+
payload: { output: "", segmentLimit: "max_turns" },
|
|
407
|
+
},
|
|
408
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
409
|
+
],
|
|
410
|
+
turnStatus: "completed",
|
|
411
|
+
sessionStatus: "idle",
|
|
412
|
+
activeTurnId: null,
|
|
413
|
+
}))
|
|
414
|
+
) {
|
|
415
|
+
return claimedResult({ status: "cancelled" });
|
|
416
|
+
}
|
|
417
|
+
control.turnMetricOutcome = "completed";
|
|
418
|
+
await recordUsageEvent(db, {
|
|
419
|
+
accountId: input.accountId,
|
|
420
|
+
workspaceId: input.workspaceId,
|
|
421
|
+
eventType: "agent_run.completed",
|
|
422
|
+
quantity: 1,
|
|
423
|
+
unit: "run",
|
|
424
|
+
sourceResourceType: "session_turn",
|
|
425
|
+
sourceResourceId: attempt.turnId,
|
|
426
|
+
sessionId: input.sessionId,
|
|
427
|
+
turnId: attempt.turnId,
|
|
428
|
+
turnAttemptId: input.attemptId,
|
|
429
|
+
idempotencyKey: `usage:agent_run.completed:${attempt.turnId}`,
|
|
430
|
+
});
|
|
431
|
+
control.activityStatus = "idle";
|
|
432
|
+
return claimedResult({ status: "idle" });
|
|
433
|
+
}
|
|
434
|
+
const settleLostCodexAttempt = async (
|
|
435
|
+
lostTurnId: string,
|
|
436
|
+
holderId: string,
|
|
437
|
+
generation: number,
|
|
438
|
+
historyCheckpointDurable = false,
|
|
439
|
+
): Promise<RunAgentTurnResult> => {
|
|
440
|
+
let checkpointDurable = historyCheckpointDurable;
|
|
441
|
+
try {
|
|
442
|
+
if (!historyCheckpointDurable) {
|
|
443
|
+
await flushRuntimeBatcher();
|
|
444
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
445
|
+
}
|
|
446
|
+
checkpointDurable = true;
|
|
447
|
+
} catch {
|
|
448
|
+
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
449
|
+
errorClass: "CodexCheckpointOperationError",
|
|
450
|
+
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
451
|
+
origin: "worker",
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const settlement = await settleCodexCredentialLeaseLoss(db, {
|
|
456
|
+
accountId: input.accountId,
|
|
457
|
+
workspaceId: input.workspaceId,
|
|
458
|
+
sessionId: input.sessionId,
|
|
459
|
+
turnId: lostTurnId,
|
|
460
|
+
attemptId: input.attemptId,
|
|
461
|
+
holderId,
|
|
462
|
+
generation,
|
|
463
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
464
|
+
checkpointDurable,
|
|
465
|
+
recoveryPayload: {
|
|
466
|
+
triggerEventId: attempt.triggerEventId!,
|
|
467
|
+
reason: "codex_lease_lost",
|
|
468
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
469
|
+
},
|
|
470
|
+
failedPayload: {
|
|
471
|
+
error:
|
|
472
|
+
"The Codex credential lease was lost and the latest conversation checkpoint could not be persisted. Automatic replay was refused.",
|
|
473
|
+
code: "codex_lease_checkpoint_failed",
|
|
474
|
+
retryable: false,
|
|
475
|
+
},
|
|
476
|
+
});
|
|
477
|
+
leases.codex.held = false;
|
|
478
|
+
observability.incrementCounter({
|
|
479
|
+
name: "opengeni_codex_lease_loss_settlements_total",
|
|
480
|
+
help: "Fenced Codex lease-loss settlements by outcome.",
|
|
481
|
+
labels: {
|
|
482
|
+
workspace_key: codexWorkspaceKey,
|
|
483
|
+
outcome: settlement.action,
|
|
484
|
+
},
|
|
485
|
+
});
|
|
486
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, settlement.events);
|
|
487
|
+
control.activityError = error;
|
|
488
|
+
if (settlement.action === "failed") {
|
|
489
|
+
control.activityStatus = "failed";
|
|
490
|
+
control.turnMetricOutcome = "failed";
|
|
491
|
+
await deliverFailedChildTurnToParent(
|
|
492
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
493
|
+
input.workspaceId,
|
|
494
|
+
input.sessionId,
|
|
495
|
+
lostTurnId,
|
|
496
|
+
);
|
|
497
|
+
return claimedResult({ status: "failed" });
|
|
498
|
+
}
|
|
499
|
+
control.activityStatus = "recovering";
|
|
500
|
+
control.turnMetricOutcome = "recovering";
|
|
501
|
+
return claimedResult({ status: "recovering" });
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
// A missing/expired/superseded lease is an execution-ownership failure,
|
|
505
|
+
// not a provider failure. Settle it before credential quarantine or the
|
|
506
|
+
// generic terminal path: the DB transaction marks a still-current turn
|
|
507
|
+
// recoverable, but a successor attempt or worker recovery makes this activity
|
|
508
|
+
// stale and unable to clobber the shared turn/session.
|
|
509
|
+
if (
|
|
510
|
+
leases.codex.lost &&
|
|
511
|
+
settings.codexCredentialLeasingEnabled &&
|
|
512
|
+
billingState.isCodexTurn &&
|
|
513
|
+
eventing.publish &&
|
|
514
|
+
attempt.turnId &&
|
|
515
|
+
eventing.turnStartedPublished &&
|
|
516
|
+
leases.codex.holderId &&
|
|
517
|
+
leases.codex.generation !== null
|
|
518
|
+
) {
|
|
519
|
+
return await settleLostCodexAttempt(
|
|
520
|
+
attempt.turnId,
|
|
521
|
+
leases.codex.holderId,
|
|
522
|
+
leases.codex.generation,
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
if (
|
|
526
|
+
leases.xai.lost &&
|
|
527
|
+
billingState.isXaiTurn &&
|
|
528
|
+
eventing.publish &&
|
|
529
|
+
attempt.turnId &&
|
|
530
|
+
eventing.turnStartedPublished
|
|
531
|
+
) {
|
|
532
|
+
await flushRuntimeBatcher();
|
|
533
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
534
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
535
|
+
sessionId: input.sessionId,
|
|
536
|
+
turnId: attempt.turnId,
|
|
537
|
+
triggerEventId: attempt.triggerEventId!,
|
|
538
|
+
attemptId: input.attemptId,
|
|
539
|
+
reason: "xai_lease_lost",
|
|
540
|
+
detail: { provider: "supergrok-subscription" },
|
|
541
|
+
});
|
|
542
|
+
if (recovery.action === "stale") {
|
|
543
|
+
acknowledgeLostAttemptOwnership();
|
|
544
|
+
control.activityStatus = "cancelled";
|
|
545
|
+
control.turnMetricOutcome = "cancelled";
|
|
546
|
+
return claimedResult({ status: "cancelled" });
|
|
547
|
+
}
|
|
548
|
+
acknowledgeRecoveryQuiescence();
|
|
549
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
550
|
+
control.activityStatus = "recovering";
|
|
551
|
+
control.turnMetricOutcome = "recovering";
|
|
552
|
+
return claimedResult({ status: "recovering" });
|
|
553
|
+
}
|
|
554
|
+
// Definitive Codex credential/account refusals are the only provider
|
|
555
|
+
// errors that may walk the pool. This is an explicit checkpoint + SAME
|
|
556
|
+
// turn recovery, never an SDK/Temporal blind retry. A network break,
|
|
557
|
+
// malformed/partial 200 stream, invalid content, prompt 4xx, or provider
|
|
558
|
+
// 5xx does not classify here and therefore cannot consume another
|
|
559
|
+
// subscription or duplicate a side effect.
|
|
560
|
+
const codexCredentialFailure =
|
|
561
|
+
settings.codexCredentialLeasingEnabled &&
|
|
562
|
+
billingState.isCodexTurn &&
|
|
563
|
+
providerTurn.effectiveCodexCredentialId
|
|
564
|
+
? classifyCodexCredentialFailure(error)
|
|
565
|
+
: null;
|
|
566
|
+
if (
|
|
567
|
+
codexCredentialFailure &&
|
|
568
|
+
providerTurn.effectiveCodexCredentialId &&
|
|
569
|
+
eventing.publish &&
|
|
570
|
+
attempt.turnId &&
|
|
571
|
+
eventing.turnStartedPublished
|
|
572
|
+
) {
|
|
573
|
+
observability.incrementCounter({
|
|
574
|
+
name: "opengeni_codex_credential_failures_total",
|
|
575
|
+
help: "Definitive Codex credential failures classified for safe failover.",
|
|
576
|
+
labels: {
|
|
577
|
+
workspace_key: codexWorkspaceKey,
|
|
578
|
+
kind: codexCredentialFailure.kind,
|
|
579
|
+
outcome: "classified",
|
|
580
|
+
},
|
|
581
|
+
});
|
|
582
|
+
const failoverStartedAt = performance.now();
|
|
583
|
+
let checkpointDurable = false;
|
|
584
|
+
try {
|
|
585
|
+
await flushRuntimeBatcher();
|
|
586
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
587
|
+
checkpointDurable = true;
|
|
588
|
+
} catch {
|
|
589
|
+
observability.incrementCounter({
|
|
590
|
+
name: "opengeni_codex_failover_checkpoints_total",
|
|
591
|
+
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
592
|
+
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
|
|
593
|
+
});
|
|
594
|
+
observability.warn("Codex failover checkpoint failed; refusing automatic replay", {
|
|
595
|
+
errorClass: "CodexCheckpointOperationError",
|
|
596
|
+
errorCode: "codex_failover_checkpoint_failed",
|
|
597
|
+
origin: "worker",
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
if (checkpointDurable) {
|
|
602
|
+
observability.incrementCounter({
|
|
603
|
+
name: "opengeni_codex_failover_checkpoints_total",
|
|
604
|
+
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
605
|
+
labels: { workspace_key: codexWorkspaceKey, outcome: "completed" },
|
|
606
|
+
});
|
|
607
|
+
const now = new Date();
|
|
608
|
+
const before = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
|
|
609
|
+
const servingCached = before.find(
|
|
610
|
+
(account) => account.id === providerTurn.effectiveCodexCredentialId,
|
|
611
|
+
);
|
|
612
|
+
const usageSnapshot = providerTurn.latestCodexUsage as CodexUsageHeaderSnapshot | null;
|
|
613
|
+
const serving = servingCached
|
|
614
|
+
? {
|
|
615
|
+
...servingCached,
|
|
616
|
+
...(usageSnapshot
|
|
617
|
+
? {
|
|
618
|
+
primaryUsedPercent: usageSnapshot.primaryUsedPercent,
|
|
619
|
+
primaryResetAt: usageSnapshot.primaryResetAt,
|
|
620
|
+
secondaryUsedPercent: usageSnapshot.secondaryUsedPercent,
|
|
621
|
+
secondaryResetAt: usageSnapshot.secondaryResetAt,
|
|
622
|
+
}
|
|
623
|
+
: {}),
|
|
624
|
+
}
|
|
625
|
+
: null;
|
|
626
|
+
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
627
|
+
const statePersisted =
|
|
628
|
+
leases.codex.holderId && leases.codex.generation !== null
|
|
629
|
+
? await quarantineCodexCredentialForLease(db, {
|
|
630
|
+
accountId: input.accountId,
|
|
631
|
+
workspaceId: input.workspaceId,
|
|
632
|
+
turnId: attempt.turnId,
|
|
633
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
634
|
+
holderId: leases.codex.holderId,
|
|
635
|
+
generation: leases.codex.generation,
|
|
636
|
+
quarantine:
|
|
637
|
+
codexCredentialFailure.kind === "auth"
|
|
638
|
+
? {
|
|
639
|
+
kind: "status",
|
|
640
|
+
status: "needs_relogin",
|
|
641
|
+
lastError: "model request remained unauthorized after refresh",
|
|
642
|
+
}
|
|
643
|
+
: codexCredentialFailure.kind === "forbidden"
|
|
644
|
+
? {
|
|
645
|
+
kind: "status",
|
|
646
|
+
status: "error",
|
|
647
|
+
lastError: "model request was forbidden for this credential",
|
|
648
|
+
}
|
|
649
|
+
: { kind: "cooldown", until: cooldownUntil! },
|
|
650
|
+
})
|
|
651
|
+
: false;
|
|
652
|
+
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
653
|
+
leases.codex.lost = true;
|
|
654
|
+
return await settleLostCodexAttempt(
|
|
655
|
+
attempt.turnId,
|
|
656
|
+
leases.codex.holderId,
|
|
657
|
+
leases.codex.generation,
|
|
658
|
+
true,
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
const [rotation, accounts] = await Promise.all([
|
|
662
|
+
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
663
|
+
listCodexAccountStatuses(db, input.workspaceId).catch(() => []),
|
|
664
|
+
]);
|
|
665
|
+
const decision = rotation
|
|
666
|
+
? chooseRotationActive({
|
|
667
|
+
rotationStrategy: rotation.rotationStrategy as CodexRotationStrategy,
|
|
668
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
669
|
+
priorCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
670
|
+
accounts,
|
|
671
|
+
now: new Date(),
|
|
672
|
+
})
|
|
673
|
+
: ({ kind: "none" } as const);
|
|
674
|
+
const candidateAvailable =
|
|
675
|
+
statePersisted &&
|
|
676
|
+
Boolean(rotation?.rotationEnabled && rotation?.leaseRotationEnabled) &&
|
|
677
|
+
decision.kind === "active" &&
|
|
678
|
+
decision.credentialId !== providerTurn.effectiveCodexCredentialId;
|
|
679
|
+
|
|
680
|
+
if (candidateAvailable && leases.codex.holderId && leases.codex.generation !== null) {
|
|
681
|
+
const settlement = await settleCodexCredentialFailover(db, {
|
|
682
|
+
accountId: input.accountId,
|
|
683
|
+
workspaceId: input.workspaceId,
|
|
684
|
+
sessionId: input.sessionId,
|
|
685
|
+
turnId: attempt.turnId,
|
|
686
|
+
attemptId: input.attemptId,
|
|
687
|
+
holderId: leases.codex.holderId,
|
|
688
|
+
generation: leases.codex.generation,
|
|
689
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
690
|
+
maxFailovers: Math.max(1, accounts.length),
|
|
691
|
+
recoveryPayload: {
|
|
692
|
+
triggerEventId: attempt.triggerEventId!,
|
|
693
|
+
reason: "codex_credential_failover",
|
|
694
|
+
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
695
|
+
failureKind: codexCredentialFailure.kind,
|
|
696
|
+
...(cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}),
|
|
697
|
+
},
|
|
698
|
+
});
|
|
699
|
+
observability.incrementCounter({
|
|
700
|
+
name: "opengeni_codex_failover_settlements_total",
|
|
701
|
+
help: "Atomic Codex failover settlements by outcome.",
|
|
702
|
+
labels: {
|
|
703
|
+
workspace_key: codexWorkspaceKey,
|
|
704
|
+
outcome: settlement.action,
|
|
705
|
+
},
|
|
706
|
+
});
|
|
707
|
+
if (settlement.action === "recovering") {
|
|
708
|
+
leases.codex.held = false;
|
|
709
|
+
await publishDurableSessionEvents(
|
|
710
|
+
bus,
|
|
711
|
+
input.workspaceId,
|
|
712
|
+
input.sessionId,
|
|
713
|
+
settlement.events,
|
|
714
|
+
);
|
|
715
|
+
observability.observeHistogram({
|
|
716
|
+
name: "opengeni_codex_failover_recovery_seconds",
|
|
717
|
+
help: "Time from credential refusal to durable same-turn recovery.",
|
|
718
|
+
labels: {
|
|
719
|
+
workspace_key: codexWorkspaceKey,
|
|
720
|
+
kind: codexCredentialFailure.kind,
|
|
721
|
+
},
|
|
722
|
+
value: Math.max(0, (performance.now() - failoverStartedAt) / 1000),
|
|
723
|
+
});
|
|
724
|
+
control.activityStatus = "recovering";
|
|
725
|
+
control.turnMetricOutcome = "recovering";
|
|
726
|
+
return claimedResult({ status: "recovering" });
|
|
727
|
+
}
|
|
728
|
+
if (settlement.action === "stale") {
|
|
729
|
+
// One transaction proves both exact-holder recovery (including a
|
|
730
|
+
// just-expired or reaped lease row) and successor/control-gate
|
|
731
|
+
// rejection. Cross the hard tool fence so a control-gate loss can
|
|
732
|
+
// write its quiescence receipt; a successor-only loss is a no-op.
|
|
733
|
+
acknowledgeLostAttemptOwnership();
|
|
734
|
+
control.activityStatus = "cancelled";
|
|
735
|
+
control.turnMetricOutcome = "cancelled";
|
|
736
|
+
return claimedResult({ status: "recovering" });
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
const xaiCredentialFailure =
|
|
742
|
+
billingState.isXaiTurn && providerTurn.effectiveXaiCredentialId
|
|
743
|
+
? classifyXaiCredentialFailure(error)
|
|
744
|
+
: null;
|
|
745
|
+
if (
|
|
746
|
+
xaiCredentialFailure &&
|
|
747
|
+
providerTurn.effectiveXaiCredentialId &&
|
|
748
|
+
providerTurn.xaiAuthoritySnapshot &&
|
|
749
|
+
leases.xai.subjectId &&
|
|
750
|
+
leases.xai.holderId &&
|
|
751
|
+
leases.xai.generation !== null &&
|
|
752
|
+
eventing.publish &&
|
|
753
|
+
attempt.turnId &&
|
|
754
|
+
eventing.turnStartedPublished
|
|
755
|
+
) {
|
|
756
|
+
await flushRuntimeBatcher();
|
|
757
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
758
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
759
|
+
const activeGoal = goal?.status === "active" ? goal : null;
|
|
760
|
+
const now = new Date();
|
|
761
|
+
const cooldownUntil =
|
|
762
|
+
xaiCredentialFailure.kind === "rate_limit"
|
|
763
|
+
? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
|
|
764
|
+
: null;
|
|
765
|
+
const failurePayload = {
|
|
766
|
+
error:
|
|
767
|
+
xaiCredentialFailure.kind === "auth"
|
|
768
|
+
? "The serving SuperGrok account requires reconnection"
|
|
769
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
770
|
+
? "The serving SuperGrok account is not authorized for this request"
|
|
771
|
+
: "The serving SuperGrok account is temporarily rate limited",
|
|
772
|
+
code:
|
|
773
|
+
xaiCredentialFailure.kind === "auth"
|
|
774
|
+
? "xai_relogin_required"
|
|
775
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
776
|
+
? "xai_account_forbidden"
|
|
777
|
+
: "xai_account_rate_limited",
|
|
778
|
+
detail: "the same accepted turn is waiting for another eligible account",
|
|
779
|
+
};
|
|
780
|
+
const armed = await armXaiCapacityWait(db, {
|
|
781
|
+
accountId: input.accountId,
|
|
782
|
+
workspaceId: input.workspaceId,
|
|
783
|
+
subjectId: leases.xai.subjectId,
|
|
784
|
+
sessionId: input.sessionId,
|
|
785
|
+
turnId: attempt.turnId,
|
|
786
|
+
attemptId: input.attemptId,
|
|
787
|
+
workflowId: input.workflowId,
|
|
788
|
+
authoritySnapshot: providerTurn.xaiAuthoritySnapshot,
|
|
789
|
+
goalId: activeGoal?.id ?? null,
|
|
790
|
+
goalVersion: activeGoal?.version ?? null,
|
|
791
|
+
earliestResetAt: cooldownUntil,
|
|
792
|
+
failurePayload,
|
|
793
|
+
leaseFence: {
|
|
794
|
+
holderId: leases.xai.holderId,
|
|
795
|
+
generation: leases.xai.generation,
|
|
796
|
+
},
|
|
797
|
+
credentialQuarantine:
|
|
798
|
+
xaiCredentialFailure.kind === "auth"
|
|
799
|
+
? {
|
|
800
|
+
kind: "status",
|
|
801
|
+
status: "needs_relogin",
|
|
802
|
+
lastError: "model request remained unauthorized after refresh",
|
|
803
|
+
}
|
|
804
|
+
: xaiCredentialFailure.kind === "forbidden"
|
|
805
|
+
? {
|
|
806
|
+
kind: "status",
|
|
807
|
+
status: "error",
|
|
808
|
+
lastError: "model request was forbidden for this credential",
|
|
809
|
+
}
|
|
810
|
+
: { kind: "cooldown", until: cooldownUntil! },
|
|
811
|
+
now,
|
|
812
|
+
});
|
|
813
|
+
if (armed.action === "waiting") {
|
|
814
|
+
leases.xai.held = false;
|
|
815
|
+
providerTurn.xaiCredentialQuarantined = true;
|
|
816
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
817
|
+
const evaluated = await reconcileXaiCapacityWait(db, {
|
|
818
|
+
accountId: input.accountId,
|
|
819
|
+
workspaceId: input.workspaceId,
|
|
820
|
+
sessionId: input.sessionId,
|
|
821
|
+
waiterId: armed.waiter.id,
|
|
822
|
+
generation: armed.waiter.generation,
|
|
823
|
+
now,
|
|
824
|
+
});
|
|
825
|
+
if (evaluated.events.length > 0) {
|
|
826
|
+
await publishDurableSessionEvents(
|
|
827
|
+
bus,
|
|
828
|
+
input.workspaceId,
|
|
829
|
+
input.sessionId,
|
|
830
|
+
evaluated.events,
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
control.activityError = error;
|
|
834
|
+
if (evaluated.action === "resumed") {
|
|
835
|
+
control.activityStatus = "recovering";
|
|
836
|
+
control.turnMetricOutcome = "recovering";
|
|
837
|
+
return claimedResult({ status: "recovering" });
|
|
838
|
+
}
|
|
839
|
+
if (evaluated.action === "waiting") {
|
|
840
|
+
control.activityStatus = "waiting_capacity";
|
|
841
|
+
control.turnMetricOutcome = "recovering";
|
|
842
|
+
return claimedResult({
|
|
843
|
+
status: "waiting_capacity",
|
|
844
|
+
capacityWait: {
|
|
845
|
+
provider: "xai",
|
|
846
|
+
waiterId: evaluated.waiter.id,
|
|
847
|
+
generation: evaluated.waiter.generation,
|
|
848
|
+
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
849
|
+
wakeRevision: evaluated.waiter.wakeRevision,
|
|
850
|
+
},
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
acknowledgeLostAttemptOwnership();
|
|
854
|
+
control.activityStatus = "cancelled";
|
|
855
|
+
control.turnMetricOutcome = "cancelled";
|
|
856
|
+
return claimedResult({ status: "cancelled" });
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
860
|
+
sessionId: input.sessionId,
|
|
861
|
+
turnId: attempt.turnId,
|
|
862
|
+
triggerEventId: attempt.triggerEventId!,
|
|
863
|
+
attemptId: input.attemptId,
|
|
864
|
+
reason: "xai_credential_recheck",
|
|
865
|
+
detail: failurePayload,
|
|
866
|
+
});
|
|
867
|
+
if (recovery.action === "stale") {
|
|
868
|
+
acknowledgeLostAttemptOwnership();
|
|
869
|
+
control.activityStatus = "cancelled";
|
|
870
|
+
control.turnMetricOutcome = "cancelled";
|
|
871
|
+
return claimedResult({ status: "cancelled" });
|
|
872
|
+
}
|
|
873
|
+
acknowledgeRecoveryQuiescence();
|
|
874
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
875
|
+
control.activityStatus = "recovering";
|
|
876
|
+
control.turnMetricOutcome = "recovering";
|
|
877
|
+
control.activityError = error;
|
|
878
|
+
return claimedResult({ status: "recovering" });
|
|
879
|
+
}
|
|
880
|
+
// A ChatGPT/Codex usage cap (429 usage_limit_reached) is account state,
|
|
881
|
+
// NOT an agent failure: surface the precise, actionable message (so the
|
|
882
|
+
// user sees the reset window) but idle the session — never go terminal,
|
|
883
|
+
// which would reject the user's next message after the cap lifts. The
|
|
884
|
+
// payload is retryable:false so the generic provider-backpressure auto-retry
|
|
885
|
+
// does not loop. For an active goal we hold the continuation for the reported
|
|
886
|
+
// reset window (capped) so it resumes itself when access returns, instead of
|
|
887
|
+
// hammering the capped backend.
|
|
888
|
+
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
889
|
+
if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
890
|
+
const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
891
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
892
|
+
await flushRuntimeBatcher();
|
|
893
|
+
await historySink.reconcileConversationTruth();
|
|
894
|
+
// --- P3 reactive rotation (gated; re-fetch fresh state on this already-failed,
|
|
895
|
+
// already-idling path). Mark THIS account cooling until its reset, then CONSULT the
|
|
896
|
+
// engine over fresh accounts to decide continueDelayMs: a fast 0-delay re-dispatch
|
|
897
|
+
// when another account is available, or idle-until-earliest when all are capped. The
|
|
898
|
+
// catch deliberately does NOT move the active pointer — the re-dispatched turn's
|
|
899
|
+
// proactive seam (turn-start) is the single authoritative pointer-move + strip site.
|
|
900
|
+
let rotated = false;
|
|
901
|
+
let rotationResumeMs: number | null = null; // 0 ⇒ a candidate is available; re-dispatch now
|
|
902
|
+
let rotationResumeIdleUntilReset = false; // circuit-breaker fall (Finding 1b) ⇒ MANDATORY hold
|
|
903
|
+
let allCappedResetAt: Date | null = null; // set ⇒ every account capped; idle until this
|
|
904
|
+
let capacityAuthoritativeResetAt: Date | null = null;
|
|
905
|
+
if (providerTurn.effectiveCodexCredentialId) {
|
|
906
|
+
const [rotation, sessionCodex] = await Promise.all([
|
|
907
|
+
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
908
|
+
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null),
|
|
909
|
+
]);
|
|
910
|
+
const reactiveStrategy = (rotation?.rotationStrategy ??
|
|
911
|
+
"most_remaining") as CodexRotationStrategy;
|
|
912
|
+
const reactiveDisposition = classifyCodexPin({
|
|
913
|
+
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
914
|
+
pinSource: sessionCodex?.pinSource ?? null,
|
|
915
|
+
strategy: reactiveStrategy,
|
|
916
|
+
rotationEnabled: Boolean(rotation?.rotationEnabled),
|
|
917
|
+
});
|
|
918
|
+
const reactiveSharded = reactiveDisposition === "sharded";
|
|
919
|
+
const rotating =
|
|
920
|
+
Boolean(rotation?.rotationEnabled || rotation?.leaseRotationEnabled) &&
|
|
921
|
+
reactiveDisposition !== "manual";
|
|
922
|
+
if (rotating && rotation) {
|
|
923
|
+
const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
|
|
924
|
+
const serving =
|
|
925
|
+
accounts.find((a) => a.id === providerTurn.effectiveCodexCredentialId) ?? null;
|
|
926
|
+
// Both provider allowance windows bind. Use the same canonical
|
|
927
|
+
// quarantine calculation as the fenced failover path so a short
|
|
928
|
+
// five-hour reset can never overwrite a later weekly reset.
|
|
929
|
+
const until = codexCredentialCooldownUntil(
|
|
930
|
+
{ kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds },
|
|
931
|
+
serving,
|
|
932
|
+
new Date(),
|
|
933
|
+
)!;
|
|
934
|
+
// Finding 1a: INSPECT the cooldown-write result. A swallowed best-effort
|
|
935
|
+
// write whose failure went unnoticed is exactly what lets the next proactive
|
|
936
|
+
// rank re-pick this just-capped account (stale-low cached usedPercent, not
|
|
937
|
+
// cooling) — so capture whether it PERSISTED and feed it into the resume floor.
|
|
938
|
+
const cooldownMutation = await setCodexCredentialExhaustedWithWakeTargets(
|
|
939
|
+
db,
|
|
940
|
+
input.workspaceId,
|
|
941
|
+
providerTurn.effectiveCodexCredentialId,
|
|
942
|
+
until,
|
|
943
|
+
).catch(() => null);
|
|
944
|
+
const cooldownPersisted = cooldownMutation?.result ?? false;
|
|
945
|
+
if (cooldownMutation) {
|
|
946
|
+
await signalCodexCapacityWakeTargets(
|
|
947
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
948
|
+
cooldownMutation.wakeTargets,
|
|
949
|
+
);
|
|
950
|
+
}
|
|
951
|
+
// Re-rank over the fresh accounts; the in-memory list predates the cooldown
|
|
952
|
+
// write, so stamp the just-cooled account so the engine excludes it now. The
|
|
953
|
+
// serving account is thus walked AT MOST ONCE per turn (invariant 4: bounded).
|
|
954
|
+
const fresh = accounts.map((a) =>
|
|
955
|
+
a.id === providerTurn.effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a,
|
|
956
|
+
);
|
|
957
|
+
if (reactiveSharded) {
|
|
958
|
+
// AM-5: RE-SHARD over the healthy survivors (the just-capped serving account is
|
|
959
|
+
// marked cooling in `fresh` → excluded) so sessions sharing a capped account
|
|
960
|
+
// spread across the pool rather than re-concentrating on one first-eligible
|
|
961
|
+
// failover. AM-3: DURABLY REWRITE the session's POLICY pin to the new home —
|
|
962
|
+
// selectCodexCredentialForTurn returns a cooling pinned account with NO
|
|
963
|
+
// exhaustion check, so a pointer-only move would leave the re-dispatched turn on
|
|
964
|
+
// the capped pin. Like the classic path we do NOT touch the workspace active
|
|
965
|
+
// pointer; the session pin is the sharded home.
|
|
966
|
+
const newHome = shardCredentialForSession({
|
|
967
|
+
sessionId: input.sessionId,
|
|
968
|
+
accounts: fresh,
|
|
969
|
+
now: new Date(),
|
|
970
|
+
});
|
|
971
|
+
if (newHome) {
|
|
972
|
+
rotated = true;
|
|
973
|
+
const pinMutation = await withSessionCodexCapacityMutation(
|
|
974
|
+
db,
|
|
975
|
+
{
|
|
976
|
+
workspaceId: input.workspaceId,
|
|
977
|
+
reason: "codex_policy_pin_resharded",
|
|
978
|
+
},
|
|
979
|
+
async (tx) => {
|
|
980
|
+
const changed = await setSessionCodexPinInTransaction(
|
|
981
|
+
tx,
|
|
982
|
+
input.workspaceId,
|
|
983
|
+
input.sessionId,
|
|
984
|
+
newHome,
|
|
985
|
+
"policy",
|
|
986
|
+
{
|
|
987
|
+
expected: {
|
|
988
|
+
pinnedCredentialId: sessionCodex?.pinnedCredentialId ?? null,
|
|
989
|
+
pinSource: sessionCodex?.pinSource ?? null,
|
|
990
|
+
},
|
|
991
|
+
},
|
|
992
|
+
);
|
|
993
|
+
return { result: changed, changed };
|
|
994
|
+
},
|
|
995
|
+
).catch(() => null);
|
|
996
|
+
if (pinMutation) {
|
|
997
|
+
await signalCodexCapacityWakeTargets(
|
|
998
|
+
{ signalCodexCapacityWorkflow, wakeSessionWorkflow },
|
|
999
|
+
pinMutation.wakeTargets,
|
|
1000
|
+
);
|
|
1001
|
+
}
|
|
1002
|
+
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
1003
|
+
db,
|
|
1004
|
+
input.workspaceId,
|
|
1005
|
+
input.sessionId,
|
|
1006
|
+
).catch(() => 0);
|
|
1007
|
+
const resume = computeReactiveRotationResume({
|
|
1008
|
+
cooldownPersisted,
|
|
1009
|
+
priorConsecutiveRotations,
|
|
1010
|
+
connectedAccountCount: accounts.length,
|
|
1011
|
+
});
|
|
1012
|
+
rotationResumeMs = resume.continueDelayMs;
|
|
1013
|
+
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
1014
|
+
} else {
|
|
1015
|
+
// Every account capped/cooling → idle until the earliest reset across all.
|
|
1016
|
+
rotated = true;
|
|
1017
|
+
allCappedResetAt = earliestCodexReset(fresh, new Date());
|
|
1018
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1019
|
+
}
|
|
1020
|
+
} else {
|
|
1021
|
+
const decision = chooseRotationActive({
|
|
1022
|
+
rotationStrategy: reactiveStrategy,
|
|
1023
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
1024
|
+
priorCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
1025
|
+
accounts: fresh,
|
|
1026
|
+
now: new Date(),
|
|
1027
|
+
});
|
|
1028
|
+
if (decision.kind === "active") {
|
|
1029
|
+
rotated = true;
|
|
1030
|
+
// Finding 1: a live candidate normally re-dispatches NOW (0). Two second-order
|
|
1031
|
+
// faults would turn that 0 into a hot loop, so bound it. Count the consecutive
|
|
1032
|
+
// reactive failovers since the last successful turn (this one is not yet
|
|
1033
|
+
// published) and combine with the cooldown-persistence result.
|
|
1034
|
+
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(
|
|
1035
|
+
db,
|
|
1036
|
+
input.workspaceId,
|
|
1037
|
+
input.sessionId,
|
|
1038
|
+
).catch(() => 0);
|
|
1039
|
+
const resume = computeReactiveRotationResume({
|
|
1040
|
+
cooldownPersisted,
|
|
1041
|
+
priorConsecutiveRotations,
|
|
1042
|
+
connectedAccountCount: accounts.length,
|
|
1043
|
+
});
|
|
1044
|
+
rotationResumeMs = resume.continueDelayMs; // 0 (happy path), a slow-retry floor, or the circuit-breaker idle
|
|
1045
|
+
rotationResumeIdleUntilReset = resume.idleUntilReset; // true only on the circuit-breaker fall (MANDATORY hold)
|
|
1046
|
+
} else if (decision.kind === "allCapped") {
|
|
1047
|
+
rotated = true;
|
|
1048
|
+
allCappedResetAt = decision.earliestResetAt;
|
|
1049
|
+
capacityAuthoritativeResetAt = authoritativeCodexCapacityResetAt(fresh, new Date());
|
|
1050
|
+
}
|
|
1051
|
+
// kind:"none" → fall through to today's single-account idle.
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
const failurePayload = allCappedResetAt
|
|
1057
|
+
? codexUsageLimitFailurePayload(
|
|
1058
|
+
{
|
|
1059
|
+
resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1000),
|
|
1060
|
+
},
|
|
1061
|
+
error instanceof Error ? error.message : String(error),
|
|
1062
|
+
{ allAccounts: true },
|
|
1063
|
+
)
|
|
1064
|
+
: codexUsageLimitFailurePayload(
|
|
1065
|
+
usageLimit,
|
|
1066
|
+
error instanceof Error ? error.message : String(error),
|
|
1067
|
+
);
|
|
1068
|
+
// A live alternate is still handled by the existing immediate,
|
|
1069
|
+
// same-policy continuation path. When no alternate exists (all capped,
|
|
1070
|
+
// or a single non-rotating account), persist the native capacity wait
|
|
1071
|
+
// instead of an in-memory delay/user-message recovery.
|
|
1072
|
+
if (rotationResumeMs === null) {
|
|
1073
|
+
const providerResetAt =
|
|
1074
|
+
capacityAuthoritativeResetAt ??
|
|
1075
|
+
(usageLimit.resetsInSeconds !== null &&
|
|
1076
|
+
Number.isFinite(usageLimit.resetsInSeconds) &&
|
|
1077
|
+
usageLimit.resetsInSeconds > 0
|
|
1078
|
+
? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1000)
|
|
1079
|
+
: null);
|
|
1080
|
+
const armed = await armCodexCapacityWait(db, {
|
|
1081
|
+
accountId: input.accountId,
|
|
1082
|
+
workspaceId: input.workspaceId,
|
|
1083
|
+
sessionId: input.sessionId,
|
|
1084
|
+
turnId: attempt.turnId,
|
|
1085
|
+
attemptId: input.attemptId,
|
|
1086
|
+
workflowId: input.workflowId,
|
|
1087
|
+
goalId: goalActive && goal ? goal.id : null,
|
|
1088
|
+
goalVersion: goalActive && goal ? goal.version : null,
|
|
1089
|
+
earliestResetAt: providerResetAt,
|
|
1090
|
+
resetKind: providerResetAt ? "authoritative" : "bounded_refresh",
|
|
1091
|
+
failurePayload,
|
|
1092
|
+
...(leases.codex.holderId && leases.codex.generation !== null
|
|
1093
|
+
? {
|
|
1094
|
+
leaseFence: {
|
|
1095
|
+
holderId: leases.codex.holderId,
|
|
1096
|
+
generation: leases.codex.generation,
|
|
1097
|
+
},
|
|
1098
|
+
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
1099
|
+
}
|
|
1100
|
+
: {}),
|
|
1101
|
+
});
|
|
1102
|
+
if (armed.action === "waiting") {
|
|
1103
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
1104
|
+
control.turnMetricOutcome = "recovering";
|
|
1105
|
+
control.activityStatus = "waiting_capacity";
|
|
1106
|
+
control.activityError = error;
|
|
1107
|
+
return claimedResult({
|
|
1108
|
+
status: "waiting_capacity",
|
|
1109
|
+
capacityWait: {
|
|
1110
|
+
waiterId: armed.waiter.id,
|
|
1111
|
+
generation: armed.waiter.generation,
|
|
1112
|
+
nextCheckAt: armed.waiter.nextCheckAt.toISOString(),
|
|
1113
|
+
wakeRevision: armed.waiter.wakeRevision,
|
|
1114
|
+
},
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
if (
|
|
1119
|
+
!(await eventing.settle!({
|
|
1120
|
+
events: [
|
|
1121
|
+
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
1122
|
+
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
1123
|
+
{
|
|
1124
|
+
type: "turn.failed",
|
|
1125
|
+
payload: {
|
|
1126
|
+
...failurePayload,
|
|
1127
|
+
recovery: goalActive ? "goal_continuation" : "user_message",
|
|
1128
|
+
...(rotated ? { rotated: true } : {}),
|
|
1129
|
+
},
|
|
1130
|
+
},
|
|
1131
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1132
|
+
],
|
|
1133
|
+
turnStatus: "failed",
|
|
1134
|
+
sessionStatus: "idle",
|
|
1135
|
+
activeTurnId: null,
|
|
1136
|
+
}))
|
|
1137
|
+
) {
|
|
1138
|
+
return claimedResult({ status: "cancelled" });
|
|
1139
|
+
}
|
|
1140
|
+
control.turnMetricOutcome = "failed";
|
|
1141
|
+
control.activityStatus = "idle";
|
|
1142
|
+
control.activityError = error;
|
|
1143
|
+
if (goalActive) {
|
|
1144
|
+
// Rotation: a candidate is available → continue NOW (0). All-capped → idle until the
|
|
1145
|
+
// earliest reset across all accounts (capped at 1h). Else the unchanged single-account idle.
|
|
1146
|
+
if (rotationResumeMs !== null) {
|
|
1147
|
+
// A candidate IS available. Normally the just-failed account is now cooling so
|
|
1148
|
+
// the ranker cannot re-pick it → 0 (re-dispatch NOW, the legitimate skip-the-hold
|
|
1149
|
+
// case). Finding 1 bounds the two exceptions: a persistence fault yields a positive
|
|
1150
|
+
// slow-retry floor, and once consecutive failovers exceed the account count + margin
|
|
1151
|
+
// the circuit breaker returns a fixed MANDATORY idle (idleUntilReset) — never a 0-delay
|
|
1152
|
+
// hot loop against a capped backend + DB.
|
|
1153
|
+
return claimedResult({
|
|
1154
|
+
status: "idle",
|
|
1155
|
+
continueDelayMs: rotationResumeMs,
|
|
1156
|
+
...(rotationResumeIdleUntilReset ? { idleUntilReset: true } : {}),
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
// All-capped: clamp to [MIN_IDLE_MS, max] — a POSITIVE, BOUNDED hold (never 0,
|
|
1160
|
+
// so session.ts can never tight-loop). The post-idle continuation re-dispatch
|
|
1161
|
+
// hits the proactive seam, which refreshes usage and self-heals.
|
|
1162
|
+
const resumeMs = allCappedResetAt
|
|
1163
|
+
? computeIdleDelayMs(allCappedResetAt, new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1164
|
+
: usageLimit.resetsInSeconds !== null &&
|
|
1165
|
+
Number.isFinite(usageLimit.resetsInSeconds) &&
|
|
1166
|
+
usageLimit.resetsInSeconds > 0
|
|
1167
|
+
? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1000, CODEX_USAGE_LIMIT_MAX_RESUME_MS)
|
|
1168
|
+
: CODEX_USAGE_LIMIT_MAX_RESUME_MS;
|
|
1169
|
+
return claimedResult({
|
|
1170
|
+
status: "idle",
|
|
1171
|
+
continueDelayMs: resumeMs,
|
|
1172
|
+
...(allCappedResetAt ? { idleUntilReset: true } : {}),
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
return claimedResult({ status: "idle" });
|
|
1176
|
+
}
|
|
1177
|
+
// Budget/limit exhaustion between model calls is account state, not an
|
|
1178
|
+
// agent failure: idle the session for goal-bearing and goal-less runs
|
|
1179
|
+
// alike (a failed session would reject the user's next message after a
|
|
1180
|
+
// top-up). An active goal pauses visibly with reason "limits" at the
|
|
1181
|
+
// next continuation evaluation, without consuming continuation budget.
|
|
1182
|
+
if (
|
|
1183
|
+
error instanceof BudgetExhaustedError &&
|
|
1184
|
+
eventing.publish &&
|
|
1185
|
+
attempt.turnId &&
|
|
1186
|
+
eventing.turnStartedPublished
|
|
1187
|
+
) {
|
|
1188
|
+
await flushRuntimeBatcher();
|
|
1189
|
+
await historySink.reconcileConversationTruth();
|
|
1190
|
+
if (
|
|
1191
|
+
!(await eventing.settle!({
|
|
1192
|
+
events: [
|
|
1193
|
+
{
|
|
1194
|
+
type: "turn.completed",
|
|
1195
|
+
payload: {
|
|
1196
|
+
output: "",
|
|
1197
|
+
segmentLimit: "budget_exhausted",
|
|
1198
|
+
detail: error.message,
|
|
1199
|
+
},
|
|
1200
|
+
},
|
|
1201
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1202
|
+
],
|
|
1203
|
+
turnStatus: "completed",
|
|
1204
|
+
sessionStatus: "idle",
|
|
1205
|
+
activeTurnId: null,
|
|
1206
|
+
}))
|
|
1207
|
+
) {
|
|
1208
|
+
return claimedResult({ status: "cancelled" });
|
|
1209
|
+
}
|
|
1210
|
+
control.turnMetricOutcome = "completed";
|
|
1211
|
+
await recordUsageEvent(db, {
|
|
1212
|
+
accountId: input.accountId,
|
|
1213
|
+
workspaceId: input.workspaceId,
|
|
1214
|
+
eventType: "agent_run.completed",
|
|
1215
|
+
quantity: 1,
|
|
1216
|
+
unit: "run",
|
|
1217
|
+
sourceResourceType: "session_turn",
|
|
1218
|
+
sourceResourceId: attempt.turnId,
|
|
1219
|
+
sessionId: input.sessionId,
|
|
1220
|
+
turnId: attempt.turnId,
|
|
1221
|
+
turnAttemptId: input.attemptId,
|
|
1222
|
+
idempotencyKey: `usage:agent_run.completed:${attempt.turnId}`,
|
|
1223
|
+
});
|
|
1224
|
+
control.activityStatus = "idle";
|
|
1225
|
+
return claimedResult({ status: "idle" });
|
|
1226
|
+
}
|
|
1227
|
+
// The Codex backend can reject an opaque reasoning artifact that it
|
|
1228
|
+
// minted on the immediately preceding successful request, even when the
|
|
1229
|
+
// credential-row UUID is unchanged. HTTP 400 + the exact provider
|
|
1230
|
+
// semantic proves this request never entered inference. Atomically mark
|
|
1231
|
+
// the active opaque artifacts rejected and recover the SAME logical turn
|
|
1232
|
+
// from durable history; messages, tool calls/results, readable reasoning,
|
|
1233
|
+
// and the original audit rows remain intact. Opaque remote compaction has
|
|
1234
|
+
// no portable plaintext representation. If no artifact can be invalidated,
|
|
1235
|
+
// fall through to the terminal path rather than resend an equivalent
|
|
1236
|
+
// request forever.
|
|
1237
|
+
const encryptedArtifactRejection =
|
|
1238
|
+
billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
|
|
1239
|
+
? classifyCodexEncryptedArtifactRejection(error)
|
|
1240
|
+
: null;
|
|
1241
|
+
if (
|
|
1242
|
+
encryptedArtifactRejection &&
|
|
1243
|
+
providerTurn.effectiveCodexCredentialId &&
|
|
1244
|
+
eventing.publish &&
|
|
1245
|
+
attempt.turnId &&
|
|
1246
|
+
eventing.turnStartedPublished
|
|
1247
|
+
) {
|
|
1248
|
+
await flushRuntimeBatcher();
|
|
1249
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1250
|
+
const activeHistory = await getActiveSessionHistoryItemsPaged(
|
|
1251
|
+
db,
|
|
1252
|
+
input.workspaceId,
|
|
1253
|
+
input.sessionId,
|
|
1254
|
+
);
|
|
1255
|
+
const rejectedHistoryItemIds = selectRejectedProviderArtifactHistoryIds(
|
|
1256
|
+
activeHistory,
|
|
1257
|
+
historySink.providerArtifactCandidates,
|
|
1258
|
+
providerTurn.lastCodexRequestOpaqueArtifacts,
|
|
1259
|
+
);
|
|
1260
|
+
const rejectedRunStateId =
|
|
1261
|
+
historySink.providerArtifactCandidates.runStateId &&
|
|
1262
|
+
providerTurn.lastCodexRequestOpaqueArtifacts.length > 0
|
|
1263
|
+
? historySink.providerArtifactCandidates.runStateId
|
|
1264
|
+
: undefined;
|
|
1265
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
1266
|
+
sessionId: input.sessionId,
|
|
1267
|
+
turnId: attempt.turnId,
|
|
1268
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1269
|
+
attemptId: input.attemptId,
|
|
1270
|
+
reason: encryptedArtifactRejection.kind,
|
|
1271
|
+
detail: {
|
|
1272
|
+
code: encryptedArtifactRejection.kind,
|
|
1273
|
+
retryable: true,
|
|
1274
|
+
},
|
|
1275
|
+
providerArtifactInvalidation: {
|
|
1276
|
+
historyItemIds: rejectedHistoryItemIds,
|
|
1277
|
+
...(rejectedRunStateId ? { runStateId: rejectedRunStateId } : {}),
|
|
1278
|
+
reason: encryptedArtifactRejection.kind,
|
|
1279
|
+
},
|
|
1280
|
+
});
|
|
1281
|
+
if (recovery.action === "stale") {
|
|
1282
|
+
acknowledgeLostAttemptOwnership();
|
|
1283
|
+
control.activityStatus = "cancelled";
|
|
1284
|
+
control.turnMetricOutcome = "cancelled";
|
|
1285
|
+
return claimedResult({ status: "cancelled" });
|
|
1286
|
+
}
|
|
1287
|
+
if (recovery.action === "recovering") {
|
|
1288
|
+
acknowledgeRecoveryQuiescence();
|
|
1289
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
1290
|
+
control.turnMetricOutcome = "recovering";
|
|
1291
|
+
control.activityStatus = "recovering";
|
|
1292
|
+
control.activityError = error;
|
|
1293
|
+
return claimedResult({ status: "recovering" });
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
// A retryable provider/MCP failure is transient external backpressure,
|
|
1297
|
+
// not a session or goal failure. The in-client retry budget is already
|
|
1298
|
+
// exhausted by the time the error reaches here. Checkpoint conversation
|
|
1299
|
+
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
1300
|
+
// it after a pacing delay. This is independent of goal state and never
|
|
1301
|
+
// relies on a synthetic continuation prompt.
|
|
1302
|
+
let failure = agentRunFailurePayload(error, {
|
|
1303
|
+
isCodexTurn: billingState.isCodexTurn,
|
|
1304
|
+
}) as ReturnType<typeof agentRunFailurePayload>;
|
|
1305
|
+
if (isSessionEventPersistenceError(error)) {
|
|
1306
|
+
// Preserve the exact source message in the internal runtime diagnostic;
|
|
1307
|
+
// SQLSTATE/catalog facts remain separate classification attributes.
|
|
1308
|
+
observability.error("session event persistence failed", {
|
|
1309
|
+
accountId: input.accountId,
|
|
1310
|
+
workspaceId: input.workspaceId,
|
|
1311
|
+
sessionId: input.sessionId,
|
|
1312
|
+
turnId: attempt.turnId,
|
|
1313
|
+
attemptId: input.attemptId,
|
|
1314
|
+
code: error.details.code,
|
|
1315
|
+
sqlState: error.details.sqlState ?? "unknown",
|
|
1316
|
+
stage: error.details.stage,
|
|
1317
|
+
eventTypes: error.details.eventTypes.join(","),
|
|
1318
|
+
correlationId: error.details.correlationId,
|
|
1319
|
+
attempts: error.details.attempts,
|
|
1320
|
+
retryOutcome: error.details.retryOutcome,
|
|
1321
|
+
dbSeverity: error.details.database.severity,
|
|
1322
|
+
dbSchema: error.details.database.schema,
|
|
1323
|
+
dbTable: error.details.database.table,
|
|
1324
|
+
dbColumn: error.details.database.column,
|
|
1325
|
+
dbDataType: error.details.database.dataType,
|
|
1326
|
+
dbConstraint: error.details.database.constraint,
|
|
1327
|
+
dbRoutine: error.details.database.routine,
|
|
1328
|
+
error: error.message,
|
|
1329
|
+
});
|
|
1330
|
+
}
|
|
1331
|
+
if (failure.retryable && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
1332
|
+
const nextProviderRecoveryCount = attempt.providerRecoveryCount + 1;
|
|
1333
|
+
const recoveryResult = providerRecoveryResult({
|
|
1334
|
+
failureCode: failure.code,
|
|
1335
|
+
attemptNumber: nextProviderRecoveryCount,
|
|
1336
|
+
retryAfterMs: providerRetryAfterMs(error),
|
|
1337
|
+
});
|
|
1338
|
+
try {
|
|
1339
|
+
if (recoveryResult.status === "recovering") {
|
|
1340
|
+
await flushRuntimeBatcher();
|
|
1341
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1342
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
1343
|
+
sessionId: input.sessionId,
|
|
1344
|
+
turnId: attempt.turnId,
|
|
1345
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1346
|
+
attemptId: input.attemptId,
|
|
1347
|
+
reason: failure.code ?? "provider_unavailable",
|
|
1348
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1349
|
+
detail: {
|
|
1350
|
+
...failure,
|
|
1351
|
+
continueDelayMs: recoveryResult.continueDelayMs,
|
|
1352
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1353
|
+
},
|
|
1354
|
+
});
|
|
1355
|
+
if (recovery.action === "stale") {
|
|
1356
|
+
acknowledgeLostAttemptOwnership();
|
|
1357
|
+
control.activityStatus = "cancelled";
|
|
1358
|
+
control.turnMetricOutcome = "cancelled";
|
|
1359
|
+
return claimedResult({ status: "cancelled" });
|
|
1360
|
+
}
|
|
1361
|
+
acknowledgeRecoveryQuiescence();
|
|
1362
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
1363
|
+
control.turnMetricOutcome = "recovering";
|
|
1364
|
+
control.activityStatus = "recovering";
|
|
1365
|
+
control.activityError = error;
|
|
1366
|
+
return claimedResult(recoveryResult);
|
|
1367
|
+
}
|
|
1368
|
+
failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
|
|
1369
|
+
} catch (recoveryError) {
|
|
1370
|
+
const escaped =
|
|
1371
|
+
recoveryResult.status === "recovering"
|
|
1372
|
+
? escapedMcpTimeoutRecoveryFailure({
|
|
1373
|
+
failureCode: failure.code,
|
|
1374
|
+
modelRequestStarted: attempt.modelRequestStarted,
|
|
1375
|
+
detail: {
|
|
1376
|
+
turnId: attempt.turnId,
|
|
1377
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1378
|
+
executionGeneration: attempt.executionGeneration,
|
|
1379
|
+
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1380
|
+
continueDelayMs: recoveryResult.continueDelayMs,
|
|
1381
|
+
},
|
|
1382
|
+
})
|
|
1383
|
+
: null;
|
|
1384
|
+
if (escaped) {
|
|
1385
|
+
control.activityStatus = "recovering";
|
|
1386
|
+
control.turnMetricOutcome = "recovering";
|
|
1387
|
+
control.activityError = error;
|
|
1388
|
+
throw escaped;
|
|
1389
|
+
}
|
|
1390
|
+
throw recoveryError;
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
control.activityStatus = "failed";
|
|
1394
|
+
control.activityError = error;
|
|
1395
|
+
if (!attempt.turnId) {
|
|
1396
|
+
throw preClaimAdmissionFailure(error);
|
|
1397
|
+
}
|
|
1398
|
+
if (!eventing.publish || !eventing.turnStartedPublished) {
|
|
1399
|
+
throw error;
|
|
1400
|
+
}
|
|
1401
|
+
// A partial/malformed stream may have emitted assistant/tool items (and
|
|
1402
|
+
// external side effects) before its terminal error. Persist every item the
|
|
1403
|
+
// SDK state observed before marking the turn failed so a later user revive
|
|
1404
|
+
// never replays work from an incomplete history. This does not retry or
|
|
1405
|
+
// rotate the ambiguous request.
|
|
1406
|
+
await flushRuntimeBatcher();
|
|
1407
|
+
await historySink.reconcileConversationTruth();
|
|
1408
|
+
if (
|
|
1409
|
+
!(await eventing.settle!({
|
|
1410
|
+
events: [
|
|
1411
|
+
{ type: "turn.failed", payload: failure },
|
|
1412
|
+
{ type: "session.status.changed", payload: { status: "failed" } },
|
|
1413
|
+
],
|
|
1414
|
+
turnStatus: "failed",
|
|
1415
|
+
sessionStatus: "failed",
|
|
1416
|
+
activeTurnId: null,
|
|
1417
|
+
}))
|
|
1418
|
+
) {
|
|
1419
|
+
return claimedResult({ status: "cancelled" });
|
|
1420
|
+
}
|
|
1421
|
+
control.turnMetricOutcome = "failed";
|
|
1422
|
+
// The common failure path ends here: runAgentTurn marks the session
|
|
1423
|
+
// failed and returns "failed", and the session workflow then exits
|
|
1424
|
+
// WITHOUT calling failSession/markSessionIdle. Wake a spawned worker's
|
|
1425
|
+
// parent here too, so a manager learns of a worker that died inside its
|
|
1426
|
+
// turn (not just one failed by the workflow's failSession path). Turn
|
|
1427
|
+
// settlement already owns the durable outbox payload; this call only
|
|
1428
|
+
// delivers that exact turn-scoped row.
|
|
1429
|
+
await deliverFailedChildTurnToParent(
|
|
1430
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
1431
|
+
input.workspaceId,
|
|
1432
|
+
input.sessionId,
|
|
1433
|
+
attempt.turnId,
|
|
1434
|
+
);
|
|
1435
|
+
return claimedResult({ status: "failed" });
|
|
1436
|
+
}
|