@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +17 -4
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync.d.ts +4 -0
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +9 -10
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12701
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +147 -49
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync.ts +178 -37
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +102 -142
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-BJSDRSIP.js.map +0 -1
- package/dist/activities-turn-J5YO7XXX.js +0 -17243
- package/dist/activities-turn-J5YO7XXX.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -0,0 +1,1068 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ActiveSessionHistoryLimitExceededError,
|
|
3
|
+
ApprovalRunStateLimitExceededError,
|
|
4
|
+
isRetryableDatabaseTransportFailure,
|
|
5
|
+
isSessionEventPersistenceError,
|
|
6
|
+
} from "@opengeni/db";
|
|
7
|
+
import {
|
|
8
|
+
ActiveBackendUnresolvableError,
|
|
9
|
+
CompactionProviderResponseError,
|
|
10
|
+
EmptyCompactionSummaryError,
|
|
11
|
+
isMcpRequestTimeoutError,
|
|
12
|
+
isMcpTransportConnectivityError,
|
|
13
|
+
RoutingWorkspaceRootChangedError,
|
|
14
|
+
SelfhostedWorkspaceRootChangedError,
|
|
15
|
+
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
16
|
+
} from "@opengeni/runtime";
|
|
17
|
+
import {
|
|
18
|
+
mcpTransportRequestFailureDiagnostic,
|
|
19
|
+
type McpTransportRequestFailureDiagnostic,
|
|
20
|
+
} from "@opengeni/runtime/mcp-network";
|
|
21
|
+
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
22
|
+
import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
|
|
23
|
+
import type { CodexAccountStatus } from "@opengeni/db";
|
|
24
|
+
import {
|
|
25
|
+
CodexReloginRequired,
|
|
26
|
+
classifyCodexResponseTimeoutError,
|
|
27
|
+
classifyCodexUsageLimitError,
|
|
28
|
+
isCodexTransportError,
|
|
29
|
+
} from "@opengeni/codex";
|
|
30
|
+
import {
|
|
31
|
+
classifyXaiSubscriptionStreamingTerminalError,
|
|
32
|
+
classifyXaiSubscriptionStreamIdleTimeoutError,
|
|
33
|
+
isXaiSubscriptionHostedToolContinuationError,
|
|
34
|
+
isXaiSubscriptionRateLimitDiagnostic,
|
|
35
|
+
isXaiSubscriptionTransportError,
|
|
36
|
+
XaiSubscriptionReloginRequired,
|
|
37
|
+
} from "@opengeni/xai-subscription";
|
|
38
|
+
import type { EscapedMcpTimeoutRecoveryDetail, PreClaimFailureDetail } from "../types";
|
|
39
|
+
import {
|
|
40
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
41
|
+
ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
42
|
+
PRE_CLAIM_FAILURE_MESSAGE,
|
|
43
|
+
PRE_CLAIM_FAILURE_TYPE,
|
|
44
|
+
} from "../types";
|
|
45
|
+
import {
|
|
46
|
+
MandatoryHistoryPersistenceError,
|
|
47
|
+
type MandatoryHistoryPersistenceStage,
|
|
48
|
+
} from "./quiescence";
|
|
49
|
+
|
|
50
|
+
// Retryable provider connectivity/5xx failures start quickly and back off to
|
|
51
|
+
// this ceiling. Explicit rate limits retain the minute-granular fallback.
|
|
52
|
+
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
53
|
+
export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
54
|
+
export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
55
|
+
|
|
56
|
+
export type ProviderRecoveryResult =
|
|
57
|
+
| {
|
|
58
|
+
status: "recovering";
|
|
59
|
+
continueDelayMs: number;
|
|
60
|
+
}
|
|
61
|
+
| {
|
|
62
|
+
status: "exhausted";
|
|
63
|
+
providerRecoveryCount: number;
|
|
64
|
+
maxProviderRecoveryCount: number;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export function providerRecoveryResult(input: {
|
|
68
|
+
failureCode: string | undefined;
|
|
69
|
+
attemptNumber: number;
|
|
70
|
+
retryAfterMs?: number | null;
|
|
71
|
+
}): ProviderRecoveryResult {
|
|
72
|
+
if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
|
|
73
|
+
return {
|
|
74
|
+
status: "exhausted",
|
|
75
|
+
providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
76
|
+
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
const providerDelay =
|
|
80
|
+
input.retryAfterMs !== null &&
|
|
81
|
+
input.retryAfterMs !== undefined &&
|
|
82
|
+
Number.isFinite(input.retryAfterMs) &&
|
|
83
|
+
input.retryAfterMs > 0
|
|
84
|
+
? Math.ceil(input.retryAfterMs)
|
|
85
|
+
: null;
|
|
86
|
+
const continueDelayMs =
|
|
87
|
+
input.failureCode === "provider_rate_limited"
|
|
88
|
+
? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
|
|
89
|
+
: input.failureCode === "provider_unavailable" ||
|
|
90
|
+
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
91
|
+
input.failureCode === "mcp_transport_timeout" ||
|
|
92
|
+
input.failureCode === "mcp_transport_unavailable"
|
|
93
|
+
? Math.max(
|
|
94
|
+
providerDelay ?? 0,
|
|
95
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS[
|
|
96
|
+
Math.min(
|
|
97
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
98
|
+
PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1,
|
|
99
|
+
)
|
|
100
|
+
]!,
|
|
101
|
+
)
|
|
102
|
+
: PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
103
|
+
return {
|
|
104
|
+
status: "recovering",
|
|
105
|
+
continueDelayMs,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function providerRecoveryExhaustedFailure<
|
|
110
|
+
T extends Record<string, unknown> & { error: string },
|
|
111
|
+
>(
|
|
112
|
+
failure: T,
|
|
113
|
+
recovery: Extract<ProviderRecoveryResult, { status: "exhausted" }>,
|
|
114
|
+
): T & {
|
|
115
|
+
retryable: false;
|
|
116
|
+
recoveryExhausted: true;
|
|
117
|
+
providerRecoveryCount: number;
|
|
118
|
+
maxProviderRecoveryCount: number;
|
|
119
|
+
lastRetryableError: string;
|
|
120
|
+
} {
|
|
121
|
+
return {
|
|
122
|
+
...failure,
|
|
123
|
+
error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
|
|
124
|
+
retryable: false,
|
|
125
|
+
recoveryExhausted: true,
|
|
126
|
+
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
127
|
+
maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
|
|
128
|
+
lastRetryableError: failure.error,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
|
|
133
|
+
const value = metadata.providerRecoveryCount;
|
|
134
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function headerValue(headers: unknown, name: string): string | null {
|
|
138
|
+
if (!headers || typeof headers !== "object") return null;
|
|
139
|
+
const getter = (headers as { get?: unknown }).get;
|
|
140
|
+
if (typeof getter === "function") {
|
|
141
|
+
const value = getter.call(headers, name);
|
|
142
|
+
return typeof value === "string" ? value : null;
|
|
143
|
+
}
|
|
144
|
+
const target = name.toLowerCase();
|
|
145
|
+
const entry = Object.entries(headers as Record<string, unknown>).find(
|
|
146
|
+
([key, value]) => key.toLowerCase() === target && typeof value === "string",
|
|
147
|
+
);
|
|
148
|
+
return typeof entry?.[1] === "string" ? entry[1] : null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Read a provider Retry-After hint without retaining response headers/body. */
|
|
152
|
+
export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number | null {
|
|
153
|
+
let current: unknown = error;
|
|
154
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
155
|
+
const value = current as Record<string, unknown>;
|
|
156
|
+
const body =
|
|
157
|
+
value.error && typeof value.error === "object"
|
|
158
|
+
? (value.error as Record<string, unknown>)
|
|
159
|
+
: null;
|
|
160
|
+
const directSeconds = Number(
|
|
161
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
162
|
+
);
|
|
163
|
+
const header =
|
|
164
|
+
headerValue(value.headers, "retry-after") ??
|
|
165
|
+
headerValue(value.responseHeaders, "retry-after") ??
|
|
166
|
+
headerValue(body?.headers, "retry-after");
|
|
167
|
+
const headerSeconds = header === null ? Number.NaN : Number(header);
|
|
168
|
+
const headerDate =
|
|
169
|
+
header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
|
|
170
|
+
const seconds = Number.isFinite(directSeconds)
|
|
171
|
+
? directSeconds
|
|
172
|
+
: Number.isFinite(headerSeconds)
|
|
173
|
+
? headerSeconds
|
|
174
|
+
: Number.isFinite(headerDate)
|
|
175
|
+
? Math.max(0, (headerDate - nowMs) / 1_000)
|
|
176
|
+
: Number.NaN;
|
|
177
|
+
if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
|
|
178
|
+
current = value.cause;
|
|
179
|
+
}
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Tell the model when durable MCP policy references are unavailable for this
|
|
185
|
+
* exact turn. The policy projection already bounds and validates these ids;
|
|
186
|
+
* this notice prevents a graceful runtime drop from becoming a silent source-
|
|
187
|
+
* of-truth substitution or a false claim that the disconnected system was read.
|
|
188
|
+
*/
|
|
189
|
+
export function unavailableMcpOperationalContext(input: {
|
|
190
|
+
droppedIds: readonly string[];
|
|
191
|
+
droppedCount: number;
|
|
192
|
+
}): string | undefined {
|
|
193
|
+
if (input.droppedCount <= 0) {
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
|
|
197
|
+
const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
|
|
198
|
+
const inventory = listed
|
|
199
|
+
? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}`
|
|
200
|
+
: `${input.droppedCount} unavailable server(s)`;
|
|
201
|
+
return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Preserve one precise recovery obligation across Temporal's activity boundary.
|
|
206
|
+
* This is intentionally narrower than the ordinary retryable-provider path:
|
|
207
|
+
* only a recovered turn (generation > 1) whose MCP timeout happened before a
|
|
208
|
+
* model request may ask the workflow's DB-only control activity to finish the
|
|
209
|
+
* same-turn checkpoint. The original transport/recovery errors are excluded
|
|
210
|
+
* from details so raw MCP response data can never enter workflow history.
|
|
211
|
+
*/
|
|
212
|
+
export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
213
|
+
failureCode: string | undefined;
|
|
214
|
+
modelRequestStarted: boolean;
|
|
215
|
+
detail: EscapedMcpTimeoutRecoveryDetail;
|
|
216
|
+
}): ApplicationFailure | null {
|
|
217
|
+
if (
|
|
218
|
+
input.failureCode !== "mcp_transport_timeout" ||
|
|
219
|
+
input.modelRequestStarted ||
|
|
220
|
+
!Number.isSafeInteger(input.detail.executionGeneration) ||
|
|
221
|
+
input.detail.executionGeneration <= 1 ||
|
|
222
|
+
!Number.isSafeInteger(input.detail.providerRecoveryCount) ||
|
|
223
|
+
input.detail.providerRecoveryCount <= 0 ||
|
|
224
|
+
input.detail.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
|
|
225
|
+
!Number.isSafeInteger(input.detail.continueDelayMs) ||
|
|
226
|
+
input.detail.continueDelayMs <= 0
|
|
227
|
+
) {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
return ApplicationFailure.create({
|
|
231
|
+
message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
|
|
232
|
+
type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
|
|
233
|
+
nonRetryable: true,
|
|
234
|
+
details: [input.detail],
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Convert the atomic claim transaction's failure into a small, stable
|
|
240
|
+
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
241
|
+
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
242
|
+
* workflow history. Contention and operational database unavailability are
|
|
243
|
+
* safe to re-read after backoff because the claim transaction contains no
|
|
244
|
+
* model/tool effects; permanent database and state failures require terminal
|
|
245
|
+
* settlement.
|
|
246
|
+
*/
|
|
247
|
+
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
248
|
+
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
249
|
+
const sqlState = persistenceFailure?.details.sqlState ?? null;
|
|
250
|
+
const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
|
|
251
|
+
// The database transaction itself retries only the two contention failures
|
|
252
|
+
// proven safe for immediate replay. Once the activity has failed, the
|
|
253
|
+
// workflow may also retry operational outages after a durable re-read and
|
|
254
|
+
// bounded delay. Unknown driver/database failures stay recoverable because
|
|
255
|
+
// they commonly represent a lost connection; known constraint, auth, and
|
|
256
|
+
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
257
|
+
const retryable = Boolean(
|
|
258
|
+
rawTransportFailure ||
|
|
259
|
+
(persistenceFailure &&
|
|
260
|
+
(sqlState === null ||
|
|
261
|
+
sqlState.startsWith("08") ||
|
|
262
|
+
sqlState.startsWith("40") ||
|
|
263
|
+
sqlState.startsWith("53") ||
|
|
264
|
+
sqlState === "55P03" ||
|
|
265
|
+
sqlState === "57014" ||
|
|
266
|
+
sqlState === "57P01" ||
|
|
267
|
+
sqlState === "57P02" ||
|
|
268
|
+
sqlState === "57P03" ||
|
|
269
|
+
sqlState.startsWith("58"))),
|
|
270
|
+
);
|
|
271
|
+
const detail: PreClaimFailureDetail = {
|
|
272
|
+
disposition: retryable ? "retryable" : "permanent",
|
|
273
|
+
code:
|
|
274
|
+
persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
|
|
275
|
+
};
|
|
276
|
+
return ApplicationFailure.create({
|
|
277
|
+
message: PRE_CLAIM_FAILURE_MESSAGE,
|
|
278
|
+
type: PRE_CLAIM_FAILURE_TYPE,
|
|
279
|
+
nonRetryable: true,
|
|
280
|
+
details: [detail],
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Resolve which Codex account a turn runs on (multi-account P1): session-pin >
|
|
286
|
+
* workspace-active. No rotation in P1. The selected id must still be in the
|
|
287
|
+
* connected set — a disconnected pin was FK-nulled, so a stale id can't appear,
|
|
288
|
+
* but we guard anyway. Returns null when there is no usable account (the turn
|
|
289
|
+
* then fails with the existing relogin error path).
|
|
290
|
+
*/
|
|
291
|
+
export function isWorkerShutdownCancellation(error: unknown): boolean {
|
|
292
|
+
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Recognize the one active-route transition that cannot finish inside its
|
|
297
|
+
* originating attempt. A Modal-home session may start on a Connected Machine
|
|
298
|
+
* without creating or leasing its managed home box. When an explicit attach
|
|
299
|
+
* clears the active pointer back to home, the pointer commit is authoritative,
|
|
300
|
+
* but this attempt has no home session to serve the next sandbox operation.
|
|
301
|
+
*
|
|
302
|
+
* The Agents SDK may retain the typed routing error directly, through `cause`,
|
|
303
|
+
* or inside an AggregateError from a parallel function-tool batch. Traverse
|
|
304
|
+
* only those structural error links with a strict bound; never classify from
|
|
305
|
+
* message text, which could originate in model or tool content.
|
|
306
|
+
*/
|
|
307
|
+
export function sandboxRouteTransitionCode(
|
|
308
|
+
error: unknown,
|
|
309
|
+
): "home_unavailable_this_turn" | "workspace_root_changed_this_turn" | null {
|
|
310
|
+
const pending: unknown[] = [error];
|
|
311
|
+
const seen = new WeakSet<object>();
|
|
312
|
+
let inspected = 0;
|
|
313
|
+
|
|
314
|
+
while (pending.length > 0 && inspected < 64) {
|
|
315
|
+
const current = pending.shift();
|
|
316
|
+
inspected += 1;
|
|
317
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
318
|
+
seen.add(current);
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
const record = current as Record<string, unknown>;
|
|
322
|
+
if (
|
|
323
|
+
(current instanceof ActiveBackendUnresolvableError ||
|
|
324
|
+
record.name === "ActiveBackendUnresolvableError") &&
|
|
325
|
+
record.code === "home_unavailable_this_turn"
|
|
326
|
+
) {
|
|
327
|
+
return "home_unavailable_this_turn";
|
|
328
|
+
}
|
|
329
|
+
if (
|
|
330
|
+
current instanceof RoutingWorkspaceRootChangedError ||
|
|
331
|
+
current instanceof SelfhostedWorkspaceRootChangedError ||
|
|
332
|
+
record.name === "RoutingWorkspaceRootChangedError" ||
|
|
333
|
+
record.name === "SelfhostedWorkspaceRootChangedError"
|
|
334
|
+
) {
|
|
335
|
+
return "workspace_root_changed_this_turn";
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
for (const key of ["cause", "error"] as const) {
|
|
339
|
+
const nested = record[key];
|
|
340
|
+
if (nested && typeof nested === "object") pending.push(nested);
|
|
341
|
+
}
|
|
342
|
+
if (Array.isArray(record.errors)) {
|
|
343
|
+
pending.push(...record.errors.slice(0, 32));
|
|
344
|
+
}
|
|
345
|
+
} catch {
|
|
346
|
+
// A hostile proxy or getter is not a route-transition proof.
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
return null;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
export function isSandboxRouteTransitionError(error: unknown): boolean {
|
|
354
|
+
return sandboxRouteTransitionCode(error) !== null;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/** Backward-compatible name for callers/tests that only exercised the original
|
|
358
|
+
* machine-to-home transition. */
|
|
359
|
+
export function isHomeSandboxTurnTransitionError(error: unknown): boolean {
|
|
360
|
+
return sandboxRouteTransitionCode(error) === "home_unavailable_this_turn";
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Review captures and protective snapshots are cache/persistence housekeeping,
|
|
365
|
+
* never part of cancellation correctness. A control-fenced or Temporal-cancelled
|
|
366
|
+
* attempt must release its physical activity promptly so Steer/Pause can advance.
|
|
367
|
+
*/
|
|
368
|
+
export function compactionFailureReason(reason: string): string {
|
|
369
|
+
return reason.startsWith("compaction summarization failed:")
|
|
370
|
+
? reason
|
|
371
|
+
: `compaction summarization failed: ${reason}`;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
export type SafeErrorDiagnostic = {
|
|
375
|
+
errorClass: "WorkerOperationError";
|
|
376
|
+
errorCode: "worker_operation_failed";
|
|
377
|
+
status?: number;
|
|
378
|
+
origin: "worker";
|
|
379
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
380
|
+
};
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Produce the only exception shape allowed in worker logs. It deliberately
|
|
384
|
+
* excludes the arbitrary source message, stack, cause, response/request
|
|
385
|
+
* bodies, and enumerable properties. Exact failure content belongs in the
|
|
386
|
+
* permission-controlled session event, not stdout or telemetry.
|
|
387
|
+
*/
|
|
388
|
+
export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
|
|
389
|
+
const diagnostic: SafeErrorDiagnostic = {
|
|
390
|
+
errorClass: "WorkerOperationError",
|
|
391
|
+
errorCode: "worker_operation_failed",
|
|
392
|
+
origin: "worker",
|
|
393
|
+
};
|
|
394
|
+
try {
|
|
395
|
+
let statusSource = error;
|
|
396
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
397
|
+
diagnostic.historyPersistenceStage = error.stage;
|
|
398
|
+
statusSource = error.cause;
|
|
399
|
+
}
|
|
400
|
+
if (statusSource && typeof statusSource === "object") {
|
|
401
|
+
const status = Number(
|
|
402
|
+
(statusSource as { status?: unknown; statusCode?: unknown }).status ??
|
|
403
|
+
(statusSource as { statusCode?: unknown }).statusCode,
|
|
404
|
+
);
|
|
405
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
406
|
+
diagnostic.status = status;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
} catch {
|
|
410
|
+
// Public diagnostics are best-effort and must never replace the exact
|
|
411
|
+
// internal worker failure.
|
|
412
|
+
}
|
|
413
|
+
return diagnostic;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
export function safeErrorForTelemetry(error: unknown): Error {
|
|
417
|
+
const diagnostic = safeErrorDiagnostic(error);
|
|
418
|
+
const safe = new Error("worker operation failed") as Error & {
|
|
419
|
+
code?: string;
|
|
420
|
+
status?: number;
|
|
421
|
+
origin?: string;
|
|
422
|
+
};
|
|
423
|
+
safe.name = "WorkerOperationError";
|
|
424
|
+
safe.code = diagnostic.errorCode;
|
|
425
|
+
if (diagnostic.status !== undefined) safe.status = diagnostic.status;
|
|
426
|
+
safe.origin = diagnostic.origin;
|
|
427
|
+
return safe;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
export function compactionFailureReasonFromError(error: unknown): string {
|
|
431
|
+
if (
|
|
432
|
+
error instanceof CompactionProviderResponseError ||
|
|
433
|
+
error instanceof EmptyCompactionSummaryError
|
|
434
|
+
) {
|
|
435
|
+
return compactionFailureReason(error.message);
|
|
436
|
+
}
|
|
437
|
+
const errorName = error instanceof Error && error.name ? error.name : "unknown error";
|
|
438
|
+
return compactionFailureReason(`unexpected ${errorName}`);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
export function isCompactionSummaryFailure(error: unknown): boolean {
|
|
442
|
+
return (
|
|
443
|
+
error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
|
|
444
|
+
);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
|
|
448
|
+
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
449
|
+
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
450
|
+
return agentRunFailurePayload(error).retryable === true;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
export function classifyContextWindowOverflowError(
|
|
454
|
+
error: unknown,
|
|
455
|
+
): { message: string; code?: string; detail?: string } | null {
|
|
456
|
+
const fields = collectErrorStrings(error);
|
|
457
|
+
const matched = fields.find(
|
|
458
|
+
(value) =>
|
|
459
|
+
/context[_\s-]*length[_\s-]*exceeded/i.test(value) ||
|
|
460
|
+
/exceeds?\s+(?:the\s+)?context\s+window/i.test(value) ||
|
|
461
|
+
/maximum\s+context\s+length/i.test(value) ||
|
|
462
|
+
/context\s+window[^.]*exceed/i.test(value),
|
|
463
|
+
);
|
|
464
|
+
if (!matched) {
|
|
465
|
+
return null;
|
|
466
|
+
}
|
|
467
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
468
|
+
const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
|
|
469
|
+
return {
|
|
470
|
+
message,
|
|
471
|
+
...(code ? { code } : {}),
|
|
472
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* Recognize an MCP transport/request timeout that escaped the SDK's per-tool
|
|
478
|
+
* `mcpConfig.errorFunction` boundary. A thrown tool invocation is normally
|
|
479
|
+
* converted to an `{isError:true}` tool output; however, connect/tools-list or
|
|
480
|
+
* next-loop transport work can reject the stream iterator after a prior tool
|
|
481
|
+
* output was already published. That is transient external backpressure, not a
|
|
482
|
+
* terminal session error. Match MCP-qualified timeout text only: an unrelated
|
|
483
|
+
* sandbox/model timeout and MCP's application-defined Authentication required signal must
|
|
484
|
+
* retain their existing semantics.
|
|
485
|
+
*/
|
|
486
|
+
export function classifyMcpTransportTimeoutError(
|
|
487
|
+
error: unknown,
|
|
488
|
+
): { message: string; detail?: string } | null {
|
|
489
|
+
const fields = collectErrorStrings(error);
|
|
490
|
+
const matchedText = fields.find(
|
|
491
|
+
(value) =>
|
|
492
|
+
/\bmcp\b/i.test(value) &&
|
|
493
|
+
/(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
|
|
494
|
+
value,
|
|
495
|
+
) &&
|
|
496
|
+
!/authentication\s+required/i.test(value),
|
|
497
|
+
);
|
|
498
|
+
const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
|
|
499
|
+
if (!matchedText && !sanitizedSdkTimeout) {
|
|
500
|
+
return null;
|
|
501
|
+
}
|
|
502
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
503
|
+
const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
|
|
504
|
+
return {
|
|
505
|
+
message,
|
|
506
|
+
...(matched && matched !== message ? { detail: matched } : {}),
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
|
|
511
|
+
if (typeof value === "string") {
|
|
512
|
+
return [value];
|
|
513
|
+
}
|
|
514
|
+
if (!value || typeof value !== "object") {
|
|
515
|
+
return [];
|
|
516
|
+
}
|
|
517
|
+
if (seen.has(value)) {
|
|
518
|
+
return [];
|
|
519
|
+
}
|
|
520
|
+
seen.add(value);
|
|
521
|
+
const out: string[] = [];
|
|
522
|
+
const record = value as Record<string, unknown>;
|
|
523
|
+
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
524
|
+
const field = record[key];
|
|
525
|
+
if (typeof field === "string" && field.length > 0) {
|
|
526
|
+
out.push(field);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
for (const key of ["error", "cause", "response", "data"]) {
|
|
530
|
+
out.push(...collectErrorStrings(record[key], seen));
|
|
531
|
+
}
|
|
532
|
+
return out;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Compute the conversation-truth rows a reconcile pass should append, given the
|
|
537
|
+
* SDK's current `state.history` and the count already persisted.
|
|
538
|
+
*
|
|
539
|
+
* `state.history` is a computed getter that runs the SDK's orphan-tool-call
|
|
540
|
+
* pruning on every access, so it is non-monotonic: a `function_call` with no
|
|
541
|
+
* settling result yet is transiently absent and a later access yields a
|
|
542
|
+
* different, possibly shorter/reordered list. The old code sliced this list by
|
|
543
|
+
* a blind length watermark and appended at fixed positions with
|
|
544
|
+
* onConflictDoNothing, which could freeze a position with one shape and later
|
|
545
|
+
* persist a `function_call_result` whose `function_call` had been pruned away in
|
|
546
|
+
* an earlier slice — the orphaned tool output that 400s the Responses API and
|
|
547
|
+
* bricks the session on every replay.
|
|
548
|
+
*
|
|
549
|
+
* Defending: structurally repair the full current history into an API-valid sequence (the
|
|
550
|
+
* same pure rules the read path uses), then append only the new tail beyond the
|
|
551
|
+
* watermark. A trailing dangling call is dropped here and re-evaluated next
|
|
552
|
+
* pass once its result lands, so a call and its result are written together at
|
|
553
|
+
* consecutive positions and a result is never persisted without its call. The
|
|
554
|
+
* watermark advances to the repaired length — never past anything unwritten —
|
|
555
|
+
* so a non-monotonic history can never desync it. When previously-persisted
|
|
556
|
+
* rows already exceed the repaired length (e.g. legacy orphans written before
|
|
557
|
+
* this fix), nothing new is appended and the watermark holds steady.
|
|
558
|
+
*/
|
|
559
|
+
/**
|
|
560
|
+
* Stable+unique usage source key for one model call, used to build the per-call
|
|
561
|
+
* idempotency key (`usage:model.tokens:${turnId}:${sourceKey}`). The turnId is
|
|
562
|
+
* shared across a new attempt of the SAME turn (recovery, approval
|
|
563
|
+
* rerun, activity retry), so the sourceKey alone must distinguish calls.
|
|
564
|
+
*
|
|
565
|
+
* - A provider responseId is globally stable+unique, so reuse it verbatim: a
|
|
566
|
+
* true activity retry that re-emits the same responseId correctly DEDUPES
|
|
567
|
+
* (one charge), while two distinct calls get distinct ids.
|
|
568
|
+
* - Without a responseId the old synthesized key was only POSITIONAL ("response-1",
|
|
569
|
+
* "aggregate"), which collides across a re-dispatch — dispatch B's first
|
|
570
|
+
* call reuses dispatch A's "response-1" key and its charge is silently
|
|
571
|
+
* dropped (undercharge). Qualifying the synthesized key with the
|
|
572
|
+
* per-execution dispatch id (the Temporal activityId, unique per scheduled
|
|
573
|
+
* execution) makes re-dispatched calls distinct while still deduping a
|
|
574
|
+
* same-execution retry.
|
|
575
|
+
*/
|
|
576
|
+
export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
|
|
577
|
+
"unable to connect. is the computer able to access the url?";
|
|
578
|
+
|
|
579
|
+
export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
|
|
580
|
+
return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
export function isTransientProviderError(error: unknown): boolean {
|
|
584
|
+
const status =
|
|
585
|
+
typeof error === "object" && error !== null
|
|
586
|
+
? Number(
|
|
587
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
588
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
589
|
+
)
|
|
590
|
+
: undefined;
|
|
591
|
+
// A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
|
|
592
|
+
// (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
|
|
593
|
+
// a request fault that must NOT auto-retry — even if its body happens to read like
|
|
594
|
+
// "connection error" or "overloaded". The code/message heuristics below apply ONLY
|
|
595
|
+
// when no status survived: a network fault or an SDK-rethrown bare Error.
|
|
596
|
+
if (status !== undefined && Number.isFinite(status)) {
|
|
597
|
+
return status >= 500 && status < 600;
|
|
598
|
+
}
|
|
599
|
+
const code =
|
|
600
|
+
typeof error === "object" && error !== null && "code" in error
|
|
601
|
+
? String((error as { code?: unknown }).code)
|
|
602
|
+
: undefined;
|
|
603
|
+
if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
|
|
604
|
+
return true;
|
|
605
|
+
}
|
|
606
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
607
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
608
|
+
return true;
|
|
609
|
+
}
|
|
610
|
+
return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
|
|
611
|
+
message,
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
export type XaiCredentialFailure = {
|
|
616
|
+
kind: "auth" | "forbidden" | "rate_limit";
|
|
617
|
+
cooldownMs: number | null;
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Only definitive SuperGrok account refusals may move the same logical turn to
|
|
622
|
+
* another credential. A marked HTTP 401/403/429, or an HTTP 200 SSE terminal
|
|
623
|
+
* that is a rate-limit/capacity diagnostic, proves inference was refused
|
|
624
|
+
* without an accepted model response; refresh relogin is equally definitive.
|
|
625
|
+
*/
|
|
626
|
+
export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
|
|
627
|
+
let relogin: unknown = error;
|
|
628
|
+
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
629
|
+
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
630
|
+
return { kind: "auth", cooldownMs: null };
|
|
631
|
+
}
|
|
632
|
+
relogin = (relogin as Record<string, unknown>).cause;
|
|
633
|
+
}
|
|
634
|
+
if (!isXaiSubscriptionTransportError(error)) return null;
|
|
635
|
+
let current: unknown = error;
|
|
636
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
637
|
+
const value = current as Record<string, unknown>;
|
|
638
|
+
const body =
|
|
639
|
+
value.error && typeof value.error === "object"
|
|
640
|
+
? (value.error as Record<string, unknown>)
|
|
641
|
+
: null;
|
|
642
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
643
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
644
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
645
|
+
return { kind: "auth", cooldownMs: null };
|
|
646
|
+
}
|
|
647
|
+
if (status === 403) {
|
|
648
|
+
return { kind: "forbidden", cooldownMs: null };
|
|
649
|
+
}
|
|
650
|
+
const message = String(value.message ?? body?.message ?? "");
|
|
651
|
+
if (
|
|
652
|
+
isXaiSubscriptionRateLimitDiagnostic({
|
|
653
|
+
code,
|
|
654
|
+
message,
|
|
655
|
+
status: Number.isInteger(status) ? status : null,
|
|
656
|
+
})
|
|
657
|
+
) {
|
|
658
|
+
return {
|
|
659
|
+
kind: "rate_limit",
|
|
660
|
+
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
current = value.cause;
|
|
664
|
+
}
|
|
665
|
+
return null;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
export function agentRunFailurePayload(
|
|
669
|
+
error: unknown,
|
|
670
|
+
options: { isCodexTurn?: boolean } = {},
|
|
671
|
+
): {
|
|
672
|
+
error: string;
|
|
673
|
+
code?: string;
|
|
674
|
+
retryable?: boolean;
|
|
675
|
+
detail?: string;
|
|
676
|
+
timeoutClass?: string;
|
|
677
|
+
responseObserved?: boolean;
|
|
678
|
+
requestId?: string;
|
|
679
|
+
eventCount?: number;
|
|
680
|
+
lastEventType?: string;
|
|
681
|
+
silenceDurationMs?: number;
|
|
682
|
+
correlationId?: string;
|
|
683
|
+
stage?: string;
|
|
684
|
+
sqlState?: string | null;
|
|
685
|
+
attempts?: number;
|
|
686
|
+
retryOutcome?: string;
|
|
687
|
+
database?: Record<string, string>;
|
|
688
|
+
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
689
|
+
mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
|
|
690
|
+
} {
|
|
691
|
+
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
692
|
+
const underlying = isSessionEventPersistenceError(error.cause)
|
|
693
|
+
? agentRunFailurePayload(error.cause, options)
|
|
694
|
+
: {
|
|
695
|
+
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
696
|
+
};
|
|
697
|
+
return {
|
|
698
|
+
...underlying,
|
|
699
|
+
historyPersistenceStage: error.stage,
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
703
|
+
const status =
|
|
704
|
+
typeof error === "object" && error !== null
|
|
705
|
+
? Number(
|
|
706
|
+
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
707
|
+
(error as { statusCode?: unknown }).statusCode,
|
|
708
|
+
)
|
|
709
|
+
: undefined;
|
|
710
|
+
const code =
|
|
711
|
+
typeof error === "object" && error !== null && "code" in error
|
|
712
|
+
? String((error as { code?: unknown }).code)
|
|
713
|
+
: undefined;
|
|
714
|
+
if (error instanceof ActiveSessionHistoryLimitExceededError) {
|
|
715
|
+
return {
|
|
716
|
+
error:
|
|
717
|
+
"The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
|
|
718
|
+
code: error.code,
|
|
719
|
+
retryable: false,
|
|
720
|
+
detail: error.message,
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
if (error instanceof ApprovalRunStateLimitExceededError) {
|
|
724
|
+
return {
|
|
725
|
+
error:
|
|
726
|
+
"The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
|
|
727
|
+
code: error.code,
|
|
728
|
+
retryable: false,
|
|
729
|
+
detail: error.message,
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
// An accepted Codex stream with no terminal response is malformed/partial,
|
|
733
|
+
// not provider backpressure. Replaying the same accepted turn could repeat
|
|
734
|
+
// model or tool effects, so this marked transport failure must outrank the
|
|
735
|
+
// generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
|
|
736
|
+
if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
|
|
737
|
+
return {
|
|
738
|
+
error: "The Codex response stream ended without a terminal response",
|
|
739
|
+
code: "invalid_sse_terminal",
|
|
740
|
+
retryable: false,
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
if (isXaiSubscriptionHostedToolContinuationError(error)) {
|
|
744
|
+
return {
|
|
745
|
+
error:
|
|
746
|
+
"SuperGrok stopped responding after its hosted search completed. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
|
|
747
|
+
code: "xai_hosted_tool_continuation_stalled",
|
|
748
|
+
retryable: false,
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
const xaiStreamTerminal = classifyXaiSubscriptionStreamingTerminalError(error);
|
|
752
|
+
if (xaiStreamTerminal) {
|
|
753
|
+
return {
|
|
754
|
+
error: xaiStreamTerminal.message,
|
|
755
|
+
code: xaiStreamTerminal.code,
|
|
756
|
+
retryable: xaiStreamTerminal.status === 429,
|
|
757
|
+
lastEventType: xaiStreamTerminal.eventType,
|
|
758
|
+
...(xaiStreamTerminal.requestId ? { requestId: xaiStreamTerminal.requestId } : {}),
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
const xaiStreamTimeout = classifyXaiSubscriptionStreamIdleTimeoutError(error);
|
|
762
|
+
if (xaiStreamTimeout) {
|
|
763
|
+
return {
|
|
764
|
+
error:
|
|
765
|
+
"SuperGrok stopped sending valid response events. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
|
|
766
|
+
code: "xai_response_stream_idle_timeout",
|
|
767
|
+
retryable: false,
|
|
768
|
+
responseObserved: xaiStreamTimeout.responseObserved,
|
|
769
|
+
eventCount: xaiStreamTimeout.eventCount,
|
|
770
|
+
silenceDurationMs: xaiStreamTimeout.silenceDurationMs,
|
|
771
|
+
...(xaiStreamTimeout.requestId ? { requestId: xaiStreamTimeout.requestId } : {}),
|
|
772
|
+
...(xaiStreamTimeout.lastEventType ? { lastEventType: xaiStreamTimeout.lastEventType } : {}),
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
if (isSessionEventPersistenceError(error)) {
|
|
776
|
+
const { details } = error;
|
|
777
|
+
return {
|
|
778
|
+
error: error.message,
|
|
779
|
+
code: details.code,
|
|
780
|
+
detail:
|
|
781
|
+
details.retryOutcome === "exhausted"
|
|
782
|
+
? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
|
|
783
|
+
: "The database rejected the idempotent persistence transaction.",
|
|
784
|
+
correlationId: details.correlationId,
|
|
785
|
+
stage: details.stage,
|
|
786
|
+
sqlState: details.sqlState,
|
|
787
|
+
attempts: details.attempts,
|
|
788
|
+
retryOutcome: details.retryOutcome,
|
|
789
|
+
...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
793
|
+
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
794
|
+
// goal against a capped backend). Surface a precise, actionable message with
|
|
795
|
+
// the humanized reset window and code, non-retryable. Checked BEFORE the
|
|
796
|
+
// generic 429 branch below (a usage cap is also a 429).
|
|
797
|
+
// This terminal payload classifier may receive a plain SDK-shaped error in
|
|
798
|
+
// tests or after wrapper metadata was stripped. An explicit
|
|
799
|
+
// `usage_limit_reached` shape must still outrank generic 429 retryability.
|
|
800
|
+
// Credential quarantine/failover remains separately provenance-gated by
|
|
801
|
+
// `isCodexTransportError`; this branch only chooses the truthful user payload.
|
|
802
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
803
|
+
if (usageLimit) {
|
|
804
|
+
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
805
|
+
}
|
|
806
|
+
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
807
|
+
allowLegacyRequestTimeout: options.isCodexTurn === true,
|
|
808
|
+
});
|
|
809
|
+
if (codexTimeout) {
|
|
810
|
+
return {
|
|
811
|
+
error: codexTimeout.responseObserved
|
|
812
|
+
? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
|
|
813
|
+
: "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
|
|
814
|
+
code: "codex_response_timeout",
|
|
815
|
+
retryable: false,
|
|
816
|
+
timeoutClass: codexTimeout.timeoutClass,
|
|
817
|
+
responseObserved: codexTimeout.responseObserved,
|
|
818
|
+
...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
|
|
819
|
+
...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
const mcpTimeout = classifyMcpTransportTimeoutError(error);
|
|
823
|
+
if (mcpTimeout) {
|
|
824
|
+
const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
|
|
825
|
+
return {
|
|
826
|
+
error:
|
|
827
|
+
"An MCP server request timed out. Any completed tool output was checkpointed; the session can continue safely.",
|
|
828
|
+
code: "mcp_transport_timeout",
|
|
829
|
+
retryable: true,
|
|
830
|
+
...(mcpTimeout.detail || mcpTimeout.message
|
|
831
|
+
? { detail: mcpTimeout.detail ?? mcpTimeout.message }
|
|
832
|
+
: {}),
|
|
833
|
+
...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
if (isMcpTransportConnectivityError(error)) {
|
|
837
|
+
const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
|
|
838
|
+
return {
|
|
839
|
+
error:
|
|
840
|
+
"A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
|
|
841
|
+
code: "mcp_transport_unavailable",
|
|
842
|
+
retryable: true,
|
|
843
|
+
detail: message,
|
|
844
|
+
...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
|
|
845
|
+
};
|
|
846
|
+
}
|
|
847
|
+
if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
|
|
848
|
+
return {
|
|
849
|
+
error:
|
|
850
|
+
"The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
|
|
851
|
+
code: UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
852
|
+
retryable: true,
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
if (
|
|
856
|
+
status === 429 ||
|
|
857
|
+
code === "rate_limit_exceeded" ||
|
|
858
|
+
/(?:too many requests|rate.?limit|\b429\b)/i.test(message)
|
|
859
|
+
) {
|
|
860
|
+
return {
|
|
861
|
+
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
862
|
+
code: "provider_rate_limited",
|
|
863
|
+
retryable: true,
|
|
864
|
+
...(message && message !== "Too Many Requests" ? { detail: message } : {}),
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
// Transient upstream backpressure (5xx / overloaded / dropped connection): keep
|
|
868
|
+
// the provider's own message (it is already user-meaningful) but mark it
|
|
869
|
+
// retryable so a goal-bearing session idles and auto-continues instead of going
|
|
870
|
+
// terminal on a provider's bad minute. See isTransientProviderError.
|
|
871
|
+
if (isTransientProviderError(error)) {
|
|
872
|
+
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
873
|
+
return {
|
|
874
|
+
error:
|
|
875
|
+
"OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
876
|
+
code: "upstream_connectivity_unavailable",
|
|
877
|
+
retryable: true,
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
return { error: message, code: "provider_unavailable", retryable: true };
|
|
881
|
+
}
|
|
882
|
+
return { error: message };
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
export type CodexCredentialFailure = {
|
|
886
|
+
kind: "auth" | "forbidden" | "rate_limit" | "quota";
|
|
887
|
+
cooldownSeconds: number | null;
|
|
888
|
+
};
|
|
889
|
+
|
|
890
|
+
export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* Resolve a deterministic quarantine end. Generic request throttling honors
|
|
894
|
+
* provider retry-after (or one minute); allowance/quota refusal waits for the
|
|
895
|
+
* LAST of provider reset and every still-binding cached window (five-hour and
|
|
896
|
+
* weekly both bind), falling back to one complete five-hour window when no reset
|
|
897
|
+
* metadata exists.
|
|
898
|
+
*/
|
|
899
|
+
export function codexCredentialCooldownUntil(
|
|
900
|
+
failure: CodexCredentialFailure,
|
|
901
|
+
account: Pick<
|
|
902
|
+
CodexAccountStatus,
|
|
903
|
+
"primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
|
|
904
|
+
> | null,
|
|
905
|
+
now: Date,
|
|
906
|
+
): Date | null {
|
|
907
|
+
if (failure.kind === "auth" || failure.kind === "forbidden") {
|
|
908
|
+
return null;
|
|
909
|
+
}
|
|
910
|
+
const providerReset =
|
|
911
|
+
failure.cooldownSeconds !== null &&
|
|
912
|
+
Number.isFinite(failure.cooldownSeconds) &&
|
|
913
|
+
failure.cooldownSeconds > 0
|
|
914
|
+
? new Date(now.getTime() + Math.ceil(failure.cooldownSeconds) * 1000)
|
|
915
|
+
: null;
|
|
916
|
+
if (failure.kind === "rate_limit") {
|
|
917
|
+
return providerReset ?? new Date(now.getTime() + PROVIDER_BACKPRESSURE_DELAY_MS);
|
|
918
|
+
}
|
|
919
|
+
const blockingResets = account
|
|
920
|
+
? [
|
|
921
|
+
{ used: account.primaryUsedPercent, reset: account.primaryResetAt },
|
|
922
|
+
{ used: account.secondaryUsedPercent, reset: account.secondaryResetAt },
|
|
923
|
+
]
|
|
924
|
+
.filter(
|
|
925
|
+
(window): window is { used: number; reset: Date } =>
|
|
926
|
+
(window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
|
|
927
|
+
window.reset instanceof Date &&
|
|
928
|
+
window.reset.getTime() > now.getTime(),
|
|
929
|
+
)
|
|
930
|
+
.map((window) => window.reset)
|
|
931
|
+
: [];
|
|
932
|
+
const quotaResets = providerReset ? [...blockingResets, providerReset] : blockingResets;
|
|
933
|
+
if (quotaResets.length === 0) {
|
|
934
|
+
return new Date(now.getTime() + CODEX_ALLOWANCE_FALLBACK_MS);
|
|
935
|
+
}
|
|
936
|
+
return quotaResets.reduce((latest, reset) =>
|
|
937
|
+
reset.getTime() > latest.getTime() ? reset : latest,
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* Only definitive credential/account refusals are safe rotation signals.
|
|
943
|
+
* Ambiguous network failures, malformed/partial streams, invalid model content,
|
|
944
|
+
* prompt 4xx, and provider 5xx may already have consumed tokens or persisted
|
|
945
|
+
* progress and therefore MUST NOT walk the credential pool automatically.
|
|
946
|
+
*/
|
|
947
|
+
export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
|
|
948
|
+
// A permanent OAuth refresh failure is definitive and the shared resolver has
|
|
949
|
+
// already fenced/stamped the exact credential version. The OpenAI client can
|
|
950
|
+
// wrap a rejection from its custom fetch in APIConnectionError, so recognize
|
|
951
|
+
// the typed exception through the same bounded cause chain used below.
|
|
952
|
+
let refreshError: unknown = error;
|
|
953
|
+
for (let depth = 0; depth < 6 && refreshError && typeof refreshError === "object"; depth += 1) {
|
|
954
|
+
if (refreshError instanceof CodexReloginRequired) {
|
|
955
|
+
return { kind: "auth", cooldownSeconds: null };
|
|
956
|
+
}
|
|
957
|
+
refreshError = (refreshError as Record<string, unknown>).cause;
|
|
958
|
+
}
|
|
959
|
+
// The activity catch also receives sandbox, MCP, storage, and tool failures.
|
|
960
|
+
// Their HTTP status codes are not Codex account state and must never walk the
|
|
961
|
+
// subscription pool or replay a tool on another credential.
|
|
962
|
+
if (!isCodexTransportError(error)) {
|
|
963
|
+
return null;
|
|
964
|
+
}
|
|
965
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
966
|
+
if (usageLimit) {
|
|
967
|
+
return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
|
|
968
|
+
}
|
|
969
|
+
let cur: unknown = error;
|
|
970
|
+
for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
|
|
971
|
+
const value = cur as Record<string, unknown>;
|
|
972
|
+
const body =
|
|
973
|
+
value.error && typeof value.error === "object"
|
|
974
|
+
? (value.error as Record<string, unknown>)
|
|
975
|
+
: null;
|
|
976
|
+
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
977
|
+
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
978
|
+
const directRetryAfter = Number(
|
|
979
|
+
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
980
|
+
);
|
|
981
|
+
const retryAfterHeader =
|
|
982
|
+
headerValue(value.headers, "retry-after") ??
|
|
983
|
+
headerValue(value.responseHeaders, "retry-after") ??
|
|
984
|
+
headerValue(body?.headers, "retry-after");
|
|
985
|
+
const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
|
|
986
|
+
const retryAfterDate =
|
|
987
|
+
retryAfterHeader !== null && !Number.isFinite(retryAfterNumber)
|
|
988
|
+
? Date.parse(retryAfterHeader)
|
|
989
|
+
: Number.NaN;
|
|
990
|
+
const retryAfter = Number.isFinite(directRetryAfter)
|
|
991
|
+
? directRetryAfter
|
|
992
|
+
: Number.isFinite(retryAfterNumber)
|
|
993
|
+
? retryAfterNumber
|
|
994
|
+
: Number.isFinite(retryAfterDate)
|
|
995
|
+
? Math.max(0, (retryAfterDate - Date.now()) / 1000)
|
|
996
|
+
: Number.NaN;
|
|
997
|
+
const cooldownSeconds =
|
|
998
|
+
Number.isFinite(retryAfter) && retryAfter > 0 ? Math.ceil(retryAfter) : null;
|
|
999
|
+
// Provider quota codes are more specific than their HTTP transport status.
|
|
1000
|
+
// A permanent allowance refusal commonly arrives as HTTP 429; classify it
|
|
1001
|
+
// before generic backpressure so it receives the binding-window cooldown.
|
|
1002
|
+
if (
|
|
1003
|
+
code === "insufficient_quota" ||
|
|
1004
|
+
code === "quota_exceeded" ||
|
|
1005
|
+
code === "billing_hard_limit_reached"
|
|
1006
|
+
) {
|
|
1007
|
+
return { kind: "quota", cooldownSeconds };
|
|
1008
|
+
}
|
|
1009
|
+
if (status === 401 || code === "unauthorized" || code === "invalid_api_key") {
|
|
1010
|
+
return { kind: "auth", cooldownSeconds };
|
|
1011
|
+
}
|
|
1012
|
+
if (status === 403) {
|
|
1013
|
+
return { kind: "forbidden", cooldownSeconds };
|
|
1014
|
+
}
|
|
1015
|
+
if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
|
|
1016
|
+
return { kind: "rate_limit", cooldownSeconds };
|
|
1017
|
+
}
|
|
1018
|
+
cur = value.cause;
|
|
1019
|
+
}
|
|
1020
|
+
return null;
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/** Humanize a seconds duration into a short "2h 5m" / "9m" / "in under a minute" string. */
|
|
1024
|
+
export function humanizeResetWindow(resetsInSeconds: number | null): string {
|
|
1025
|
+
if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
|
|
1026
|
+
return "shortly";
|
|
1027
|
+
}
|
|
1028
|
+
const total = Math.ceil(resetsInSeconds);
|
|
1029
|
+
if (total < 60) {
|
|
1030
|
+
return "in under a minute";
|
|
1031
|
+
}
|
|
1032
|
+
const hours = Math.floor(total / 3600);
|
|
1033
|
+
const minutes = Math.round((total % 3600) / 60);
|
|
1034
|
+
if (hours > 0) {
|
|
1035
|
+
return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
|
|
1036
|
+
}
|
|
1037
|
+
return `in about ${minutes}m`;
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/**
|
|
1041
|
+
* Build the turn.failed payload for a ChatGPT/Codex usage cap: a precise,
|
|
1042
|
+
* actionable message naming the reset window, the stable `codex_usage_limit_reached`
|
|
1043
|
+
* code, and retryable:false (an auto-retry would just re-hit the cap).
|
|
1044
|
+
*/
|
|
1045
|
+
export function codexUsageLimitFailurePayload(
|
|
1046
|
+
info: { resetsInSeconds: number | null },
|
|
1047
|
+
detail: string,
|
|
1048
|
+
opts?: { allAccounts?: boolean },
|
|
1049
|
+
): { error: string; code: string; retryable: boolean; detail?: string } {
|
|
1050
|
+
// P3: when EVERY connected subscription is rate-limited the message names the
|
|
1051
|
+
// earliest reset across accounts; the single-account message is unchanged.
|
|
1052
|
+
const error = opts?.allAccounts
|
|
1053
|
+
? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. ` +
|
|
1054
|
+
`You can switch this session to a different model in the meantime, or wait for a subscription to reset.`
|
|
1055
|
+
: `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. ` +
|
|
1056
|
+
`You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
|
|
1057
|
+
return {
|
|
1058
|
+
error,
|
|
1059
|
+
code: "codex_usage_limit_reached",
|
|
1060
|
+
retryable: false,
|
|
1061
|
+
...(detail ? { detail } : {}),
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// A usage cap that won't reset for a long time should not pin a Temporal timer
|
|
1066
|
+
// open indefinitely for a goal-bearing session; cap the continuation hold so the
|
|
1067
|
+
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
1068
|
+
export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|