@opengeni/worker-bundle 0.20.3 → 0.22.1-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +52 -0
  2. package/dist/activities/agent-turn/agent-build.d.ts +89 -0
  3. package/dist/activities/agent-turn/claim.d.ts +67 -0
  4. package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
  5. package/dist/activities/agent-turn/codex.d.ts +32 -0
  6. package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
  7. package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
  8. package/dist/activities/agent-turn/errors.d.ts +253 -0
  9. package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
  10. package/dist/activities/agent-turn/file-resources.d.ts +37 -0
  11. package/dist/activities/agent-turn/finalization.d.ts +42 -0
  12. package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
  13. package/dist/activities/agent-turn/governance-model.d.ts +66 -0
  14. package/dist/activities/agent-turn/history-sink.d.ts +52 -0
  15. package/dist/activities/agent-turn/history.d.ts +63 -0
  16. package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
  17. package/dist/activities/agent-turn/model-usage.d.ts +218 -0
  18. package/dist/activities/agent-turn/quiescence.d.ts +135 -0
  19. package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
  20. package/dist/activities/agent-turn/run.d.ts +3 -0
  21. package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
  22. package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
  23. package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
  24. package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
  25. package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
  26. package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
  27. package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
  28. package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
  29. package/dist/activities/agent-turn/turn-context.d.ts +144 -0
  30. package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
  31. package/dist/activities/agent-turn.d.ts +12 -954
  32. package/dist/activities/automations.d.ts +16 -0
  33. package/dist/activities/common.d.ts +1 -1
  34. package/dist/activities/connector-attachments.d.ts +13 -0
  35. package/dist/activities/documents.d.ts +1 -0
  36. package/dist/activities/environment.d.ts +26 -15
  37. package/dist/activities/generated-images.d.ts +1 -1
  38. package/dist/activities/git-credential-renewal.d.ts +3 -1
  39. package/dist/activities/goals.d.ts +4 -1
  40. package/dist/activities/google-drive-provider.d.ts +29 -0
  41. package/dist/activities/google-drive-publication.d.ts +17 -4
  42. package/dist/activities/incident-telemetry-authority.d.ts +52 -0
  43. package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
  44. package/dist/activities/knowledge-source-sync.d.ts +4 -0
  45. package/dist/activities/mcp-credentials.d.ts +9 -3
  46. package/dist/activities/open-suffix-resume.d.ts +77 -0
  47. package/dist/activities/packs.d.ts +3 -11
  48. package/dist/activities/parent-wake.d.ts +19 -0
  49. package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
  50. package/dist/activities/retained-screenshots.d.ts +1 -1
  51. package/dist/activities/rig-verification.d.ts +1 -1
  52. package/dist/activities/run-input.d.ts +9 -10
  53. package/dist/activities/sandbox-images.d.ts +1 -14
  54. package/dist/activities/sandbox-lease.d.ts +21 -3
  55. package/dist/activities/scheduled-tasks.d.ts +47 -0
  56. package/dist/activities/types.d.ts +23 -2
  57. package/dist/activities/video-media-validation.d.ts +1 -1
  58. package/dist/activities/video-output-retention.d.ts +1 -2
  59. package/dist/activities/xai-auth.d.ts +3 -0
  60. package/dist/{activities-control-BJSDRSIP.js → activities-control-DKG22ONJ.js} +2646 -551
  61. package/dist/activities-control-DKG22ONJ.js.map +1 -0
  62. package/dist/activities-control.d.ts +6 -2
  63. package/dist/activities-turn-4IMC3MAA.js +21326 -0
  64. package/dist/activities-turn-4IMC3MAA.js.map +1 -0
  65. package/dist/activities.d.ts +7 -3
  66. package/dist/artifact-materializer-entry.js +41 -22
  67. package/dist/artifact-materializer-entry.js.map +1 -1
  68. package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
  69. package/dist/chunk-CE7TPKBS.js.map +1 -0
  70. package/dist/chunk-CNYGVRKF.js +4698 -0
  71. package/dist/chunk-CNYGVRKF.js.map +1 -0
  72. package/dist/connection-provider-use.d.ts +18 -0
  73. package/dist/editable-artifact-materializer.d.ts +4 -3
  74. package/dist/github-rest-mcp.d.ts +26 -0
  75. package/dist/index.d.ts +10 -2
  76. package/dist/index.js +58 -14
  77. package/dist/index.js.map +1 -1
  78. package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
  79. package/dist/model-context-contributions.d.ts +8 -4
  80. package/dist/observability-metrics.d.ts +63 -4
  81. package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
  82. package/dist/personal-github-git-credentials.d.ts +9 -0
  83. package/dist/pr-review-credentials.d.ts +6 -0
  84. package/dist/sandbox-archive-storage.d.ts +45 -0
  85. package/dist/sandbox-resume.d.ts +65 -6
  86. package/dist/sandbox-routing.d.ts +38 -2
  87. package/dist/scheduled-alert-occurrence.d.ts +33 -0
  88. package/dist/workflow-bundle.js +87 -28
  89. package/dist/workflows/activities.d.ts +1 -0
  90. package/dist/workflows/automations.d.ts +6 -0
  91. package/dist/workflows/document-index.d.ts +1 -0
  92. package/dist/workflows.d.ts +1 -0
  93. package/package.json +17 -17
  94. package/src/activities/agent-turn/admission.ts +292 -0
  95. package/src/activities/agent-turn/agent-build.ts +802 -0
  96. package/src/activities/agent-turn/claim.ts +590 -0
  97. package/src/activities/agent-turn/codex-capacity.ts +720 -0
  98. package/src/activities/agent-turn/codex.ts +94 -0
  99. package/src/activities/agent-turn/compaction-prep.ts +722 -0
  100. package/src/activities/agent-turn/credential-leases.ts +240 -0
  101. package/src/activities/agent-turn/errors.ts +1068 -0
  102. package/src/activities/agent-turn/failure-settlement.ts +1436 -0
  103. package/src/activities/agent-turn/file-resources.ts +227 -0
  104. package/src/activities/agent-turn/finalization.ts +754 -0
  105. package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
  106. package/src/activities/agent-turn/governance-model.ts +465 -0
  107. package/src/activities/agent-turn/history-sink.ts +139 -0
  108. package/src/activities/agent-turn/history.ts +200 -0
  109. package/src/activities/agent-turn/media-artifacts.ts +359 -0
  110. package/src/activities/agent-turn/model-usage.ts +823 -0
  111. package/src/activities/agent-turn/quiescence.ts +353 -0
  112. package/src/activities/agent-turn/run-credentials.ts +761 -0
  113. package/src/activities/agent-turn/run.ts +1530 -0
  114. package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
  115. package/src/activities/agent-turn/sandbox-provision.ts +569 -0
  116. package/src/activities/agent-turn/sandbox-route.ts +358 -0
  117. package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
  118. package/src/activities/agent-turn/stream-attempt.ts +1670 -0
  119. package/src/activities/agent-turn/tool-environment.ts +729 -0
  120. package/src/activities/agent-turn/tool-policy.ts +216 -0
  121. package/src/activities/agent-turn/tool-result-spill.ts +265 -0
  122. package/src/activities/agent-turn/turn-context.ts +281 -0
  123. package/src/activities/agent-turn/xai-capacity.ts +229 -0
  124. package/src/activities/agent-turn.ts +12 -12701
  125. package/src/activities/api-integrations.ts +3 -0
  126. package/src/activities/automations.ts +214 -0
  127. package/src/activities/codemode-dispatcher.ts +12 -1
  128. package/src/activities/common.ts +1 -5
  129. package/src/activities/connector-attachments.ts +168 -0
  130. package/src/activities/context-compaction.ts +18 -0
  131. package/src/activities/environment.ts +151 -55
  132. package/src/activities/generated-images.ts +26 -11
  133. package/src/activities/git-credential-renewal.ts +13 -3
  134. package/src/activities/goals.ts +85 -19
  135. package/src/activities/google-drive-provider.ts +178 -0
  136. package/src/activities/google-drive-publication.ts +147 -49
  137. package/src/activities/incident-telemetry-authority.ts +327 -0
  138. package/src/activities/incident-telemetry-preflight.ts +317 -0
  139. package/src/activities/knowledge-source-sync.ts +178 -37
  140. package/src/activities/mcp-credentials.ts +191 -17
  141. package/src/activities/open-suffix-resume.ts +422 -0
  142. package/src/activities/packs.ts +12 -60
  143. package/src/activities/parent-wake.ts +64 -14
  144. package/src/activities/retained-screenshot-reaper.ts +6 -3
  145. package/src/activities/retained-screenshots.ts +15 -47
  146. package/src/activities/rig-verification.ts +1 -1
  147. package/src/activities/run-input.ts +102 -142
  148. package/src/activities/sandbox-images.ts +7 -166
  149. package/src/activities/sandbox-lease.ts +406 -15
  150. package/src/activities/scheduled-tasks.ts +2103 -411
  151. package/src/activities/session-state.ts +10 -2
  152. package/src/activities/types.ts +32 -2
  153. package/src/activities/video-media-validation.ts +3 -3
  154. package/src/activities/video-output-retention.ts +2 -57
  155. package/src/activities/xai-auth.ts +8 -0
  156. package/src/activities-control.ts +2 -0
  157. package/src/activities.ts +3 -0
  158. package/src/activity-services.ts +13 -4
  159. package/src/connection-provider-use.ts +36 -0
  160. package/src/editable-artifact-materializer-subprocess.ts +25 -14
  161. package/src/editable-artifact-materializer.ts +29 -18
  162. package/src/github-rest-mcp.ts +713 -0
  163. package/src/http.ts +16 -6
  164. package/src/index.ts +40 -5
  165. package/src/knowledge-source-sync-activity-policy.ts +2 -0
  166. package/src/model-context-contributions.ts +33 -10
  167. package/src/observability-metrics.ts +357 -0
  168. package/src/opensandbox-kubernetes-inventory.ts +292 -0
  169. package/src/personal-github-git-credentials.ts +331 -0
  170. package/src/pr-review-credentials.ts +163 -0
  171. package/src/sandbox-archive-storage.ts +159 -0
  172. package/src/sandbox-resume.ts +449 -63
  173. package/src/sandbox-routing.ts +314 -16
  174. package/src/scheduled-alert-occurrence.ts +176 -0
  175. package/src/workflows/activities.ts +18 -2
  176. package/src/workflows/automations.ts +11 -0
  177. package/src/workflows/session.ts +13 -3
  178. package/src/workflows.ts +1 -0
  179. package/dist/activities-control-BJSDRSIP.js.map +0 -1
  180. package/dist/activities-turn-J5YO7XXX.js +0 -17243
  181. package/dist/activities-turn-J5YO7XXX.js.map +0 -1
  182. package/dist/chunk-N5TJIFCE.js.map +0 -1
  183. package/dist/chunk-P6Z2SLAA.js +0 -3092
  184. package/dist/chunk-P6Z2SLAA.js.map +0 -1
@@ -0,0 +1,1068 @@
1
+ import {
2
+ ActiveSessionHistoryLimitExceededError,
3
+ ApprovalRunStateLimitExceededError,
4
+ isRetryableDatabaseTransportFailure,
5
+ isSessionEventPersistenceError,
6
+ } from "@opengeni/db";
7
+ import {
8
+ ActiveBackendUnresolvableError,
9
+ CompactionProviderResponseError,
10
+ EmptyCompactionSummaryError,
11
+ isMcpRequestTimeoutError,
12
+ isMcpTransportConnectivityError,
13
+ RoutingWorkspaceRootChangedError,
14
+ SelfhostedWorkspaceRootChangedError,
15
+ UNKNOWN_MODEL_FINISH_REASON_CODE,
16
+ } from "@opengeni/runtime";
17
+ import {
18
+ mcpTransportRequestFailureDiagnostic,
19
+ type McpTransportRequestFailureDiagnostic,
20
+ } from "@opengeni/runtime/mcp-network";
21
+ import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
22
+ import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
23
+ import type { CodexAccountStatus } from "@opengeni/db";
24
+ import {
25
+ CodexReloginRequired,
26
+ classifyCodexResponseTimeoutError,
27
+ classifyCodexUsageLimitError,
28
+ isCodexTransportError,
29
+ } from "@opengeni/codex";
30
+ import {
31
+ classifyXaiSubscriptionStreamingTerminalError,
32
+ classifyXaiSubscriptionStreamIdleTimeoutError,
33
+ isXaiSubscriptionHostedToolContinuationError,
34
+ isXaiSubscriptionRateLimitDiagnostic,
35
+ isXaiSubscriptionTransportError,
36
+ XaiSubscriptionReloginRequired,
37
+ } from "@opengeni/xai-subscription";
38
+ import type { EscapedMcpTimeoutRecoveryDetail, PreClaimFailureDetail } from "../types";
39
+ import {
40
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
41
+ ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
42
+ PRE_CLAIM_FAILURE_MESSAGE,
43
+ PRE_CLAIM_FAILURE_TYPE,
44
+ } from "../types";
45
+ import {
46
+ MandatoryHistoryPersistenceError,
47
+ type MandatoryHistoryPersistenceStage,
48
+ } from "./quiescence";
49
+
50
+ // Retryable provider connectivity/5xx failures start quickly and back off to
51
+ // this ceiling. Explicit rate limits retain the minute-granular fallback.
52
+ export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
53
+ export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
54
+ export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
55
+
56
+ export type ProviderRecoveryResult =
57
+ | {
58
+ status: "recovering";
59
+ continueDelayMs: number;
60
+ }
61
+ | {
62
+ status: "exhausted";
63
+ providerRecoveryCount: number;
64
+ maxProviderRecoveryCount: number;
65
+ };
66
+
67
+ export function providerRecoveryResult(input: {
68
+ failureCode: string | undefined;
69
+ attemptNumber: number;
70
+ retryAfterMs?: number | null;
71
+ }): ProviderRecoveryResult {
72
+ if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
73
+ return {
74
+ status: "exhausted",
75
+ providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
76
+ maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
77
+ };
78
+ }
79
+ const providerDelay =
80
+ input.retryAfterMs !== null &&
81
+ input.retryAfterMs !== undefined &&
82
+ Number.isFinite(input.retryAfterMs) &&
83
+ input.retryAfterMs > 0
84
+ ? Math.ceil(input.retryAfterMs)
85
+ : null;
86
+ const continueDelayMs =
87
+ input.failureCode === "provider_rate_limited"
88
+ ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
89
+ : input.failureCode === "provider_unavailable" ||
90
+ input.failureCode === "upstream_connectivity_unavailable" ||
91
+ input.failureCode === "mcp_transport_timeout" ||
92
+ input.failureCode === "mcp_transport_unavailable"
93
+ ? Math.max(
94
+ providerDelay ?? 0,
95
+ PROVIDER_CONNECTIVITY_BACKOFF_MS[
96
+ Math.min(
97
+ Math.max(Math.trunc(input.attemptNumber) - 1, 0),
98
+ PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1,
99
+ )
100
+ ]!,
101
+ )
102
+ : PROVIDER_BACKPRESSURE_DELAY_MS;
103
+ return {
104
+ status: "recovering",
105
+ continueDelayMs,
106
+ };
107
+ }
108
+
109
+ export function providerRecoveryExhaustedFailure<
110
+ T extends Record<string, unknown> & { error: string },
111
+ >(
112
+ failure: T,
113
+ recovery: Extract<ProviderRecoveryResult, { status: "exhausted" }>,
114
+ ): T & {
115
+ retryable: false;
116
+ recoveryExhausted: true;
117
+ providerRecoveryCount: number;
118
+ maxProviderRecoveryCount: number;
119
+ lastRetryableError: string;
120
+ } {
121
+ return {
122
+ ...failure,
123
+ error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
124
+ retryable: false,
125
+ recoveryExhausted: true,
126
+ providerRecoveryCount: recovery.providerRecoveryCount,
127
+ maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
128
+ lastRetryableError: failure.error,
129
+ };
130
+ }
131
+
132
+ export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
133
+ const value = metadata.providerRecoveryCount;
134
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
135
+ }
136
+
137
+ export function headerValue(headers: unknown, name: string): string | null {
138
+ if (!headers || typeof headers !== "object") return null;
139
+ const getter = (headers as { get?: unknown }).get;
140
+ if (typeof getter === "function") {
141
+ const value = getter.call(headers, name);
142
+ return typeof value === "string" ? value : null;
143
+ }
144
+ const target = name.toLowerCase();
145
+ const entry = Object.entries(headers as Record<string, unknown>).find(
146
+ ([key, value]) => key.toLowerCase() === target && typeof value === "string",
147
+ );
148
+ return typeof entry?.[1] === "string" ? entry[1] : null;
149
+ }
150
+
151
+ /** Read a provider Retry-After hint without retaining response headers/body. */
152
+ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number | null {
153
+ let current: unknown = error;
154
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
155
+ const value = current as Record<string, unknown>;
156
+ const body =
157
+ value.error && typeof value.error === "object"
158
+ ? (value.error as Record<string, unknown>)
159
+ : null;
160
+ const directSeconds = Number(
161
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
162
+ );
163
+ const header =
164
+ headerValue(value.headers, "retry-after") ??
165
+ headerValue(value.responseHeaders, "retry-after") ??
166
+ headerValue(body?.headers, "retry-after");
167
+ const headerSeconds = header === null ? Number.NaN : Number(header);
168
+ const headerDate =
169
+ header !== null && !Number.isFinite(headerSeconds) ? Date.parse(header) : Number.NaN;
170
+ const seconds = Number.isFinite(directSeconds)
171
+ ? directSeconds
172
+ : Number.isFinite(headerSeconds)
173
+ ? headerSeconds
174
+ : Number.isFinite(headerDate)
175
+ ? Math.max(0, (headerDate - nowMs) / 1_000)
176
+ : Number.NaN;
177
+ if (Number.isFinite(seconds) && seconds > 0) return Math.ceil(seconds * 1_000);
178
+ current = value.cause;
179
+ }
180
+ return null;
181
+ }
182
+
183
+ /**
184
+ * Tell the model when durable MCP policy references are unavailable for this
185
+ * exact turn. The policy projection already bounds and validates these ids;
186
+ * this notice prevents a graceful runtime drop from becoming a silent source-
187
+ * of-truth substitution or a false claim that the disconnected system was read.
188
+ */
189
+ export function unavailableMcpOperationalContext(input: {
190
+ droppedIds: readonly string[];
191
+ droppedCount: number;
192
+ }): string | undefined {
193
+ if (input.droppedCount <= 0) {
194
+ return undefined;
195
+ }
196
+ const omittedCount = Math.max(0, input.droppedCount - input.droppedIds.length);
197
+ const listed = input.droppedIds.map((id) => `"${id}"`).join(", ");
198
+ const inventory = listed
199
+ ? `${listed}${omittedCount > 0 ? `, plus ${omittedCount} additional unavailable server(s)` : ""}`
200
+ : `${input.droppedCount} unavailable server(s)`;
201
+ return `MCP capability availability for this turn: the following session-selected server(s) are disconnected or no longer registered and were skipped: ${inventory}. Do not claim to have read or updated those systems. If the task depends on one as a source of truth, explain the limitation and ask the user to reconnect it or select another authoritative source; continue with unaffected work only when safe.`;
202
+ }
203
+
204
+ /**
205
+ * Preserve one precise recovery obligation across Temporal's activity boundary.
206
+ * This is intentionally narrower than the ordinary retryable-provider path:
207
+ * only a recovered turn (generation > 1) whose MCP timeout happened before a
208
+ * model request may ask the workflow's DB-only control activity to finish the
209
+ * same-turn checkpoint. The original transport/recovery errors are excluded
210
+ * from details so raw MCP response data can never enter workflow history.
211
+ */
212
+ export function escapedMcpTimeoutRecoveryFailure(input: {
213
+ failureCode: string | undefined;
214
+ modelRequestStarted: boolean;
215
+ detail: EscapedMcpTimeoutRecoveryDetail;
216
+ }): ApplicationFailure | null {
217
+ if (
218
+ input.failureCode !== "mcp_transport_timeout" ||
219
+ input.modelRequestStarted ||
220
+ !Number.isSafeInteger(input.detail.executionGeneration) ||
221
+ input.detail.executionGeneration <= 1 ||
222
+ !Number.isSafeInteger(input.detail.providerRecoveryCount) ||
223
+ input.detail.providerRecoveryCount <= 0 ||
224
+ input.detail.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES ||
225
+ !Number.isSafeInteger(input.detail.continueDelayMs) ||
226
+ input.detail.continueDelayMs <= 0
227
+ ) {
228
+ return null;
229
+ }
230
+ return ApplicationFailure.create({
231
+ message: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE,
232
+ type: ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE,
233
+ nonRetryable: true,
234
+ details: [input.detail],
235
+ });
236
+ }
237
+
238
+ /**
239
+ * Convert the atomic claim transaction's failure into a small, stable
240
+ * Temporal wire contract. The original error remains in activity diagnostics,
241
+ * but SQL text, parameters, and arbitrary invariant messages never enter
242
+ * workflow history. Contention and operational database unavailability are
243
+ * safe to re-read after backoff because the claim transaction contains no
244
+ * model/tool effects; permanent database and state failures require terminal
245
+ * settlement.
246
+ */
247
+ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
248
+ const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
249
+ const sqlState = persistenceFailure?.details.sqlState ?? null;
250
+ const rawTransportFailure = sqlState === null && isRetryableDatabaseTransportFailure(error);
251
+ // The database transaction itself retries only the two contention failures
252
+ // proven safe for immediate replay. Once the activity has failed, the
253
+ // workflow may also retry operational outages after a durable re-read and
254
+ // bounded delay. Unknown driver/database failures stay recoverable because
255
+ // they commonly represent a lost connection; known constraint, auth, and
256
+ // application SQLSTATEs are permanent and must not create an infinite loop.
257
+ const retryable = Boolean(
258
+ rawTransportFailure ||
259
+ (persistenceFailure &&
260
+ (sqlState === null ||
261
+ sqlState.startsWith("08") ||
262
+ sqlState.startsWith("40") ||
263
+ sqlState.startsWith("53") ||
264
+ sqlState === "55P03" ||
265
+ sqlState === "57014" ||
266
+ sqlState === "57P01" ||
267
+ sqlState === "57P02" ||
268
+ sqlState === "57P03" ||
269
+ sqlState.startsWith("58"))),
270
+ );
271
+ const detail: PreClaimFailureDetail = {
272
+ disposition: retryable ? "retryable" : "permanent",
273
+ code:
274
+ persistenceFailure?.details.code ?? (rawTransportFailure ? "db_failure" : "claim_invariant"),
275
+ };
276
+ return ApplicationFailure.create({
277
+ message: PRE_CLAIM_FAILURE_MESSAGE,
278
+ type: PRE_CLAIM_FAILURE_TYPE,
279
+ nonRetryable: true,
280
+ details: [detail],
281
+ });
282
+ }
283
+
284
+ /**
285
+ * Resolve which Codex account a turn runs on (multi-account P1): session-pin >
286
+ * workspace-active. No rotation in P1. The selected id must still be in the
287
+ * connected set — a disconnected pin was FK-nulled, so a stale id can't appear,
288
+ * but we guard anyway. Returns null when there is no usable account (the turn
289
+ * then fails with the existing relogin error path).
290
+ */
291
+ export function isWorkerShutdownCancellation(error: unknown): boolean {
292
+ return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
293
+ }
294
+
295
+ /**
296
+ * Recognize the one active-route transition that cannot finish inside its
297
+ * originating attempt. A Modal-home session may start on a Connected Machine
298
+ * without creating or leasing its managed home box. When an explicit attach
299
+ * clears the active pointer back to home, the pointer commit is authoritative,
300
+ * but this attempt has no home session to serve the next sandbox operation.
301
+ *
302
+ * The Agents SDK may retain the typed routing error directly, through `cause`,
303
+ * or inside an AggregateError from a parallel function-tool batch. Traverse
304
+ * only those structural error links with a strict bound; never classify from
305
+ * message text, which could originate in model or tool content.
306
+ */
307
+ export function sandboxRouteTransitionCode(
308
+ error: unknown,
309
+ ): "home_unavailable_this_turn" | "workspace_root_changed_this_turn" | null {
310
+ const pending: unknown[] = [error];
311
+ const seen = new WeakSet<object>();
312
+ let inspected = 0;
313
+
314
+ while (pending.length > 0 && inspected < 64) {
315
+ const current = pending.shift();
316
+ inspected += 1;
317
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
318
+ seen.add(current);
319
+
320
+ try {
321
+ const record = current as Record<string, unknown>;
322
+ if (
323
+ (current instanceof ActiveBackendUnresolvableError ||
324
+ record.name === "ActiveBackendUnresolvableError") &&
325
+ record.code === "home_unavailable_this_turn"
326
+ ) {
327
+ return "home_unavailable_this_turn";
328
+ }
329
+ if (
330
+ current instanceof RoutingWorkspaceRootChangedError ||
331
+ current instanceof SelfhostedWorkspaceRootChangedError ||
332
+ record.name === "RoutingWorkspaceRootChangedError" ||
333
+ record.name === "SelfhostedWorkspaceRootChangedError"
334
+ ) {
335
+ return "workspace_root_changed_this_turn";
336
+ }
337
+
338
+ for (const key of ["cause", "error"] as const) {
339
+ const nested = record[key];
340
+ if (nested && typeof nested === "object") pending.push(nested);
341
+ }
342
+ if (Array.isArray(record.errors)) {
343
+ pending.push(...record.errors.slice(0, 32));
344
+ }
345
+ } catch {
346
+ // A hostile proxy or getter is not a route-transition proof.
347
+ }
348
+ }
349
+
350
+ return null;
351
+ }
352
+
353
+ export function isSandboxRouteTransitionError(error: unknown): boolean {
354
+ return sandboxRouteTransitionCode(error) !== null;
355
+ }
356
+
357
+ /** Backward-compatible name for callers/tests that only exercised the original
358
+ * machine-to-home transition. */
359
+ export function isHomeSandboxTurnTransitionError(error: unknown): boolean {
360
+ return sandboxRouteTransitionCode(error) === "home_unavailable_this_turn";
361
+ }
362
+
363
+ /**
364
+ * Review captures and protective snapshots are cache/persistence housekeeping,
365
+ * never part of cancellation correctness. A control-fenced or Temporal-cancelled
366
+ * attempt must release its physical activity promptly so Steer/Pause can advance.
367
+ */
368
+ export function compactionFailureReason(reason: string): string {
369
+ return reason.startsWith("compaction summarization failed:")
370
+ ? reason
371
+ : `compaction summarization failed: ${reason}`;
372
+ }
373
+
374
+ export type SafeErrorDiagnostic = {
375
+ errorClass: "WorkerOperationError";
376
+ errorCode: "worker_operation_failed";
377
+ status?: number;
378
+ origin: "worker";
379
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
380
+ };
381
+
382
+ /**
383
+ * Produce the only exception shape allowed in worker logs. It deliberately
384
+ * excludes the arbitrary source message, stack, cause, response/request
385
+ * bodies, and enumerable properties. Exact failure content belongs in the
386
+ * permission-controlled session event, not stdout or telemetry.
387
+ */
388
+ export function safeErrorDiagnostic(error: unknown): SafeErrorDiagnostic {
389
+ const diagnostic: SafeErrorDiagnostic = {
390
+ errorClass: "WorkerOperationError",
391
+ errorCode: "worker_operation_failed",
392
+ origin: "worker",
393
+ };
394
+ try {
395
+ let statusSource = error;
396
+ if (error instanceof MandatoryHistoryPersistenceError) {
397
+ diagnostic.historyPersistenceStage = error.stage;
398
+ statusSource = error.cause;
399
+ }
400
+ if (statusSource && typeof statusSource === "object") {
401
+ const status = Number(
402
+ (statusSource as { status?: unknown; statusCode?: unknown }).status ??
403
+ (statusSource as { statusCode?: unknown }).statusCode,
404
+ );
405
+ if (Number.isInteger(status) && status >= 100 && status <= 599) {
406
+ diagnostic.status = status;
407
+ }
408
+ }
409
+ } catch {
410
+ // Public diagnostics are best-effort and must never replace the exact
411
+ // internal worker failure.
412
+ }
413
+ return diagnostic;
414
+ }
415
+
416
+ export function safeErrorForTelemetry(error: unknown): Error {
417
+ const diagnostic = safeErrorDiagnostic(error);
418
+ const safe = new Error("worker operation failed") as Error & {
419
+ code?: string;
420
+ status?: number;
421
+ origin?: string;
422
+ };
423
+ safe.name = "WorkerOperationError";
424
+ safe.code = diagnostic.errorCode;
425
+ if (diagnostic.status !== undefined) safe.status = diagnostic.status;
426
+ safe.origin = diagnostic.origin;
427
+ return safe;
428
+ }
429
+
430
+ export function compactionFailureReasonFromError(error: unknown): string {
431
+ if (
432
+ error instanceof CompactionProviderResponseError ||
433
+ error instanceof EmptyCompactionSummaryError
434
+ ) {
435
+ return compactionFailureReason(error.message);
436
+ }
437
+ const errorName = error instanceof Error && error.name ? error.name : "unknown error";
438
+ return compactionFailureReason(`unexpected ${errorName}`);
439
+ }
440
+
441
+ export function isCompactionSummaryFailure(error: unknown): boolean {
442
+ return (
443
+ error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
444
+ );
445
+ }
446
+
447
+ export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
448
+ if (!(error instanceof CompactionProviderResponseError)) return false;
449
+ if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
450
+ return agentRunFailurePayload(error).retryable === true;
451
+ }
452
+
453
+ export function classifyContextWindowOverflowError(
454
+ error: unknown,
455
+ ): { message: string; code?: string; detail?: string } | null {
456
+ const fields = collectErrorStrings(error);
457
+ const matched = fields.find(
458
+ (value) =>
459
+ /context[_\s-]*length[_\s-]*exceeded/i.test(value) ||
460
+ /exceeds?\s+(?:the\s+)?context\s+window/i.test(value) ||
461
+ /maximum\s+context\s+length/i.test(value) ||
462
+ /context\s+window[^.]*exceed/i.test(value),
463
+ );
464
+ if (!matched) {
465
+ return null;
466
+ }
467
+ const message = error instanceof Error ? error.message : String(error);
468
+ const code = fields.find((value) => /context[_\s-]*length[_\s-]*exceeded/i.test(value));
469
+ return {
470
+ message,
471
+ ...(code ? { code } : {}),
472
+ ...(matched && matched !== message ? { detail: matched } : {}),
473
+ };
474
+ }
475
+
476
+ /**
477
+ * Recognize an MCP transport/request timeout that escaped the SDK's per-tool
478
+ * `mcpConfig.errorFunction` boundary. A thrown tool invocation is normally
479
+ * converted to an `{isError:true}` tool output; however, connect/tools-list or
480
+ * next-loop transport work can reject the stream iterator after a prior tool
481
+ * output was already published. That is transient external backpressure, not a
482
+ * terminal session error. Match MCP-qualified timeout text only: an unrelated
483
+ * sandbox/model timeout and MCP's application-defined Authentication required signal must
484
+ * retain their existing semantics.
485
+ */
486
+ export function classifyMcpTransportTimeoutError(
487
+ error: unknown,
488
+ ): { message: string; detail?: string } | null {
489
+ const fields = collectErrorStrings(error);
490
+ const matchedText = fields.find(
491
+ (value) =>
492
+ /\bmcp\b/i.test(value) &&
493
+ /(?:request\s+timed\s+out|request\s+timeout|\btimed\s+out\b|\btimeout\b|ETIMEDOUT)/i.test(
494
+ value,
495
+ ) &&
496
+ !/authentication\s+required/i.test(value),
497
+ );
498
+ const sanitizedSdkTimeout = isMcpRequestTimeoutError(error);
499
+ if (!matchedText && !sanitizedSdkTimeout) {
500
+ return null;
501
+ }
502
+ const message = error instanceof Error ? error.message : String(error);
503
+ const matched = matchedText ?? fields.find((value) => /\bmcp\b/i.test(value));
504
+ return {
505
+ message,
506
+ ...(matched && matched !== message ? { detail: matched } : {}),
507
+ };
508
+ }
509
+
510
+ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()): string[] {
511
+ if (typeof value === "string") {
512
+ return [value];
513
+ }
514
+ if (!value || typeof value !== "object") {
515
+ return [];
516
+ }
517
+ if (seen.has(value)) {
518
+ return [];
519
+ }
520
+ seen.add(value);
521
+ const out: string[] = [];
522
+ const record = value as Record<string, unknown>;
523
+ for (const key of ["message", "code", "type", "name", "param"]) {
524
+ const field = record[key];
525
+ if (typeof field === "string" && field.length > 0) {
526
+ out.push(field);
527
+ }
528
+ }
529
+ for (const key of ["error", "cause", "response", "data"]) {
530
+ out.push(...collectErrorStrings(record[key], seen));
531
+ }
532
+ return out;
533
+ }
534
+
535
+ /**
536
+ * Compute the conversation-truth rows a reconcile pass should append, given the
537
+ * SDK's current `state.history` and the count already persisted.
538
+ *
539
+ * `state.history` is a computed getter that runs the SDK's orphan-tool-call
540
+ * pruning on every access, so it is non-monotonic: a `function_call` with no
541
+ * settling result yet is transiently absent and a later access yields a
542
+ * different, possibly shorter/reordered list. The old code sliced this list by
543
+ * a blind length watermark and appended at fixed positions with
544
+ * onConflictDoNothing, which could freeze a position with one shape and later
545
+ * persist a `function_call_result` whose `function_call` had been pruned away in
546
+ * an earlier slice — the orphaned tool output that 400s the Responses API and
547
+ * bricks the session on every replay.
548
+ *
549
+ * Defending: structurally repair the full current history into an API-valid sequence (the
550
+ * same pure rules the read path uses), then append only the new tail beyond the
551
+ * watermark. A trailing dangling call is dropped here and re-evaluated next
552
+ * pass once its result lands, so a call and its result are written together at
553
+ * consecutive positions and a result is never persisted without its call. The
554
+ * watermark advances to the repaired length — never past anything unwritten —
555
+ * so a non-monotonic history can never desync it. When previously-persisted
556
+ * rows already exceed the repaired length (e.g. legacy orphans written before
557
+ * this fix), nothing new is appended and the watermark holds steady.
558
+ */
559
+ /**
560
+ * Stable+unique usage source key for one model call, used to build the per-call
561
+ * idempotency key (`usage:model.tokens:${turnId}:${sourceKey}`). The turnId is
562
+ * shared across a new attempt of the SAME turn (recovery, approval
563
+ * rerun, activity retry), so the sourceKey alone must distinguish calls.
564
+ *
565
+ * - A provider responseId is globally stable+unique, so reuse it verbatim: a
566
+ * true activity retry that re-emits the same responseId correctly DEDUPES
567
+ * (one charge), while two distinct calls get distinct ids.
568
+ * - Without a responseId the old synthesized key was only POSITIONAL ("response-1",
569
+ * "aggregate"), which collides across a re-dispatch — dispatch B's first
570
+ * call reuses dispatch A's "response-1" key and its charge is silently
571
+ * dropped (undercharge). Qualifying the synthesized key with the
572
+ * per-execution dispatch id (the Temporal activityId, unique per scheduled
573
+ * execution) makes re-dispatched calls distinct while still deduping a
574
+ * same-execution retry.
575
+ */
576
+ export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
577
+ "unable to connect. is the computer able to access the url?";
578
+
579
+ export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
580
+ return message.trim().toLowerCase() === STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE;
581
+ }
582
+
583
+ export function isTransientProviderError(error: unknown): boolean {
584
+ const status =
585
+ typeof error === "object" && error !== null
586
+ ? Number(
587
+ (error as { status?: unknown; statusCode?: unknown }).status ??
588
+ (error as { statusCode?: unknown }).statusCode,
589
+ )
590
+ : undefined;
591
+ // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
592
+ // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
593
+ // a request fault that must NOT auto-retry — even if its body happens to read like
594
+ // "connection error" or "overloaded". The code/message heuristics below apply ONLY
595
+ // when no status survived: a network fault or an SDK-rethrown bare Error.
596
+ if (status !== undefined && Number.isFinite(status)) {
597
+ return status >= 500 && status < 600;
598
+ }
599
+ const code =
600
+ typeof error === "object" && error !== null && "code" in error
601
+ ? String((error as { code?: unknown }).code)
602
+ : undefined;
603
+ if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
604
+ return true;
605
+ }
606
+ const message = error instanceof Error ? error.message : String(error);
607
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
608
+ return true;
609
+ }
610
+ return /overloaded|an error occurred while processing your request|connection error|service unavailable|bad gateway|gateway timeout/i.test(
611
+ message,
612
+ );
613
+ }
614
+
615
+ export type XaiCredentialFailure = {
616
+ kind: "auth" | "forbidden" | "rate_limit";
617
+ cooldownMs: number | null;
618
+ };
619
+
620
+ /**
621
+ * Only definitive SuperGrok account refusals may move the same logical turn to
622
+ * another credential. A marked HTTP 401/403/429, or an HTTP 200 SSE terminal
623
+ * that is a rate-limit/capacity diagnostic, proves inference was refused
624
+ * without an accepted model response; refresh relogin is equally definitive.
625
+ */
626
+ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailure | null {
627
+ let relogin: unknown = error;
628
+ for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
629
+ if (relogin instanceof XaiSubscriptionReloginRequired) {
630
+ return { kind: "auth", cooldownMs: null };
631
+ }
632
+ relogin = (relogin as Record<string, unknown>).cause;
633
+ }
634
+ if (!isXaiSubscriptionTransportError(error)) return null;
635
+ let current: unknown = error;
636
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
637
+ const value = current as Record<string, unknown>;
638
+ const body =
639
+ value.error && typeof value.error === "object"
640
+ ? (value.error as Record<string, unknown>)
641
+ : null;
642
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
643
+ const code = String(value.code ?? body?.code ?? "").toLowerCase();
644
+ if (status === 401 || code === "unauthorized" || code === "invalid_token") {
645
+ return { kind: "auth", cooldownMs: null };
646
+ }
647
+ if (status === 403) {
648
+ return { kind: "forbidden", cooldownMs: null };
649
+ }
650
+ const message = String(value.message ?? body?.message ?? "");
651
+ if (
652
+ isXaiSubscriptionRateLimitDiagnostic({
653
+ code,
654
+ message,
655
+ status: Number.isInteger(status) ? status : null,
656
+ })
657
+ ) {
658
+ return {
659
+ kind: "rate_limit",
660
+ cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
661
+ };
662
+ }
663
+ current = value.cause;
664
+ }
665
+ return null;
666
+ }
667
+
668
+ export function agentRunFailurePayload(
669
+ error: unknown,
670
+ options: { isCodexTurn?: boolean } = {},
671
+ ): {
672
+ error: string;
673
+ code?: string;
674
+ retryable?: boolean;
675
+ detail?: string;
676
+ timeoutClass?: string;
677
+ responseObserved?: boolean;
678
+ requestId?: string;
679
+ eventCount?: number;
680
+ lastEventType?: string;
681
+ silenceDurationMs?: number;
682
+ correlationId?: string;
683
+ stage?: string;
684
+ sqlState?: string | null;
685
+ attempts?: number;
686
+ retryOutcome?: string;
687
+ database?: Record<string, string>;
688
+ historyPersistenceStage?: MandatoryHistoryPersistenceStage;
689
+ mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
690
+ } {
691
+ if (error instanceof MandatoryHistoryPersistenceError) {
692
+ const underlying = isSessionEventPersistenceError(error.cause)
693
+ ? agentRunFailurePayload(error.cause, options)
694
+ : {
695
+ error: error.cause instanceof Error ? error.cause.message : String(error.cause),
696
+ };
697
+ return {
698
+ ...underlying,
699
+ historyPersistenceStage: error.stage,
700
+ };
701
+ }
702
+ const message = error instanceof Error ? error.message : String(error);
703
+ const status =
704
+ typeof error === "object" && error !== null
705
+ ? Number(
706
+ (error as { status?: unknown; statusCode?: unknown }).status ??
707
+ (error as { statusCode?: unknown }).statusCode,
708
+ )
709
+ : undefined;
710
+ const code =
711
+ typeof error === "object" && error !== null && "code" in error
712
+ ? String((error as { code?: unknown }).code)
713
+ : undefined;
714
+ if (error instanceof ActiveSessionHistoryLimitExceededError) {
715
+ return {
716
+ error:
717
+ "The session's active conversation history exceeds the worker's safe materialization envelope. Clear the session context before retrying; an oversized history cannot be compacted safely in a serving worker.",
718
+ code: error.code,
719
+ retryable: false,
720
+ detail: error.message,
721
+ };
722
+ }
723
+ if (error instanceof ApprovalRunStateLimitExceededError) {
724
+ return {
725
+ error:
726
+ "The saved approval state exceeds the worker's safe materialization envelope. Clear the pending approval context before retrying.",
727
+ code: error.code,
728
+ retryable: false,
729
+ detail: error.message,
730
+ };
731
+ }
732
+ // An accepted Codex stream with no terminal response is malformed/partial,
733
+ // not provider backpressure. Replaying the same accepted turn could repeat
734
+ // model or tool effects, so this marked transport failure must outrank the
735
+ // generic 5xx retry classifier (CodexStreamingTerminalError uses status 502).
736
+ if (isCodexTransportError(error) && code === "invalid_sse_terminal") {
737
+ return {
738
+ error: "The Codex response stream ended without a terminal response",
739
+ code: "invalid_sse_terminal",
740
+ retryable: false,
741
+ };
742
+ }
743
+ if (isXaiSubscriptionHostedToolContinuationError(error)) {
744
+ return {
745
+ error:
746
+ "SuperGrok stopped responding after its hosted search completed. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
747
+ code: "xai_hosted_tool_continuation_stalled",
748
+ retryable: false,
749
+ };
750
+ }
751
+ const xaiStreamTerminal = classifyXaiSubscriptionStreamingTerminalError(error);
752
+ if (xaiStreamTerminal) {
753
+ return {
754
+ error: xaiStreamTerminal.message,
755
+ code: xaiStreamTerminal.code,
756
+ retryable: xaiStreamTerminal.status === 429,
757
+ lastEventType: xaiStreamTerminal.eventType,
758
+ ...(xaiStreamTerminal.requestId ? { requestId: xaiStreamTerminal.requestId } : {}),
759
+ };
760
+ }
761
+ const xaiStreamTimeout = classifyXaiSubscriptionStreamIdleTimeoutError(error);
762
+ if (xaiStreamTimeout) {
763
+ return {
764
+ error:
765
+ "SuperGrok stopped sending valid response events. Partial output was preserved; automatic replay is disabled because the accepted response may still have provider-side effects.",
766
+ code: "xai_response_stream_idle_timeout",
767
+ retryable: false,
768
+ responseObserved: xaiStreamTimeout.responseObserved,
769
+ eventCount: xaiStreamTimeout.eventCount,
770
+ silenceDurationMs: xaiStreamTimeout.silenceDurationMs,
771
+ ...(xaiStreamTimeout.requestId ? { requestId: xaiStreamTimeout.requestId } : {}),
772
+ ...(xaiStreamTimeout.lastEventType ? { lastEventType: xaiStreamTimeout.lastEventType } : {}),
773
+ };
774
+ }
775
+ if (isSessionEventPersistenceError(error)) {
776
+ const { details } = error;
777
+ return {
778
+ error: error.message,
779
+ code: details.code,
780
+ detail:
781
+ details.retryOutcome === "exhausted"
782
+ ? `The idempotent persistence transaction failed after ${details.attempts} attempts.`
783
+ : "The database rejected the idempotent persistence transaction.",
784
+ correlationId: details.correlationId,
785
+ stage: details.stage,
786
+ sqlState: details.sqlState,
787
+ attempts: details.attempts,
788
+ retryOutcome: details.retryOutcome,
789
+ ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
790
+ };
791
+ }
792
+ // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
793
+ // must NOT be reported as a generic, retryable rate-limit (which would loop a
794
+ // goal against a capped backend). Surface a precise, actionable message with
795
+ // the humanized reset window and code, non-retryable. Checked BEFORE the
796
+ // generic 429 branch below (a usage cap is also a 429).
797
+ // This terminal payload classifier may receive a plain SDK-shaped error in
798
+ // tests or after wrapper metadata was stripped. An explicit
799
+ // `usage_limit_reached` shape must still outrank generic 429 retryability.
800
+ // Credential quarantine/failover remains separately provenance-gated by
801
+ // `isCodexTransportError`; this branch only chooses the truthful user payload.
802
+ const usageLimit = classifyCodexUsageLimitError(error);
803
+ if (usageLimit) {
804
+ return codexUsageLimitFailurePayload(usageLimit, message);
805
+ }
806
+ const codexTimeout = classifyCodexResponseTimeoutError(error, {
807
+ allowLegacyRequestTimeout: options.isCodexTurn === true,
808
+ });
809
+ if (codexTimeout) {
810
+ return {
811
+ error: codexTimeout.responseObserved
812
+ ? "The Codex response timed out after streaming began. Observed output was checkpointed; automatic replay is disabled because the upstream operation may still be active."
813
+ : "The Codex response timed out before any response was observed. Upstream acceptance is unknown, so automatic replay is disabled.",
814
+ code: "codex_response_timeout",
815
+ retryable: false,
816
+ timeoutClass: codexTimeout.timeoutClass,
817
+ responseObserved: codexTimeout.responseObserved,
818
+ ...(codexTimeout.requestId ? { requestId: codexTimeout.requestId } : {}),
819
+ ...(codexTimeout.message ? { detail: codexTimeout.message } : {}),
820
+ };
821
+ }
822
+ const mcpTimeout = classifyMcpTransportTimeoutError(error);
823
+ if (mcpTimeout) {
824
+ const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
825
+ return {
826
+ error:
827
+ "An MCP server request timed out. Any completed tool output was checkpointed; the session can continue safely.",
828
+ code: "mcp_transport_timeout",
829
+ retryable: true,
830
+ ...(mcpTimeout.detail || mcpTimeout.message
831
+ ? { detail: mcpTimeout.detail ?? mcpTimeout.message }
832
+ : {}),
833
+ ...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
834
+ };
835
+ }
836
+ if (isMcpTransportConnectivityError(error)) {
837
+ const mcpTransportDiagnostic = mcpTransportRequestFailureDiagnostic(error);
838
+ return {
839
+ error:
840
+ "A required MCP server was temporarily unreachable. The same turn will retry after a short delay.",
841
+ code: "mcp_transport_unavailable",
842
+ retryable: true,
843
+ detail: message,
844
+ ...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
845
+ };
846
+ }
847
+ if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
848
+ return {
849
+ error:
850
+ "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
851
+ code: UNKNOWN_MODEL_FINISH_REASON_CODE,
852
+ retryable: true,
853
+ };
854
+ }
855
+ if (
856
+ status === 429 ||
857
+ code === "rate_limit_exceeded" ||
858
+ /(?:too many requests|rate.?limit|\b429\b)/i.test(message)
859
+ ) {
860
+ return {
861
+ error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
862
+ code: "provider_rate_limited",
863
+ retryable: true,
864
+ ...(message && message !== "Too Many Requests" ? { detail: message } : {}),
865
+ };
866
+ }
867
+ // Transient upstream backpressure (5xx / overloaded / dropped connection): keep
868
+ // the provider's own message (it is already user-meaningful) but mark it
869
+ // retryable so a goal-bearing session idles and auto-continues instead of going
870
+ // terminal on a provider's bad minute. See isTransientProviderError.
871
+ if (isTransientProviderError(error)) {
872
+ if (isExactStatuslessUpstreamConnectivityMessage(message)) {
873
+ return {
874
+ error:
875
+ "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
876
+ code: "upstream_connectivity_unavailable",
877
+ retryable: true,
878
+ };
879
+ }
880
+ return { error: message, code: "provider_unavailable", retryable: true };
881
+ }
882
+ return { error: message };
883
+ }
884
+
885
+ export type CodexCredentialFailure = {
886
+ kind: "auth" | "forbidden" | "rate_limit" | "quota";
887
+ cooldownSeconds: number | null;
888
+ };
889
+
890
+ export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
891
+
892
+ /**
893
+ * Resolve a deterministic quarantine end. Generic request throttling honors
894
+ * provider retry-after (or one minute); allowance/quota refusal waits for the
895
+ * LAST of provider reset and every still-binding cached window (five-hour and
896
+ * weekly both bind), falling back to one complete five-hour window when no reset
897
+ * metadata exists.
898
+ */
899
+ export function codexCredentialCooldownUntil(
900
+ failure: CodexCredentialFailure,
901
+ account: Pick<
902
+ CodexAccountStatus,
903
+ "primaryUsedPercent" | "primaryResetAt" | "secondaryUsedPercent" | "secondaryResetAt"
904
+ > | null,
905
+ now: Date,
906
+ ): Date | null {
907
+ if (failure.kind === "auth" || failure.kind === "forbidden") {
908
+ return null;
909
+ }
910
+ const providerReset =
911
+ failure.cooldownSeconds !== null &&
912
+ Number.isFinite(failure.cooldownSeconds) &&
913
+ failure.cooldownSeconds > 0
914
+ ? new Date(now.getTime() + Math.ceil(failure.cooldownSeconds) * 1000)
915
+ : null;
916
+ if (failure.kind === "rate_limit") {
917
+ return providerReset ?? new Date(now.getTime() + PROVIDER_BACKPRESSURE_DELAY_MS);
918
+ }
919
+ const blockingResets = account
920
+ ? [
921
+ { used: account.primaryUsedPercent, reset: account.primaryResetAt },
922
+ { used: account.secondaryUsedPercent, reset: account.secondaryResetAt },
923
+ ]
924
+ .filter(
925
+ (window): window is { used: number; reset: Date } =>
926
+ (window.used ?? 0) >= CODEX_USAGE_EXHAUSTED_PCT &&
927
+ window.reset instanceof Date &&
928
+ window.reset.getTime() > now.getTime(),
929
+ )
930
+ .map((window) => window.reset)
931
+ : [];
932
+ const quotaResets = providerReset ? [...blockingResets, providerReset] : blockingResets;
933
+ if (quotaResets.length === 0) {
934
+ return new Date(now.getTime() + CODEX_ALLOWANCE_FALLBACK_MS);
935
+ }
936
+ return quotaResets.reduce((latest, reset) =>
937
+ reset.getTime() > latest.getTime() ? reset : latest,
938
+ );
939
+ }
940
+
941
+ /**
942
+ * Only definitive credential/account refusals are safe rotation signals.
943
+ * Ambiguous network failures, malformed/partial streams, invalid model content,
944
+ * prompt 4xx, and provider 5xx may already have consumed tokens or persisted
945
+ * progress and therefore MUST NOT walk the credential pool automatically.
946
+ */
947
+ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
948
+ // A permanent OAuth refresh failure is definitive and the shared resolver has
949
+ // already fenced/stamped the exact credential version. The OpenAI client can
950
+ // wrap a rejection from its custom fetch in APIConnectionError, so recognize
951
+ // the typed exception through the same bounded cause chain used below.
952
+ let refreshError: unknown = error;
953
+ for (let depth = 0; depth < 6 && refreshError && typeof refreshError === "object"; depth += 1) {
954
+ if (refreshError instanceof CodexReloginRequired) {
955
+ return { kind: "auth", cooldownSeconds: null };
956
+ }
957
+ refreshError = (refreshError as Record<string, unknown>).cause;
958
+ }
959
+ // The activity catch also receives sandbox, MCP, storage, and tool failures.
960
+ // Their HTTP status codes are not Codex account state and must never walk the
961
+ // subscription pool or replay a tool on another credential.
962
+ if (!isCodexTransportError(error)) {
963
+ return null;
964
+ }
965
+ const usageLimit = classifyCodexUsageLimitError(error);
966
+ if (usageLimit) {
967
+ return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
968
+ }
969
+ let cur: unknown = error;
970
+ for (let depth = 0; depth < 6 && cur && typeof cur === "object"; depth++) {
971
+ const value = cur as Record<string, unknown>;
972
+ const body =
973
+ value.error && typeof value.error === "object"
974
+ ? (value.error as Record<string, unknown>)
975
+ : null;
976
+ const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
977
+ const code = String(value.code ?? body?.code ?? "").toLowerCase();
978
+ const directRetryAfter = Number(
979
+ value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
980
+ );
981
+ const retryAfterHeader =
982
+ headerValue(value.headers, "retry-after") ??
983
+ headerValue(value.responseHeaders, "retry-after") ??
984
+ headerValue(body?.headers, "retry-after");
985
+ const retryAfterNumber = retryAfterHeader === null ? Number.NaN : Number(retryAfterHeader);
986
+ const retryAfterDate =
987
+ retryAfterHeader !== null && !Number.isFinite(retryAfterNumber)
988
+ ? Date.parse(retryAfterHeader)
989
+ : Number.NaN;
990
+ const retryAfter = Number.isFinite(directRetryAfter)
991
+ ? directRetryAfter
992
+ : Number.isFinite(retryAfterNumber)
993
+ ? retryAfterNumber
994
+ : Number.isFinite(retryAfterDate)
995
+ ? Math.max(0, (retryAfterDate - Date.now()) / 1000)
996
+ : Number.NaN;
997
+ const cooldownSeconds =
998
+ Number.isFinite(retryAfter) && retryAfter > 0 ? Math.ceil(retryAfter) : null;
999
+ // Provider quota codes are more specific than their HTTP transport status.
1000
+ // A permanent allowance refusal commonly arrives as HTTP 429; classify it
1001
+ // before generic backpressure so it receives the binding-window cooldown.
1002
+ if (
1003
+ code === "insufficient_quota" ||
1004
+ code === "quota_exceeded" ||
1005
+ code === "billing_hard_limit_reached"
1006
+ ) {
1007
+ return { kind: "quota", cooldownSeconds };
1008
+ }
1009
+ if (status === 401 || code === "unauthorized" || code === "invalid_api_key") {
1010
+ return { kind: "auth", cooldownSeconds };
1011
+ }
1012
+ if (status === 403) {
1013
+ return { kind: "forbidden", cooldownSeconds };
1014
+ }
1015
+ if (status === 429 || code === "rate_limit_exceeded" || code === "too_many_requests") {
1016
+ return { kind: "rate_limit", cooldownSeconds };
1017
+ }
1018
+ cur = value.cause;
1019
+ }
1020
+ return null;
1021
+ }
1022
+
1023
+ /** Humanize a seconds duration into a short "2h 5m" / "9m" / "in under a minute" string. */
1024
+ export function humanizeResetWindow(resetsInSeconds: number | null): string {
1025
+ if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
1026
+ return "shortly";
1027
+ }
1028
+ const total = Math.ceil(resetsInSeconds);
1029
+ if (total < 60) {
1030
+ return "in under a minute";
1031
+ }
1032
+ const hours = Math.floor(total / 3600);
1033
+ const minutes = Math.round((total % 3600) / 60);
1034
+ if (hours > 0) {
1035
+ return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
1036
+ }
1037
+ return `in about ${minutes}m`;
1038
+ }
1039
+
1040
+ /**
1041
+ * Build the turn.failed payload for a ChatGPT/Codex usage cap: a precise,
1042
+ * actionable message naming the reset window, the stable `codex_usage_limit_reached`
1043
+ * code, and retryable:false (an auto-retry would just re-hit the cap).
1044
+ */
1045
+ export function codexUsageLimitFailurePayload(
1046
+ info: { resetsInSeconds: number | null },
1047
+ detail: string,
1048
+ opts?: { allAccounts?: boolean },
1049
+ ): { error: string; code: string; retryable: boolean; detail?: string } {
1050
+ // P3: when EVERY connected subscription is rate-limited the message names the
1051
+ // earliest reset across accounts; the single-account message is unchanged.
1052
+ const error = opts?.allAccounts
1053
+ ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. ` +
1054
+ `You can switch this session to a different model in the meantime, or wait for a subscription to reset.`
1055
+ : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. ` +
1056
+ `You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
1057
+ return {
1058
+ error,
1059
+ code: "codex_usage_limit_reached",
1060
+ retryable: false,
1061
+ ...(detail ? { detail } : {}),
1062
+ };
1063
+ }
1064
+
1065
+ // A usage cap that won't reset for a long time should not pin a Temporal timer
1066
+ // open indefinitely for a goal-bearing session; cap the continuation hold so the
1067
+ // goal re-evaluates at most this far out (it will re-pause if still capped).
1068
+ export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h