@opengeni/worker-bundle 1.4.2 → 1.4.5-canary.37458084490001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +75 -2
  2. package/dist/activities/agent-turn/governance-model.d.ts +2 -1
  3. package/dist/activities/agent-turn/model-usage.d.ts +22 -0
  4. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  5. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  6. package/dist/activities/agent-turn/turn-context.d.ts +9 -0
  7. package/dist/activities/capabilities.d.ts +2 -0
  8. package/dist/activities/sandbox-lease.d.ts +5 -0
  9. package/dist/activities/types.d.ts +3 -0
  10. package/dist/{activities-control-AKJZHDTY.js → activities-control-KBOSUF5Y.js} +45 -15
  11. package/dist/activities-control-KBOSUF5Y.js.map +1 -0
  12. package/dist/{activities-turn-AOXNE6BM.js → activities-turn-QOR26CTC.js} +230 -39
  13. package/dist/activities-turn-QOR26CTC.js.map +1 -0
  14. package/dist/artifact-materializer-entry.js +0 -0
  15. package/dist/artifact-outbox-entry.js +0 -0
  16. package/dist/{chunk-DCQ7XTXW.js → chunk-CMIUQ2D3.js} +114 -13
  17. package/dist/chunk-CMIUQ2D3.js.map +1 -0
  18. package/dist/{chunk-WVZQJBGD.js → chunk-WX5PX2C6.js} +245 -20
  19. package/dist/chunk-WX5PX2C6.js.map +1 -0
  20. package/dist/index.js +5 -5
  21. package/dist/index.js.map +1 -1
  22. package/dist/observability-metrics.d.ts +61 -6
  23. package/dist/workflow-bundle.js +72 -39
  24. package/package.json +21 -21
  25. package/src/activities/agent-turn/admission.ts +7 -8
  26. package/src/activities/agent-turn/claim.ts +20 -1
  27. package/src/activities/agent-turn/errors.ts +296 -13
  28. package/src/activities/agent-turn/failure-settlement.ts +19 -7
  29. package/src/activities/agent-turn/governance-model.ts +3 -0
  30. package/src/activities/agent-turn/model-usage.ts +114 -3
  31. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  32. package/src/activities/agent-turn/provider-recovery-metrics.ts +1 -1
  33. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  34. package/src/activities/agent-turn/run.ts +14 -1
  35. package/src/activities/agent-turn/stream-attempt.ts +68 -1
  36. package/src/activities/agent-turn/turn-context.ts +5 -0
  37. package/src/activities/automations.ts +5 -2
  38. package/src/activities/capabilities.ts +58 -2
  39. package/src/activities/sandbox-lease.ts +17 -3
  40. package/src/activities/session-state.ts +41 -5
  41. package/src/activities/types.ts +3 -1
  42. package/src/index.ts +2 -2
  43. package/src/observability-metrics.ts +186 -17
  44. package/src/workflows/session.ts +22 -12
  45. package/dist/activities-control-AKJZHDTY.js.map +0 -1
  46. package/dist/activities-turn-AOXNE6BM.js.map +0 -1
  47. package/dist/chunk-DCQ7XTXW.js.map +0 -1
  48. package/dist/chunk-WVZQJBGD.js.map +0 -1
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  observabilityEventBusOptions
3
- } from "./chunk-DCQ7XTXW.js";
3
+ } from "./chunk-CMIUQ2D3.js";
4
4
 
5
5
  // src/activity-services.ts
6
6
  import {
@@ -4373,7 +4373,7 @@ async function confluenceFooterComments(source, pageId, authorization) {
4373
4373
  }
4374
4374
  return comments;
4375
4375
  }
4376
- async function atlassianJsonRequest(url, authorization, label) {
4376
+ async function atlassianJsonRequest(url, authorization, label2) {
4377
4377
  const response = await sourceFetch(url, {
4378
4378
  headers: { authorization, accept: "application/json" },
4379
4379
  redirect: "error",
@@ -4389,7 +4389,7 @@ async function atlassianJsonRequest(url, authorization, label) {
4389
4389
  response.status >= 500 || response.status === 429
4390
4390
  );
4391
4391
  }
4392
- return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES, label);
4392
+ return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES, label2);
4393
4393
  }
4394
4394
  function confluenceCursor(value) {
4395
4395
  const next = optionalString(value);
@@ -4466,8 +4466,8 @@ ${children.trimEnd()}
4466
4466
  function objectRecord(value) {
4467
4467
  return value && typeof value === "object" && !Array.isArray(value) ? value : {};
4468
4468
  }
4469
- function requiredAtlassianString(value, label) {
4470
- if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${label}`);
4469
+ function requiredAtlassianString(value, label2) {
4470
+ if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${label2}`);
4471
4471
  return value;
4472
4472
  }
4473
4473
  function deterministicUuid(value) {
@@ -4481,8 +4481,8 @@ function safeFilename(value) {
4481
4481
  const normalized = value.normalize("NFKC").replace(/[\\/:*?"<>|\0-\x1f]/g, "_").trim();
4482
4482
  return (normalized || "document").slice(0, 240);
4483
4483
  }
4484
- function requiredString(value, label) {
4485
- if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${label}`);
4484
+ function requiredString(value, label2) {
4485
+ if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${label2}`);
4486
4486
  return value;
4487
4487
  }
4488
4488
  function optionalString(value) {
@@ -8023,7 +8023,7 @@ function incidentTelemetryAuthorityFence(input) {
8023
8023
  input.alertOccurrenceLabels !== null
8024
8024
  );
8025
8025
  if (declaration.action !== "required" || !input.alertOccurrenceLabels) return null;
8026
- const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((label) => [label, input.alertOccurrenceLabels?.[label]]).sort(([left], [right]) => left.localeCompare(right));
8026
+ const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((label2) => [label2, input.alertOccurrenceLabels?.[label2]]).sort(([left], [right]) => left.localeCompare(right));
8027
8027
  if (selectorEntries.some(([, value]) => typeof value !== "string" || value.length === 0)) {
8028
8028
  return null;
8029
8029
  }
@@ -8081,15 +8081,15 @@ function seriesMetadataSatisfies(dataSource) {
8081
8081
  (required) => dataSource.availableSeries.some((available) => {
8082
8082
  if (available.metric !== required.metric) return false;
8083
8083
  const labels = new Set(available.labels);
8084
- return labels.has(dataSource.workspaceLabel) && required.labels.every((label) => labels.has(label));
8084
+ return labels.has(dataSource.workspaceLabel) && required.labels.every((label2) => labels.has(label2));
8085
8085
  })
8086
8086
  );
8087
8087
  }
8088
8088
  function alertSelectorsSatisfy(dataSource, occurrenceLabels) {
8089
8089
  if (!occurrenceLabels) return false;
8090
8090
  return dataSource.alertSelectorLabels.every(
8091
- (label) => label !== dataSource.workspaceLabel && typeof occurrenceLabels[label] === "string" && occurrenceLabels[label].length > 0 && dataSource.requiredSeries.every((series) => series.labels.includes(label)) && dataSource.availableSeries.some(
8092
- (series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(label)
8091
+ (label2) => label2 !== dataSource.workspaceLabel && typeof occurrenceLabels[label2] === "string" && occurrenceLabels[label2].length > 0 && dataSource.requiredSeries.every((series) => series.labels.includes(label2)) && dataSource.availableSeries.some(
8092
+ (series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(label2)
8093
8093
  )
8094
8094
  );
8095
8095
  }
@@ -9113,6 +9113,19 @@ async function finalizeDurableTurnOpStreams(sessions, fallback, toolCallIds) {
9113
9113
  }
9114
9114
  }
9115
9115
 
9116
+ // src/activities/agent-turn/provider-recovery-policy.ts
9117
+ var PROVIDER_OVERLOAD_RECOVERY_CODE = "provider_overloaded";
9118
+ var MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES = 15;
9119
+ var PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS = 15 * 6e4;
9120
+ function validProviderOverloadRecoveryDelay(code, delay) {
9121
+ if (delay === void 0) return code !== PROVIDER_OVERLOAD_RECOVERY_CODE;
9122
+ return code === PROVIDER_OVERLOAD_RECOVERY_CODE && typeof delay === "number" && Number.isSafeInteger(delay) && delay > 0 && delay < PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS;
9123
+ }
9124
+ function readProviderRecoveryStartedAt(metadata) {
9125
+ const value = typeof metadata.providerRecoveryStartedAt === "string" ? Date.parse(metadata.providerRecoveryStartedAt) : NaN;
9126
+ return Number.isFinite(value) ? value : void 0;
9127
+ }
9128
+
9116
9129
  // src/activities/agent-turn/errors.ts
9117
9130
  import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
9118
9131
 
@@ -9179,6 +9192,7 @@ import {
9179
9192
  RoutingWorkspaceRootChangedError,
9180
9193
  RoutingBackendRecoveryRequiredError as RoutingBackendRecoveryRequiredError2,
9181
9194
  ResponsesStreamingTerminalError,
9195
+ classifyModelStreamIdleTimeoutError,
9182
9196
  SandboxMaterializationVerificationError,
9183
9197
  materializationVerificationDiagnostic,
9184
9198
  PROVIDER_QUOTA_EXHAUSTED_CODE,
@@ -9728,8 +9742,8 @@ import {
9728
9742
  var CODEX_PLAN_ENTITLEMENT_CODE = "codex_plan_entitlement";
9729
9743
  var CODEX_REQUEST_REJECTED_CODE = "codex_request_rejected";
9730
9744
  function codexAccountDisplayLabel(account) {
9731
- const label = account?.label?.trim() || null;
9732
- return label ? label.slice(0, 120) : null;
9745
+ const label2 = account?.label?.trim() || null;
9746
+ return label2 ? label2.slice(0, 120) : null;
9733
9747
  }
9734
9748
  function accountPhrase(accountLabel) {
9735
9749
  return accountLabel ? `The ChatGPT account "${accountLabel}"` : "The connected ChatGPT account";
@@ -9839,10 +9853,58 @@ var POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE = "Agent turn database recovery
9839
9853
  var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
9840
9854
  var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
9841
9855
 
9856
+ // src/activities/agent-turn/provider-recovery-copy.ts
9857
+ var MODEL_PROVIDER_RECOVERY_CODES = /* @__PURE__ */ new Set([
9858
+ "provider_unavailable",
9859
+ "provider_rate_limited",
9860
+ "provider_unknown_finish_reason",
9861
+ "post_compaction_continuation_empty"
9862
+ ]);
9863
+ function label(value) {
9864
+ const trimmed = typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
9865
+ return trimmed ? trimmed.slice(0, 120) : null;
9866
+ }
9867
+ function providerRecoverySubject(input) {
9868
+ const model = label(input.modelLabel) ?? "The model";
9869
+ const provider = label(input.providerLabel);
9870
+ const at = provider ? ` at the provider (${provider})` : " at the provider";
9871
+ switch (input.code) {
9872
+ case "provider_rate_limited":
9873
+ return `${model} is rate limited${at}`;
9874
+ case "provider_unavailable":
9875
+ return input.providerCondition === "overloaded" ? `${model} is overloaded${at}` : input.providerCondition === "unresponsive" ? `${model} stopped responding${at}` : `${model} is temporarily unavailable${at}`;
9876
+ case "provider_unknown_finish_reason":
9877
+ return `${model} ended its response unexpectedly`;
9878
+ case "post_compaction_continuation_empty":
9879
+ return `${model} stopped right after compacting the conversation`;
9880
+ case "upstream_connectivity_unavailable":
9881
+ return "Opengeni can't reach an upstream service";
9882
+ case "mcp_transport_timeout":
9883
+ return "A required MCP server isn't responding";
9884
+ case "mcp_transport_unavailable":
9885
+ return "A required MCP server is unreachable";
9886
+ case "sandbox_command_start_unavailable":
9887
+ return "The sandbox isn't ready yet";
9888
+ case "turn_execution_policy_definition_mismatch":
9889
+ return "Opengeni is applying a configuration update";
9890
+ default:
9891
+ return "A service this turn depends on is temporarily unavailable";
9892
+ }
9893
+ }
9894
+ function providerRecoveryExhaustedMessage(input) {
9895
+ const count = input.providerRecoveryCount;
9896
+ const retried = `Opengeni retried ${count} ${count === 1 ? "time" : "times"} without success.`;
9897
+ const remedy = MODEL_PROVIDER_RECOVERY_CODES.has(input.code ?? "") ? "Try again in a few minutes, or switch to another model." : "Try again in a few minutes.";
9898
+ return `${providerRecoverySubject(input)}. ${retried} ${remedy}`;
9899
+ }
9900
+
9842
9901
  // src/activities/agent-turn/errors.ts
9843
9902
  var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
9844
9903
  var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
9845
9904
  var MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
9905
+ function providerRecoveryLimit(failureCode) {
9906
+ return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES : MAX_AUTOMATIC_PROVIDER_RECOVERIES;
9907
+ }
9846
9908
  var PROVIDER_RATE_LIMIT_BACKOFF_MS = [1e4, 2e4, 4e4, 6e4, 12e4];
9847
9909
  function providerRecoveryJitterMs(delayMs, sample) {
9848
9910
  const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
@@ -9857,13 +9919,27 @@ var PostCompactionContinuationEmptyError = class extends Error {
9857
9919
  }
9858
9920
  };
9859
9921
  function providerRecoveryResult(input) {
9860
- if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
9922
+ const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
9923
+ const limit = providerRecoveryLimit(input.failureCode);
9924
+ const exhausted = (reason) => ({
9925
+ status: "exhausted",
9926
+ providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
9927
+ maxProviderRecoveryCount: limit,
9928
+ providerRecoveryExhaustedReason: reason
9929
+ });
9930
+ if (input.attemptNumber > limit) {
9931
+ if (overload) return exhausted("retry_limit");
9861
9932
  return {
9862
9933
  status: "exhausted",
9863
9934
  providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
9864
9935
  maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES
9865
9936
  };
9866
9937
  }
9938
+ const now = input.now ?? Date.now();
9939
+ const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
9940
+ if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
9941
+ return exhausted("invalid_clock");
9942
+ }
9867
9943
  const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
9868
9944
  const continueDelayMs = input.failureCode === "provider_rate_limited" ? Math.max(
9869
9945
  providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
@@ -9871,29 +9947,93 @@ function providerRecoveryResult(input) {
9871
9947
  Math.max(Math.trunc(input.attemptNumber) - 1, 0),
9872
9948
  PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1
9873
9949
  )]
9874
- ) : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "sandbox_command_start_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" || input.failureCode === "turn_execution_policy_definition_mismatch" || input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE ? Math.max(
9950
+ ) : overload || input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "sandbox_command_start_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" || input.failureCode === "turn_execution_policy_definition_mismatch" || input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE ? Math.max(
9875
9951
  providerDelay ?? 0,
9876
9952
  PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
9877
9953
  Math.max(Math.trunc(input.attemptNumber) - 1, 0),
9878
9954
  PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
9879
9955
  )]
9880
9956
  ) : PROVIDER_BACKPRESSURE_DELAY_MS;
9957
+ const delayMs = continueDelayMs + (overload || input.failureCode === "provider_rate_limited" || input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0) : 0);
9958
+ if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
9959
+ return exhausted("deadline");
9960
+ }
9881
9961
  return {
9882
9962
  status: "recovering",
9883
- continueDelayMs: continueDelayMs + (input.failureCode === "provider_rate_limited" || input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0) : 0)
9963
+ continueDelayMs: delayMs,
9964
+ ...overload ? { maxProviderRecoveryCount: limit } : {}
9884
9965
  };
9885
9966
  }
9967
+ var ProviderOverloadRecoveryExpiredError = class extends Error {
9968
+ failure;
9969
+ constructor(count, reason) {
9970
+ super("Confirmed provider overload recovery window expired before dispatch");
9971
+ this.name = "ProviderOverloadRecoveryExpiredError";
9972
+ this.failure = providerRecoveryExhaustedFailure(
9973
+ {
9974
+ error: this.message,
9975
+ code: "provider_unavailable",
9976
+ providerCondition: "overloaded"
9977
+ },
9978
+ {
9979
+ status: "exhausted",
9980
+ // The queued replacement has not dispatched; count only prior retries.
9981
+ providerRecoveryCount: Math.min(
9982
+ MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
9983
+ Math.max(0, count - 1)
9984
+ ),
9985
+ maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
9986
+ providerRecoveryExhaustedReason: reason
9987
+ }
9988
+ );
9989
+ }
9990
+ };
9991
+ function assertProviderOverloadRecoveryActive(input) {
9992
+ if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
9993
+ return;
9994
+ }
9995
+ if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
9996
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
9997
+ }
9998
+ const now = input.now ?? Date.now();
9999
+ const startedAt = input.recoveryStartedAt;
10000
+ if (startedAt === void 0 || !Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now) {
10001
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
10002
+ }
10003
+ if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
10004
+ throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
10005
+ }
10006
+ }
9886
10007
  function providerRecoveryExhaustedFailure(failure, recovery) {
9887
10008
  return {
9888
10009
  ...failure,
9889
- error: `Automatic same-turn recovery stopped after ${recovery.providerRecoveryCount} retries because the upstream dependency remained unavailable. Send a new message to retry after the dependency recovers.`,
10010
+ error: providerRecoveryExhaustedMessage({
10011
+ code: typeof failure.code === "string" ? failure.code : null,
10012
+ providerCondition: isProviderCondition(failure.providerCondition) ? failure.providerCondition : null,
10013
+ modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
10014
+ providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
10015
+ providerRecoveryCount: recovery.providerRecoveryCount
10016
+ }),
9890
10017
  retryable: false,
9891
10018
  recoveryExhausted: true,
9892
10019
  providerRecoveryCount: recovery.providerRecoveryCount,
9893
10020
  maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
10021
+ ...recovery.providerRecoveryExhaustedReason ? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason } : {},
9894
10022
  lastRetryableError: failure.error
9895
10023
  };
9896
10024
  }
10025
+ function isProviderCondition(value) {
10026
+ return value === "overloaded" || value === "unavailable" || value === "unresponsive" || value === "rate_limited";
10027
+ }
10028
+ function withModelRoutePresentation(failure, route) {
10029
+ if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
10030
+ return {
10031
+ ...failure,
10032
+ model: route.model,
10033
+ modelLabel: route.modelLabel,
10034
+ providerLabel: route.providerLabel
10035
+ };
10036
+ }
9897
10037
  function providerRecoveryCountFromMetadata(metadata) {
9898
10038
  const value = metadata.providerRecoveryCount;
9899
10039
  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
@@ -10103,7 +10243,10 @@ function postClaimDatabaseRecoveryFailure(input) {
10103
10243
  if (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted || (input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) && input.providerRecovery) {
10104
10244
  return null;
10105
10245
  }
10106
- if (input.providerRecovery && (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) || input.providerRecovery.providerRecoveryCount <= 0 || input.providerRecovery.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode))) {
10246
+ if (input.providerRecovery && (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) || input.providerRecovery.providerRecoveryCount <= 0 || input.providerRecovery.providerRecoveryCount > providerRecoveryLimit(input.providerRecovery.failureCode) || !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) || !validProviderOverloadRecoveryDelay(
10247
+ input.providerRecovery.failureCode,
10248
+ input.providerRecovery.continueDelayMs
10249
+ ))) {
10107
10250
  return null;
10108
10251
  }
10109
10252
  const detail = {
@@ -10115,7 +10258,8 @@ function postClaimDatabaseRecoveryFailure(input) {
10115
10258
  ...input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
10116
10259
  ...input.providerRecovery ? {
10117
10260
  providerFailureCode: input.providerRecovery.failureCode,
10118
- providerRecoveryCount: input.providerRecovery.providerRecoveryCount
10261
+ providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
10262
+ ...input.providerRecovery.continueDelayMs !== void 0 ? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs } : {}
10119
10263
  } : {}
10120
10264
  };
10121
10265
  return ApplicationFailure.create({
@@ -10394,6 +10538,20 @@ function providerHttpStatus(error) {
10394
10538
  }
10395
10539
  return void 0;
10396
10540
  }
10541
+ function isTransportTimeoutError(error) {
10542
+ let current = error;
10543
+ const seen = /* @__PURE__ */ new Set();
10544
+ for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
10545
+ if (seen.has(current)) break;
10546
+ seen.add(current);
10547
+ const value = current;
10548
+ if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
10549
+ return true;
10550
+ }
10551
+ current = value.cause;
10552
+ }
10553
+ return false;
10554
+ }
10397
10555
  function isTransientProviderError(error) {
10398
10556
  if (error instanceof ResponsesStreamingTerminalError) {
10399
10557
  return error.category === "unavailable";
@@ -10491,6 +10649,42 @@ function isRawProviderCommandOutcomeUnknown(error) {
10491
10649
  }
10492
10650
  }
10493
10651
  function agentRunFailurePayload(error, options = {}) {
10652
+ return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
10653
+ }
10654
+ function withProviderCondition(error, failure) {
10655
+ if (failure.code === "provider_rate_limited") {
10656
+ return { ...failure, providerCondition: "rate_limited" };
10657
+ }
10658
+ if (failure.code !== "provider_unavailable") return failure;
10659
+ if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
10660
+ return {
10661
+ ...failure,
10662
+ providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable"
10663
+ };
10664
+ }
10665
+ function isProviderOverloadError(error, failure = {}) {
10666
+ if (providerHttpStatus(error) === 529) return true;
10667
+ const anthropic = anthropicRequestDiagnostic(error);
10668
+ return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
10669
+ (value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value)
10670
+ );
10671
+ }
10672
+ function providerRecoveryCode(error, failure) {
10673
+ if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
10674
+ const status = providerHttpStatus(error);
10675
+ let current = error;
10676
+ const seen = /* @__PURE__ */ new Set();
10677
+ for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
10678
+ if (seen.has(current)) break;
10679
+ seen.add(current);
10680
+ if (current instanceof AnthropicRequestError && status !== void 0 && status >= 500 && status < 600 && (status === 529 || current.errorType === "overloaded_error")) {
10681
+ return PROVIDER_OVERLOAD_RECOVERY_CODE;
10682
+ }
10683
+ current = current.cause;
10684
+ }
10685
+ return failure.code;
10686
+ }
10687
+ function classifyAgentRunFailurePayload(error, options = {}) {
10494
10688
  const graph = structuredRecoveryCauseGraph(error);
10495
10689
  const nodes = graph ? [...graph.keys()] : [];
10496
10690
  const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
@@ -10505,6 +10699,8 @@ function agentRunFailurePayload(error, options = {}) {
10505
10699
  retryable: false
10506
10700
  };
10507
10701
  }
10702
+ const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
10703
+ if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
10508
10704
  const failure = baseAgentRunFailurePayload(error, options);
10509
10705
  const diagnostic = materializationVerificationDiagnostic(error);
10510
10706
  const anthropic = anthropicRequestDiagnostic(error);
@@ -10757,6 +10953,28 @@ function baseAgentRunFailurePayload(error, options = {}) {
10757
10953
  ...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
10758
10954
  };
10759
10955
  }
10956
+ const streamIdle = classifyModelStreamIdleTimeoutError(error);
10957
+ if (streamIdle) {
10958
+ const seconds = streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1e3) : null;
10959
+ const window = seconds !== null ? ` for ${seconds}s` : "";
10960
+ return {
10961
+ error: streamIdle.kind === "progress" ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.` : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
10962
+ code: "provider_unavailable",
10963
+ retryable: true,
10964
+ timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
10965
+ responseObserved: true,
10966
+ detail: streamIdle.message
10967
+ };
10968
+ }
10969
+ if (status === void 0 && isTransportTimeoutError(error)) {
10970
+ return {
10971
+ error: "The model provider connection timed out. The same turn will retry after a short delay.",
10972
+ code: "provider_unavailable",
10973
+ retryable: true,
10974
+ timeoutClass: "transport",
10975
+ detail: message
10976
+ };
10977
+ }
10760
10978
  if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
10761
10979
  return {
10762
10980
  error: "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
@@ -11015,10 +11233,16 @@ export {
11015
11233
  turnFinalizerCancellationSignal,
11016
11234
  runMandatoryHistoryPersistenceStep,
11017
11235
  finalizeDurableTurnOpStreams,
11236
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
11237
+ validProviderOverloadRecoveryDelay,
11238
+ readProviderRecoveryStartedAt,
11018
11239
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
11240
+ providerRecoveryLimit,
11019
11241
  PostCompactionContinuationEmptyError,
11020
11242
  providerRecoveryResult,
11243
+ assertProviderOverloadRecoveryActive,
11021
11244
  providerRecoveryExhaustedFailure,
11245
+ withModelRoutePresentation,
11022
11246
  providerRecoveryCountFromMetadata,
11023
11247
  providerRecoveryCountAfterModelRequestPhase,
11024
11248
  providerRetryAfterMs,
@@ -11040,6 +11264,7 @@ export {
11040
11264
  classifyContextWindowOverflowError,
11041
11265
  classifyXaiCredentialFailure,
11042
11266
  agentRunFailurePayload,
11267
+ providerRecoveryCode,
11043
11268
  agentRunRecoveryFailurePayload,
11044
11269
  codexCredentialCooldownUntil,
11045
11270
  classifyCodexCredentialFailure,
@@ -11047,4 +11272,4 @@ export {
11047
11272
  CODEX_USAGE_LIMIT_MAX_RESUME_MS,
11048
11273
  classifyClaudeCredentialFailure
11049
11274
  };
11050
- //# sourceMappingURL=chunk-WVZQJBGD.js.map
11275
+ //# sourceMappingURL=chunk-WX5PX2C6.js.map