@opengeni/worker-bundle 1.4.2 → 1.4.5-canary.37458084490001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +75 -2
- package/dist/activities/agent-turn/governance-model.d.ts +2 -1
- package/dist/activities/agent-turn/model-usage.d.ts +22 -0
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -0
- package/dist/activities/capabilities.d.ts +2 -0
- package/dist/activities/sandbox-lease.d.ts +5 -0
- package/dist/activities/types.d.ts +3 -0
- package/dist/{activities-control-AKJZHDTY.js → activities-control-KBOSUF5Y.js} +45 -15
- package/dist/activities-control-KBOSUF5Y.js.map +1 -0
- package/dist/{activities-turn-AOXNE6BM.js → activities-turn-QOR26CTC.js} +230 -39
- package/dist/activities-turn-QOR26CTC.js.map +1 -0
- package/dist/artifact-materializer-entry.js +0 -0
- package/dist/artifact-outbox-entry.js +0 -0
- package/dist/{chunk-DCQ7XTXW.js → chunk-CMIUQ2D3.js} +114 -13
- package/dist/chunk-CMIUQ2D3.js.map +1 -0
- package/dist/{chunk-WVZQJBGD.js → chunk-WX5PX2C6.js} +245 -20
- package/dist/chunk-WX5PX2C6.js.map +1 -0
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +61 -6
- package/dist/workflow-bundle.js +72 -39
- package/package.json +21 -21
- package/src/activities/agent-turn/admission.ts +7 -8
- package/src/activities/agent-turn/claim.ts +20 -1
- package/src/activities/agent-turn/errors.ts +296 -13
- package/src/activities/agent-turn/failure-settlement.ts +19 -7
- package/src/activities/agent-turn/governance-model.ts +3 -0
- package/src/activities/agent-turn/model-usage.ts +114 -3
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +1 -1
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/run.ts +14 -1
- package/src/activities/agent-turn/stream-attempt.ts +68 -1
- package/src/activities/agent-turn/turn-context.ts +5 -0
- package/src/activities/automations.ts +5 -2
- package/src/activities/capabilities.ts +58 -2
- package/src/activities/sandbox-lease.ts +17 -3
- package/src/activities/session-state.ts +41 -5
- package/src/activities/types.ts +3 -1
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +186 -17
- package/src/workflows/session.ts +22 -12
- package/dist/activities-control-AKJZHDTY.js.map +0 -1
- package/dist/activities-turn-AOXNE6BM.js.map +0 -1
- package/dist/chunk-DCQ7XTXW.js.map +0 -1
- package/dist/chunk-WVZQJBGD.js.map +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
observabilityEventBusOptions
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-CMIUQ2D3.js";
|
|
4
4
|
|
|
5
5
|
// src/activity-services.ts
|
|
6
6
|
import {
|
|
@@ -4373,7 +4373,7 @@ async function confluenceFooterComments(source, pageId, authorization) {
|
|
|
4373
4373
|
}
|
|
4374
4374
|
return comments;
|
|
4375
4375
|
}
|
|
4376
|
-
async function atlassianJsonRequest(url, authorization,
|
|
4376
|
+
async function atlassianJsonRequest(url, authorization, label2) {
|
|
4377
4377
|
const response = await sourceFetch(url, {
|
|
4378
4378
|
headers: { authorization, accept: "application/json" },
|
|
4379
4379
|
redirect: "error",
|
|
@@ -4389,7 +4389,7 @@ async function atlassianJsonRequest(url, authorization, label) {
|
|
|
4389
4389
|
response.status >= 500 || response.status === 429
|
|
4390
4390
|
);
|
|
4391
4391
|
}
|
|
4392
|
-
return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES,
|
|
4392
|
+
return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES, label2);
|
|
4393
4393
|
}
|
|
4394
4394
|
function confluenceCursor(value) {
|
|
4395
4395
|
const next = optionalString(value);
|
|
@@ -4466,8 +4466,8 @@ ${children.trimEnd()}
|
|
|
4466
4466
|
function objectRecord(value) {
|
|
4467
4467
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
4468
4468
|
}
|
|
4469
|
-
function requiredAtlassianString(value,
|
|
4470
|
-
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${
|
|
4469
|
+
function requiredAtlassianString(value, label2) {
|
|
4470
|
+
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${label2}`);
|
|
4471
4471
|
return value;
|
|
4472
4472
|
}
|
|
4473
4473
|
function deterministicUuid(value) {
|
|
@@ -4481,8 +4481,8 @@ function safeFilename(value) {
|
|
|
4481
4481
|
const normalized = value.normalize("NFKC").replace(/[\\/:*?"<>|\0-\x1f]/g, "_").trim();
|
|
4482
4482
|
return (normalized || "document").slice(0, 240);
|
|
4483
4483
|
}
|
|
4484
|
-
function requiredString(value,
|
|
4485
|
-
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${
|
|
4484
|
+
function requiredString(value, label2) {
|
|
4485
|
+
if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${label2}`);
|
|
4486
4486
|
return value;
|
|
4487
4487
|
}
|
|
4488
4488
|
function optionalString(value) {
|
|
@@ -8023,7 +8023,7 @@ function incidentTelemetryAuthorityFence(input) {
|
|
|
8023
8023
|
input.alertOccurrenceLabels !== null
|
|
8024
8024
|
);
|
|
8025
8025
|
if (declaration.action !== "required" || !input.alertOccurrenceLabels) return null;
|
|
8026
|
-
const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((
|
|
8026
|
+
const selectorEntries = declaration.preflight.dataSource.alertSelectorLabels.map((label2) => [label2, input.alertOccurrenceLabels?.[label2]]).sort(([left], [right]) => left.localeCompare(right));
|
|
8027
8027
|
if (selectorEntries.some(([, value]) => typeof value !== "string" || value.length === 0)) {
|
|
8028
8028
|
return null;
|
|
8029
8029
|
}
|
|
@@ -8081,15 +8081,15 @@ function seriesMetadataSatisfies(dataSource) {
|
|
|
8081
8081
|
(required) => dataSource.availableSeries.some((available) => {
|
|
8082
8082
|
if (available.metric !== required.metric) return false;
|
|
8083
8083
|
const labels = new Set(available.labels);
|
|
8084
|
-
return labels.has(dataSource.workspaceLabel) && required.labels.every((
|
|
8084
|
+
return labels.has(dataSource.workspaceLabel) && required.labels.every((label2) => labels.has(label2));
|
|
8085
8085
|
})
|
|
8086
8086
|
);
|
|
8087
8087
|
}
|
|
8088
8088
|
function alertSelectorsSatisfy(dataSource, occurrenceLabels) {
|
|
8089
8089
|
if (!occurrenceLabels) return false;
|
|
8090
8090
|
return dataSource.alertSelectorLabels.every(
|
|
8091
|
-
(
|
|
8092
|
-
(series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(
|
|
8091
|
+
(label2) => label2 !== dataSource.workspaceLabel && typeof occurrenceLabels[label2] === "string" && occurrenceLabels[label2].length > 0 && dataSource.requiredSeries.every((series) => series.labels.includes(label2)) && dataSource.availableSeries.some(
|
|
8092
|
+
(series) => dataSource.requiredSeries.some((required) => required.metric === series.metric) && series.labels.includes(dataSource.workspaceLabel) && series.labels.includes(label2)
|
|
8093
8093
|
)
|
|
8094
8094
|
);
|
|
8095
8095
|
}
|
|
@@ -9113,6 +9113,19 @@ async function finalizeDurableTurnOpStreams(sessions, fallback, toolCallIds) {
|
|
|
9113
9113
|
}
|
|
9114
9114
|
}
|
|
9115
9115
|
|
|
9116
|
+
// src/activities/agent-turn/provider-recovery-policy.ts
|
|
9117
|
+
var PROVIDER_OVERLOAD_RECOVERY_CODE = "provider_overloaded";
|
|
9118
|
+
var MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES = 15;
|
|
9119
|
+
var PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS = 15 * 6e4;
|
|
9120
|
+
function validProviderOverloadRecoveryDelay(code, delay) {
|
|
9121
|
+
if (delay === void 0) return code !== PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
9122
|
+
return code === PROVIDER_OVERLOAD_RECOVERY_CODE && typeof delay === "number" && Number.isSafeInteger(delay) && delay > 0 && delay < PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS;
|
|
9123
|
+
}
|
|
9124
|
+
function readProviderRecoveryStartedAt(metadata) {
|
|
9125
|
+
const value = typeof metadata.providerRecoveryStartedAt === "string" ? Date.parse(metadata.providerRecoveryStartedAt) : NaN;
|
|
9126
|
+
return Number.isFinite(value) ? value : void 0;
|
|
9127
|
+
}
|
|
9128
|
+
|
|
9116
9129
|
// src/activities/agent-turn/errors.ts
|
|
9117
9130
|
import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
|
|
9118
9131
|
|
|
@@ -9179,6 +9192,7 @@ import {
|
|
|
9179
9192
|
RoutingWorkspaceRootChangedError,
|
|
9180
9193
|
RoutingBackendRecoveryRequiredError as RoutingBackendRecoveryRequiredError2,
|
|
9181
9194
|
ResponsesStreamingTerminalError,
|
|
9195
|
+
classifyModelStreamIdleTimeoutError,
|
|
9182
9196
|
SandboxMaterializationVerificationError,
|
|
9183
9197
|
materializationVerificationDiagnostic,
|
|
9184
9198
|
PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
@@ -9728,8 +9742,8 @@ import {
|
|
|
9728
9742
|
var CODEX_PLAN_ENTITLEMENT_CODE = "codex_plan_entitlement";
|
|
9729
9743
|
var CODEX_REQUEST_REJECTED_CODE = "codex_request_rejected";
|
|
9730
9744
|
function codexAccountDisplayLabel(account) {
|
|
9731
|
-
const
|
|
9732
|
-
return
|
|
9745
|
+
const label2 = account?.label?.trim() || null;
|
|
9746
|
+
return label2 ? label2.slice(0, 120) : null;
|
|
9733
9747
|
}
|
|
9734
9748
|
function accountPhrase(accountLabel) {
|
|
9735
9749
|
return accountLabel ? `The ChatGPT account "${accountLabel}"` : "The connected ChatGPT account";
|
|
@@ -9839,10 +9853,58 @@ var POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE = "Agent turn database recovery
|
|
|
9839
9853
|
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_TYPE = "EscapedMcpTimeoutRecoveryFailure";
|
|
9840
9854
|
var ESCAPED_MCP_TIMEOUT_RECOVERY_FAILURE_MESSAGE = "MCP request timeout recovery checkpoint failed before model request";
|
|
9841
9855
|
|
|
9856
|
+
// src/activities/agent-turn/provider-recovery-copy.ts
|
|
9857
|
+
var MODEL_PROVIDER_RECOVERY_CODES = /* @__PURE__ */ new Set([
|
|
9858
|
+
"provider_unavailable",
|
|
9859
|
+
"provider_rate_limited",
|
|
9860
|
+
"provider_unknown_finish_reason",
|
|
9861
|
+
"post_compaction_continuation_empty"
|
|
9862
|
+
]);
|
|
9863
|
+
function label(value) {
|
|
9864
|
+
const trimmed = typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
|
|
9865
|
+
return trimmed ? trimmed.slice(0, 120) : null;
|
|
9866
|
+
}
|
|
9867
|
+
function providerRecoverySubject(input) {
|
|
9868
|
+
const model = label(input.modelLabel) ?? "The model";
|
|
9869
|
+
const provider = label(input.providerLabel);
|
|
9870
|
+
const at = provider ? ` at the provider (${provider})` : " at the provider";
|
|
9871
|
+
switch (input.code) {
|
|
9872
|
+
case "provider_rate_limited":
|
|
9873
|
+
return `${model} is rate limited${at}`;
|
|
9874
|
+
case "provider_unavailable":
|
|
9875
|
+
return input.providerCondition === "overloaded" ? `${model} is overloaded${at}` : input.providerCondition === "unresponsive" ? `${model} stopped responding${at}` : `${model} is temporarily unavailable${at}`;
|
|
9876
|
+
case "provider_unknown_finish_reason":
|
|
9877
|
+
return `${model} ended its response unexpectedly`;
|
|
9878
|
+
case "post_compaction_continuation_empty":
|
|
9879
|
+
return `${model} stopped right after compacting the conversation`;
|
|
9880
|
+
case "upstream_connectivity_unavailable":
|
|
9881
|
+
return "Opengeni can't reach an upstream service";
|
|
9882
|
+
case "mcp_transport_timeout":
|
|
9883
|
+
return "A required MCP server isn't responding";
|
|
9884
|
+
case "mcp_transport_unavailable":
|
|
9885
|
+
return "A required MCP server is unreachable";
|
|
9886
|
+
case "sandbox_command_start_unavailable":
|
|
9887
|
+
return "The sandbox isn't ready yet";
|
|
9888
|
+
case "turn_execution_policy_definition_mismatch":
|
|
9889
|
+
return "Opengeni is applying a configuration update";
|
|
9890
|
+
default:
|
|
9891
|
+
return "A service this turn depends on is temporarily unavailable";
|
|
9892
|
+
}
|
|
9893
|
+
}
|
|
9894
|
+
function providerRecoveryExhaustedMessage(input) {
|
|
9895
|
+
const count = input.providerRecoveryCount;
|
|
9896
|
+
const retried = `Opengeni retried ${count} ${count === 1 ? "time" : "times"} without success.`;
|
|
9897
|
+
const remedy = MODEL_PROVIDER_RECOVERY_CODES.has(input.code ?? "") ? "Try again in a few minutes, or switch to another model." : "Try again in a few minutes.";
|
|
9898
|
+
return `${providerRecoverySubject(input)}. ${retried} ${remedy}`;
|
|
9899
|
+
}
|
|
9900
|
+
|
|
9842
9901
|
// src/activities/agent-turn/errors.ts
|
|
9843
9902
|
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
9844
9903
|
var PROVIDER_CONNECTIVITY_BACKOFF_MS = [2e3, 5e3, 15e3, 3e4, 6e4];
|
|
9845
9904
|
var MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
9905
|
+
function providerRecoveryLimit(failureCode) {
|
|
9906
|
+
return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES : MAX_AUTOMATIC_PROVIDER_RECOVERIES;
|
|
9907
|
+
}
|
|
9846
9908
|
var PROVIDER_RATE_LIMIT_BACKOFF_MS = [1e4, 2e4, 4e4, 6e4, 12e4];
|
|
9847
9909
|
function providerRecoveryJitterMs(delayMs, sample) {
|
|
9848
9910
|
const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
|
|
@@ -9857,13 +9919,27 @@ var PostCompactionContinuationEmptyError = class extends Error {
|
|
|
9857
9919
|
}
|
|
9858
9920
|
};
|
|
9859
9921
|
function providerRecoveryResult(input) {
|
|
9860
|
-
|
|
9922
|
+
const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
9923
|
+
const limit = providerRecoveryLimit(input.failureCode);
|
|
9924
|
+
const exhausted = (reason) => ({
|
|
9925
|
+
status: "exhausted",
|
|
9926
|
+
providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
|
|
9927
|
+
maxProviderRecoveryCount: limit,
|
|
9928
|
+
providerRecoveryExhaustedReason: reason
|
|
9929
|
+
});
|
|
9930
|
+
if (input.attemptNumber > limit) {
|
|
9931
|
+
if (overload) return exhausted("retry_limit");
|
|
9861
9932
|
return {
|
|
9862
9933
|
status: "exhausted",
|
|
9863
9934
|
providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
9864
9935
|
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES
|
|
9865
9936
|
};
|
|
9866
9937
|
}
|
|
9938
|
+
const now = input.now ?? Date.now();
|
|
9939
|
+
const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
|
|
9940
|
+
if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
|
|
9941
|
+
return exhausted("invalid_clock");
|
|
9942
|
+
}
|
|
9867
9943
|
const providerDelay = input.retryAfterMs !== null && input.retryAfterMs !== void 0 && Number.isFinite(input.retryAfterMs) && input.retryAfterMs > 0 ? Math.ceil(input.retryAfterMs) : null;
|
|
9868
9944
|
const continueDelayMs = input.failureCode === "provider_rate_limited" ? Math.max(
|
|
9869
9945
|
providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
@@ -9871,29 +9947,93 @@ function providerRecoveryResult(input) {
|
|
|
9871
9947
|
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
9872
9948
|
PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1
|
|
9873
9949
|
)]
|
|
9874
|
-
) : input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "sandbox_command_start_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" || input.failureCode === "turn_execution_policy_definition_mismatch" || input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE ? Math.max(
|
|
9950
|
+
) : overload || input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" || input.failureCode === "sandbox_command_start_unavailable" || input.failureCode === "mcp_transport_timeout" || input.failureCode === "mcp_transport_unavailable" || input.failureCode === "turn_execution_policy_definition_mismatch" || input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE ? Math.max(
|
|
9875
9951
|
providerDelay ?? 0,
|
|
9876
9952
|
PROVIDER_CONNECTIVITY_BACKOFF_MS[Math.min(
|
|
9877
9953
|
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
9878
9954
|
PROVIDER_CONNECTIVITY_BACKOFF_MS.length - 1
|
|
9879
9955
|
)]
|
|
9880
9956
|
) : PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
9957
|
+
const delayMs = continueDelayMs + (overload || input.failureCode === "provider_rate_limited" || input.failureCode === "provider_unavailable" || input.failureCode === "upstream_connectivity_unavailable" ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0) : 0);
|
|
9958
|
+
if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
9959
|
+
return exhausted("deadline");
|
|
9960
|
+
}
|
|
9881
9961
|
return {
|
|
9882
9962
|
status: "recovering",
|
|
9883
|
-
continueDelayMs:
|
|
9963
|
+
continueDelayMs: delayMs,
|
|
9964
|
+
...overload ? { maxProviderRecoveryCount: limit } : {}
|
|
9884
9965
|
};
|
|
9885
9966
|
}
|
|
9967
|
+
var ProviderOverloadRecoveryExpiredError = class extends Error {
|
|
9968
|
+
failure;
|
|
9969
|
+
constructor(count, reason) {
|
|
9970
|
+
super("Confirmed provider overload recovery window expired before dispatch");
|
|
9971
|
+
this.name = "ProviderOverloadRecoveryExpiredError";
|
|
9972
|
+
this.failure = providerRecoveryExhaustedFailure(
|
|
9973
|
+
{
|
|
9974
|
+
error: this.message,
|
|
9975
|
+
code: "provider_unavailable",
|
|
9976
|
+
providerCondition: "overloaded"
|
|
9977
|
+
},
|
|
9978
|
+
{
|
|
9979
|
+
status: "exhausted",
|
|
9980
|
+
// The queued replacement has not dispatched; count only prior retries.
|
|
9981
|
+
providerRecoveryCount: Math.min(
|
|
9982
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
9983
|
+
Math.max(0, count - 1)
|
|
9984
|
+
),
|
|
9985
|
+
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
9986
|
+
providerRecoveryExhaustedReason: reason
|
|
9987
|
+
}
|
|
9988
|
+
);
|
|
9989
|
+
}
|
|
9990
|
+
};
|
|
9991
|
+
function assertProviderOverloadRecoveryActive(input) {
|
|
9992
|
+
if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
|
|
9993
|
+
return;
|
|
9994
|
+
}
|
|
9995
|
+
if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
|
|
9996
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
|
|
9997
|
+
}
|
|
9998
|
+
const now = input.now ?? Date.now();
|
|
9999
|
+
const startedAt = input.recoveryStartedAt;
|
|
10000
|
+
if (startedAt === void 0 || !Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now) {
|
|
10001
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
|
|
10002
|
+
}
|
|
10003
|
+
if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
10004
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
|
|
10005
|
+
}
|
|
10006
|
+
}
|
|
9886
10007
|
function providerRecoveryExhaustedFailure(failure, recovery) {
|
|
9887
10008
|
return {
|
|
9888
10009
|
...failure,
|
|
9889
|
-
error:
|
|
10010
|
+
error: providerRecoveryExhaustedMessage({
|
|
10011
|
+
code: typeof failure.code === "string" ? failure.code : null,
|
|
10012
|
+
providerCondition: isProviderCondition(failure.providerCondition) ? failure.providerCondition : null,
|
|
10013
|
+
modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
|
|
10014
|
+
providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
|
|
10015
|
+
providerRecoveryCount: recovery.providerRecoveryCount
|
|
10016
|
+
}),
|
|
9890
10017
|
retryable: false,
|
|
9891
10018
|
recoveryExhausted: true,
|
|
9892
10019
|
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
9893
10020
|
maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
|
|
10021
|
+
...recovery.providerRecoveryExhaustedReason ? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason } : {},
|
|
9894
10022
|
lastRetryableError: failure.error
|
|
9895
10023
|
};
|
|
9896
10024
|
}
|
|
10025
|
+
function isProviderCondition(value) {
|
|
10026
|
+
return value === "overloaded" || value === "unavailable" || value === "unresponsive" || value === "rate_limited";
|
|
10027
|
+
}
|
|
10028
|
+
function withModelRoutePresentation(failure, route) {
|
|
10029
|
+
if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
|
|
10030
|
+
return {
|
|
10031
|
+
...failure,
|
|
10032
|
+
model: route.model,
|
|
10033
|
+
modelLabel: route.modelLabel,
|
|
10034
|
+
providerLabel: route.providerLabel
|
|
10035
|
+
};
|
|
10036
|
+
}
|
|
9897
10037
|
function providerRecoveryCountFromMetadata(metadata) {
|
|
9898
10038
|
const value = metadata.providerRecoveryCount;
|
|
9899
10039
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
@@ -10103,7 +10243,10 @@ function postClaimDatabaseRecoveryFailure(input) {
|
|
|
10103
10243
|
if (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted || (input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) && input.providerRecovery) {
|
|
10104
10244
|
return null;
|
|
10105
10245
|
}
|
|
10106
|
-
if (input.providerRecovery && (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) || input.providerRecovery.providerRecoveryCount <= 0 || input.providerRecovery.providerRecoveryCount >
|
|
10246
|
+
if (input.providerRecovery && (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) || input.providerRecovery.providerRecoveryCount <= 0 || input.providerRecovery.providerRecoveryCount > providerRecoveryLimit(input.providerRecovery.failureCode) || !/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) || !validProviderOverloadRecoveryDelay(
|
|
10247
|
+
input.providerRecovery.failureCode,
|
|
10248
|
+
input.providerRecovery.continueDelayMs
|
|
10249
|
+
))) {
|
|
10107
10250
|
return null;
|
|
10108
10251
|
}
|
|
10109
10252
|
const detail = {
|
|
@@ -10115,7 +10258,8 @@ function postClaimDatabaseRecoveryFailure(input) {
|
|
|
10115
10258
|
...input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
|
|
10116
10259
|
...input.providerRecovery ? {
|
|
10117
10260
|
providerFailureCode: input.providerRecovery.failureCode,
|
|
10118
|
-
providerRecoveryCount: input.providerRecovery.providerRecoveryCount
|
|
10261
|
+
providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
|
|
10262
|
+
...input.providerRecovery.continueDelayMs !== void 0 ? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs } : {}
|
|
10119
10263
|
} : {}
|
|
10120
10264
|
};
|
|
10121
10265
|
return ApplicationFailure.create({
|
|
@@ -10394,6 +10538,20 @@ function providerHttpStatus(error) {
|
|
|
10394
10538
|
}
|
|
10395
10539
|
return void 0;
|
|
10396
10540
|
}
|
|
10541
|
+
function isTransportTimeoutError(error) {
|
|
10542
|
+
let current = error;
|
|
10543
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10544
|
+
for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
|
|
10545
|
+
if (seen.has(current)) break;
|
|
10546
|
+
seen.add(current);
|
|
10547
|
+
const value = current;
|
|
10548
|
+
if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
|
|
10549
|
+
return true;
|
|
10550
|
+
}
|
|
10551
|
+
current = value.cause;
|
|
10552
|
+
}
|
|
10553
|
+
return false;
|
|
10554
|
+
}
|
|
10397
10555
|
function isTransientProviderError(error) {
|
|
10398
10556
|
if (error instanceof ResponsesStreamingTerminalError) {
|
|
10399
10557
|
return error.category === "unavailable";
|
|
@@ -10491,6 +10649,42 @@ function isRawProviderCommandOutcomeUnknown(error) {
|
|
|
10491
10649
|
}
|
|
10492
10650
|
}
|
|
10493
10651
|
function agentRunFailurePayload(error, options = {}) {
|
|
10652
|
+
return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
|
|
10653
|
+
}
|
|
10654
|
+
function withProviderCondition(error, failure) {
|
|
10655
|
+
if (failure.code === "provider_rate_limited") {
|
|
10656
|
+
return { ...failure, providerCondition: "rate_limited" };
|
|
10657
|
+
}
|
|
10658
|
+
if (failure.code !== "provider_unavailable") return failure;
|
|
10659
|
+
if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
|
|
10660
|
+
return {
|
|
10661
|
+
...failure,
|
|
10662
|
+
providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable"
|
|
10663
|
+
};
|
|
10664
|
+
}
|
|
10665
|
+
function isProviderOverloadError(error, failure = {}) {
|
|
10666
|
+
if (providerHttpStatus(error) === 529) return true;
|
|
10667
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
10668
|
+
return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
|
|
10669
|
+
(value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value)
|
|
10670
|
+
);
|
|
10671
|
+
}
|
|
10672
|
+
function providerRecoveryCode(error, failure) {
|
|
10673
|
+
if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
|
|
10674
|
+
const status = providerHttpStatus(error);
|
|
10675
|
+
let current = error;
|
|
10676
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10677
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
10678
|
+
if (seen.has(current)) break;
|
|
10679
|
+
seen.add(current);
|
|
10680
|
+
if (current instanceof AnthropicRequestError && status !== void 0 && status >= 500 && status < 600 && (status === 529 || current.errorType === "overloaded_error")) {
|
|
10681
|
+
return PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
10682
|
+
}
|
|
10683
|
+
current = current.cause;
|
|
10684
|
+
}
|
|
10685
|
+
return failure.code;
|
|
10686
|
+
}
|
|
10687
|
+
function classifyAgentRunFailurePayload(error, options = {}) {
|
|
10494
10688
|
const graph = structuredRecoveryCauseGraph(error);
|
|
10495
10689
|
const nodes = graph ? [...graph.keys()] : [];
|
|
10496
10690
|
const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
|
|
@@ -10505,6 +10699,8 @@ function agentRunFailurePayload(error, options = {}) {
|
|
|
10505
10699
|
retryable: false
|
|
10506
10700
|
};
|
|
10507
10701
|
}
|
|
10702
|
+
const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
|
|
10703
|
+
if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
|
|
10508
10704
|
const failure = baseAgentRunFailurePayload(error, options);
|
|
10509
10705
|
const diagnostic = materializationVerificationDiagnostic(error);
|
|
10510
10706
|
const anthropic = anthropicRequestDiagnostic(error);
|
|
@@ -10757,6 +10953,28 @@ function baseAgentRunFailurePayload(error, options = {}) {
|
|
|
10757
10953
|
...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
|
|
10758
10954
|
};
|
|
10759
10955
|
}
|
|
10956
|
+
const streamIdle = classifyModelStreamIdleTimeoutError(error);
|
|
10957
|
+
if (streamIdle) {
|
|
10958
|
+
const seconds = streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1e3) : null;
|
|
10959
|
+
const window = seconds !== null ? ` for ${seconds}s` : "";
|
|
10960
|
+
return {
|
|
10961
|
+
error: streamIdle.kind === "progress" ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.` : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
|
|
10962
|
+
code: "provider_unavailable",
|
|
10963
|
+
retryable: true,
|
|
10964
|
+
timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
|
|
10965
|
+
responseObserved: true,
|
|
10966
|
+
detail: streamIdle.message
|
|
10967
|
+
};
|
|
10968
|
+
}
|
|
10969
|
+
if (status === void 0 && isTransportTimeoutError(error)) {
|
|
10970
|
+
return {
|
|
10971
|
+
error: "The model provider connection timed out. The same turn will retry after a short delay.",
|
|
10972
|
+
code: "provider_unavailable",
|
|
10973
|
+
retryable: true,
|
|
10974
|
+
timeoutClass: "transport",
|
|
10975
|
+
detail: message
|
|
10976
|
+
};
|
|
10977
|
+
}
|
|
10760
10978
|
if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
|
|
10761
10979
|
return {
|
|
10762
10980
|
error: "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
|
|
@@ -11015,10 +11233,16 @@ export {
|
|
|
11015
11233
|
turnFinalizerCancellationSignal,
|
|
11016
11234
|
runMandatoryHistoryPersistenceStep,
|
|
11017
11235
|
finalizeDurableTurnOpStreams,
|
|
11236
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
11237
|
+
validProviderOverloadRecoveryDelay,
|
|
11238
|
+
readProviderRecoveryStartedAt,
|
|
11018
11239
|
MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
11240
|
+
providerRecoveryLimit,
|
|
11019
11241
|
PostCompactionContinuationEmptyError,
|
|
11020
11242
|
providerRecoveryResult,
|
|
11243
|
+
assertProviderOverloadRecoveryActive,
|
|
11021
11244
|
providerRecoveryExhaustedFailure,
|
|
11245
|
+
withModelRoutePresentation,
|
|
11022
11246
|
providerRecoveryCountFromMetadata,
|
|
11023
11247
|
providerRecoveryCountAfterModelRequestPhase,
|
|
11024
11248
|
providerRetryAfterMs,
|
|
@@ -11040,6 +11264,7 @@ export {
|
|
|
11040
11264
|
classifyContextWindowOverflowError,
|
|
11041
11265
|
classifyXaiCredentialFailure,
|
|
11042
11266
|
agentRunFailurePayload,
|
|
11267
|
+
providerRecoveryCode,
|
|
11043
11268
|
agentRunRecoveryFailurePayload,
|
|
11044
11269
|
codexCredentialCooldownUntil,
|
|
11045
11270
|
classifyCodexCredentialFailure,
|
|
@@ -11047,4 +11272,4 @@ export {
|
|
|
11047
11272
|
CODEX_USAGE_LIMIT_MAX_RESUME_MS,
|
|
11048
11273
|
classifyClaudeCredentialFailure
|
|
11049
11274
|
};
|
|
11050
|
-
//# sourceMappingURL=chunk-
|
|
11275
|
+
//# sourceMappingURL=chunk-WX5PX2C6.js.map
|