@opengeni/worker-bundle 1.4.4 → 1.4.5-canary.37458084490001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +69 -2
  2. package/dist/activities/agent-turn/governance-model.d.ts +2 -1
  3. package/dist/activities/agent-turn/model-usage.d.ts +22 -0
  4. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  5. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  6. package/dist/activities/agent-turn/turn-context.d.ts +9 -0
  7. package/dist/activities/capabilities.d.ts +2 -0
  8. package/dist/activities/types.d.ts +3 -0
  9. package/dist/{activities-control-VRFS65FT.js → activities-control-KBOSUF5Y.js} +36 -12
  10. package/dist/activities-control-KBOSUF5Y.js.map +1 -0
  11. package/dist/{activities-turn-P2YAW2QW.js → activities-turn-QOR26CTC.js} +230 -39
  12. package/dist/activities-turn-QOR26CTC.js.map +1 -0
  13. package/dist/artifact-materializer-entry.js +0 -0
  14. package/dist/artifact-outbox-entry.js +0 -0
  15. package/dist/{chunk-HVNNCWEU.js → chunk-CMIUQ2D3.js} +114 -13
  16. package/dist/chunk-CMIUQ2D3.js.map +1 -0
  17. package/dist/{chunk-BICGFGO7.js → chunk-WX5PX2C6.js} +208 -20
  18. package/dist/chunk-WX5PX2C6.js.map +1 -0
  19. package/dist/index.js +5 -5
  20. package/dist/index.js.map +1 -1
  21. package/dist/observability-metrics.d.ts +59 -4
  22. package/dist/workflow-bundle.js +72 -39
  23. package/package.json +21 -21
  24. package/src/activities/agent-turn/admission.ts +7 -8
  25. package/src/activities/agent-turn/claim.ts +20 -1
  26. package/src/activities/agent-turn/errors.ts +242 -13
  27. package/src/activities/agent-turn/failure-settlement.ts +19 -7
  28. package/src/activities/agent-turn/governance-model.ts +3 -0
  29. package/src/activities/agent-turn/model-usage.ts +114 -3
  30. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  31. package/src/activities/agent-turn/provider-recovery-metrics.ts +1 -1
  32. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  33. package/src/activities/agent-turn/run.ts +14 -1
  34. package/src/activities/agent-turn/stream-attempt.ts +68 -1
  35. package/src/activities/agent-turn/turn-context.ts +5 -0
  36. package/src/activities/automations.ts +5 -2
  37. package/src/activities/capabilities.ts +58 -2
  38. package/src/activities/session-state.ts +41 -5
  39. package/src/activities/types.ts +3 -1
  40. package/src/index.ts +2 -2
  41. package/src/observability-metrics.ts +184 -15
  42. package/src/workflows/session.ts +22 -12
  43. package/dist/activities-control-VRFS65FT.js.map +0 -1
  44. package/dist/activities-turn-P2YAW2QW.js.map +0 -1
  45. package/dist/chunk-BICGFGO7.js.map +0 -1
  46. package/dist/chunk-HVNNCWEU.js.map +0 -1
@@ -4,9 +4,12 @@ import { ApplicationFailure } from "@temporalio/activity";
4
4
  import type { CodexAccountStatus } from "@opengeni/db";
5
5
  import type { EscapedMcpTimeoutRecoveryDetail } from "../types.js";
6
6
  import { type MandatoryHistoryPersistenceStage } from "./quiescence.js";
7
+ import { type ProviderCondition } from "./provider-recovery-copy.js";
8
+ export { PROVIDER_OVERLOAD_RECOVERY_CODE, MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES, PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS, } from "./provider-recovery-policy.js";
7
9
  export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
8
10
  export declare const PROVIDER_CONNECTIVITY_BACKOFF_MS: readonly [2000, 5000, 15000, 30000, 60000];
9
11
  export declare const MAX_AUTOMATIC_PROVIDER_RECOVERIES: 5;
12
+ export declare function providerRecoveryLimit(failureCode: unknown): number;
10
13
  /**
11
14
  * Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
12
15
  * answer a per-minute token limit with a `retry-after` of about a second, which
@@ -23,17 +26,44 @@ export declare class PostCompactionContinuationEmptyError extends Error {
23
26
  export type ProviderRecoveryResult = {
24
27
  status: "recovering";
25
28
  continueDelayMs: number;
29
+ maxProviderRecoveryCount?: number;
26
30
  } | {
27
31
  status: "exhausted";
28
32
  providerRecoveryCount: number;
29
33
  maxProviderRecoveryCount: number;
34
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
30
35
  };
31
36
  export declare function providerRecoveryResult(input: {
32
37
  failureCode: string | undefined;
33
38
  attemptNumber: number;
34
39
  retryAfterMs?: number | null;
35
40
  jitterSample?: number;
41
+ /** Durable first-failure clock, never a new clock on each replacement attempt. */
42
+ recoveryStartedAt?: number | undefined;
43
+ now?: number;
36
44
  }): ProviderRecoveryResult;
45
+ /** A timer/queue delay cannot authorize a provider call after the recovery window. */
46
+ export declare class ProviderOverloadRecoveryExpiredError extends Error {
47
+ readonly failure: {
48
+ error: string;
49
+ code: string;
50
+ providerCondition: "overloaded";
51
+ } & {
52
+ retryable: false;
53
+ recoveryExhausted: true;
54
+ providerRecoveryCount: number;
55
+ maxProviderRecoveryCount: number;
56
+ lastRetryableError: string;
57
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
58
+ };
59
+ constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock");
60
+ }
61
+ export declare function assertProviderOverloadRecoveryActive(input: {
62
+ failureCode: unknown;
63
+ providerRecoveryCount: number;
64
+ recoveryStartedAt?: number | undefined;
65
+ now?: number;
66
+ }): void;
37
67
  export declare function providerRecoveryExhaustedFailure<T extends Record<string, unknown> & {
38
68
  error: string;
39
69
  }>(failure: T, recovery: Extract<ProviderRecoveryResult, {
@@ -44,6 +74,23 @@ export declare function providerRecoveryExhaustedFailure<T extends Record<string
44
74
  providerRecoveryCount: number;
45
75
  maxProviderRecoveryCount: number;
46
76
  lastRetryableError: string;
77
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
78
+ };
79
+ /**
80
+ * Name the exact accepted model route on model-provider recovery evidence so a
81
+ * person learns which model is affected. Display labels only: the route's
82
+ * credentials, base URL and upstream ids never enter the event.
83
+ */
84
+ export declare function withModelRoutePresentation<T extends {
85
+ code?: string | undefined;
86
+ }>(failure: T, route: {
87
+ model: string;
88
+ modelLabel: string;
89
+ providerLabel: string;
90
+ } | undefined): T & {
91
+ model?: string;
92
+ modelLabel?: string;
93
+ providerLabel?: string;
47
94
  };
48
95
  export declare function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number;
49
96
  export declare function providerRecoveryCountAfterModelRequestPhase(currentCount: number, phase: string): number;
@@ -100,6 +147,7 @@ export declare function postClaimDatabaseRecoveryFailure(input: {
100
147
  providerRecovery?: {
101
148
  failureCode: string;
102
149
  providerRecoveryCount: number;
150
+ continueDelayMs?: number;
103
151
  };
104
152
  }): ApplicationFailure | null;
105
153
  /**
@@ -276,6 +324,16 @@ export declare function classifyXaiCredentialFailure(error: unknown): XaiCredent
276
324
  export declare function agentRunFailurePayload(error: unknown, options?: {
277
325
  isCodexTurn?: boolean;
278
326
  }): ReturnType<typeof baseAgentRunFailurePayload>;
327
+ /** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
328
+ export declare function isProviderOverloadError(error: unknown, failure?: {
329
+ error?: string;
330
+ detail?: string;
331
+ }): boolean;
332
+ /** Retry authority must not depend on the looser presentation-only overload label. */
333
+ export declare function providerRecoveryCode(error: unknown, failure: {
334
+ code?: string;
335
+ retryable?: boolean;
336
+ }): string | undefined;
279
337
  /** Keep Anthropic provider text on terminal failures, never recovery events. */
280
338
  export declare function agentRunRecoveryFailurePayload(error: unknown, failure: ReturnType<typeof agentRunFailurePayload>): ReturnType<typeof agentRunFailurePayload>;
281
339
  declare function baseAgentRunFailurePayload(error: unknown, options?: {
@@ -303,8 +361,18 @@ declare function baseAgentRunFailurePayload(error: unknown, options?: {
303
361
  quotaScope?: ProviderQuotaScope;
304
362
  /** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
305
363
  planType?: string | null;
306
- /** Product model id a `codex_plan_entitlement` failure refers to. */
364
+ /** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
307
365
  model?: string | null;
366
+ /** Display label of the turn's model on model-provider recovery evidence. */
367
+ modelLabel?: string;
368
+ /** Display label of the serving provider on model-provider recovery evidence. */
369
+ providerLabel?: string;
370
+ /** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
371
+ providerCondition?: ProviderCondition;
372
+ recoveryExhausted?: boolean;
373
+ providerRecoveryCount?: number;
374
+ maxProviderRecoveryCount?: number;
375
+ providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
308
376
  };
309
377
  export type CodexCredentialFailure = {
310
378
  /**
@@ -352,4 +420,3 @@ export declare const CODEX_USAGE_LIMIT_MAX_RESUME_MS: number;
352
420
  export declare function classifyClaudeCredentialFailure(error: unknown): (XaiCredentialFailure & {
353
421
  requestId?: string;
354
422
  }) | null;
355
- export {};
@@ -1,4 +1,4 @@
1
- import { materializeRigVersionForAttempt } from "@opengeni/db";
1
+ import { materializeRigVersionForAttempt, freezeAgentLearningPolicy } from "@opengeni/db";
2
2
  import { renderWorkspaceGovernanceContext, type OpenGeniRuntime } from "@opengeni/runtime";
3
3
  import { type Settings } from "@opengeni/config";
4
4
  import { createModelHistoryAttachmentProjector } from "../run-input.js";
@@ -9,6 +9,7 @@ import { lazyToolTransportForTurn, openAiHostedImageProviderBindingForTurn, mode
9
9
  import type { ClaimTurnOk } from "./claim.js";
10
10
  import type { EventingState, WorkspaceRefState } from "./turn-context.js";
11
11
  export type GovernanceModelDeps = {
12
+ learningPolicy: Awaited<ReturnType<typeof freezeAgentLearningPolicy>>;
12
13
  input: RunAgentTurnInput;
13
14
  db: ActivityServices["db"];
14
15
  observability: ActivityServices["observability"];
@@ -227,7 +227,29 @@ export declare function recordModelUsageAndDebitCredits(settings: Settings, db:
227
227
  sourceKey: string;
228
228
  latencyMode?: LatencyMode;
229
229
  observability?: ActivityServices["observability"];
230
+ /** Configured provider id for metric labels; never billing authority. */
231
+ metricProvider?: string;
230
232
  }): Promise<ModelUsageBillingRecord | null>;
233
+ /**
234
+ * The bounded `model` metric label: the deployment catalog product id for a
235
+ * catalog model, or `custom` for a workspace-owned model (workspace gateway or
236
+ * a customer-key provider) and for anything the deployment catalog does not
237
+ * list. Metrics only; never billing or routing authority.
238
+ */
239
+ export declare function modelMetricProductId(settings: Settings, provider: string | undefined, model: string): string;
240
+ /**
241
+ * Per-model usage and estimated provider cost for one authoritative response.
242
+ * Call only after the durable usage event was accepted as current (the same
243
+ * fence as the Insights fact). Best-effort: never throws.
244
+ */
245
+ export declare function recordAuthoritativeModelUsageMetrics(input: {
246
+ observability: ActivityServices["observability"];
247
+ settings: Settings;
248
+ provider: string;
249
+ model: string;
250
+ externallyBilled: boolean;
251
+ billing: ModelUsageBillingRecord;
252
+ }): void;
231
253
  /** Soft-fail Insights fact write — never throws into the billing/emit path. */
232
254
  export declare function recordAuthoritativeModelCallFact(input: {
233
255
  db: ActivityServices["db"];
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Person-facing copy for finite same-turn provider recovery. Presentation only:
3
+ * retry authority, pacing and the recovery budget stay in `errors.ts`.
4
+ *
5
+ * `@opengeni/react` (`lib/provider-recovery.ts`) composes the same sentences
6
+ * from the structured event fields, so live retry status and historical
7
+ * failures read identically in every surface. Keep the two in step.
8
+ */
9
+ /** Closed provider condition recorded on model-provider recovery payloads. */
10
+ export type ProviderCondition = "overloaded" | "unavailable" | "unresponsive" | "rate_limited";
11
+ /** Failure codes whose transient condition belongs to the turn's model route. */
12
+ export declare const MODEL_PROVIDER_RECOVERY_CODES: ReadonlySet<string>;
13
+ export type ProviderRecoveryCopyInput = {
14
+ code?: string | null | undefined;
15
+ providerCondition?: ProviderCondition | null | undefined;
16
+ modelLabel?: string | null | undefined;
17
+ providerLabel?: string | null | undefined;
18
+ };
19
+ /** One present-tense clause naming what is unavailable, without a trailing period. */
20
+ export declare function providerRecoverySubject(input: ProviderRecoveryCopyInput): string;
21
+ /** Terminal copy once the finite automatic recovery budget is spent. */
22
+ export declare function providerRecoveryExhaustedMessage(input: ProviderRecoveryCopyInput & {
23
+ providerRecoveryCount: number;
24
+ }): string;
@@ -0,0 +1,6 @@
1
+ /** Closed policy facts shared with the workflow's content-free recovery wire. */
2
+ export declare const PROVIDER_OVERLOAD_RECOVERY_CODE = "provider_overloaded";
3
+ export declare const MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES = 15;
4
+ export declare const PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS: number;
5
+ export declare function validProviderOverloadRecoveryDelay(code: unknown, delay: unknown): boolean;
6
+ export declare function readProviderRecoveryStartedAt(metadata: Record<string, unknown>): number | undefined;
@@ -43,11 +43,20 @@ export type AttemptIdentityState = {
43
43
  triggerEventId: string | undefined;
44
44
  executionGeneration: number;
45
45
  providerRecoveryCount: number;
46
+ /** Durable recovery policy reason; never inferred from display wording. */
47
+ providerRecoveryPolicyCode?: string | undefined;
48
+ providerRecoveryStartedAt?: number | undefined;
46
49
  providerRecoveryObservation?: import("./provider-recovery-metrics.js").ProviderRecoveryObservation | undefined;
47
50
  modelMetricRoute?: {
48
51
  provider: string;
49
52
  model: string;
50
53
  };
54
+ /** Public display labels of the accepted model route, for recovery copy only. */
55
+ modelRoutePresentation?: {
56
+ model: string;
57
+ modelLabel: string;
58
+ providerLabel: string;
59
+ };
51
60
  claudeAuthRecovery?: {
52
61
  credentialId: string;
53
62
  credentialVersion: number;
@@ -24,4 +24,6 @@ export declare function withCodexProvider(settings: Settings): Settings;
24
24
  export declare function withXaiSubscriptionProvider(settings: Settings): Settings;
25
25
  export declare function settingsWithWorkspaceGatewayCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
26
26
  export declare function settingsWithWorkspaceOpenRouterCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
27
+ /** Worker-only: overlay the workspace Opper catalog and, when active, its decrypted key. */
28
+ export declare function settingsWithWorkspaceOpperCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
27
29
  export declare function settingsWithOrganizationProviderCredentials(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
@@ -226,6 +226,7 @@ export type FailSessionAttemptResult = {
226
226
  action: "failed";
227
227
  } | {
228
228
  action: "recovering";
229
+ continueDelayMs?: number;
229
230
  } | {
230
231
  action: "unclaimed";
231
232
  } | {
@@ -254,6 +255,8 @@ export type PostClaimDatabaseRecoveryDetail = {
254
255
  providerRecoveryCount?: number;
255
256
  /** Safe classified provider cause paired with providerRecoveryCount. */
256
257
  providerFailureCode?: string;
258
+ /** Preserve a confirmed-overload Retry-After through checkpoint DB outages. */
259
+ providerRecoveryContinueDelayMs?: number;
257
260
  /** Persist the no-replay setup marker, not a new provider/setup attempt. */
258
261
  sandboxSetupOutcomeUnknown?: true;
259
262
  /** Preserve the exhausted budget for a command proven not dispatched. */
@@ -8,6 +8,7 @@ import {
8
8
  import {
9
9
  INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
10
10
  MAX_AUTOMATIC_PROVIDER_RECOVERIES,
11
+ PROVIDER_OVERLOAD_RECOVERY_CODE,
11
12
  assertReusableSessionRevivable,
12
13
  captureConnectedCommandOutput,
13
14
  createCodexCapacityActivities,
@@ -23,6 +24,7 @@ import {
23
24
  notifyParentOfChildIdle,
24
25
  persistWorkspaceArchiveCandidate,
25
26
  providerRecoveryCountFromMetadata,
27
+ providerRecoveryLimit,
26
28
  publishDurableSessionEvents,
27
29
  putVersion1TarArchiveOrInline,
28
30
  reconcileAutomaticSessionTitleFanout,
@@ -31,10 +33,11 @@ import {
31
33
  resolveIncidentTelemetryResponderMetadata,
32
34
  rigProviderImageSourceImage,
33
35
  scheduledUserMessagePayload,
36
+ validProviderOverloadRecoveryDelay,
34
37
  warnDrainSnapshotFailure,
35
38
  withFirstPartyTools,
36
39
  workflowIdForSession
37
- } from "./chunk-BICGFGO7.js";
40
+ } from "./chunk-WX5PX2C6.js";
38
41
  import {
39
42
  inspectOpenSandboxKubernetesInventory,
40
43
  recordActiveUserGauges,
@@ -64,7 +67,7 @@ import {
64
67
  recordVerifiedSignupTrialSwitchGauge,
65
68
  recordWorkerDeathRecoveryMetrics,
66
69
  runtimeMetricsHooksForObservability
67
- } from "./chunk-HVNNCWEU.js";
70
+ } from "./chunk-CMIUQ2D3.js";
68
71
  import "./chunk-PZ5AY32C.js";
69
72
 
70
73
  // src/activities/knowledge-indexing.ts
@@ -712,7 +715,8 @@ function createBrowserDeadlineCheckpointActivities(services, connectController =
712
715
  import {
713
716
  resolveTurnExecutionPolicyV1,
714
717
  WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
715
- WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
718
+ WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
719
+ WORKSPACE_OPPER_MODEL_ID_PREFIX
716
720
  } from "@opengeni/config";
717
721
  import {
718
722
  applySessionAgentConfigWriteThrough,
@@ -800,7 +804,7 @@ function createAutomationActivities(services, options = {}) {
800
804
  return { action: "failed", reason: "credential_restriction_violation" };
801
805
  }
802
806
  const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
803
- const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
807
+ const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
804
808
  accountId: input.accountId,
805
809
  workspaceId: input.workspaceId,
806
810
  retainedProductModelId: requestedModel
@@ -950,7 +954,7 @@ function createAutomationActivities(services, options = {}) {
950
954
  firstPartyMcpPermissions: template.firstPartyMcpPermissions,
951
955
  firstPartyMcpTools: agentWriteThrough.firstPartyMcpTools,
952
956
  ...agentWriteThrough.config ? { agentConfig: agentWriteThrough.config } : {},
953
- retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
957
+ retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX),
954
958
  createIdempotencyKey: `automation-run:${run.id}`,
955
959
  subjectId: accepted.serviceSubjectId
956
960
  });
@@ -8172,6 +8176,7 @@ import {
8172
8176
  } from "@opengeni/db";
8173
8177
  import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
8174
8178
  import { CancelledFailure } from "@temporalio/activity";
8179
+ import { interactionInterventionMetricObserver } from "@opengeni/observability";
8175
8180
  var WORKER_DEATH_MAX_REDISPATCHES = 3;
8176
8181
  function createSessionStateActivities(services, overrides = {}) {
8177
8182
  const settleSessionAttemptInterruptionsFn = overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
@@ -8286,13 +8291,14 @@ function createSessionStateActivities(services, overrides = {}) {
8286
8291
  const postClaimIdentityMatches = Boolean(
8287
8292
  postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
8288
8293
  );
8289
- if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
8294
+ if (postClaimRecovery && !postClaimIdentityMatches) {
8290
8295
  return { action: "stale" };
8291
8296
  }
8292
8297
  const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
8293
8298
  const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
8294
8299
  const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
8295
8300
  const hasProviderFailureCode = providerFailureCode !== void 0;
8301
+ const providerRecoveryContinueDelayMs = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryContinueDelayMs : void 0;
8296
8302
  const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
8297
8303
  const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
8298
8304
  if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
@@ -8301,7 +8307,10 @@ function createSessionStateActivities(services, overrides = {}) {
8301
8307
  if (hasProviderRecoveryCount !== hasProviderFailureCode) {
8302
8308
  return { action: "stale" };
8303
8309
  }
8304
- if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
8310
+ if (!validProviderOverloadRecoveryDelay(providerFailureCode, providerRecoveryContinueDelayMs)) {
8311
+ return { action: "stale" };
8312
+ }
8313
+ if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > providerRecoveryLimit(providerFailureCode) || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
8305
8314
  return { action: "stale" };
8306
8315
  }
8307
8316
  const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
@@ -8316,7 +8325,7 @@ function createSessionStateActivities(services, overrides = {}) {
8316
8325
  ...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
8317
8326
  ...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
8318
8327
  detail: {
8319
- code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
8328
+ code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? "provider_unavailable" : providerFailureCode ?? recoveredClaimCode,
8320
8329
  retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
8321
8330
  ...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
8322
8331
  ...setupRecoveryExhausted ? {
@@ -8327,7 +8336,12 @@ function createSessionStateActivities(services, overrides = {}) {
8327
8336
  } : {},
8328
8337
  ...providerRecoveryCount !== void 0 ? {
8329
8338
  databaseFailureCode: recoveredClaimCode,
8330
- providerRecoveryCount
8339
+ providerRecoveryCount,
8340
+ ...providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? {
8341
+ providerCondition: "overloaded",
8342
+ maxProviderRecoveryCount: providerRecoveryLimit(providerFailureCode),
8343
+ continueDelayMs: providerRecoveryContinueDelayMs
8344
+ } : {}
8331
8345
  } : {},
8332
8346
  recoverySource: "workflow_activity_failure"
8333
8347
  },
@@ -8341,7 +8355,10 @@ function createSessionStateActivities(services, overrides = {}) {
8341
8355
  countQueuedTurnsFn,
8342
8356
  recordTurnsQueuedGaugeFn
8343
8357
  );
8344
- return { action: "recovering" };
8358
+ return {
8359
+ action: "recovering",
8360
+ ...providerRecoveryContinueDelayMs !== void 0 ? { continueDelayMs: providerRecoveryContinueDelayMs } : {}
8361
+ };
8345
8362
  }
8346
8363
  const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
8347
8364
  const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
@@ -8659,8 +8676,15 @@ function createSessionStateActivities(services, overrides = {}) {
8659
8676
  };
8660
8677
  }
8661
8678
  async function expireSessionInteractionIntervention(input) {
8662
- const { db, bus } = await services();
8679
+ const { db, bus, observability } = await services();
8663
8680
  const result = await expireSessionInteractionInterventionFn(db, input);
8681
+ if (result.action === "expired" && result.intervention) {
8682
+ interactionInterventionMetricObserver(observability)({
8683
+ kind: result.intervention.kind,
8684
+ outcome: "expired",
8685
+ waitMs: Date.now() - result.intervention.createdAt.getTime()
8686
+ });
8687
+ }
8664
8688
  if (result.events.length > 0) {
8665
8689
  await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
8666
8690
  }
@@ -8859,4 +8883,4 @@ export {
8859
8883
  createControlActivities,
8860
8884
  createControlActivitiesFromServices
8861
8885
  };
8862
- //# sourceMappingURL=activities-control-VRFS65FT.js.map
8886
+ //# sourceMappingURL=activities-control-KBOSUF5Y.js.map