@opengeni/worker-bundle 2.3.1-canary.37062299158001 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/activities/agent-turn/claude-usage-observer.d.ts +6 -30
  2. package/dist/activities/agent-turn/credential-leases.d.ts +32 -5
  3. package/dist/activities/agent-turn/errors.d.ts +0 -3
  4. package/dist/activities/agent-turn/finalization-monitor.d.ts +1 -1
  5. package/dist/activities/agent-turn/finalization.d.ts +0 -2
  6. package/dist/activities/agent-turn/quiescence.d.ts +4 -3
  7. package/dist/activities/agent-turn/turn-context.d.ts +1 -1
  8. package/dist/activities/session-state.d.ts +2 -2
  9. package/dist/activities/types.d.ts +0 -7
  10. package/dist/{activities-control-TTGQGLCJ.js → activities-control-3HAH4FMG.js} +32 -83
  11. package/dist/activities-control-3HAH4FMG.js.map +1 -0
  12. package/dist/activities-control.d.ts +4 -4
  13. package/dist/{activities-turn-TTNAVGKN.js → activities-turn-KU25J3VQ.js} +308 -263
  14. package/dist/activities-turn-KU25J3VQ.js.map +1 -0
  15. package/dist/activities.d.ts +4 -4
  16. package/dist/{chunk-PA7Z55P4.js → chunk-3K3ULWEE.js} +17 -242
  17. package/dist/chunk-3K3ULWEE.js.map +1 -0
  18. package/dist/{chunk-YOI3Z3GF.js → chunk-YG27NEMX.js} +1 -9
  19. package/dist/chunk-YG27NEMX.js.map +1 -0
  20. package/dist/http.d.ts +1 -1
  21. package/dist/index.d.ts +2 -5
  22. package/dist/index.js +16 -82
  23. package/dist/index.js.map +1 -1
  24. package/dist/observability-metrics.d.ts +1 -6
  25. package/dist/sandbox-resume.d.ts +0 -3
  26. package/dist/worker-service-lifecycle.d.ts +0 -12
  27. package/dist/workflow-bundle.js +2 -2
  28. package/package.json +21 -21
  29. package/src/activities/agent-turn/claim.ts +0 -13
  30. package/src/activities/agent-turn/claude-usage-observer.ts +8 -99
  31. package/src/activities/agent-turn/credential-leases.ts +213 -86
  32. package/src/activities/agent-turn/errors.ts +1 -61
  33. package/src/activities/agent-turn/failure-settlement.ts +5 -59
  34. package/src/activities/agent-turn/finalization-monitor.ts +4 -3
  35. package/src/activities/agent-turn/finalization.ts +3 -8
  36. package/src/activities/agent-turn/quiescence.ts +4 -3
  37. package/src/activities/agent-turn/run.ts +20 -24
  38. package/src/activities/agent-turn/sandbox-establish.ts +0 -2
  39. package/src/activities/agent-turn/sandbox-runtime.ts +1 -5
  40. package/src/activities/agent-turn/skill-selection.ts +0 -2
  41. package/src/activities/agent-turn/turn-context.ts +4 -1
  42. package/src/activities/codex-rotation.ts +12 -2
  43. package/src/activities/sandbox-lease.ts +8 -59
  44. package/src/activities/session-state.ts +4 -29
  45. package/src/activities/types.ts +0 -7
  46. package/src/activities-turn.ts +0 -5
  47. package/src/activities.ts +0 -6
  48. package/src/http.ts +2 -4
  49. package/src/index.ts +9 -39
  50. package/src/observability-metrics.ts +0 -27
  51. package/src/sandbox-resume.ts +0 -9
  52. package/src/sandbox-routing.ts +0 -77
  53. package/src/worker-service-lifecycle.ts +0 -53
  54. package/src/workflows/session.ts +1 -6
  55. package/dist/activities/agent-turn/subscription-lease.d.ts +0 -38
  56. package/dist/activities-control-TTGQGLCJ.js.map +0 -1
  57. package/dist/activities-turn-TTNAVGKN.js.map +0 -1
  58. package/dist/chunk-PA7Z55P4.js.map +0 -1
  59. package/dist/chunk-YOI3Z3GF.js.map +0 -1
  60. package/dist/session-event-fanout.d.ts +0 -6
  61. package/src/activities/agent-turn/subscription-lease.ts +0 -115
  62. package/src/session-event-fanout.ts +0 -23
@@ -1,17 +1,6 @@
1
1
  import { type parseModelProvidersJson, type ClaudeUsageObservation } from "@opengeni/config";
2
2
  type Scope = "workspace" | "organization";
3
- type ClaudeRequestCredential = {
4
- token: string;
5
- connectionId: string;
6
- credentialVersion: number;
7
- };
8
- export declare class ClaudeSubscriptionConnectionUnavailable extends Error {
9
- readonly status = 409;
10
- readonly code = "claude_subscription_connection_changed";
11
- constructor();
12
- }
13
3
  export type CapturedClaudeUsage = {
14
- scope: Scope;
15
4
  token: string;
16
5
  expectedConnectionId: string;
17
6
  expectedCredentialVersion: number;
@@ -22,29 +11,16 @@ export type CapturedClaudeUsage = {
22
11
  };
23
12
  };
24
13
  /** Capture the exact credential before requests; replacement fences late responses. */
25
- export declare function createClaudeUsageObserver(providers: ReturnType<typeof parseModelProvidersJson>, latest: Map<string, CapturedClaudeUsage>, readCredential: (scope: Scope) => Promise<{
14
+ export declare function createClaudeUsageObserver(providers: ReturnType<typeof parseModelProvidersJson>, latest: Map<Scope, CapturedClaudeUsage>, readCredential: (scope: Scope) => Promise<{
26
15
  token: string;
27
16
  connectionId: string;
28
17
  credentialVersion: number;
29
- } | null>): Promise<((providerId: string, response: Response, upstreamModelId?: string, requestToken?: string | null) => void) & {
30
- prepareRequestWithObserver: (providerId: string, headers: Headers, resolve: (binding: {
31
- scope: Scope;
32
- expectedConnectionId: string;
33
- expectedCredentialVersion: number;
34
- }) => Promise<ClaudeRequestCredential | null>) => Promise<{
35
- headers: Headers;
36
- observe: (providerId: string, response: Response, upstreamModelId?: string, requestToken?: string | null) => void;
37
- }>;
38
- prepareRequest(providerId: string, headers: Headers, resolve: (binding: {
39
- scope: Scope;
40
- expectedConnectionId: string;
41
- expectedCredentialVersion: number;
42
- }) => Promise<ClaudeRequestCredential | null>): Promise<Headers>;
18
+ } | null>): Promise<((providerId: string, response: Response) => void) & {
43
19
  binding(providerId: string): {
44
- scope: Scope;
45
- token: string;
46
- expectedConnectionId: string;
47
- expectedCredentialVersion: number;
20
+ readonly scope: Scope;
21
+ readonly token: string;
22
+ readonly expectedConnectionId: string;
23
+ readonly expectedCredentialVersion: number;
48
24
  } | undefined;
49
25
  renew(providerId: string, credential: {
50
26
  token: string;
@@ -1,6 +1,6 @@
1
1
  import type { SharedActivityServices } from "../types.js";
2
- import { SubscriptionTurnLease, type LeaseRenewReason, type LeaseLossReason } from "./subscription-lease.js";
3
- export type { LeaseRenewReason, LeaseLossReason } from "./subscription-lease.js";
2
+ export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
3
+ export type LeaseLossReason = "deadline" | "not_found";
4
4
  export declare class CodexCredentialLeaseLostError extends Error {
5
5
  readonly reason: LeaseLossReason;
6
6
  readonly code = "codex_credential_lease_lost";
@@ -19,13 +19,40 @@ export type TurnCredentialLeaseDeps = {
19
19
  * cross-replica fairness primitive; the heartbeat here only extends its short
20
20
  * TTL. A killed worker stops heartbeating and the holder self-expires.
21
21
  */
22
- export declare class CodexTurnLease extends SubscriptionTurnLease {
22
+ export declare class CodexTurnLease {
23
+ private readonly deps;
24
+ held: boolean;
25
+ lost: boolean;
26
+ lossReason: LeaseLossReason | null;
27
+ holderId: string | null;
28
+ generation: number | null;
29
+ confirmedUntilMs: number | null;
30
+ private heartbeatInFlight;
31
+ private heartbeatTimer;
23
32
  constructor(deps: TurnCredentialLeaseDeps);
33
+ markLost: (reason: LeaseLossReason) => void;
34
+ /** Fail closed at the last worker-confirmed lease deadline before dispatch. */
35
+ assertUsable: () => void;
36
+ renew: (reason: LeaseRenewReason) => Promise<void>;
37
+ startHeartbeat: () => void;
38
+ stopHeartbeat: () => void;
24
39
  }
25
- /** SuperGrok uses the same deadline and holder fences as Codex. */
26
- export declare class XaiTurnLease extends SubscriptionTurnLease {
40
+ /** The xAI/SuperGrok credential holder for one running turn; same shape. */
41
+ export declare class XaiTurnLease {
42
+ private readonly deps;
43
+ held: boolean;
44
+ lost: boolean;
27
45
  subjectId: string | null;
46
+ holderId: string | null;
47
+ generation: number | null;
48
+ confirmedUntilMs: number | null;
49
+ private heartbeatInFlight;
50
+ private heartbeatTimer;
28
51
  constructor(deps: TurnCredentialLeaseDeps);
52
+ markLost: (reason: LeaseLossReason) => void;
53
+ renew: () => Promise<void>;
54
+ startHeartbeat: () => void;
55
+ stopHeartbeat: () => void;
29
56
  }
30
57
  /** Both serving-credential leases for one turn attempt plus their composite views. */
31
58
  export type TurnCredentialLeases = {
@@ -90,7 +90,6 @@ export declare function postClaimDatabaseRecoveryFailure(input: {
90
90
  triggerEventId: string;
91
91
  executionGeneration: number;
92
92
  sandboxSetupOutcomeUnknown?: true;
93
- sandboxSetupRecoveryExhausted?: true;
94
93
  providerRecovery?: {
95
94
  failureCode: string;
96
95
  providerRecoveryCount: number;
@@ -267,8 +266,6 @@ export declare function classifyXaiCredentialFailure(error: unknown): XaiCredent
267
266
  export declare function agentRunFailurePayload(error: unknown, options?: {
268
267
  isCodexTurn?: boolean;
269
268
  }): ReturnType<typeof baseAgentRunFailurePayload>;
270
- /** Keep Anthropic provider text on terminal failures, never recovery events. */
271
- export declare function agentRunRecoveryFailurePayload(error: unknown, failure: ReturnType<typeof agentRunFailurePayload>): ReturnType<typeof agentRunFailurePayload>;
272
269
  declare function baseAgentRunFailurePayload(error: unknown, options?: {
273
270
  isCodexTurn?: boolean;
274
271
  }): {
@@ -8,7 +8,7 @@ export declare function startTurnFinalizationMonitor(input: {
8
8
  observability: Observability;
9
9
  details: TurnHeartbeatDetails;
10
10
  heartbeat: (details: TurnHeartbeatDetails) => void;
11
- requestWorkerDrain: () => void;
11
+ terminateWorker: () => void;
12
12
  timeoutMs?: number;
13
13
  slowAfterMs?: number;
14
14
  }): {
@@ -16,8 +16,6 @@ export type TurnFinalizationDeps = {
16
16
  wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
17
17
  signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
18
18
  signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
19
- requestWorkerDrain: ActivityServices["requestWorkerDrain"];
20
- turnFinalizationTimeoutMs: ActivityServices["turnFinalizationTimeoutMs"];
21
19
  cancellationSignal: AbortSignal | undefined;
22
20
  sandboxResumeController: AbortController;
23
21
  activityContext: ReturnType<typeof currentActivityContext>;
@@ -49,9 +49,10 @@ export declare const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
49
49
  export declare const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5000;
50
50
  export declare const TURN_QUIESCENCE_WATCHDOG_MS: number;
51
51
  export declare const TURN_FINALIZER_STEP_TIMEOUT_MS = 30000;
52
- /** A fenced activity must never heartbeat forever while physical cleanup is
53
- * stuck. The callback requests host-owned containment, which drains peer turns
54
- * before applying its final exit policy. A timeout is never quiescence proof. */
52
+ /** A fenced activity must never heartbeat forever while its physical writer
53
+ * drain is stuck. The last-resort worker exit stops every in-process writer;
54
+ * Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
55
+ * then commits the exact quiescence receipt. */
55
56
  export declare function armTurnQuiescenceWatchdog(input: {
56
57
  enabled: boolean;
57
58
  timeoutMs?: number;
@@ -146,7 +146,7 @@ export type ProviderTurnState = {
146
146
  xaiCredentialQuarantined: boolean;
147
147
  priorSessionCodexCredentialId: string | null;
148
148
  latestCodexUsage: CodexUsageHeaderSnapshot | null;
149
- latestClaudeUsage: Map<string, import("./claude-usage-observer.js").CapturedClaudeUsage>;
149
+ latestClaudeUsage: Map<"workspace" | "organization", import("./claude-usage-observer.js").CapturedClaudeUsage>;
150
150
  lastCodexRequestOpaqueArtifacts: readonly string[];
151
151
  };
152
152
  export type TurnContext = {
@@ -53,8 +53,8 @@ export declare function createSessionStateActivities(services: () => Promise<Con
53
53
  admissionFence?: import("@opengeni/db").SessionAdmissionFence;
54
54
  } | {
55
55
  kind: "admission-blocked";
56
- reason?: "sandbox_setup_outcome_unknown" | "sandbox_setup_recovery_exhausted";
57
- ref?: import("@opengeni/db").SandboxSetupOutcomeUnknown | import("@opengeni/db").SandboxSetupRecoveryExhausted;
56
+ reason?: "sandbox_setup_outcome_unknown";
57
+ ref?: import("@opengeni/db").SandboxSetupOutcomeUnknown;
58
58
  } | {
59
59
  kind: "sandbox-lifecycle-wait";
60
60
  ref: import("@opengeni/db").SandboxLifecycleWait;
@@ -87,11 +87,6 @@ export type SharedActivityServices = {
87
87
  export type ControlActivityServices = SharedActivityServices;
88
88
  /** Turn workers own the model loop and never construct document parsers. */
89
89
  export type TurnActivityServices = SharedActivityServices & {
90
- /** Host-owned containment: stop polling and checkpoint peer turns before exit.
91
- * Bare activity embedders must provide their own worker lifecycle edge. */
92
- requestWorkerDrain: () => void;
93
- /** Deterministic test clock for the cleanup-only monitor. */
94
- turnFinalizationTimeoutMs?: number | undefined;
95
90
  runtime: OpenGeniRuntime;
96
91
  /** Provider-free test/profiling seam; production injects the real runtime summarizer. */
97
92
  summarizeContextForCompaction: typeof import("@opengeni/runtime").summarizeForCompaction;
@@ -242,8 +237,6 @@ export type PostClaimDatabaseRecoveryDetail = {
242
237
  providerFailureCode?: string;
243
238
  /** Persist the no-replay setup marker, not a new provider/setup attempt. */
244
239
  sandboxSetupOutcomeUnknown?: true;
245
- /** Preserve the exhausted budget for a command proven not dispatched. */
246
- sandboxSetupRecoveryExhausted?: true;
247
240
  };
248
241
  export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";
249
242
  export declare const POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE = "Agent turn database recovery required after attempt claim.";
@@ -24,7 +24,6 @@ import {
24
24
  notifyParentOfChildIdle,
25
25
  persistWorkspaceArchiveCandidate,
26
26
  providerRecoveryCountFromMetadata,
27
- publishDurableSessionEvents,
28
27
  putVersion1TarArchiveOrInline,
29
28
  reconcileAutomaticSessionTitleFanout,
30
29
  reconcilePendingParentSystemUpdates,
@@ -35,7 +34,7 @@ import {
35
34
  warnDrainSnapshotFailure,
36
35
  withFirstPartyTools,
37
36
  workflowIdForSession
38
- } from "./chunk-PA7Z55P4.js";
37
+ } from "./chunk-3K3ULWEE.js";
39
38
  import {
40
39
  inspectOpenSandboxKubernetesInventory,
41
40
  recordActiveUserGauges,
@@ -48,7 +47,6 @@ import {
48
47
  recordRetainedProcessReconciliation,
49
48
  recordSandboxCheckpointArtifactGauges,
50
49
  recordSandboxCheckpointArtifactOutcome,
51
- recordSandboxCommandContainment,
52
50
  recordSandboxDeadlineRotationsRequested,
53
51
  recordSandboxInventoryProjectionFailure,
54
52
  recordSandboxInventoryProjectionSuccess,
@@ -62,7 +60,7 @@ import {
62
60
  recordVerifiedSignupTrialSwitchGauge,
63
61
  recordWorkerDeathRecoveryMetrics,
64
62
  runtimeMetricsHooksForObservability
65
- } from "./chunk-YOI3Z3GF.js";
63
+ } from "./chunk-YG27NEMX.js";
66
64
  import "./chunk-PZ5AY32C.js";
67
65
 
68
66
  // src/activities/knowledge-indexing.ts
@@ -2723,7 +2721,6 @@ import {
2723
2721
  workspaceArchiveCaptureDeadlineElapsed,
2724
2722
  retainedProcessReconciliationProof,
2725
2723
  retainedProcessSettlementIdentity,
2726
- recoverManagedSessionBackgroundCommand,
2727
2724
  settleClaimedConnectedMachineBackgroundCommand,
2728
2725
  settleSandboxCheckpointArtifactGc,
2729
2726
  rlsContextForWorkspace,
@@ -2890,18 +2887,14 @@ function createSandboxLeaseActivities(services, options = {}) {
2890
2887
  try {
2891
2888
  const { db, settings, observability } = await services();
2892
2889
  const timing = sandboxDrainTiming(settings);
2893
- const commandContainment = {
2894
- idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
2895
- onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
2896
- onCommandContainmentError: (error) => {
2897
- observability.warn("sandbox reaper: retained command containment inspection failed", {
2898
- error: error instanceof Error ? error.message : String(error)
2899
- });
2900
- }
2890
+ const onUnobservableCommandDrainError = (error) => {
2891
+ observability.warn("sandbox reaper: unobservable command drain inspection failed", {
2892
+ error: error instanceof Error ? error.message : String(error)
2893
+ });
2901
2894
  };
2902
2895
  if (!settings.sandboxOwnershipEnabled) {
2903
2896
  const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
2904
- ...commandContainment,
2897
+ onUnobservableCommandDrainError,
2905
2898
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
2906
2899
  turnHolderTtlMs: settings.sandboxLeaseTtlMs,
2907
2900
  interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -2928,7 +2921,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2928
2921
  });
2929
2922
  recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
2930
2923
  const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
2931
- ...commandContainment,
2924
+ onUnobservableCommandDrainError,
2932
2925
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
2933
2926
  // Dead-worker turn holders: a live holder is touched every 10s from the
2934
2927
  // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -2969,7 +2962,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2969
2962
  instanceId: input.target.instanceId
2970
2963
  });
2971
2964
  try {
2972
- const { db, settings, observability, objectStorage, bus } = await services();
2965
+ const { db, settings, observability, objectStorage } = await services();
2973
2966
  assertSandboxDrainInputTiming(input);
2974
2967
  const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
2975
2968
  const row = input.target;
@@ -2989,8 +2982,7 @@ function createSandboxLeaseActivities(services, options = {}) {
2989
2982
  terminateBox,
2990
2983
  probeDrainableProvider,
2991
2984
  captureAttempt,
2992
- objectStorage,
2993
- bus
2985
+ objectStorage
2994
2986
  );
2995
2987
  return { status: drainedCold ? "terminated" : "skipped" };
2996
2988
  } catch (error) {
@@ -3625,24 +3617,6 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
3625
3617
  sessionId: process.sessionId,
3626
3618
  processId: process.id
3627
3619
  };
3628
- if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
3629
- try {
3630
- const command = await recoverManagedSessionBackgroundCommand(db, {
3631
- ...processScope,
3632
- expected,
3633
- reconciliationClaimId: claim.claimId
3634
- });
3635
- if (command) {
3636
- claim.ownerState = `background_${command.state}`;
3637
- recordRetainedProcessReconciliation(observability, "background_recovered");
3638
- }
3639
- } catch (error) {
3640
- observability.warn("sandbox reaper: retained-command background recovery failed", {
3641
- processId: process.id,
3642
- error: error instanceof Error ? error.message : String(error)
3643
- });
3644
- }
3645
- }
3646
3620
  const storedCommandPersistence = retainedProviderCommandPersistence(
3647
3621
  db,
3648
3622
  processScope,
@@ -4634,7 +4608,7 @@ function sandboxDrainReason(lease) {
4634
4608
  if (lease.rotationRequestedAt === null) return "idle";
4635
4609
  return lease.rotationReason ?? "rotation_requested";
4636
4610
  }
4637
- async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
4611
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
4638
4612
  const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
4639
4613
  const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
4640
4614
  if (!lease) {
@@ -4948,31 +4922,18 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
4948
4922
  }
4949
4923
  }
4950
4924
  const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
4951
- const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
4925
+ const { wentCold } = await confirmDrainCold(db, {
4952
4926
  accountId,
4953
4927
  workspaceId: row.workspaceId,
4954
4928
  sandboxGroupId: row.sandboxGroupId,
4955
4929
  expectedEpoch: row.leaseEpoch,
4956
4930
  ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
4957
- providerMissingBeforeCapture: providerMissing,
4958
- idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
4959
- });
4960
- await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
4961
- observability.warn("sandbox reaper: contained command event fanout failed", {
4962
- sandboxGroupId: row.sandboxGroupId,
4963
- error: error instanceof Error ? error.message : String(error)
4964
- });
4931
+ providerMissingBeforeCapture: providerMissing
4965
4932
  });
4966
4933
  if (wentCold) {
4967
4934
  if (providerMissing) {
4968
4935
  recordSandboxProviderMissingBeforeCapture(observability, backend);
4969
4936
  }
4970
- if (lease.unobservableCommandDrainIds?.length) {
4971
- recordSandboxCommandContainment(
4972
- observability,
4973
- providerMissing ? "provider_missing" : "contained"
4974
- );
4975
- }
4976
4937
  try {
4977
4938
  await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
4978
4939
  type: "sandbox.box.terminated",
@@ -5356,7 +5317,7 @@ import {
5356
5317
  upsertScheduledSessionGoalForRun,
5357
5318
  withSessionActivityRlsContext
5358
5319
  } from "@opengeni/db";
5359
- import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
5320
+ import { publishDurableSessionEvents } from "@opengeni/events";
5360
5321
  import {
5361
5322
  allowedFirstPartyMcpToolsForSession,
5362
5323
  resolveFirstPartyMcpToolPolicy,
@@ -6497,7 +6458,7 @@ function createScheduledTaskActivities(services) {
6497
6458
  events: started.events
6498
6459
  });
6499
6460
  } else {
6500
- await publishDurableSessionEvents2(
6461
+ await publishDurableSessionEvents(
6501
6462
  bus,
6502
6463
  task.workspaceId,
6503
6464
  session.id,
@@ -6540,7 +6501,7 @@ function createScheduledTaskActivities(services) {
6540
6501
  if (deferPublications) {
6541
6502
  deferredEvents.push({ sessionId: session.id, events: started.events });
6542
6503
  } else {
6543
- await publishDurableSessionEvents2(
6504
+ await publishDurableSessionEvents(
6544
6505
  bus,
6545
6506
  task.workspaceId,
6546
6507
  session.id,
@@ -6569,7 +6530,7 @@ function createScheduledTaskActivities(services) {
6569
6530
  if (deferPublications) {
6570
6531
  deferredEvents.push({ sessionId: session.id, events: titleEvents });
6571
6532
  } else {
6572
- await publishDurableSessionEvents2(bus, task.workspaceId, session.id, titleEvents);
6533
+ await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
6573
6534
  }
6574
6535
  }
6575
6536
  const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -6656,7 +6617,7 @@ function createScheduledTaskActivities(services) {
6656
6617
  events: scheduledUpdate.events
6657
6618
  });
6658
6619
  } else {
6659
- await publishDurableSessionEvents2(
6620
+ await publishDurableSessionEvents(
6660
6621
  bus,
6661
6622
  task.workspaceId,
6662
6623
  session.id,
@@ -6783,7 +6744,7 @@ function createScheduledTaskActivities(services) {
6783
6744
  events: bundled.events
6784
6745
  });
6785
6746
  } else {
6786
- await publishDurableSessionEvents2(
6747
+ await publishDurableSessionEvents(
6787
6748
  bus,
6788
6749
  task.workspaceId,
6789
6750
  session.id,
@@ -6865,7 +6826,7 @@ function createScheduledTaskActivities(services) {
6865
6826
  });
6866
6827
  }
6867
6828
  for (const deferred of deferredEvents) {
6868
- await publishDurableSessionEvents2(
6829
+ await publishDurableSessionEvents(
6869
6830
  bus,
6870
6831
  task.workspaceId,
6871
6832
  deferred.sessionId,
@@ -7097,7 +7058,7 @@ async function seedScheduledGeneratedSessionRoute(input) {
7097
7058
  if (input.deferPublications) {
7098
7059
  input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
7099
7060
  } else {
7100
- await publishDurableSessionEvents2(
7061
+ await publishDurableSessionEvents(
7101
7062
  input.bus,
7102
7063
  input.task.workspaceId,
7103
7064
  input.session.id,
@@ -7367,7 +7328,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
7367
7328
  });
7368
7329
  workflowId = started.temporalWorkflowId;
7369
7330
  if (started.events.length > 0) {
7370
- await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, started.events);
7331
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
7371
7332
  }
7372
7333
  if (task.runMode === "reusable_session") {
7373
7334
  await materializeScheduledTaskReusableSessionFromRun(input.db, {
@@ -7386,7 +7347,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
7386
7347
  taskName: task.name
7387
7348
  });
7388
7349
  if (titleEvents.length > 0) {
7389
- await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, titleEvents);
7350
+ await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
7390
7351
  }
7391
7352
  }
7392
7353
  const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -7479,7 +7440,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
7479
7440
  recordUserMessageAccepted(input.observability, { surface: "scheduled" });
7480
7441
  }
7481
7442
  if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
7482
- await publishDurableSessionEvents2(
7443
+ await publishDurableSessionEvents(
7483
7444
  input.bus,
7484
7445
  task.workspaceId,
7485
7446
  session.id,
@@ -7835,10 +7796,9 @@ import {
7835
7796
  expireSessionHumanInputRequest,
7836
7797
  markSessionAttemptQuiesced,
7837
7798
  requireSession as requireSession2,
7838
- settleSessionIdleWithParentOutbox,
7839
- SANDBOX_SETUP_RECOVERY_LIMIT
7799
+ settleSessionIdleWithParentOutbox
7840
7800
  } from "@opengeni/db";
7841
- import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
7801
+ import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
7842
7802
  import { CancelledFailure } from "@temporalio/activity";
7843
7803
  var WORKER_DEATH_MAX_REDISPATCHES = 3;
7844
7804
  function createSessionStateActivities(services, overrides = {}) {
@@ -7862,7 +7822,7 @@ function createSessionStateActivities(services, overrides = {}) {
7862
7822
  const requireSessionFn = overrides.requireSession ?? requireSession2;
7863
7823
  const settleSessionIdleWithParentOutboxFn = overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
7864
7824
  const markSessionAttemptQuiescedFn = overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
7865
- const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents3;
7825
+ const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents2;
7866
7826
  const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
7867
7827
  const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
7868
7828
  const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
@@ -7951,16 +7911,12 @@ function createSessionStateActivities(services, overrides = {}) {
7951
7911
  const postClaimIdentityMatches = Boolean(
7952
7912
  postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
7953
7913
  );
7954
- if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
7955
- return { action: "stale" };
7956
- }
7957
7914
  const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
7958
7915
  const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
7959
7916
  const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
7960
7917
  const hasProviderFailureCode = providerFailureCode !== void 0;
7961
7918
  const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
7962
- const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
7963
- if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
7919
+ if (setupOutcomeUnknown && (hasProviderRecoveryCount || hasProviderFailureCode)) {
7964
7920
  return { action: "stale" };
7965
7921
  }
7966
7922
  if (hasProviderRecoveryCount !== hasProviderFailureCode) {
@@ -7976,20 +7932,13 @@ function createSessionStateActivities(services, overrides = {}) {
7976
7932
  turnId: turn.id,
7977
7933
  triggerEventId: turn.triggerEventId,
7978
7934
  attemptId: input.attemptId,
7979
- reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? "claimed_attempt_database_failure",
7935
+ reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? "claimed_attempt_database_failure",
7980
7936
  ...setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
7981
- ...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
7982
7937
  ...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
7983
7938
  detail: {
7984
- code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
7985
- retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
7939
+ code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? recoveredClaimCode,
7940
+ retryable: !setupOutcomeUnknown,
7986
7941
  ...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
7987
- ...setupRecoveryExhausted ? {
7988
- setupOutcome: "not_started",
7989
- replay: "blocked",
7990
- recoveryExhausted: true,
7991
- providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
7992
- } : {},
7993
7942
  ...providerRecoveryCount !== void 0 ? {
7994
7943
  databaseFailureCode: recoveredClaimCode,
7995
7944
  providerRecoveryCount
@@ -8445,4 +8394,4 @@ export {
8445
8394
  createControlActivities,
8446
8395
  createControlActivitiesFromServices
8447
8396
  };
8448
- //# sourceMappingURL=activities-control-TTGQGLCJ.js.map
8397
+ //# sourceMappingURL=activities-control-3HAH4FMG.js.map