@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  2. package/dist/activities/agent-turn/claim.d.ts +7 -0
  3. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  4. package/dist/activities/agent-turn/errors.d.ts +6 -5
  5. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  6. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  7. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  8. package/dist/activities/agent-turn/quiescence.d.ts +13 -8
  9. package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
  10. package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
  11. package/dist/activities/agent-turn/turn-context.d.ts +9 -1
  12. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  13. package/dist/activities/codex-capacity.d.ts +32 -3
  14. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  15. package/dist/activities/codex-image-generation.d.ts +1 -1
  16. package/dist/activities/codex-rotation.d.ts +8 -67
  17. package/dist/activities/goals.d.ts +1 -1
  18. package/dist/activities/google-drive-publication.d.ts +2 -13
  19. package/dist/activities/image-generation-operation.d.ts +13 -4
  20. package/dist/activities/sandbox-lease.d.ts +7 -2
  21. package/dist/activities/session-state.d.ts +4 -2
  22. package/dist/activities/types.d.ts +11 -1
  23. package/dist/activities/xai-quota.d.ts +3 -0
  24. package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
  25. package/dist/activities-control-KS3ZPQVY.js.map +1 -0
  26. package/dist/activities-control.d.ts +2 -0
  27. package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
  28. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  29. package/dist/activities.d.ts +2 -0
  30. package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
  31. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  32. package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
  33. package/dist/chunk-O2LQWJIK.js.map +1 -0
  34. package/dist/github-rest-mcp.d.ts +0 -2
  35. package/dist/index.d.ts +1 -0
  36. package/dist/index.js +2 -2
  37. package/dist/index.js.map +1 -1
  38. package/dist/observability-metrics.d.ts +1 -1
  39. package/dist/sandbox-routing.d.ts +6 -0
  40. package/dist/workflow-bundle.js +94 -9
  41. package/dist/workflows/activities.d.ts +1 -1
  42. package/dist/workflows/session.d.ts +3 -0
  43. package/package.json +17 -17
  44. package/src/activities/agent-turn/agent-build.ts +21 -26
  45. package/src/activities/agent-turn/claim.ts +26 -1
  46. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  47. package/src/activities/agent-turn/credential-leases.ts +36 -0
  48. package/src/activities/agent-turn/errors.ts +73 -12
  49. package/src/activities/agent-turn/failure-settlement.ts +472 -331
  50. package/src/activities/agent-turn/history-prefix.ts +51 -0
  51. package/src/activities/agent-turn/history-sink.ts +30 -6
  52. package/src/activities/agent-turn/quiescence.ts +29 -17
  53. package/src/activities/agent-turn/run.ts +22 -5
  54. package/src/activities/agent-turn/stream-attempt.ts +35 -8
  55. package/src/activities/agent-turn/tool-environment.ts +42 -32
  56. package/src/activities/agent-turn/tool-policy.ts +14 -8
  57. package/src/activities/agent-turn/turn-context.ts +13 -0
  58. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  59. package/src/activities/codemode-dispatcher.ts +235 -59
  60. package/src/activities/codex-capacity.ts +107 -10
  61. package/src/activities/codex-fleet-shadow.ts +8 -1
  62. package/src/activities/codex-image-generation.ts +19 -2
  63. package/src/activities/codex-rotation.ts +45 -142
  64. package/src/activities/goals.ts +9 -9
  65. package/src/activities/google-drive-publication.ts +4 -81
  66. package/src/activities/image-generation-operation.ts +28 -3
  67. package/src/activities/run-input.ts +6 -7
  68. package/src/activities/sandbox-lease.ts +246 -87
  69. package/src/activities/session-state.ts +20 -1
  70. package/src/activities/types.ts +16 -8
  71. package/src/activities/xai-quota.ts +35 -0
  72. package/src/activities.ts +1 -0
  73. package/src/github-rest-mcp.ts +0 -82
  74. package/src/observability-metrics.ts +4 -0
  75. package/src/sandbox-routing.ts +186 -16
  76. package/src/workflows/activities.ts +1 -0
  77. package/src/workflows/session.ts +114 -8
  78. package/dist/activities/api-integrations.d.ts +0 -21
  79. package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
  80. package/dist/activities-turn-I4OPLQLS.js.map +0 -1
  81. package/dist/chunk-65JEWP2Q.js.map +0 -1
  82. package/dist/chunk-RQNH6CGR.js.map +0 -1
  83. package/src/activities/api-integrations.ts +0 -151
@@ -11,23 +11,16 @@ import {
11
11
  } from "@opengeni/contracts/personal-github";
12
12
  import { personalGitHubRepositoryResources } from "@opengeni/core";
13
13
  import {
14
- beginConnectorActionExecution,
15
- completeConnectorActionExecution,
16
14
  getConnectionMetadata,
17
15
  getPersonalGitHubRepositorySelectionState,
18
16
  hasAuditableGitHubInstallationAuthority,
19
17
  listGitHubInstallationAccessForWorkspace,
20
- prepareConnectorActionApproval,
21
18
  resolveAcceptedConnectionUse,
22
19
  type Database,
23
20
  type ResolveConnectionCredentialInput,
24
21
  type ResolveConnectionCredentialResult,
25
22
  type SessionTurnForExecution,
26
23
  } from "@opengeni/db";
27
- import {
28
- AttemptToolApprovalRequiredError,
29
- type AttemptToolAuthorization,
30
- } from "@opengeni/codemode";
31
24
  import { createGitHubAppInstallationTokenWithExpiry, githubAppBotIdentity } from "@opengeni/github";
32
25
  import {
33
26
  type AttemptConnectorActionBinding,
@@ -44,7 +37,6 @@ import {
44
37
  GitHubRestAuthorityError,
45
38
  githubRestConnectorActionOutcome,
46
39
  GitHubRestMcpServer,
47
- GitHubRestMutationNotExecutedError,
48
40
  type GitHubRestRepository,
49
41
  } from "@opengeni/runtime/github-rest-mcp";
50
42
 
@@ -60,7 +52,6 @@ export type GitHubRestMcpRuntime = {
60
52
  tools: ToolRef[];
61
53
  localMcpServers: LocalMcpServerRegistration[];
62
54
  connectorBindings: AttemptConnectorActionBinding[];
63
- authorizeCodemodeCall: AttemptToolAuthorization | null;
64
55
  };
65
56
 
66
57
  export async function buildGitHubRestMcpForTurn(input: {
@@ -81,7 +72,6 @@ export async function buildGitHubRestMcpForTurn(input: {
81
72
  tools: input.tools,
82
73
  localMcpServers: [],
83
74
  connectorBindings: [],
84
- authorizeCodemodeCall: null,
85
75
  };
86
76
  }
87
77
 
@@ -96,7 +86,6 @@ export async function buildGitHubRestMcpForTurn(input: {
96
86
  tools: input.tools,
97
87
  localMcpServers: [],
98
88
  connectorBindings: [],
99
- authorizeCodemodeCall: null,
100
89
  };
101
90
  }
102
91
  const configuredIds = new Set(input.settings.mcpServers.map((server) => server.id));
@@ -112,7 +101,6 @@ export async function buildGitHubRestMcpForTurn(input: {
112
101
  const localMcpServers: LocalMcpServerRegistration[] = [];
113
102
  const toolRefs: ToolRef[] = [];
114
103
  const connectorBindings: AttemptConnectorActionBinding[] = [];
115
- const codemodeActions = createCodemodeActionLifecycle(input, connectorBindings);
116
104
 
117
105
  if (appRepositories.length > 0) {
118
106
  const appResolver = workspaceAppAuthorityResolver(input, appRepositories);
@@ -123,7 +111,6 @@ export async function buildGitHubRestMcpForTurn(input: {
123
111
  authorityKind: "workspace_app",
124
112
  repositories: appRepositories,
125
113
  resolveAuthority: appResolver,
126
- completeCodemodeAction: codemodeActions.complete,
127
114
  }),
128
115
  });
129
116
  configs.push(internalMcpConfig(GITHUB_REST_MCP_APP_SERVER_ID, "GitHub — OpenGeni bot"));
@@ -141,7 +128,6 @@ export async function buildGitHubRestMcpForTurn(input: {
141
128
  authorityKind: "personal_oauth",
142
129
  repositories: personalRepositories,
143
130
  resolveAuthority: personalAuthorityResolver(input),
144
- completeCodemodeAction: codemodeActions.complete,
145
131
  }),
146
132
  });
147
133
  configs.push(internalMcpConfig(GITHUB_REST_MCP_PERSONAL_SERVER_ID, "GitHub — My account"));
@@ -159,74 +145,6 @@ export async function buildGitHubRestMcpForTurn(input: {
159
145
  tools: mergeToolRefs(input.tools, toolRefs),
160
146
  localMcpServers,
161
147
  connectorBindings,
162
- authorizeCodemodeCall: codemodeActions.authorize,
163
- };
164
- }
165
-
166
- function createCodemodeActionLifecycle(
167
- input: {
168
- db: Database;
169
- accountId: string;
170
- workspaceId: string;
171
- sessionId: string;
172
- attemptId: string;
173
- turn: SessionTurnForExecution;
174
- },
175
- bindings: AttemptConnectorActionBinding[],
176
- ): {
177
- authorize: AttemptToolAuthorization;
178
- complete: NonNullable<
179
- ConstructorParameters<typeof GitHubRestMcpServer>[0]["completeCodemodeAction"]
180
- >;
181
- } {
182
- const identity = {
183
- accountId: input.accountId,
184
- workspaceId: input.workspaceId,
185
- sessionId: input.sessionId,
186
- turnId: input.turn.id,
187
- attemptId: input.attemptId,
188
- executionGeneration: input.turn.executionGeneration,
189
- initiator: input.turn.initiator,
190
- };
191
- const executing = new Map<string, string>();
192
- return {
193
- authorize: async ({ call, entry }) => {
194
- if (call.caller.kind !== "codemode") return;
195
- const binding = bindings.find((candidate) => candidate.modelName === entry.modelName);
196
- if (!binding) return;
197
- const invocation = binding.call(call.operationId, call.arguments);
198
- const policy = await prepareConnectorActionApproval(input.db, identity, invocation);
199
- if (policy.managed && policy.decision === "ask") {
200
- throw new AttemptToolApprovalRequiredError();
201
- }
202
- if (policy.managed && policy.decision === "block") {
203
- throw new GitHubRestMutationNotExecutedError(
204
- "GitHub connector action is blocked by policy",
205
- );
206
- }
207
- const admission = await beginConnectorActionExecution(input.db, identity, invocation);
208
- if (!admission.allowed) {
209
- if (admission.reason === "approval_required") {
210
- throw new AttemptToolApprovalRequiredError();
211
- }
212
- throw new GitHubRestMutationNotExecutedError(
213
- `GitHub connector action was not executed: ${admission.reason}`,
214
- );
215
- }
216
- if (admission.managed) executing.set(call.operationId, admission.requestId);
217
- },
218
- complete: async ({ operationId, outcome }) => {
219
- const requestId = executing.get(operationId);
220
- if (!requestId) return;
221
- executing.delete(operationId);
222
- await completeConnectorActionExecution(input.db, {
223
- accountId: input.accountId,
224
- workspaceId: input.workspaceId,
225
- requestId,
226
- attemptId: input.attemptId,
227
- outcome,
228
- });
229
- },
230
148
  };
231
149
  }
232
150
 
@@ -1360,6 +1360,7 @@ export type TurnStartupPhase =
1360
1360
  | "tool_optional_connect"
1361
1361
  | "tool_attempt_catalog_build"
1362
1362
  | "tool_attempt_catalog_persist"
1363
+ | "tool_workspace_gateway_catalog_build"
1363
1364
  | "post_tool_preparation"
1364
1365
  | "agent_construction"
1365
1366
  | "post_agent_preparation"
@@ -1408,6 +1409,9 @@ export type TurnStartupPhase =
1408
1409
  | "model_prepare_runner_before_mcp_tools"
1409
1410
  | "model_prepare_mcp_tools_snapshot"
1410
1411
  | "model_prepare_mcp_tools_before_input_filter"
1412
+ | "model_prepare_mcp_tools_before_repository_skill_discovery"
1413
+ | "model_prepare_repository_skill_discovery"
1414
+ | "model_prepare_repository_skill_discovery_before_input_filter"
1411
1415
  | "model_prepare_input_filter_base"
1412
1416
  | "model_prepare_input_filter_genesis"
1413
1417
  | "model_prepare_input_filter_host"
@@ -17,11 +17,13 @@
17
17
  import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  adoptConnectedMachineSessionBackgroundCommand,
20
+ adoptManagedSessionBackgroundCommand,
20
21
  advanceWorkspaceGenerationForRetainedProcess,
21
22
  advanceWorkspaceGeneration,
22
23
  getRetainedProcess,
23
24
  retainWorkspaceMutationProcess,
24
25
  retainedProcessSettlementIdentity,
26
+ settleConnectedMachineSessionBackgroundCommand,
25
27
  settleRetainedProcess,
26
28
  verifyRetainedProcessMutationSettlement,
27
29
  verifyWorkspaceMutationSettlement,
@@ -36,10 +38,9 @@ import {
36
38
  type EnrollmentRecord,
37
39
  type SandboxWorkspaceMutationAdmission,
38
40
  } from "@opengeni/db";
39
- import {
40
- settleConnectedMachineSessionBackgroundCommand,
41
- settleSessionBackgroundCommandForRetainedProcess,
42
- } from "@opengeni/db/session-background-commands";
41
+ import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-background-commands";
42
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
43
+ import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
43
44
  import type { EventBus } from "@opengeni/events";
44
45
  import {
45
46
  buildSelfhostedBackendSession,
@@ -515,10 +516,6 @@ function afterPersistableHomeMutation(
515
516
  admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
516
517
  operation: op,
517
518
  providerBinding: boundAdmission.providerBinding ?? null,
518
- backgroundCommand: {
519
- commandId: retainedProcess.id,
520
- command: op,
521
- },
522
519
  owner: {
523
520
  kind: "turn",
524
521
  turnId: fence.turnId,
@@ -642,6 +639,76 @@ function afterRetainedProcessMutation(
642
639
  };
643
640
  }
644
641
 
642
+ export function captureConnectedCommandOutput(
643
+ db: Database,
644
+ ids: { accountId: string; workspaceId: string; sessionId: string },
645
+ bus?: EventBus,
646
+ ) {
647
+ return async (commandId: string, frames: OpStreamOutputFrame[]) => {
648
+ for (const frame of frames) {
649
+ const events = await appendSessionCommandOutput(db, {
650
+ ...ids,
651
+ commandId,
652
+ chunkId: `op-frame:${frame.sequence}`,
653
+ stream: frame.stream,
654
+ chunk: frame.chunk,
655
+ });
656
+ if (events.length && bus)
657
+ await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
658
+ }
659
+ };
660
+ }
661
+
662
+ function captureRetainedProcessOutputForTurn(
663
+ services: RoutingWiringServices,
664
+ ids: RoutingWiringIds,
665
+ ) {
666
+ const accountId = ids.workspaceMutationFence?.accountId;
667
+ if (!accountId) return undefined;
668
+ return async ({
669
+ process,
670
+ chunkId,
671
+ chunk,
672
+ stream,
673
+ streamFidelity,
674
+ }: {
675
+ process: RoutingRetainedProcess;
676
+ chunkId: string;
677
+ chunk: string;
678
+ stream: "stdout" | "stderr";
679
+ streamFidelity: "separate" | "merged";
680
+ }) => {
681
+ const events = await appendSessionCommandOutput(services.db, {
682
+ accountId,
683
+ workspaceId: ids.workspaceId,
684
+ sessionId: ids.sessionId,
685
+ commandId: process.id,
686
+ chunkId,
687
+ chunk,
688
+ stream,
689
+ streamFidelity,
690
+ });
691
+ if (events.length && services.bus)
692
+ await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => undefined);
693
+ };
694
+ }
695
+
696
+ function observeRetainedProcessTerminalForTurn(
697
+ services: RoutingWiringServices,
698
+ ids: RoutingWiringIds,
699
+ ) {
700
+ const fence = ids.workspaceMutationFence;
701
+ if (!fence) return undefined;
702
+ return async ({ process }: { process: RoutingRetainedProcess }) => {
703
+ await observeSessionBackgroundCommandCompletion(services.db, {
704
+ accountId: fence.accountId,
705
+ workspaceId: ids.workspaceId,
706
+ sessionId: ids.sessionId,
707
+ commandId: process.id,
708
+ });
709
+ };
710
+ }
711
+
645
712
  function settleRetainedProcessForTurn(
646
713
  services: RoutingWiringServices,
647
714
  ids: RoutingWiringIds,
@@ -686,14 +753,50 @@ function settleRetainedProcessForTurn(
686
753
  if (settlement.process.state === "active") {
687
754
  throw new Error("Retained-process settlement returned an active durable process");
688
755
  }
689
- await settleSessionBackgroundCommandForRetainedProcess(services.db, {
756
+ if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
757
+ await services.bus
758
+ .publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents)
759
+ .catch(() => undefined);
760
+ }
761
+ };
762
+ }
763
+
764
+ function adoptRetainedProcessAsBackgroundCommandForTurn(
765
+ services: RoutingWiringServices,
766
+ ids: RoutingWiringIds,
767
+ ):
768
+ | ((input: { backend: ResolvedActiveBackend; process: RoutingRetainedProcess }) => Promise<void>)
769
+ | undefined {
770
+ const fence = ids.workspaceMutationFence;
771
+ if (!fence) return undefined;
772
+ return async ({ backend, process }) => {
773
+ const durable = await getRetainedProcess(services.db, {
774
+ workspaceId: ids.workspaceId,
775
+ sessionId: ids.sessionId,
776
+ processId: process.id,
777
+ });
778
+ if (
779
+ !durable ||
780
+ durable.providerSessionId !== process.providerSessionId ||
781
+ durable.providerBackend !== backend.kind ||
782
+ durable.providerInstanceId !== backend.providerInstanceId ||
783
+ durable.leaseEpoch !== backend.leaseEpoch ||
784
+ durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
785
+ durable.routeTargetId !== backend.sandboxId ||
786
+ durable.routeEpoch !== backend.activeEpoch
787
+ ) {
788
+ throw new Error("Background command adoption lost its exact retained-process identity");
789
+ }
790
+ await adoptManagedSessionBackgroundCommand(services.db, {
690
791
  accountId: fence.accountId,
691
792
  workspaceId: ids.workspaceId,
692
793
  sessionId: ids.sessionId,
693
- retainedProcessId: process.id,
694
- outcome: settlement.process.state,
695
- exitCode: settlement.process.exitCode,
696
- reason: settlement.process.settlementReason ?? proof.reason,
794
+ turnId: fence.turnId,
795
+ executionGeneration: fence.executionGeneration,
796
+ attemptId: fence.attemptId,
797
+ processId: process.id,
798
+ expected: retainedProcessSettlementIdentity(durable),
799
+ command: "execCommand",
697
800
  });
698
801
  };
699
802
  }
@@ -736,6 +839,10 @@ export function wrapTurnBoxWithRouting(
736
839
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
737
840
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
738
841
  const settleProcess = settleRetainedProcessForTurn(services, ids);
842
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
843
+ services,
844
+ ids,
845
+ );
739
846
  const resolver = makeActiveBackendResolver({
740
847
  workspaceId: ids.workspaceId,
741
848
  defaultBackend: established.session as RoutableBackendSession,
@@ -794,7 +901,7 @@ export function wrapTurnBoxWithRouting(
794
901
  return { commandId: adopted.id };
795
902
  },
796
903
  selfhostedSettleBackgroundCommand: async (command) => {
797
- await settleConnectedMachineSessionBackgroundCommand(db, {
904
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
798
905
  accountId: backgroundCommandAccountId,
799
906
  workspaceId: ids.workspaceId,
800
907
  sessionId: ids.sessionId,
@@ -806,8 +913,23 @@ export function wrapTurnBoxWithRouting(
806
913
  outcome: command.outcome,
807
914
  exitCode: command.exitCode,
808
915
  reason: command.reason,
916
+ ...(command.failure ? { failure: command.failure } : {}),
809
917
  });
918
+ if (settlement && bus) {
919
+ await bus
920
+ .publish(ids.workspaceId, ids.sessionId, settlement.events)
921
+ .catch(() => undefined);
922
+ }
810
923
  },
924
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
925
+ db,
926
+ {
927
+ accountId: backgroundCommandAccountId,
928
+ workspaceId: ids.workspaceId,
929
+ sessionId: ids.sessionId,
930
+ },
931
+ bus,
932
+ ),
811
933
  }
812
934
  : {}),
813
935
  // The turn's declared environment → a selfhosted swap target's manifest, so the
@@ -892,6 +1014,13 @@ export function wrapTurnBoxWithRouting(
892
1014
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
893
1015
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
894
1016
  ...(settleProcess ? { settleProcess } : {}),
1017
+ ...(ids.workspaceMutationFence
1018
+ ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
1019
+ : {}),
1020
+ ...(ids.workspaceMutationFence
1021
+ ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
1022
+ : {}),
1023
+ ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
895
1024
  ...(ids.homeLease
896
1025
  ? {
897
1026
  onDefaultBackendError: async ({
@@ -970,6 +1099,10 @@ export function wrapLazyTurnBoxWithRouting(
970
1099
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
971
1100
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
972
1101
  const settleProcess = settleRetainedProcessForTurn(services, ids);
1102
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
1103
+ services,
1104
+ ids,
1105
+ );
973
1106
  const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
974
1107
  const defaultSupportsPty = defaultBackend
975
1108
  ? selectBackend(defaultBackend).capabilities.Terminal.pty
@@ -1038,7 +1171,7 @@ export function wrapLazyTurnBoxWithRouting(
1038
1171
  return { commandId: adopted.id };
1039
1172
  },
1040
1173
  selfhostedSettleBackgroundCommand: async (command) => {
1041
- await settleConnectedMachineSessionBackgroundCommand(db, {
1174
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
1042
1175
  accountId: backgroundCommandAccountId,
1043
1176
  workspaceId: ids.workspaceId,
1044
1177
  sessionId: ids.sessionId,
@@ -1050,8 +1183,23 @@ export function wrapLazyTurnBoxWithRouting(
1050
1183
  outcome: command.outcome,
1051
1184
  exitCode: command.exitCode,
1052
1185
  reason: command.reason,
1186
+ ...(command.failure ? { failure: command.failure } : {}),
1053
1187
  });
1188
+ if (settlement && bus) {
1189
+ await bus
1190
+ .publish(ids.workspaceId, ids.sessionId, settlement.events)
1191
+ .catch(() => undefined);
1192
+ }
1054
1193
  },
1194
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
1195
+ db,
1196
+ {
1197
+ accountId: backgroundCommandAccountId,
1198
+ workspaceId: ids.workspaceId,
1199
+ sessionId: ids.sessionId,
1200
+ },
1201
+ bus,
1202
+ ),
1055
1203
  }
1056
1204
  : {}),
1057
1205
  ...(ids.environment !== undefined ? { environment: ids.environment } : {}),
@@ -1105,6 +1253,13 @@ export function wrapLazyTurnBoxWithRouting(
1105
1253
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
1106
1254
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
1107
1255
  ...(settleProcess ? { settleProcess } : {}),
1256
+ ...(ids.workspaceMutationFence
1257
+ ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
1258
+ : {}),
1259
+ ...(ids.workspaceMutationFence
1260
+ ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids)! }
1261
+ : {}),
1262
+ ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
1108
1263
  ...(args.homeLeaseIdentity
1109
1264
  ? {
1110
1265
  onDefaultBackendError: async ({
@@ -1382,8 +1537,17 @@ export async function establishSelfhostedTurnSession(
1382
1537
  },
1383
1538
  }
1384
1539
  : {}),
1540
+ captureBackgroundCommandOutput: captureConnectedCommandOutput(
1541
+ db,
1542
+ {
1543
+ accountId: args.accountId,
1544
+ workspaceId: args.workspaceId,
1545
+ sessionId: args.sessionId,
1546
+ },
1547
+ bus,
1548
+ ),
1385
1549
  settleBackgroundCommand: async (command) => {
1386
- await settleConnectedMachineSessionBackgroundCommand(db, {
1550
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
1387
1551
  accountId: args.accountId,
1388
1552
  workspaceId: args.workspaceId,
1389
1553
  sessionId: args.sessionId,
@@ -1395,7 +1559,13 @@ export async function establishSelfhostedTurnSession(
1395
1559
  outcome: command.outcome,
1396
1560
  exitCode: command.exitCode,
1397
1561
  reason: command.reason,
1562
+ ...(command.failure ? { failure: command.failure } : {}),
1398
1563
  });
1564
+ if (settlement && bus) {
1565
+ await bus
1566
+ .publish(args.workspaceId, args.sessionId, settlement.events)
1567
+ .catch(() => undefined);
1568
+ }
1399
1569
  },
1400
1570
  // Meter every control op (out-of-band telemetry) — no-op when unwired.
1401
1571
  ...(onOp !== undefined ? { onOp } : {}),
@@ -20,6 +20,7 @@ type WorkflowControlActivities = Pick<
20
20
  | "recoverDispatch"
21
21
  | "recoverEscapedMcpTimeout"
22
22
  | "settleSessionInterruptions"
23
+ | "settleSessionInputWait"
23
24
  >;
24
25
 
25
26
  /**
@@ -95,6 +95,14 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
95
95
  return currentWakeups !== entryWakeups;
96
96
  }
97
97
 
98
+ /** A typed activity cancellation is recoverable unless its exact-attempt
99
+ * redispatch budget was atomically exhausted. */
100
+ export function cancelledAttemptRecoveryMayContinue(
101
+ action: activities.RecoverDispatchResult["action"],
102
+ ): boolean {
103
+ return action !== "exceeded";
104
+ }
105
+
98
106
  /**
99
107
  * Bound repeated failures that happen before an attempt row exists. The
100
108
  * activity mirrors this deterministic delay into the durable wake outbox, so
@@ -340,8 +348,13 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
340
348
  // every new run records v2 and uses the activity-owned receipt contract.
341
349
  const receiptGatedCancellation = patched("session-attempt-quiescence-v2");
342
350
  const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
351
+ const preserveQuiescenceWake = patched("session-quiescence-reconciliation-wake-v1");
343
352
  const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
344
353
  const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
354
+ // PR #2208 changed a typed-cancelled result from a plain re-peek into a
355
+ // recoverDispatch activity. Version that new command so histories which
356
+ // already recorded the legacy re-peek remain deterministic on replay.
357
+ const cancelledAttemptRecovery = patched("session-cancelled-attempt-recovery-v1");
345
358
  const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
346
359
  let approvalWakeups = 0;
347
360
  let interruptionWakeups = 0;
@@ -572,6 +585,8 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
572
585
  continue;
573
586
  }
574
587
  if (peek.kind === "cancellation-wait") {
588
+ // The receipt wake can arrive while reconciliation returns an older pending result.
589
+ const beforeReconciliationSignalVersion = signalVersion;
575
590
  if (writerSetQuiescenceRecovery) {
576
591
  const reconciliation = await activity.reconcileSessionAttemptQuiescence({
577
592
  accountId: input.accountId,
@@ -588,7 +603,9 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
588
603
  // genuinely slow, close this workflow run rather than consuming a turn
589
604
  // slot or churning control activities; the outbox uses signalWithStart to
590
605
  // restart this exact workflow after the receipt commits.
591
- const seenSignalVersion = signalVersion;
606
+ const seenSignalVersion = preserveQuiescenceWake
607
+ ? beforeReconciliationSignalVersion
608
+ : signalVersion;
592
609
  const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
593
610
  if (woke) continue;
594
611
  // Close only against the same signal snapshot. A proof signal accepted
@@ -601,6 +618,29 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
601
618
  await waitForProviderCapacity(peek.ref);
602
619
  continue;
603
620
  }
621
+ if (peek.kind === "sandbox-lifecycle-wait") {
622
+ // The recovering turn carries an exact group/epoch marker. Do not reserve
623
+ // another turn-worker slot while the same draining lease still owns that
624
+ // transition. The draining->cold commit enqueues a durable workflow wake
625
+ // for this exact marker; keep only the standard five-second close-race
626
+ // window so a signal or an already-completed transition is not lost.
627
+ const seenSignalVersion = signalVersion;
628
+ const woke = await condition(() => signalVersion !== seenSignalVersion, "5s");
629
+ if (woke) continue;
630
+ const finalPeek = await activity.peekSessionWork({
631
+ workspaceId: input.workspaceId,
632
+ sessionId: input.sessionId,
633
+ });
634
+ if (
635
+ finalPeek.kind !== "sandbox-lifecycle-wait" ||
636
+ finalPeek.ref.sandboxGroupId !== peek.ref.sandboxGroupId ||
637
+ finalPeek.ref.leaseEpoch !== peek.ref.leaseEpoch
638
+ ) {
639
+ continue;
640
+ }
641
+ if (signalVersion !== closeSignalVersion) continue;
642
+ return;
643
+ }
604
644
  if (peek.kind === "approval-wait") {
605
645
  const seenApprovalWakeups = approvalWakeups;
606
646
  const seenWakeups = wakeups;
@@ -644,6 +684,48 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
644
684
  }
645
685
  continue;
646
686
  }
687
+ if (peek.kind === "input-wait") {
688
+ const settlement = await activity.settleSessionInputWait({
689
+ accountId: input.accountId,
690
+ workspaceId: input.workspaceId,
691
+ sessionId: input.sessionId,
692
+ waitTurnId: peek.waitTurnId,
693
+ disposition: peek.disposition,
694
+ });
695
+ if (settlement.action !== "held") continue;
696
+
697
+ // The durable outbox owns the long deadline. Keep this workflow run open
698
+ // only for the same bounded close-race window used by ordinary idle; any
699
+ // signal is a hint to re-peek PostgreSQL truth.
700
+ const seenWakeups = wakeups;
701
+ const seenApprovalWakeups = approvalWakeups;
702
+ const seenInterruptionWakeups = interruptionWakeups;
703
+ const woke = await condition(
704
+ () =>
705
+ interruptionWakeups !== seenInterruptionWakeups ||
706
+ wakeups !== seenWakeups ||
707
+ approvalWakeups !== seenApprovalWakeups,
708
+ "5s",
709
+ );
710
+ if (woke) continue;
711
+ const finalPeek = await activity.peekSessionWork({
712
+ workspaceId: input.workspaceId,
713
+ sessionId: input.sessionId,
714
+ });
715
+ if (
716
+ finalPeek.kind !== "input-wait" ||
717
+ finalPeek.disposition !== "held" ||
718
+ finalPeek.waitTurnId !== peek.waitTurnId
719
+ ) {
720
+ continue;
721
+ }
722
+ await activity.markSessionIdle({
723
+ workspaceId: input.workspaceId,
724
+ sessionId: input.sessionId,
725
+ });
726
+ if (signalVersion !== closeSignalVersion) continue;
727
+ return;
728
+ }
647
729
  if (peek.kind === "idle") {
648
730
  let continuation: activities.MaybeContinueGoalResult = { action: "none" };
649
731
  try {
@@ -663,11 +745,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
663
745
  });
664
746
  }
665
747
  if (continuation.action === "continue" || continuation.action === "queue") continue;
666
- // `none`, `paused`, `held`, and `deferred` all close this run below. A
667
- // held or deferred (idle-backoff) goal keeps its durable obligation
668
- // armed; the delayed wake-outbox row at the hold/pacing deadline, pending
669
- // machine input, a human prompt, or any producer's signalWithStart
670
- // restarts the workflow. No Temporal timer is used for pacing.
748
+ // `none`, `paused`, and `deferred` all close this run below. A deferred
749
+ // idle-backoff goal keeps its durable obligation armed; the delayed
750
+ // wake-outbox row at the pacing deadline or any producer signal restarts
751
+ // the workflow. No Temporal timer is used for pacing.
671
752
  const seenWakeups = wakeups;
672
753
  const seenApprovalWakeups = approvalWakeups;
673
754
  const seenInterruptionWakeups = interruptionWakeups;
@@ -1052,8 +1133,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
1052
1133
  return true;
1053
1134
  }
1054
1135
 
1055
- if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
1056
- return outcome.result.status === "cancelled";
1136
+ if (outcome.result.status === "failed") {
1137
+ return false;
1138
+ }
1139
+
1140
+ if (outcome.result.status === "cancelled") {
1141
+ // Histories created before session-cancelled-attempt-recovery-v1 must
1142
+ // preserve the old command sequence. Their durable state is still safe:
1143
+ // operator recovery or a later fresh workflow run can close a stranded
1144
+ // owner, while new histories use the bounded exact-attempt transaction.
1145
+ if (!cancelledAttemptRecovery) return true;
1146
+ // A typed cancellation is normally observed through the control-signal
1147
+ // branch above, after Pause/Steer has durably fenced the attempt. A
1148
+ // worker can nevertheless return the same shape after a shutdown or a
1149
+ // stale settlement race without closing its attempt row. Blindly
1150
+ // re-peeking then retries forever against `turn.status = running` and
1151
+ // strands every later prompt. Reconcile the exact attempt through the
1152
+ // same bounded, generation-fenced redispatch transaction used after an
1153
+ // activity heartbeat loss. If the attempt actually settled meanwhile,
1154
+ // the transaction is a stale no-op and the next peek observes truth.
1155
+ const recovery = await activity.recoverDispatch({
1156
+ accountId,
1157
+ workspaceId,
1158
+ sessionId,
1159
+ attemptId: outcome.result.attemptId,
1160
+ timeoutType: "HEARTBEAT",
1161
+ });
1162
+ return cancelledAttemptRecoveryMayContinue(recovery.action);
1057
1163
  }
1058
1164
 
1059
1165
  if (outcome.result.capacityWait) {