@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/activities/agent-turn/admission.d.ts +17 -2
  2. package/dist/activities/agent-turn/agent-build.d.ts +3 -10
  3. package/dist/activities/agent-turn/claim.d.ts +8 -0
  4. package/dist/activities/agent-turn/codex.d.ts +2 -2
  5. package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
  6. package/dist/activities/agent-turn/errors.d.ts +1 -1
  7. package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
  8. package/dist/activities/agent-turn/list-models.d.ts +16 -0
  9. package/dist/activities/agent-turn/model-usage.d.ts +15 -1
  10. package/dist/activities/agent-turn/quiescence.d.ts +11 -6
  11. package/dist/activities/agent-turn/run.d.ts +9 -0
  12. package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
  13. package/dist/activities/agent-turn/session-title.d.ts +21 -0
  14. package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
  15. package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
  16. package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
  17. package/dist/activities/agent-turn/turn-context.d.ts +13 -2
  18. package/dist/activities/agent-turn.d.ts +1 -1
  19. package/dist/activities/automations.d.ts +1 -0
  20. package/dist/activities/capabilities.d.ts +3 -1
  21. package/dist/activities/codemode-dispatcher.d.ts +10 -1
  22. package/dist/activities/codex-capacity.d.ts +32 -3
  23. package/dist/activities/codex-fleet-shadow.d.ts +1 -1
  24. package/dist/activities/codex-image-generation.d.ts +1 -1
  25. package/dist/activities/codex-rotation.d.ts +8 -67
  26. package/dist/activities/context-compaction.d.ts +2 -0
  27. package/dist/activities/goals.d.ts +12 -1
  28. package/dist/activities/google-drive-publication.d.ts +2 -13
  29. package/dist/activities/image-generation-operation.d.ts +13 -4
  30. package/dist/activities/image-generation-references.d.ts +2 -1
  31. package/dist/activities/scheduled-tasks.d.ts +1 -0
  32. package/dist/activities/session-state.d.ts +4 -2
  33. package/dist/activities/types.d.ts +19 -1
  34. package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
  35. package/dist/activities-control-CTEMLTHG.js.map +1 -0
  36. package/dist/activities-control.d.ts +4 -0
  37. package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
  38. package/dist/activities-turn-S5IU347A.js.map +1 -0
  39. package/dist/activities.d.ts +4 -0
  40. package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
  41. package/dist/chunk-D4ZEFOYH.js.map +1 -0
  42. package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
  43. package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
  44. package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
  45. package/dist/chunk-TAUE3GYT.js.map +1 -0
  46. package/dist/github-rest-mcp.d.ts +0 -2
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.js +43 -7
  49. package/dist/index.js.map +1 -1
  50. package/dist/observability-metrics.d.ts +35 -8
  51. package/dist/sandbox-reaper-timeout.d.ts +2 -2
  52. package/dist/sandbox-resume.d.ts +6 -1
  53. package/dist/workflow-bundle.js +96 -10
  54. package/dist/workflows/activities.d.ts +6 -1
  55. package/dist/workflows/automations.d.ts +5 -0
  56. package/dist/workflows/session.d.ts +3 -0
  57. package/package.json +17 -17
  58. package/src/activities/agent-run-admission.ts +7 -1
  59. package/src/activities/agent-turn/admission.ts +47 -6
  60. package/src/activities/agent-turn/agent-build.ts +20 -26
  61. package/src/activities/agent-turn/claim.ts +59 -6
  62. package/src/activities/agent-turn/codex-capacity.ts +311 -234
  63. package/src/activities/agent-turn/codex.ts +2 -2
  64. package/src/activities/agent-turn/compaction-prep.ts +8 -1
  65. package/src/activities/agent-turn/credential-leases.ts +36 -0
  66. package/src/activities/agent-turn/errors.ts +29 -1
  67. package/src/activities/agent-turn/failure-settlement.ts +451 -323
  68. package/src/activities/agent-turn/finalization.ts +31 -17
  69. package/src/activities/agent-turn/list-models.ts +101 -0
  70. package/src/activities/agent-turn/model-usage.ts +120 -45
  71. package/src/activities/agent-turn/quiescence.ts +32 -17
  72. package/src/activities/agent-turn/run.ts +98 -9
  73. package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
  74. package/src/activities/agent-turn/session-title.ts +55 -2
  75. package/src/activities/agent-turn/stream-attempt.ts +281 -136
  76. package/src/activities/agent-turn/tool-environment.ts +132 -39
  77. package/src/activities/agent-turn/tool-policy.ts +71 -12
  78. package/src/activities/agent-turn/turn-context.ts +21 -2
  79. package/src/activities/agent-turn.ts +5 -1
  80. package/src/activities/automations.ts +122 -67
  81. package/src/activities/capabilities.ts +129 -2
  82. package/src/activities/codemode-dispatcher.ts +235 -59
  83. package/src/activities/codex-capacity.ts +89 -10
  84. package/src/activities/codex-fleet-shadow.ts +8 -1
  85. package/src/activities/codex-image-generation.ts +19 -2
  86. package/src/activities/codex-rotation.ts +45 -142
  87. package/src/activities/context-compaction.ts +3 -0
  88. package/src/activities/goals.ts +116 -39
  89. package/src/activities/google-drive-publication.ts +4 -81
  90. package/src/activities/image-generation-operation.ts +28 -3
  91. package/src/activities/image-generation-references.ts +9 -9
  92. package/src/activities/mcp-credentials.ts +53 -0
  93. package/src/activities/rig-verification.ts +5 -3
  94. package/src/activities/sandbox-lease.ts +29 -12
  95. package/src/activities/scheduled-tasks.ts +217 -110
  96. package/src/activities/session-state.ts +17 -0
  97. package/src/activities/site-auth-maintenance.ts +12 -12
  98. package/src/activities/types.ts +22 -8
  99. package/src/activities.ts +2 -0
  100. package/src/activity-services.ts +4 -1
  101. package/src/github-rest-mcp.ts +2 -83
  102. package/src/index.ts +34 -4
  103. package/src/observability-metrics.ts +158 -14
  104. package/src/sandbox-reaper-timeout.ts +10 -4
  105. package/src/sandbox-resume.ts +58 -8
  106. package/src/sandbox-routing.ts +71 -16
  107. package/src/workflows/activities.ts +16 -0
  108. package/src/workflows/automations.ts +21 -2
  109. package/src/workflows/session.ts +85 -7
  110. package/dist/activities/api-integrations.d.ts +0 -21
  111. package/dist/activities-control-52W5IDWM.js.map +0 -1
  112. package/dist/activities-turn-A2G2KLSY.js.map +0 -1
  113. package/dist/chunk-GFNXVZVQ.js.map +0 -1
  114. package/dist/chunk-IPVMCAW5.js.map +0 -1
  115. package/src/activities/api-integrations.ts +0 -141
@@ -709,7 +709,13 @@ export async function persistSandboxDeadlineRotationCheckpoint(
709
709
  * the user's machine IS the persistence), or a snapshot newer than the
710
710
  * interval already rides the lease.
711
711
  */
712
- export async function maybePersistWarmWorkspaceSnapshot(
712
+ export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
713
+ /** Settles only after any provider capture continuation has published or
714
+ * failed and released its exact workspace-capture admission. */
715
+ readonly settled: Promise<void>;
716
+ };
717
+
718
+ export function maybePersistWarmWorkspaceSnapshot(
713
719
  services: SandboxResumeServices,
714
720
  ids: {
715
721
  accountId: string;
@@ -723,6 +729,50 @@ export async function maybePersistWarmWorkspaceSnapshot(
723
729
  leaseEpoch: number,
724
730
  signal?: AbortSignal,
725
731
  force = false,
732
+ ): WarmWorkspaceSnapshotPromise {
733
+ let continuationRegistered = false;
734
+ let resolvePhysicalSettlement: () => void = () => undefined;
735
+ const physicalSettlement = new Promise<void>((resolve) => {
736
+ resolvePhysicalSettlement = resolve;
737
+ });
738
+ const result = persistWarmWorkspaceSnapshot(
739
+ services,
740
+ ids,
741
+ session,
742
+ leaseEpoch,
743
+ signal,
744
+ force,
745
+ (continuation) => {
746
+ continuationRegistered = true;
747
+ void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
748
+ },
749
+ );
750
+ void result.then(
751
+ () => {
752
+ if (!continuationRegistered) resolvePhysicalSettlement();
753
+ },
754
+ () => {
755
+ if (!continuationRegistered) resolvePhysicalSettlement();
756
+ },
757
+ );
758
+ return Object.assign(result, { settled: physicalSettlement });
759
+ }
760
+
761
+ async function persistWarmWorkspaceSnapshot(
762
+ services: SandboxResumeServices,
763
+ ids: {
764
+ accountId: string;
765
+ workspaceId: string;
766
+ sessionId: string;
767
+ turnId: string;
768
+ attemptId: string;
769
+ sandboxGroupId: string;
770
+ },
771
+ session: unknown,
772
+ leaseEpoch: number,
773
+ signal: AbortSignal | undefined,
774
+ force: boolean,
775
+ registerOwnedContinuation: (continuation: Promise<void>) => void,
726
776
  ): Promise<boolean> {
727
777
  const { db, settings } = services;
728
778
  const intervalMs = settings.sandboxSnapshotIntervalMs;
@@ -989,6 +1039,7 @@ export async function maybePersistWarmWorkspaceSnapshot(
989
1039
  return { kind: "settled" as const, persisted: false };
990
1040
  },
991
1041
  );
1042
+ registerOwnedContinuation(settled.then(() => undefined));
992
1043
  let cancelListener: (() => void) | undefined;
993
1044
  const outcome = await Promise.race([
994
1045
  settled,
@@ -1223,13 +1274,12 @@ export async function resumeBoxForTurn(
1223
1274
  // holder is gone / its attempt is no longer the active writer" from "the lease
1224
1275
  // could not be extended" (rotation requested, draining, another epoch). The
1225
1276
  // second is NOT a reason to release: once a provider-deadline rotation is
1226
- // requested the lease heartbeat is fenced by design while the still-running
1227
- // turn checkpoints, and the turn-side heartbeat owns that orderly
1228
- // SandboxDeadlineRotationError abort. Dropping the live holder here instead
1229
- // flips the lease to draining under the running turn, the reaper terminates
1230
- // the box, and the checkpoint then fails `attempt_fenced` (the staging
1231
- // ~61-minute box-age failure). Both loops therefore converge on
1232
- // "holder alive -> never release"; this loop keeps touching the holder.
1277
+ // requested the turn-side heartbeat immediately preempts the attempt, whose
1278
+ // finalizer retains this holder until every attempt-owned writer is physically
1279
+ // drained. Dropping it here would let the reaper capture or terminate while a
1280
+ // provider operation may still be mutating /workspace. Both loops therefore
1281
+ // converge on "holder alive -> never release"; finalization performs the
1282
+ // proof-bearing release.
1233
1283
  const releaseDeadHolder = (reason: string): void => {
1234
1284
  const message = "sandbox resume holder released: holder no longer live";
1235
1285
  const fields = {
@@ -17,11 +17,13 @@
17
17
  import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  adoptConnectedMachineSessionBackgroundCommand,
20
+ adoptManagedSessionBackgroundCommand,
20
21
  advanceWorkspaceGenerationForRetainedProcess,
21
22
  advanceWorkspaceGeneration,
22
23
  getRetainedProcess,
23
24
  retainWorkspaceMutationProcess,
24
25
  retainedProcessSettlementIdentity,
26
+ settleConnectedMachineSessionBackgroundCommand,
25
27
  settleRetainedProcess,
26
28
  verifyRetainedProcessMutationSettlement,
27
29
  verifyWorkspaceMutationSettlement,
@@ -36,10 +38,6 @@ import {
36
38
  type EnrollmentRecord,
37
39
  type SandboxWorkspaceMutationAdmission,
38
40
  } from "@opengeni/db";
39
- import {
40
- settleConnectedMachineSessionBackgroundCommand,
41
- settleSessionBackgroundCommandForRetainedProcess,
42
- } from "@opengeni/db/session-background-commands";
43
41
  import type { EventBus } from "@opengeni/events";
44
42
  import {
45
43
  buildSelfhostedBackendSession,
@@ -515,10 +513,6 @@ function afterPersistableHomeMutation(
515
513
  admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
516
514
  operation: op,
517
515
  providerBinding: boundAdmission.providerBinding ?? null,
518
- backgroundCommand: {
519
- commandId: retainedProcess.id,
520
- command: op,
521
- },
522
516
  owner: {
523
517
  kind: "turn",
524
518
  turnId: fence.turnId,
@@ -686,14 +680,50 @@ function settleRetainedProcessForTurn(
686
680
  if (settlement.process.state === "active") {
687
681
  throw new Error("Retained-process settlement returned an active durable process");
688
682
  }
689
- await settleSessionBackgroundCommandForRetainedProcess(services.db, {
683
+ if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
684
+ await services.bus
685
+ .publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents)
686
+ .catch(() => undefined);
687
+ }
688
+ };
689
+ }
690
+
691
+ function adoptRetainedProcessAsBackgroundCommandForTurn(
692
+ services: RoutingWiringServices,
693
+ ids: RoutingWiringIds,
694
+ ):
695
+ | ((input: { backend: ResolvedActiveBackend; process: RoutingRetainedProcess }) => Promise<void>)
696
+ | undefined {
697
+ const fence = ids.workspaceMutationFence;
698
+ if (!fence) return undefined;
699
+ return async ({ backend, process }) => {
700
+ const durable = await getRetainedProcess(services.db, {
701
+ workspaceId: ids.workspaceId,
702
+ sessionId: ids.sessionId,
703
+ processId: process.id,
704
+ });
705
+ if (
706
+ !durable ||
707
+ durable.providerSessionId !== process.providerSessionId ||
708
+ durable.providerBackend !== backend.kind ||
709
+ durable.providerInstanceId !== backend.providerInstanceId ||
710
+ durable.leaseEpoch !== backend.leaseEpoch ||
711
+ durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
712
+ durable.routeTargetId !== backend.sandboxId ||
713
+ durable.routeEpoch !== backend.activeEpoch
714
+ ) {
715
+ throw new Error("Background command adoption lost its exact retained-process identity");
716
+ }
717
+ await adoptManagedSessionBackgroundCommand(services.db, {
690
718
  accountId: fence.accountId,
691
719
  workspaceId: ids.workspaceId,
692
720
  sessionId: ids.sessionId,
693
- retainedProcessId: process.id,
694
- outcome: settlement.process.state,
695
- exitCode: settlement.process.exitCode,
696
- reason: settlement.process.settlementReason ?? proof.reason,
721
+ turnId: fence.turnId,
722
+ executionGeneration: fence.executionGeneration,
723
+ attemptId: fence.attemptId,
724
+ processId: process.id,
725
+ expected: retainedProcessSettlementIdentity(durable),
726
+ command: "execCommand",
697
727
  });
698
728
  };
699
729
  }
@@ -736,6 +766,10 @@ export function wrapTurnBoxWithRouting(
736
766
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
737
767
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
738
768
  const settleProcess = settleRetainedProcessForTurn(services, ids);
769
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
770
+ services,
771
+ ids,
772
+ );
739
773
  const resolver = makeActiveBackendResolver({
740
774
  workspaceId: ids.workspaceId,
741
775
  defaultBackend: established.session as RoutableBackendSession,
@@ -794,7 +828,7 @@ export function wrapTurnBoxWithRouting(
794
828
  return { commandId: adopted.id };
795
829
  },
796
830
  selfhostedSettleBackgroundCommand: async (command) => {
797
- await settleConnectedMachineSessionBackgroundCommand(db, {
831
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
798
832
  accountId: backgroundCommandAccountId,
799
833
  workspaceId: ids.workspaceId,
800
834
  sessionId: ids.sessionId,
@@ -807,6 +841,11 @@ export function wrapTurnBoxWithRouting(
807
841
  exitCode: command.exitCode,
808
842
  reason: command.reason,
809
843
  });
844
+ if (settlement && bus) {
845
+ await bus
846
+ .publish(ids.workspaceId, ids.sessionId, settlement.events)
847
+ .catch(() => undefined);
848
+ }
810
849
  },
811
850
  }
812
851
  : {}),
@@ -892,6 +931,7 @@ export function wrapTurnBoxWithRouting(
892
931
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
893
932
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
894
933
  ...(settleProcess ? { settleProcess } : {}),
934
+ ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
895
935
  ...(ids.homeLease
896
936
  ? {
897
937
  onDefaultBackendError: async ({
@@ -970,6 +1010,10 @@ export function wrapLazyTurnBoxWithRouting(
970
1010
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
971
1011
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
972
1012
  const settleProcess = settleRetainedProcessForTurn(services, ids);
1013
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
1014
+ services,
1015
+ ids,
1016
+ );
973
1017
  const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
974
1018
  const defaultSupportsPty = defaultBackend
975
1019
  ? selectBackend(defaultBackend).capabilities.Terminal.pty
@@ -1038,7 +1082,7 @@ export function wrapLazyTurnBoxWithRouting(
1038
1082
  return { commandId: adopted.id };
1039
1083
  },
1040
1084
  selfhostedSettleBackgroundCommand: async (command) => {
1041
- await settleConnectedMachineSessionBackgroundCommand(db, {
1085
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
1042
1086
  accountId: backgroundCommandAccountId,
1043
1087
  workspaceId: ids.workspaceId,
1044
1088
  sessionId: ids.sessionId,
@@ -1051,6 +1095,11 @@ export function wrapLazyTurnBoxWithRouting(
1051
1095
  exitCode: command.exitCode,
1052
1096
  reason: command.reason,
1053
1097
  });
1098
+ if (settlement && bus) {
1099
+ await bus
1100
+ .publish(ids.workspaceId, ids.sessionId, settlement.events)
1101
+ .catch(() => undefined);
1102
+ }
1054
1103
  },
1055
1104
  }
1056
1105
  : {}),
@@ -1105,6 +1154,7 @@ export function wrapLazyTurnBoxWithRouting(
1105
1154
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
1106
1155
  ...(afterProcessMutation ? { afterProcessMutation } : {}),
1107
1156
  ...(settleProcess ? { settleProcess } : {}),
1157
+ ...(adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {}),
1108
1158
  ...(args.homeLeaseIdentity
1109
1159
  ? {
1110
1160
  onDefaultBackendError: async ({
@@ -1383,7 +1433,7 @@ export async function establishSelfhostedTurnSession(
1383
1433
  }
1384
1434
  : {}),
1385
1435
  settleBackgroundCommand: async (command) => {
1386
- await settleConnectedMachineSessionBackgroundCommand(db, {
1436
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
1387
1437
  accountId: args.accountId,
1388
1438
  workspaceId: args.workspaceId,
1389
1439
  sessionId: args.sessionId,
@@ -1396,6 +1446,11 @@ export async function establishSelfhostedTurnSession(
1396
1446
  exitCode: command.exitCode,
1397
1447
  reason: command.reason,
1398
1448
  });
1449
+ if (settlement && bus) {
1450
+ await bus
1451
+ .publish(args.workspaceId, args.sessionId, settlement.events)
1452
+ .catch(() => undefined);
1453
+ }
1399
1454
  },
1400
1455
  // Meter every control op (out-of-band telemetry) — no-op when unwired.
1401
1456
  ...(onOp !== undefined ? { onOp } : {}),
@@ -20,6 +20,7 @@ type WorkflowControlActivities = Pick<
20
20
  | "recoverDispatch"
21
21
  | "recoverEscapedMcpTimeout"
22
22
  | "settleSessionInterruptions"
23
+ | "settleSessionInputWait"
23
24
  >;
24
25
 
25
26
  /**
@@ -68,6 +69,21 @@ export const automationActivity = proxyActivities<Pick<typeof activities, "dispa
68
69
  },
69
70
  );
70
71
 
72
+ /** Dispatch has a bounded retry window, but an accepted automation event must
73
+ * not remain durably `dispatching` after that window closes. This compensating
74
+ * database write is idempotent and retries without a cap until the run reaches
75
+ * a terminal state (or is observed as already dispatched). */
76
+ export const automationFailureActivity = proxyActivities<
77
+ Pick<typeof activities, "settleAutomationRunFailure">
78
+ >({
79
+ startToCloseTimeout: "2 minutes",
80
+ retry: {
81
+ initialInterval: "1 second",
82
+ backoffCoefficient: 2,
83
+ maximumInterval: "30 seconds",
84
+ },
85
+ });
86
+
71
87
  /** Goal continuation evaluates a durable Postgres obligation at an idle
72
88
  * boundary. A transient failure gets a short retry window, then records an
73
89
  * explicit delayed outbox wake instead of relying on an unrelated mutation or
@@ -1,4 +1,4 @@
1
- import { automationActivity } from "./activities";
1
+ import { automationActivity, automationFailureActivity } from "./activities";
2
2
 
3
3
  export type AutomationRunWorkflowInput = {
4
4
  accountId: string;
@@ -6,6 +6,25 @@ export type AutomationRunWorkflowInput = {
6
6
  runId: string;
7
7
  };
8
8
 
9
+ export type AutomationRunWorkflowActivities = {
10
+ dispatchAutomationRun: (input: AutomationRunWorkflowInput) => Promise<unknown>;
11
+ settleAutomationRunFailure: (input: AutomationRunWorkflowInput) => Promise<void>;
12
+ };
13
+
14
+ export async function runAutomationRunWorkflow(
15
+ input: AutomationRunWorkflowInput,
16
+ activities: AutomationRunWorkflowActivities,
17
+ ): Promise<void> {
18
+ try {
19
+ await activities.dispatchAutomationRun(input);
20
+ } catch {
21
+ await activities.settleAutomationRunFailure(input);
22
+ }
23
+ }
24
+
9
25
  export async function automationRunWorkflow(input: AutomationRunWorkflowInput): Promise<void> {
10
- await automationActivity.dispatchAutomationRun(input);
26
+ await runAutomationRunWorkflow(input, {
27
+ dispatchAutomationRun: automationActivity.dispatchAutomationRun,
28
+ settleAutomationRunFailure: automationFailureActivity.settleAutomationRunFailure,
29
+ });
11
30
  }
@@ -95,6 +95,14 @@ export function deferredResultMayContinue(entryWakeups: number, currentWakeups:
95
95
  return currentWakeups !== entryWakeups;
96
96
  }
97
97
 
98
+ /** A typed activity cancellation is recoverable unless its exact-attempt
99
+ * redispatch budget was atomically exhausted. */
100
+ export function cancelledAttemptRecoveryMayContinue(
101
+ action: activities.RecoverDispatchResult["action"],
102
+ ): boolean {
103
+ return action !== "exceeded";
104
+ }
105
+
98
106
  /**
99
107
  * Bound repeated failures that happen before an attempt row exists. The
100
108
  * activity mirrors this deterministic delay into the durable wake outbox, so
@@ -342,6 +350,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
342
350
  const writerSetQuiescenceRecovery = patched("session-attempt-writer-set-quiescence-v1");
343
351
  const staleControlSignalIsOnlyWakeHint = patched("session-control-stale-wake-v1");
344
352
  const unclaimedAttemptRecovery = patched("session-unclaimed-attempt-recovery-v1");
353
+ // PR #2208 changed a typed-cancelled result from a plain re-peek into a
354
+ // recoverDispatch activity. Version that new command so histories which
355
+ // already recorded the legacy re-peek remain deterministic on replay.
356
+ const cancelledAttemptRecovery = patched("session-cancelled-attempt-recovery-v1");
345
357
  const turnActivity = turnActivityForTaskQueue(workflowInfo().taskQueue, receiptGatedCancellation);
346
358
  let approvalWakeups = 0;
347
359
  let interruptionWakeups = 0;
@@ -644,6 +656,48 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
644
656
  }
645
657
  continue;
646
658
  }
659
+ if (peek.kind === "input-wait") {
660
+ const settlement = await activity.settleSessionInputWait({
661
+ accountId: input.accountId,
662
+ workspaceId: input.workspaceId,
663
+ sessionId: input.sessionId,
664
+ waitTurnId: peek.waitTurnId,
665
+ disposition: peek.disposition,
666
+ });
667
+ if (settlement.action !== "held") continue;
668
+
669
+ // The durable outbox owns the long deadline. Keep this workflow run open
670
+ // only for the same bounded close-race window used by ordinary idle; any
671
+ // signal is a hint to re-peek PostgreSQL truth.
672
+ const seenWakeups = wakeups;
673
+ const seenApprovalWakeups = approvalWakeups;
674
+ const seenInterruptionWakeups = interruptionWakeups;
675
+ const woke = await condition(
676
+ () =>
677
+ interruptionWakeups !== seenInterruptionWakeups ||
678
+ wakeups !== seenWakeups ||
679
+ approvalWakeups !== seenApprovalWakeups,
680
+ "5s",
681
+ );
682
+ if (woke) continue;
683
+ const finalPeek = await activity.peekSessionWork({
684
+ workspaceId: input.workspaceId,
685
+ sessionId: input.sessionId,
686
+ });
687
+ if (
688
+ finalPeek.kind !== "input-wait" ||
689
+ finalPeek.disposition !== "held" ||
690
+ finalPeek.waitTurnId !== peek.waitTurnId
691
+ ) {
692
+ continue;
693
+ }
694
+ await activity.markSessionIdle({
695
+ workspaceId: input.workspaceId,
696
+ sessionId: input.sessionId,
697
+ });
698
+ if (signalVersion !== closeSignalVersion) continue;
699
+ return;
700
+ }
647
701
  if (peek.kind === "idle") {
648
702
  let continuation: activities.MaybeContinueGoalResult = { action: "none" };
649
703
  try {
@@ -663,11 +717,10 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
663
717
  });
664
718
  }
665
719
  if (continuation.action === "continue" || continuation.action === "queue") continue;
666
- // `none`, `paused`, `held`, and `deferred` all close this run below. A
667
- // held or deferred (idle-backoff) goal keeps its durable obligation
668
- // armed; the delayed wake-outbox row at the hold/pacing deadline, pending
669
- // machine input, a human prompt, or any producer's signalWithStart
670
- // restarts the workflow. No Temporal timer is used for pacing.
720
+ // `none`, `paused`, and `deferred` all close this run below. A deferred
721
+ // idle-backoff goal keeps its durable obligation armed; the delayed
722
+ // wake-outbox row at the pacing deadline or any producer signal restarts
723
+ // the workflow. No Temporal timer is used for pacing.
671
724
  const seenWakeups = wakeups;
672
725
  const seenApprovalWakeups = approvalWakeups;
673
726
  const seenInterruptionWakeups = interruptionWakeups;
@@ -1052,8 +1105,33 @@ export async function sessionWorkflow(input: SessionWorkflowInput): Promise<void
1052
1105
  return true;
1053
1106
  }
1054
1107
 
1055
- if (outcome.result.status === "failed" || outcome.result.status === "cancelled") {
1056
- return outcome.result.status === "cancelled";
1108
+ if (outcome.result.status === "failed") {
1109
+ return false;
1110
+ }
1111
+
1112
+ if (outcome.result.status === "cancelled") {
1113
+ // Histories created before session-cancelled-attempt-recovery-v1 must
1114
+ // preserve the old command sequence. Their durable state is still safe:
1115
+ // operator recovery or a later fresh workflow run can close a stranded
1116
+ // owner, while new histories use the bounded exact-attempt transaction.
1117
+ if (!cancelledAttemptRecovery) return true;
1118
+ // A typed cancellation is normally observed through the control-signal
1119
+ // branch above, after Pause/Steer has durably fenced the attempt. A
1120
+ // worker can nevertheless return the same shape after a shutdown or a
1121
+ // stale settlement race without closing its attempt row. Blindly
1122
+ // re-peeking then retries forever against `turn.status = running` and
1123
+ // strands every later prompt. Reconcile the exact attempt through the
1124
+ // same bounded, generation-fenced redispatch transaction used after an
1125
+ // activity heartbeat loss. If the attempt actually settled meanwhile,
1126
+ // the transaction is a stale no-op and the next peek observes truth.
1127
+ const recovery = await activity.recoverDispatch({
1128
+ accountId,
1129
+ workspaceId,
1130
+ sessionId,
1131
+ attemptId: outcome.result.attemptId,
1132
+ timeoutType: "HEARTBEAT",
1133
+ });
1134
+ return cancelledAttemptRecoveryMayContinue(recovery.action);
1057
1135
  }
1058
1136
 
1059
1137
  if (outcome.result.capacityWait) {
@@ -1,21 +0,0 @@
1
- import { type IntegrationInvocationAuthority } from "@opengeni/capabilities";
2
- import type { Settings } from "@opengeni/config";
3
- import type { ToolAuthNeededPayload } from "@opengeni/contracts";
4
- import type { ApiIntegrationRuntime, ResolveConnectionCredentialInput, ResolveConnectionCredentialResult } from "@opengeni/db";
5
- import type { LocalMcpServerRegistration } from "@opengeni/runtime";
6
- import type { FetchLike } from "@opengeni/network";
7
- export type BuildApiIntegrationServersInput = {
8
- settings: Settings;
9
- integrations: readonly ApiIntegrationRuntime[];
10
- authority: Omit<IntegrationInvocationAuthority, "connectionRef">;
11
- resolveCredential: (input: ResolveConnectionCredentialInput) => Promise<ResolveConnectionCredentialResult>;
12
- onAuthNeeded?: (payload: ToolAuthNeededPayload) => Promise<void> | void;
13
- fetchImpl?: FetchLike;
14
- };
15
- /**
16
- * Compile active persisted API facets into ordinary in-process MCP servers for
17
- * one exact attempt. Connection resolution remains request-time, destination-
18
- * bound, personal-delegation-fenced, and refreshable through the same worker
19
- * resolver used by remote MCP servers.
20
- */
21
- export declare function buildApiIntegrationServersForTurn(input: BuildApiIntegrationServersInput): LocalMcpServerRegistration[];