@opengeni/worker-bundle 0.16.29 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +110 -19
  3. package/dist/activities/api-integrations.d.ts +21 -0
  4. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  5. package/dist/activities/capabilities.d.ts +6 -0
  6. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  7. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  8. package/dist/activities/codex-capacity.d.ts +6 -5
  9. package/dist/activities/codex-image-generation.d.ts +21 -0
  10. package/dist/activities/context-compaction.d.ts +1 -0
  11. package/dist/activities/documents.d.ts +3 -2
  12. package/dist/activities/environment.d.ts +17 -8
  13. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  14. package/dist/activities/file-upload-reaper.d.ts +7 -2
  15. package/dist/activities/gateway-image-generation.d.ts +39 -0
  16. package/dist/activities/gateway-video-generation.d.ts +53 -0
  17. package/dist/activities/generated-images.d.ts +112 -0
  18. package/dist/activities/git-credential-renewal.d.ts +1 -1
  19. package/dist/activities/goals.d.ts +2 -2
  20. package/dist/activities/google-drive-changes.d.ts +99 -0
  21. package/dist/activities/image-generation-operation.d.ts +74 -0
  22. package/dist/activities/image-generation-references.d.ts +25 -0
  23. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  24. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  25. package/dist/activities/packs.d.ts +21 -36
  26. package/dist/activities/parent-wake.d.ts +1 -1
  27. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  28. package/dist/activities/retained-screenshots.d.ts +36 -0
  29. package/dist/activities/rig-verification.d.ts +82 -5
  30. package/dist/activities/run-credential-renewal.d.ts +1 -1
  31. package/dist/activities/run-input.d.ts +3 -1
  32. package/dist/activities/sandbox-images.d.ts +14 -0
  33. package/dist/activities/sandbox-lease.d.ts +47 -9
  34. package/dist/activities/scheduled-tasks.d.ts +2 -2
  35. package/dist/activities/session-state.d.ts +7 -5
  36. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  37. package/dist/activities/streaming.d.ts +1 -1
  38. package/dist/activities/types.d.ts +87 -5
  39. package/dist/activities/video-generation-admission.d.ts +43 -0
  40. package/dist/activities/video-generation-credential.d.ts +24 -0
  41. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  42. package/dist/activities/video-media-validation.d.ts +34 -0
  43. package/dist/activities/video-output-retention.d.ts +36 -0
  44. package/dist/activities/video-reference-staging.d.ts +55 -0
  45. package/dist/activities/workflow-wake.d.ts +2 -2
  46. package/dist/activities/workspace-capture.d.ts +10 -1
  47. package/dist/activities/xai-auth.d.ts +23 -0
  48. package/dist/activities/xai-image-generation.d.ts +26 -0
  49. package/dist/activities/xai-video-generation.d.ts +18 -0
  50. package/dist/activities-control-XURE4VKN.js +7221 -0
  51. package/dist/activities-control-XURE4VKN.js.map +1 -0
  52. package/dist/activities-control.d.ts +297 -0
  53. package/dist/activities-turn-6EBXGGLS.js +16036 -0
  54. package/dist/activities-turn-6EBXGGLS.js.map +1 -0
  55. package/dist/activities-turn.d.ts +17 -0
  56. package/dist/activities.d.ts +138 -150
  57. package/dist/activity-services.d.ts +10 -0
  58. package/dist/artifact-materializer-entry.d.ts +6 -0
  59. package/dist/artifact-materializer-entry.js +2662 -0
  60. package/dist/artifact-materializer-entry.js.map +1 -0
  61. package/dist/artifact-outbox-entry.d.ts +2 -0
  62. package/dist/artifact-outbox-entry.js +681 -0
  63. package/dist/artifact-outbox-entry.js.map +1 -0
  64. package/dist/chunk-5GRJIP5H.js +3092 -0
  65. package/dist/chunk-5GRJIP5H.js.map +1 -0
  66. package/dist/chunk-GRFQTTSQ.js +448 -0
  67. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  68. package/dist/chunk-TYNPCG4U.js +906 -0
  69. package/dist/chunk-TYNPCG4U.js.map +1 -0
  70. package/dist/concurrency.d.ts +108 -5
  71. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  72. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  73. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  74. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  75. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  76. package/dist/editable-artifact-materializer.d.ts +262 -0
  77. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  78. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  79. package/dist/host-export.d.ts +1 -1
  80. package/dist/index.d.ts +189 -10
  81. package/dist/index.js +715 -17380
  82. package/dist/index.js.map +1 -1
  83. package/dist/memory-pressure-guard.d.ts +57 -0
  84. package/dist/model-checkpoint-memory-collector.d.ts +23 -0
  85. package/dist/model-context-contributions.d.ts +40 -0
  86. package/dist/observability-metrics.d.ts +53 -1
  87. package/dist/sandbox-reaper-contract.d.ts +40 -0
  88. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  89. package/dist/sandbox-resume.d.ts +34 -7
  90. package/dist/sandbox-routing.d.ts +6 -0
  91. package/dist/worker-service-lifecycle.d.ts +6 -2
  92. package/dist/workflow-bundle.js +1094 -370
  93. package/dist/workflows/activities.d.ts +8 -2
  94. package/dist/workflows/document-index.d.ts +1 -1
  95. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  96. package/dist/workflows/rig-verification.d.ts +1 -1
  97. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  98. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  99. package/dist/workflows/session.d.ts +2 -2
  100. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  101. package/dist/workflows/video-generation.d.ts +10 -0
  102. package/dist/workflows/workflow-wake.d.ts +1 -1
  103. package/dist/workflows.d.ts +10 -7
  104. package/package.json +35 -15
  105. package/src/activities/agent-run-admission.ts +76 -0
  106. package/src/activities/agent-turn.ts +2600 -340
  107. package/src/activities/api-integrations.ts +138 -0
  108. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  109. package/src/activities/capabilities.ts +10 -0
  110. package/src/activities/codemode-dispatcher.ts +335 -0
  111. package/src/activities/codemode-token-renewal.ts +64 -0
  112. package/src/activities/codex-capacity.ts +54 -7
  113. package/src/activities/codex-image-generation.ts +56 -0
  114. package/src/activities/context-compaction.ts +38 -3
  115. package/src/activities/documents.ts +14 -4
  116. package/src/activities/environment.ts +54 -45
  117. package/src/activities/expiring-material-renewal.ts +1 -1
  118. package/src/activities/file-upload-reaper.ts +34 -3
  119. package/src/activities/gateway-image-generation.ts +146 -0
  120. package/src/activities/gateway-video-generation.ts +298 -0
  121. package/src/activities/generated-images.ts +860 -0
  122. package/src/activities/goals.ts +8 -3
  123. package/src/activities/google-drive-changes.ts +454 -0
  124. package/src/activities/image-generation-operation.ts +333 -0
  125. package/src/activities/image-generation-references.ts +118 -0
  126. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  127. package/src/activities/knowledge-source-sync.ts +2840 -0
  128. package/src/activities/mcp-credentials.ts +1 -0
  129. package/src/activities/packs.ts +75 -152
  130. package/src/activities/parent-wake.ts +13 -1
  131. package/src/activities/retained-screenshot-reaper.ts +2 -2
  132. package/src/activities/retained-screenshots.ts +453 -28
  133. package/src/activities/rig-verification.ts +554 -43
  134. package/src/activities/run-input.ts +22 -6
  135. package/src/activities/sandbox-images.ts +167 -0
  136. package/src/activities/sandbox-lease.ts +1065 -483
  137. package/src/activities/scheduled-tasks.ts +104 -74
  138. package/src/activities/session-state.ts +22 -4
  139. package/src/activities/site-auth-maintenance.ts +269 -0
  140. package/src/activities/streaming.ts +42 -3
  141. package/src/activities/types.ts +104 -5
  142. package/src/activities/video-generation-admission.ts +303 -0
  143. package/src/activities/video-generation-credential.ts +112 -0
  144. package/src/activities/video-generation-reconciliation.ts +771 -0
  145. package/src/activities/video-media-validation.ts +332 -0
  146. package/src/activities/video-output-retention.ts +314 -0
  147. package/src/activities/video-reference-staging.ts +350 -0
  148. package/src/activities/workflow-wake.ts +2 -2
  149. package/src/activities/workspace-capture.ts +44 -10
  150. package/src/activities/xai-auth.ts +122 -0
  151. package/src/activities/xai-image-generation.ts +86 -0
  152. package/src/activities/xai-video-generation.ts +69 -0
  153. package/src/activities-control.ts +58 -0
  154. package/src/activities-turn.ts +40 -0
  155. package/src/activities.ts +53 -124
  156. package/src/activity-services.ts +62 -0
  157. package/src/artifact-materializer-entry.ts +35 -0
  158. package/src/artifact-outbox-entry.ts +29 -0
  159. package/src/concurrency.ts +484 -6
  160. package/src/editable-artifact-hint-broker.ts +235 -0
  161. package/src/editable-artifact-materializer-launcher.ts +515 -0
  162. package/src/editable-artifact-materializer-service.ts +787 -0
  163. package/src/editable-artifact-materializer-storage.ts +413 -0
  164. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  165. package/src/editable-artifact-materializer.ts +1289 -0
  166. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  167. package/src/editable-artifact-outbox-service.ts +438 -0
  168. package/src/index.ts +328 -72
  169. package/src/memory-pressure-guard.ts +489 -0
  170. package/src/model-checkpoint-memory-collector.ts +123 -0
  171. package/src/model-context-contributions.ts +159 -0
  172. package/src/observability-metrics.ts +297 -13
  173. package/src/sandbox-reaper-contract.ts +73 -0
  174. package/src/sandbox-reaper-timeout.ts +79 -0
  175. package/src/sandbox-resume.ts +294 -153
  176. package/src/sandbox-routing.ts +32 -24
  177. package/src/worker-service-lifecycle.ts +80 -14
  178. package/src/workflows/activities.ts +31 -0
  179. package/src/workflows/file-upload-reaper.ts +9 -1
  180. package/src/workflows/knowledge-source-sync.ts +83 -0
  181. package/src/workflows/sandbox-reaper.ts +183 -21
  182. package/src/workflows/scheduled-tasks.ts +42 -6
  183. package/src/workflows/session.ts +20 -6
  184. package/src/workflows/site-auth-maintenance.ts +14 -0
  185. package/src/workflows/video-generation.ts +36 -0
  186. package/src/workflows.ts +18 -1
  187. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  188. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -14,7 +14,7 @@
14
14
  // target lookup), and the selfhosted ControlRpc built over the events bus's NATS
15
15
  // request/reply connection.
16
16
 
17
- import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
17
+ import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
@@ -45,7 +45,9 @@ import {
45
45
  RoutingBackendRecoveryRequiredError,
46
46
  RoutingSandboxSession,
47
47
  resolveModalCheckpointProviderBindingForSession,
48
+ sandboxProviderInstanceIdFromEnvelope,
48
49
  sandboxBackendForSdkBackendId,
50
+ selectBackend,
49
51
  verifySandboxExecReadiness,
50
52
  type ControlRpc,
51
53
  type EstablishedSandboxSession,
@@ -100,6 +102,8 @@ export type RoutingWiringServices = {
100
102
  established: EstablishedSandboxSession;
101
103
  leaseEpoch: number;
102
104
  }) => void;
105
+ /** Cancel bounded capture waits with the owning turn activity. */
106
+ waitSignal?: AbortSignal;
103
107
  };
104
108
 
105
109
  /** Worker-turn routing may final-ack streamed command output, so it must always
@@ -137,6 +141,8 @@ export type RoutingWiringIds = {
137
141
  * manifest environment variables." Optional → the resolver defaults to `{}`.
138
142
  */
139
143
  environment?: Record<string, string>;
144
+ /** Attempt-local values projected only onto Connected Machine child execs. */
145
+ transientExecEnvironment?: () => Readonly<Record<string, string>>;
140
146
  /**
141
147
  * Stage D machine-primary: PIN the already-established turn SelfhostedSession
142
148
  * (the `established` arg's session) for THIS machine pointer `(sandboxId, epoch)`
@@ -262,27 +268,7 @@ function homeRouteRecoveryError(
262
268
  export function providerIdentityFromResumeState(
263
269
  resumeState: Record<string, unknown>,
264
270
  ): string | null {
265
- const sessionState =
266
- resumeState.sessionState && typeof resumeState.sessionState === "object"
267
- ? (resumeState.sessionState as Record<string, unknown>)
268
- : null;
269
- const providerState =
270
- sessionState?.providerState && typeof sessionState.providerState === "object"
271
- ? (sessionState.providerState as Record<string, unknown>)
272
- : null;
273
- for (const field of [
274
- "sandboxId",
275
- "instanceId",
276
- "id",
277
- "hostId",
278
- "containerId",
279
- "workspaceRootPath",
280
- "agentId",
281
- ]) {
282
- const value = providerState?.[field];
283
- if (typeof value === "string" && value.length > 0) return value;
284
- }
285
- return null;
271
+ return sandboxProviderInstanceIdFromEnvelope(resumeState);
286
272
  }
287
273
 
288
274
  /**
@@ -448,7 +434,8 @@ function beforePersistableHomeMutation(
448
434
  expectedEpoch: backend.leaseEpoch,
449
435
  expectedInstanceId: backend.providerInstanceId,
450
436
  operation: op,
451
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
437
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
438
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
452
439
  });
453
440
  return { admission, providerBinding };
454
441
  };
@@ -583,7 +570,8 @@ function beforeRetainedProcessMutation(
583
570
  sessionId: ids.sessionId,
584
571
  processId: process.id,
585
572
  operation: op,
586
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
573
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
574
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
587
575
  });
588
576
  }
589
577
 
@@ -735,6 +723,9 @@ export function wrapTurnBoxWithRouting(
735
723
  // environment variables" throw when the turn pins to a vm). Mirrors the group
736
724
  // box, which is created WITH this same environment (resumeBoxForTurn).
737
725
  ...(ids.environment !== undefined ? { environment: ids.environment } : {}),
726
+ ...(ids.transientExecEnvironment !== undefined
727
+ ? { transientExecEnvironment: ids.transientExecEnvironment }
728
+ : {}),
738
729
  // Stage D machine-primary: pin THIS established SelfhostedSession for the machine
739
730
  // pointer so the resolver returns the SAME instance (no two-instance manifest
740
731
  // divergence). `established.session` is the SelfhostedSession the establish branch
@@ -884,8 +875,17 @@ export function wrapLazyTurnBoxWithRouting(
884
875
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
885
876
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
886
877
  const settleProcess = settleRetainedProcessForTurn(services, ids);
878
+ const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
879
+ const defaultSupportsPty = defaultBackend
880
+ ? selectBackend(defaultBackend).capabilities.Terminal.pty
881
+ : false;
887
882
  const syntheticSession: RoutableBackendSession = {
888
883
  state: { manifest: args.agentDefaultManifest },
884
+ // The SDK binds shell tools synchronously before lazy provisioning. Preserve
885
+ // the authoritative descriptor's PTY shape so a Modal home exposes
886
+ // write_stdin on its first turn operation; the async cancellation transport
887
+ // below still resolves the exact live route before executing a command.
888
+ supportsPty: () => defaultSupportsPty,
889
889
  };
890
890
  const routedResolver = makeActiveBackendResolver({
891
891
  workspaceId: ids.workspaceId,
@@ -912,6 +912,9 @@ export function wrapLazyTurnBoxWithRouting(
912
912
  ...selfhostedResolverTimeouts(settings),
913
913
  ...(onOp !== undefined ? { selfhostedOnOp: onOp } : {}),
914
914
  ...(ids.environment !== undefined ? { environment: ids.environment } : {}),
915
+ ...(ids.transientExecEnvironment !== undefined
916
+ ? { transientExecEnvironment: ids.transientExecEnvironment }
917
+ : {}),
915
918
  });
916
919
 
917
920
  const proxy = new RoutingSandboxSession({
@@ -1027,6 +1030,8 @@ export type SelfhostedTurnSessionArgs = {
1027
1030
  * the manifest), threaded so the SDK's per-turn provided-session env delta is
1028
1031
  * empty. */
1029
1032
  environment: Record<string, string>;
1033
+ /** Attempt-local values projected only onto this machine's child execs. */
1034
+ transientExecEnvironment?: () => Readonly<Record<string, string>>;
1030
1035
  /** The session working directory (per-session pointer). Null ⇒ workspace_root. */
1031
1036
  workingDir: string | null;
1032
1037
  };
@@ -1083,6 +1088,9 @@ export async function establishSelfhostedTurnSession(
1083
1088
  controlRpcFactory: controlRpcFactory(bus),
1084
1089
  epoch: args.epoch,
1085
1090
  environment: args.environment,
1091
+ ...(args.transientExecEnvironment !== undefined
1092
+ ? { transientExecEnvironment: args.transientExecEnvironment }
1093
+ : {}),
1086
1094
  workingDir: args.workingDir,
1087
1095
  // Give this turn's exec ops the long deadline (control ops stay short) so a real
1088
1096
  // command is not killed at the control wall.
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
20
20
  shutdown(): void;
21
21
  };
22
22
 
23
+ /** One process may poll several disjoint Temporal queues, but it still has one
24
+ * host lifecycle. Start and stop every poller as a unit; a failed poller drains
25
+ * its siblings and preserves the original failure. */
26
+ export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
27
+ if (targets.length === 0) {
28
+ throw new Error("a worker service requires at least one Temporal poller");
29
+ }
30
+ let runPromise: Promise<void> | undefined;
31
+ const shutdown = (): void => {
32
+ let firstError: unknown;
33
+ for (const target of targets) {
34
+ try {
35
+ target.shutdown();
36
+ } catch (error) {
37
+ firstError ??= error;
38
+ }
39
+ }
40
+ if (firstError !== undefined) throw firstError;
41
+ };
42
+ return {
43
+ run: () => {
44
+ runPromise ??= (async () => {
45
+ const runs = targets.map((target) => {
46
+ try {
47
+ return target.run();
48
+ } catch (error) {
49
+ return Promise.reject(error);
50
+ }
51
+ });
52
+ try {
53
+ await Promise.all(runs);
54
+ } catch (error) {
55
+ try {
56
+ shutdown();
57
+ } catch {
58
+ // Preserve the poller failure. Every sibling still received its
59
+ // shutdown request even when one request itself failed.
60
+ }
61
+ await Promise.allSettled(runs);
62
+ throw error;
63
+ }
64
+ })();
65
+ return runPromise;
66
+ },
67
+ shutdown,
68
+ };
69
+ }
70
+
23
71
  export type WorkerServiceLifecycle = {
24
72
  state(): WorkerLifecycleState;
25
73
  run(): Promise<void>;
26
- drain(reason?: string): void;
74
+ drain(reason?: string): boolean;
27
75
  close(): Promise<void>;
28
76
  };
29
77
 
@@ -74,27 +122,37 @@ export function createWorkerServiceLifecycle(input: {
74
122
  },
75
123
  drain: (_reason = "host request") => {
76
124
  if (state === "draining" || state === "stopped" || state === "failed") {
77
- return;
125
+ return true;
78
126
  }
127
+ const previousState = state;
128
+ safeLifecycleLog(() =>
129
+ input.observability.info("OpenGeni worker draining (graceful shutdown)", {
130
+ role: input.role,
131
+ errorClass: "WorkerLifecycleOperation",
132
+ errorCode: "worker_draining",
133
+ origin: "worker-lifecycle",
134
+ }),
135
+ );
79
136
  state = "draining";
80
- input.observability.info("OpenGeni worker draining (graceful shutdown)", {
81
- role: input.role,
82
- errorClass: "WorkerLifecycleOperation",
83
- errorCode: "worker_draining",
84
- origin: "worker-lifecycle",
85
- });
86
137
  try {
87
138
  input.worker.shutdown();
139
+ return true;
88
140
  } catch {
89
- input.observability.warn("worker shutdown request failed", {
90
- errorClass: "WorkerLifecycleOperationError",
91
- errorCode: "worker_shutdown_request_failed",
92
- origin: "worker-lifecycle",
93
- });
141
+ state = previousState;
142
+ safeLifecycleLog(() =>
143
+ input.observability.warn("worker shutdown request failed", {
144
+ errorClass: "WorkerLifecycleOperationError",
145
+ errorCode: "worker_shutdown_request_failed",
146
+ origin: "worker-lifecycle",
147
+ }),
148
+ );
149
+ return false;
94
150
  }
95
151
  },
96
152
  close: async () => {
97
- lifecycle.drain("service close");
153
+ if (!lifecycle.drain("service close")) {
154
+ throw new Error("worker shutdown request failed");
155
+ }
98
156
  if (runPromise) {
99
157
  await runPromise.catch(() => undefined);
100
158
  } else {
@@ -106,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
106
164
 
107
165
  return lifecycle;
108
166
  }
167
+
168
+ function safeLifecycleLog(log: () => void): void {
169
+ try {
170
+ log();
171
+ } catch {
172
+ // Telemetry cannot change worker lifecycle state or shutdown delivery.
173
+ }
174
+ }
@@ -5,6 +5,7 @@ type WorkflowControlActivities = Pick<
5
5
  typeof activities,
6
6
  | "enqueueGoalRetryWake"
7
7
  | "expireSessionHumanInput"
8
+ | "expireSessionInteractionIntervention"
8
9
  | "failSessionAttempt"
9
10
  | "getCodexCapacityWait"
10
11
  | "markSessionIdle"
@@ -91,11 +92,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
91
92
  });
92
93
  }
93
94
 
95
+ export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
96
+ return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
97
+ taskQueue: turnTaskQueue(baseTaskQueue),
98
+ startToCloseTimeout: "20 minutes",
99
+ retry: {
100
+ initialInterval: "1 second",
101
+ backoffCoefficient: 2,
102
+ maximumInterval: "30 seconds",
103
+ maximumAttempts: 3,
104
+ },
105
+ });
106
+ }
107
+
94
108
  export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
95
109
  startToCloseTimeout: "30 minutes",
96
110
  retry: { maximumAttempts: 1 },
97
111
  });
98
112
 
113
+ /** Connector inventory/download/index batches are provider-I/O activities. The
114
+ * implementation heartbeats between bounded items and owns its resumable
115
+ * checkpoint; Temporal retries therefore repeat only idempotent source/object
116
+ * operations and never invoke the agent runtime. */
117
+ export const knowledgeSourceSyncActivity = proxyActivities<
118
+ Pick<typeof activities, "runKnowledgeSourceSyncBatch">
119
+ >({
120
+ startToCloseTimeout: "1 hour",
121
+ heartbeatTimeout: "2 minutes",
122
+ retry: {
123
+ initialInterval: "2 seconds",
124
+ backoffCoefficient: 2,
125
+ maximumInterval: "1 minute",
126
+ maximumAttempts: 5,
127
+ },
128
+ });
129
+
99
130
  export function workflowFailureMessage(error: unknown): string {
100
131
  if (error instanceof Error) {
101
132
  return error.message;
@@ -2,7 +2,13 @@ import { proxyActivities } from "@temporalio/workflow";
2
2
  import type * as activities from "../activities";
3
3
 
4
4
  const reaperActivity = proxyActivities<
5
- Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots">
5
+ Pick<
6
+ typeof activities,
7
+ | "reapExpiredFileUploads"
8
+ | "maintainRetainedScreenshots"
9
+ | "maintainBrowserStateArtifacts"
10
+ | "recoverVideoGenerationWorkflows"
11
+ >
6
12
  >({
7
13
  startToCloseTimeout: "5 minutes",
8
14
  retry: { maximumAttempts: 1 },
@@ -12,4 +18,6 @@ const reaperActivity = proxyActivities<
12
18
  export async function fileUploadReaperWorkflow(): Promise<void> {
13
19
  await reaperActivity.reapExpiredFileUploads();
14
20
  await reaperActivity.maintainRetainedScreenshots();
21
+ await reaperActivity.maintainBrowserStateArtifacts();
22
+ await reaperActivity.recoverVideoGenerationWorkflows();
15
23
  }
@@ -0,0 +1,83 @@
1
+ import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
2
+ import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
3
+ import { knowledgeSourceSyncActivity } from "./activities";
4
+
5
+ export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
6
+
7
+ export const knowledgeSourceSyncWake =
8
+ defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
9
+
10
+ type KnowledgeSourceSyncWorkflowState = {
11
+ pending: KnowledgeSourceSyncWorkflowInput | null;
12
+ batchesThisRun: number;
13
+ };
14
+
15
+ const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
16
+
17
+ /** One stable workflow ID serializes all fires for a source. Signals retain at
18
+ * most one newest pending wake; Postgres lease/checkpoint rows remain the
19
+ * authoritative overlap and resumability fence across workflow generations. */
20
+ export async function knowledgeSourceSyncWorkflow(
21
+ input: KnowledgeSourceSyncWorkflowInput,
22
+ carried?: KnowledgeSourceSyncWorkflowState,
23
+ ): Promise<void> {
24
+ let current = input;
25
+ let pending = carried?.pending ?? null;
26
+ let batchesThisRun = carried?.batchesThisRun ?? 0;
27
+
28
+ setHandler(knowledgeSourceSyncWake, (next) => {
29
+ if (
30
+ next.accountId !== input.accountId ||
31
+ next.workspaceId !== input.workspaceId ||
32
+ next.sourceId !== input.sourceId
33
+ ) {
34
+ return;
35
+ }
36
+ pending = next;
37
+ });
38
+
39
+ while (true) {
40
+ const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
41
+ batchesThisRun += 1;
42
+
43
+ if (result.action === "continue") {
44
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
45
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
46
+ pending,
47
+ batchesThisRun: 0,
48
+ });
49
+ }
50
+ continue;
51
+ }
52
+
53
+ if (
54
+ (result.action === "complete" || result.action === "failed") &&
55
+ result.bufferedScheduledTaskRunId
56
+ ) {
57
+ current = {
58
+ ...current,
59
+ scheduledTaskRunId: result.bufferedScheduledTaskRunId,
60
+ };
61
+ pending = null;
62
+ continue;
63
+ }
64
+
65
+ if (pending) {
66
+ current = pending;
67
+ pending = null;
68
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
69
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
70
+ pending: null,
71
+ batchesThisRun: 0,
72
+ });
73
+ }
74
+ continue;
75
+ }
76
+
77
+ return;
78
+ }
79
+ }
80
+
81
+ export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
82
+ return `knowledge-source-sync-${sourceId}`;
83
+ }
@@ -1,28 +1,190 @@
1
- // apps/worker/src/workflows/sandbox-reaper.ts — the ONE global reaper workflow.
2
- //
3
- // This is the action the single global reaper Temporal Schedule starts
4
- // (registered in apps/worker/src/index.ts, gated on sandboxOwnershipEnabled). It
5
- // runs on the global task queue — the SAME queue the worker pool listens on,
6
- // because the worker carries the runtime needed to resume-by-id and terminate a
7
- // provider box. There is NO per-session worker, NO per-session/per-group queue.
8
- //
9
- // The workflow is intentionally trivial: it delegates the whole sweep to the
10
- // stateless `reapSandboxLeases` activity (the sweep is DB + provider IO that
11
- // must run in an activity, not in workflow-deterministic code). The Schedule's
12
- // SKIP overlap policy means a slow sweep never overlaps itself; the activity is
13
- // idempotent regardless.
14
-
15
- import { proxyActivities } from "@temporalio/workflow";
1
+ // The global schedule performs only bounded inventory/maintenance work and
2
+ // starts one durable child per exact (workspace, group, epoch). Provider I/O is
3
+ // therefore parallel at the Temporal worker's activity-concurrency limit; one
4
+ // slow snapshot can delay only its own box, never the global reaper or a sibling.
5
+
6
+ import {
7
+ ParentClosePolicy,
8
+ WorkflowIdReusePolicy,
9
+ log,
10
+ proxyActivities,
11
+ startChild,
12
+ uuid4,
13
+ workflowInfo,
14
+ } from "@temporalio/workflow";
16
15
  import type * as activities from "../activities";
16
+ import {
17
+ SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
18
+ SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS,
19
+ SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
20
+ SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
21
+ SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
22
+ sandboxDrainWorkflowId,
23
+ sandboxLifecycleTaskQueue,
24
+ type SandboxDrainActivityInput,
25
+ type SandboxDrainWorkflowInput,
26
+ type SandboxLeaseSweepMaintenanceInput,
27
+ } from "../sandbox-reaper-contract";
28
+
29
+ function lifecycleTaskQueue(): string {
30
+ return sandboxLifecycleTaskQueue(workflowInfo().taskQueue);
31
+ }
32
+
33
+ function sweepActivities() {
34
+ return proxyActivities<Pick<typeof activities, "prepareSandboxLeaseSweep">>({
35
+ taskQueue: lifecycleTaskQueue(),
36
+ startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
37
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
38
+ // Inventory is the admission edge for every per-box child. A worker crash
39
+ // must not silently consume the only scheduled sweep.
40
+ retry: {
41
+ initialInterval: "1 second",
42
+ backoffCoefficient: 2,
43
+ maximumInterval: "30 seconds",
44
+ },
45
+ });
46
+ }
47
+
48
+ function maintenanceActivity() {
49
+ return proxyActivities<Pick<typeof activities, "maintainSandboxLeaseSweep">>({
50
+ taskQueue: lifecycleTaskQueue(),
51
+ startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
52
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
53
+ // Maintenance is ancillary and internally exact/idempotent. Never let one
54
+ // pathological pass keep the fixed-id V2 workflow open forever and thereby
55
+ // suppress all later drain inventories. The next Schedule tick retries it.
56
+ retry: { maximumAttempts: 1 },
57
+ });
58
+ }
59
+
60
+ function drainActivity(startToCloseTimeout: number) {
61
+ return proxyActivities<Pick<typeof activities, "drainSandboxLease">>({
62
+ taskQueue: lifecycleTaskQueue(),
63
+ startToCloseTimeout,
64
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
65
+ // Each retry is a new exact DB capture attempt under one logical operation.
66
+ // Heartbeat timeout moves a crashed worker to a healthy poller promptly;
67
+ // exact-attempt CAS fencing makes late callbacks harmless.
68
+ retry: {
69
+ initialInterval: "1 second",
70
+ backoffCoefficient: 2,
71
+ maximumInterval: "30 seconds",
72
+ },
73
+ });
74
+ }
17
75
 
18
- // A short, bounded reaper activity. Unlike the 30-day agent turn, the sweep is a
19
- // fast DB pass + a handful of provider terminates; cap it well under the reaper
20
- // period so a wedged sweep is abandoned rather than blocking the next fire.
21
- const reaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
22
- startToCloseTimeout: "5 minutes",
76
+ // This proxy and workflow deliberately preserve the exact pre-refactor
77
+ // ScheduleActivity command. A patch marker would let a new binary replay old
78
+ // histories, but an old binary could not replay a history first executed by a
79
+ // new worker during a rolling deployment. Routing happens inside the activity,
80
+ // where changing implementation is safe and mixed old/new workers remain valid.
81
+ const legacyReaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
82
+ startToCloseTimeout: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
23
83
  retry: { maximumAttempts: 1 },
24
84
  });
25
85
 
86
+ export async function sandboxDrainWorkflow(input: SandboxDrainWorkflowInput): Promise<void> {
87
+ const activityInput: SandboxDrainActivityInput = {
88
+ ...input,
89
+ operationId: uuid4(),
90
+ };
91
+ if (input.timeoutClass === "fast") {
92
+ await drainActivity(SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS).drainSandboxLease(activityInput);
93
+ } else {
94
+ await drainActivity(SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS).drainSandboxLease(
95
+ activityInput,
96
+ );
97
+ }
98
+ }
99
+
100
+ /** Ancillary billing/reconciliation/GC is durable but never allowed to hold the
101
+ * fixed-ID inventory workflow open behind a provider backlog. One singleton
102
+ * maintenance child may queue on the lifecycle workers while every schedule
103
+ * tick remains free to inventory and dispatch newly-drainable boxes. */
104
+ export async function sandboxReaperMaintenanceWorkflow(
105
+ input: SandboxLeaseSweepMaintenanceInput,
106
+ ): Promise<void> {
107
+ try {
108
+ await maintenanceActivity().maintainSandboxLeaseSweep(input);
109
+ } catch (error) {
110
+ // The next Schedule tick starts another singleton after this run closes.
111
+ // Drain children were already made durable before this workflow was started.
112
+ log.warn("sandbox reaper maintenance deferred to a later sweep", {
113
+ error: error instanceof Error ? error.message : String(error),
114
+ });
115
+ }
116
+ }
117
+
118
+ function alreadyRunningChild(error: unknown): boolean {
119
+ let current: unknown = error;
120
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
121
+ const record = current as { name?: unknown; cause?: unknown };
122
+ if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
123
+ current = record.cause;
124
+ }
125
+ return false;
126
+ }
127
+
26
128
  export async function sandboxReaperWorkflow(): Promise<void> {
27
- await reaperActivity.reapSandboxLeases();
129
+ await legacyReaperActivity.reapSandboxLeases();
130
+ }
131
+
132
+ export async function sandboxReaperWorkflowV2(): Promise<void> {
133
+ const sweep = sweepActivities();
134
+ const plan = await sweep.prepareSandboxLeaseSweep();
135
+ const starts = await Promise.all(
136
+ plan.drainable.map(async (target) => {
137
+ try {
138
+ await startChild(sandboxDrainWorkflow, {
139
+ workflowId: sandboxDrainWorkflowId(target),
140
+ taskQueue: lifecycleTaskQueue(),
141
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
142
+ parentClosePolicy: ParentClosePolicy.ABANDON,
143
+ args: [
144
+ {
145
+ target,
146
+ timeoutClass: plan.timeoutClass,
147
+ snapshotTimeoutMs: plan.snapshotTimeoutMs,
148
+ captureTimeoutMs: plan.captureTimeoutMs,
149
+ },
150
+ ],
151
+ });
152
+ return "started" as const;
153
+ } catch (error) {
154
+ if (alreadyRunningChild(error)) return "already_running" as const;
155
+ log.error("sandbox reaper child start failed", {
156
+ workspaceId: target.workspaceId,
157
+ sandboxGroupId: target.sandboxGroupId,
158
+ leaseEpoch: target.leaseEpoch,
159
+ instanceId: target.instanceId,
160
+ error: error instanceof Error ? error.message : String(error),
161
+ });
162
+ return "failed" as const;
163
+ }
164
+ }),
165
+ );
166
+ try {
167
+ await startChild(sandboxReaperMaintenanceWorkflow, {
168
+ workflowId: SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
169
+ taskQueue: lifecycleTaskQueue(),
170
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
171
+ parentClosePolicy: ParentClosePolicy.ABANDON,
172
+ args: [
173
+ {
174
+ examined: plan.drainable.length,
175
+ started: starts.filter((outcome) => outcome === "started").length,
176
+ alreadyRunning: starts.filter((outcome) => outcome === "already_running").length,
177
+ startFailed: starts.filter((outcome) => outcome === "failed").length,
178
+ rotationsRequested: plan.rotationsRequested,
179
+ },
180
+ ],
181
+ });
182
+ } catch (error) {
183
+ if (alreadyRunningChild(error)) return;
184
+ // Every drain child is already durable and this inventory workflow still
185
+ // completes, so the next Schedule tick can retry maintenance independently.
186
+ log.warn("sandbox reaper maintenance child start deferred to next sweep", {
187
+ error: error instanceof Error ? error.message : String(error),
188
+ });
189
+ }
28
190
  }