@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -14,7 +14,7 @@
14
14
  // target lookup), and the selfhosted ControlRpc built over the events bus's NATS
15
15
  // request/reply connection.
16
16
 
17
- import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
17
+ import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
@@ -25,6 +25,7 @@ import {
25
25
  verifyRetainedProcessMutationSettlement,
26
26
  verifyWorkspaceMutationSettlement,
27
27
  getSandbox,
28
+ getEnrollment,
28
29
  markWarmLeaseInstanceLost,
29
30
  readLease,
30
31
  readActiveSandbox,
@@ -44,7 +45,9 @@ import {
44
45
  RoutingBackendRecoveryRequiredError,
45
46
  RoutingSandboxSession,
46
47
  resolveModalCheckpointProviderBindingForSession,
48
+ sandboxProviderInstanceIdFromEnvelope,
47
49
  sandboxBackendForSdkBackendId,
50
+ selectBackend,
48
51
  verifySandboxExecReadiness,
49
52
  type ControlRpc,
50
53
  type EstablishedSandboxSession,
@@ -99,6 +102,15 @@ export type RoutingWiringServices = {
99
102
  established: EstablishedSandboxSession;
100
103
  leaseEpoch: number;
101
104
  }) => void;
105
+ /** Cancel bounded capture waits with the owning turn activity. */
106
+ waitSignal?: AbortSignal;
107
+ };
108
+
109
+ /** Worker-turn routing may final-ack streamed command output, so it must always
110
+ * carry the turn's durable frontier journal. Non-turn helpers may still omit it
111
+ * through the broader service shape above. */
112
+ type TurnRoutingWiringServices = RoutingWiringServices & {
113
+ opJournal: OpStreamJournal;
102
114
  };
103
115
 
104
116
  export type RoutingWiringIds = {
@@ -172,8 +184,8 @@ export function relayConfigFromSettings(settings: Settings): SelfhostedRelayConf
172
184
 
173
185
  /** The selfhosted CONTROL vs EXEC op deadlines for a turn, from settings. Control
174
186
  * ops (ping/fs/desktop/pty) stay on the short timeout so machine liveness is never
175
- * masked by a slow op; exec gets its own much larger budget so a real command is not
176
- * killed at the control wall. Threaded into every turn-path session build + resolver. */
187
+ * masked by a slow op; exec has its own setting and defaults to no duration wall.
188
+ * Threaded into every turn-path session build + resolver. */
177
189
  export function selfhostedTimeoutsFromSettings(settings: Settings): {
178
190
  timeoutMs: number;
179
191
  execTimeoutMs: number;
@@ -254,27 +266,7 @@ function homeRouteRecoveryError(
254
266
  export function providerIdentityFromResumeState(
255
267
  resumeState: Record<string, unknown>,
256
268
  ): string | null {
257
- const sessionState =
258
- resumeState.sessionState && typeof resumeState.sessionState === "object"
259
- ? (resumeState.sessionState as Record<string, unknown>)
260
- : null;
261
- const providerState =
262
- sessionState?.providerState && typeof sessionState.providerState === "object"
263
- ? (sessionState.providerState as Record<string, unknown>)
264
- : null;
265
- for (const field of [
266
- "sandboxId",
267
- "instanceId",
268
- "id",
269
- "hostId",
270
- "containerId",
271
- "workspaceRootPath",
272
- "agentId",
273
- ]) {
274
- const value = providerState?.[field];
275
- if (typeof value === "string" && value.length > 0) return value;
276
- }
277
- return null;
269
+ return sandboxProviderInstanceIdFromEnvelope(resumeState);
278
270
  }
279
271
 
280
272
  /**
@@ -440,7 +432,8 @@ function beforePersistableHomeMutation(
440
432
  expectedEpoch: backend.leaseEpoch,
441
433
  expectedInstanceId: backend.providerInstanceId,
442
434
  operation: op,
443
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
435
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
436
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
444
437
  });
445
438
  return { admission, providerBinding };
446
439
  };
@@ -575,7 +568,8 @@ function beforeRetainedProcessMutation(
575
568
  sessionId: ids.sessionId,
576
569
  processId: process.id,
577
570
  operation: op,
578
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
571
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
572
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
579
573
  });
580
574
  }
581
575
 
@@ -686,7 +680,7 @@ function controlRpcFactory(bus: EventBus | undefined): () => ControlRpc {
686
680
  * swap's active_epoch.
687
681
  */
688
682
  export function wrapTurnBoxWithRouting(
689
- services: RoutingWiringServices,
683
+ services: TurnRoutingWiringServices,
690
684
  ids: RoutingWiringIds,
691
685
  established: EstablishedSandboxSession,
692
686
  ): EstablishedSandboxSession {
@@ -712,6 +706,11 @@ export function wrapTurnBoxWithRouting(
712
706
  : null;
713
707
  },
714
708
  controlRpcFactory: controlRpcFactory(bus),
709
+ resolveSelfhostedOpStream: async (sandbox) => {
710
+ if (!sandbox.enrollmentId) return undefined;
711
+ const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
712
+ return opStreamDepsFor(services, enrollment?.opStream === true);
713
+ },
715
714
  relay: relayConfigFromSettings(settings),
716
715
  // A selfhosted swap target runs real commands too, so give it the same split
717
716
  // deadlines the machine-primary establish path uses (short control, long exec).
@@ -846,7 +845,7 @@ export function wrapTurnBoxWithRouting(
846
845
  }
847
846
 
848
847
  export function wrapLazyTurnBoxWithRouting(
849
- services: RoutingWiringServices,
848
+ services: TurnRoutingWiringServices,
850
849
  ids: RoutingWiringIds,
851
850
  args: {
852
851
  client: EstablishedSandboxSession["client"];
@@ -871,8 +870,17 @@ export function wrapLazyTurnBoxWithRouting(
871
870
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
872
871
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
873
872
  const settleProcess = settleRetainedProcessForTurn(services, ids);
873
+ const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
874
+ const defaultSupportsPty = defaultBackend
875
+ ? selectBackend(defaultBackend).capabilities.Terminal.pty
876
+ : false;
874
877
  const syntheticSession: RoutableBackendSession = {
875
878
  state: { manifest: args.agentDefaultManifest },
879
+ // The SDK binds shell tools synchronously before lazy provisioning. Preserve
880
+ // the authoritative descriptor's PTY shape so a Modal home exposes
881
+ // write_stdin on its first turn operation; the async cancellation transport
882
+ // below still resolves the exact live route before executing a command.
883
+ supportsPty: () => defaultSupportsPty,
876
884
  };
877
885
  const routedResolver = makeActiveBackendResolver({
878
886
  workspaceId: ids.workspaceId,
@@ -890,6 +898,11 @@ export function wrapLazyTurnBoxWithRouting(
890
898
  : null;
891
899
  },
892
900
  controlRpcFactory: controlRpcFactory(bus),
901
+ resolveSelfhostedOpStream: async (sandbox) => {
902
+ if (!sandbox.enrollmentId) return undefined;
903
+ const enrollment = await getEnrollment(db, ids.workspaceId, sandbox.enrollmentId);
904
+ return opStreamDepsFor(services, enrollment?.opStream === true);
905
+ },
893
906
  relay: relayConfigFromSettings(settings),
894
907
  ...selfhostedResolverTimeouts(settings),
895
908
  ...(onOp !== undefined ? { selfhostedOnOp: onOp } : {}),
@@ -1013,8 +1026,6 @@ export type SelfhostedTurnSessionArgs = {
1013
1026
  workingDir: string | null;
1014
1027
  };
1015
1028
 
1016
- type LegacySelfhostedTurnSessionArgs = Omit<SelfhostedTurnSessionArgs, "opStream">;
1017
-
1018
1029
  /**
1019
1030
  * Stage D machine-primary establish: bind the live SelfhostedSession for a turn
1020
1031
  * whose ACTIVE sandbox is a connected machine — WITHOUT establishing or leasing a
@@ -1034,11 +1045,10 @@ type LegacySelfhostedTurnSessionArgs = Omit<SelfhostedTurnSessionArgs, "opStream
1034
1045
  * The op-stream injection for a machine-primary turn: present iff the machine
1035
1046
  * advertised `Capabilities.op_stream` in its latest Hello AND the server flag
1036
1047
  * is on AND a bus exists to carry frames. The transport rides the SAME managed
1037
- * NATS connection as the control rpc (the bus's op-stream accessor); a bus
1038
- * without the accessor (a test double) simply yields no connection and the
1039
- * session falls back to the legacy exec on first use. Swap TARGETS resolved
1040
- * mid-turn stay legacy for now — their capability row is not at hand in the
1041
- * resolver, and legacy is always correct.
1048
+ * NATS connection as the control rpc (the bus's op-stream accessor). A bus
1049
+ * without the accessor yields no stream; an unbounded exec then fails before
1050
+ * starting instead of silently degrading to request/reply. Swap targets resolve
1051
+ * the same enrollment capability through the injected backend-resolver seam.
1042
1052
  */
1043
1053
  function opStreamDepsFor(
1044
1054
  services: RoutingWiringServices,
@@ -1056,11 +1066,11 @@ function opStreamDepsFor(
1056
1066
 
1057
1067
  export async function establishSelfhostedTurnSession(
1058
1068
  services: RoutingWiringServices,
1059
- args: SelfhostedTurnSessionArgs | LegacySelfhostedTurnSessionArgs,
1069
+ args: SelfhostedTurnSessionArgs,
1060
1070
  ): Promise<EstablishedSandboxSession> {
1061
1071
  const { settings, bus, onOp } = services;
1062
1072
  const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
1063
- const opStream = opStreamDepsFor(services, "opStream" in args && args.opStream === true);
1073
+ const opStream = opStreamDepsFor(services, args.opStream);
1064
1074
  const { client, session } = await buildSelfhostedBackendSession({
1065
1075
  workspaceId: args.workspaceId,
1066
1076
  agentId: args.agentId,
@@ -1076,8 +1086,8 @@ export async function establishSelfhostedTurnSession(
1076
1086
  // Meter every control op (out-of-band telemetry) — no-op when unwired.
1077
1087
  ...(onOp !== undefined ? { onOp } : {}),
1078
1088
  // The streaming exec transport — present iff the machine advertised the
1079
- // capability AND the server flag is on (latched per-op at OpStart; the
1080
- // legacy exec stays the permanent fallback wire form).
1089
+ // capability AND the server flag is on. It is required when execTimeoutMs=0;
1090
+ // legacy request/reply remains available only for explicitly bounded exec.
1081
1091
  ...(opStream !== undefined ? { opStream } : {}),
1082
1092
  });
1083
1093
  return {
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
20
20
  shutdown(): void;
21
21
  };
22
22
 
23
+ /** One process may poll several disjoint Temporal queues, but it still has one
24
+ * host lifecycle. Start and stop every poller as a unit; a failed poller drains
25
+ * its siblings and preserves the original failure. */
26
+ export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
27
+ if (targets.length === 0) {
28
+ throw new Error("a worker service requires at least one Temporal poller");
29
+ }
30
+ let runPromise: Promise<void> | undefined;
31
+ const shutdown = (): void => {
32
+ let firstError: unknown;
33
+ for (const target of targets) {
34
+ try {
35
+ target.shutdown();
36
+ } catch (error) {
37
+ firstError ??= error;
38
+ }
39
+ }
40
+ if (firstError !== undefined) throw firstError;
41
+ };
42
+ return {
43
+ run: () => {
44
+ runPromise ??= (async () => {
45
+ const runs = targets.map((target) => {
46
+ try {
47
+ return target.run();
48
+ } catch (error) {
49
+ return Promise.reject(error);
50
+ }
51
+ });
52
+ try {
53
+ await Promise.all(runs);
54
+ } catch (error) {
55
+ try {
56
+ shutdown();
57
+ } catch {
58
+ // Preserve the poller failure. Every sibling still received its
59
+ // shutdown request even when one request itself failed.
60
+ }
61
+ await Promise.allSettled(runs);
62
+ throw error;
63
+ }
64
+ })();
65
+ return runPromise;
66
+ },
67
+ shutdown,
68
+ };
69
+ }
70
+
23
71
  export type WorkerServiceLifecycle = {
24
72
  state(): WorkerLifecycleState;
25
73
  run(): Promise<void>;
26
- drain(reason?: string): void;
74
+ drain(reason?: string): boolean;
27
75
  close(): Promise<void>;
28
76
  };
29
77
 
@@ -72,25 +120,39 @@ export function createWorkerServiceLifecycle(input: {
72
120
  })();
73
121
  return runPromise;
74
122
  },
75
- drain: (reason = "host request") => {
123
+ drain: (_reason = "host request") => {
76
124
  if (state === "draining" || state === "stopped" || state === "failed") {
77
- return;
125
+ return true;
78
126
  }
127
+ const previousState = state;
128
+ safeLifecycleLog(() =>
129
+ input.observability.info("OpenGeni worker draining (graceful shutdown)", {
130
+ role: input.role,
131
+ errorClass: "WorkerLifecycleOperation",
132
+ errorCode: "worker_draining",
133
+ origin: "worker-lifecycle",
134
+ }),
135
+ );
79
136
  state = "draining";
80
- input.observability.info("OpenGeni worker draining (graceful shutdown)", {
81
- role: input.role,
82
- reason,
83
- });
84
137
  try {
85
138
  input.worker.shutdown();
86
- } catch (error) {
87
- input.observability.warn("worker shutdown request failed", {
88
- error: error instanceof Error ? error.message : String(error),
89
- });
139
+ return true;
140
+ } catch {
141
+ state = previousState;
142
+ safeLifecycleLog(() =>
143
+ input.observability.warn("worker shutdown request failed", {
144
+ errorClass: "WorkerLifecycleOperationError",
145
+ errorCode: "worker_shutdown_request_failed",
146
+ origin: "worker-lifecycle",
147
+ }),
148
+ );
149
+ return false;
90
150
  }
91
151
  },
92
152
  close: async () => {
93
- lifecycle.drain("service close");
153
+ if (!lifecycle.drain("service close")) {
154
+ throw new Error("worker shutdown request failed");
155
+ }
94
156
  if (runPromise) {
95
157
  await runPromise.catch(() => undefined);
96
158
  } else {
@@ -102,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
102
164
 
103
165
  return lifecycle;
104
166
  }
167
+
168
+ function safeLifecycleLog(log: () => void): void {
169
+ try {
170
+ log();
171
+ } catch {
172
+ // Telemetry cannot change worker lifecycle state or shutdown delivery.
173
+ }
174
+ }
@@ -91,11 +91,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
91
91
  });
92
92
  }
93
93
 
94
+ export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
95
+ return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
96
+ taskQueue: turnTaskQueue(baseTaskQueue),
97
+ startToCloseTimeout: "20 minutes",
98
+ retry: {
99
+ initialInterval: "1 second",
100
+ backoffCoefficient: 2,
101
+ maximumInterval: "30 seconds",
102
+ maximumAttempts: 3,
103
+ },
104
+ });
105
+ }
106
+
94
107
  export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
95
108
  startToCloseTimeout: "30 minutes",
96
109
  retry: { maximumAttempts: 1 },
97
110
  });
98
111
 
112
+ /** Connector inventory/download/index batches are provider-I/O activities. The
113
+ * implementation heartbeats between bounded items and owns its resumable
114
+ * checkpoint; Temporal retries therefore repeat only idempotent source/object
115
+ * operations and never invoke the agent runtime. */
116
+ export const knowledgeSourceSyncActivity = proxyActivities<
117
+ Pick<typeof activities, "runKnowledgeSourceSyncBatch">
118
+ >({
119
+ startToCloseTimeout: "1 hour",
120
+ heartbeatTimeout: "2 minutes",
121
+ retry: {
122
+ initialInterval: "2 seconds",
123
+ backoffCoefficient: 2,
124
+ maximumInterval: "1 minute",
125
+ maximumAttempts: 5,
126
+ },
127
+ });
128
+
99
129
  export function workflowFailureMessage(error: unknown): string {
100
130
  if (error instanceof Error) {
101
131
  return error.message;
@@ -1,7 +1,10 @@
1
1
  import { proxyActivities } from "@temporalio/workflow";
2
2
  import type * as activities from "../activities";
3
3
 
4
- const reaperActivity = proxyActivities<Pick<typeof activities, "reapExpiredFileUploads">>({
4
+ const reaperActivity = proxyActivities<
5
+ Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots"> &
6
+ Pick<typeof activities, "recoverVideoGenerationWorkflows">
7
+ >({
5
8
  startToCloseTimeout: "5 minutes",
6
9
  retry: { maximumAttempts: 1 },
7
10
  });
@@ -9,4 +12,6 @@ const reaperActivity = proxyActivities<Pick<typeof activities, "reapExpiredFileU
9
12
  /** One bounded provider/DB sweep; the Temporal Schedule owns the cadence. */
10
13
  export async function fileUploadReaperWorkflow(): Promise<void> {
11
14
  await reaperActivity.reapExpiredFileUploads();
15
+ await reaperActivity.maintainRetainedScreenshots();
16
+ await reaperActivity.recoverVideoGenerationWorkflows();
12
17
  }
@@ -0,0 +1,83 @@
1
+ import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
2
+ import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
3
+ import { knowledgeSourceSyncActivity } from "./activities";
4
+
5
+ export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
6
+
7
+ export const knowledgeSourceSyncWake =
8
+ defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
9
+
10
+ type KnowledgeSourceSyncWorkflowState = {
11
+ pending: KnowledgeSourceSyncWorkflowInput | null;
12
+ batchesThisRun: number;
13
+ };
14
+
15
+ const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
16
+
17
+ /** One stable workflow ID serializes all fires for a source. Signals retain at
18
+ * most one newest pending wake; Postgres lease/checkpoint rows remain the
19
+ * authoritative overlap and resumability fence across workflow generations. */
20
+ export async function knowledgeSourceSyncWorkflow(
21
+ input: KnowledgeSourceSyncWorkflowInput,
22
+ carried?: KnowledgeSourceSyncWorkflowState,
23
+ ): Promise<void> {
24
+ let current = input;
25
+ let pending = carried?.pending ?? null;
26
+ let batchesThisRun = carried?.batchesThisRun ?? 0;
27
+
28
+ setHandler(knowledgeSourceSyncWake, (next) => {
29
+ if (
30
+ next.accountId !== input.accountId ||
31
+ next.workspaceId !== input.workspaceId ||
32
+ next.sourceId !== input.sourceId
33
+ ) {
34
+ return;
35
+ }
36
+ pending = next;
37
+ });
38
+
39
+ while (true) {
40
+ const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
41
+ batchesThisRun += 1;
42
+
43
+ if (result.action === "continue") {
44
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
45
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
46
+ pending,
47
+ batchesThisRun: 0,
48
+ });
49
+ }
50
+ continue;
51
+ }
52
+
53
+ if (
54
+ (result.action === "complete" || result.action === "failed") &&
55
+ result.bufferedScheduledTaskRunId
56
+ ) {
57
+ current = {
58
+ ...current,
59
+ scheduledTaskRunId: result.bufferedScheduledTaskRunId,
60
+ };
61
+ pending = null;
62
+ continue;
63
+ }
64
+
65
+ if (pending) {
66
+ current = pending;
67
+ pending = null;
68
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
69
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
70
+ pending: null,
71
+ batchesThisRun: 0,
72
+ });
73
+ }
74
+ continue;
75
+ }
76
+
77
+ return;
78
+ }
79
+ }
80
+
81
+ export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
82
+ return `knowledge-source-sync-${sourceId}`;
83
+ }