@opengeni/worker-bundle 0.16.29 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/activities/agent-turn.d.ts +85 -19
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +1 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -8
  10. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +1 -1
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  25. package/dist/activities/retained-screenshots.d.ts +36 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +1 -1
  28. package/dist/activities/run-input.d.ts +3 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +10 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -150
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/concurrency.d.ts +108 -5
  64. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  65. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  66. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  67. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  68. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  69. package/dist/editable-artifact-materializer.d.ts +262 -0
  70. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  71. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  72. package/dist/host-export.d.ts +1 -1
  73. package/dist/index.d.ts +176 -10
  74. package/dist/index.js +662 -17374
  75. package/dist/index.js.map +1 -1
  76. package/dist/memory-pressure-guard.d.ts +57 -0
  77. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  78. package/dist/observability-metrics.d.ts +33 -0
  79. package/dist/sandbox-reaper-contract.d.ts +40 -0
  80. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  81. package/dist/sandbox-resume.d.ts +34 -7
  82. package/dist/sandbox-routing.d.ts +2 -0
  83. package/dist/worker-service-lifecycle.d.ts +6 -2
  84. package/dist/workflow-bundle.js +1043 -363
  85. package/dist/workflows/activities.d.ts +7 -1
  86. package/dist/workflows/document-index.d.ts +1 -1
  87. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  88. package/dist/workflows/rig-verification.d.ts +1 -1
  89. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  90. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  91. package/dist/workflows/session.d.ts +2 -2
  92. package/dist/workflows/video-generation.d.ts +10 -0
  93. package/dist/workflows/workflow-wake.d.ts +1 -1
  94. package/dist/workflows.d.ts +9 -7
  95. package/package.json +34 -15
  96. package/src/activities/agent-turn.ts +1470 -246
  97. package/src/activities/api-integrations.ts +138 -0
  98. package/src/activities/codemode-dispatcher.ts +335 -0
  99. package/src/activities/codemode-token-renewal.ts +64 -0
  100. package/src/activities/codex-capacity.ts +6 -6
  101. package/src/activities/codex-image-generation.ts +56 -0
  102. package/src/activities/context-compaction.ts +38 -3
  103. package/src/activities/documents.ts +14 -4
  104. package/src/activities/environment.ts +45 -42
  105. package/src/activities/expiring-material-renewal.ts +1 -1
  106. package/src/activities/file-upload-reaper.ts +34 -3
  107. package/src/activities/gateway-image-generation.ts +146 -0
  108. package/src/activities/gateway-video-generation.ts +298 -0
  109. package/src/activities/generated-images.ts +860 -0
  110. package/src/activities/goals.ts +6 -2
  111. package/src/activities/google-drive-changes.ts +454 -0
  112. package/src/activities/image-generation-operation.ts +333 -0
  113. package/src/activities/image-generation-references.ts +118 -0
  114. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  115. package/src/activities/knowledge-source-sync.ts +2840 -0
  116. package/src/activities/mcp-credentials.ts +1 -0
  117. package/src/activities/packs.ts +55 -55
  118. package/src/activities/retained-screenshot-reaper.ts +2 -2
  119. package/src/activities/retained-screenshots.ts +453 -28
  120. package/src/activities/rig-verification.ts +554 -43
  121. package/src/activities/run-input.ts +22 -6
  122. package/src/activities/sandbox-images.ts +167 -0
  123. package/src/activities/sandbox-lease.ts +1065 -483
  124. package/src/activities/scheduled-tasks.ts +77 -4
  125. package/src/activities/session-state.ts +4 -4
  126. package/src/activities/streaming.ts +42 -3
  127. package/src/activities/types.ts +87 -5
  128. package/src/activities/video-generation-admission.ts +270 -0
  129. package/src/activities/video-generation-credential.ts +34 -0
  130. package/src/activities/video-generation-reconciliation.ts +578 -0
  131. package/src/activities/video-media-validation.ts +326 -0
  132. package/src/activities/video-output-retention.ts +314 -0
  133. package/src/activities/video-reference-staging.ts +263 -0
  134. package/src/activities/workflow-wake.ts +2 -2
  135. package/src/activities/workspace-capture.ts +44 -10
  136. package/src/activities-control.ts +54 -0
  137. package/src/activities-turn.ts +40 -0
  138. package/src/activities.ts +48 -124
  139. package/src/activity-services.ts +62 -0
  140. package/src/artifact-materializer-entry.ts +35 -0
  141. package/src/artifact-outbox-entry.ts +29 -0
  142. package/src/concurrency.ts +484 -6
  143. package/src/editable-artifact-hint-broker.ts +235 -0
  144. package/src/editable-artifact-materializer-launcher.ts +515 -0
  145. package/src/editable-artifact-materializer-service.ts +787 -0
  146. package/src/editable-artifact-materializer-storage.ts +413 -0
  147. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  148. package/src/editable-artifact-materializer.ts +1289 -0
  149. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  150. package/src/editable-artifact-outbox-service.ts +438 -0
  151. package/src/index.ts +277 -72
  152. package/src/memory-pressure-guard.ts +489 -0
  153. package/src/model-checkpoint-memory-collector.ts +116 -0
  154. package/src/observability-metrics.ts +199 -4
  155. package/src/sandbox-reaper-contract.ts +73 -0
  156. package/src/sandbox-reaper-timeout.ts +79 -0
  157. package/src/sandbox-resume.ts +294 -153
  158. package/src/sandbox-routing.ts +19 -24
  159. package/src/worker-service-lifecycle.ts +80 -14
  160. package/src/workflows/activities.ts +30 -0
  161. package/src/workflows/file-upload-reaper.ts +3 -1
  162. package/src/workflows/knowledge-source-sync.ts +83 -0
  163. package/src/workflows/sandbox-reaper.ts +183 -21
  164. package/src/workflows/scheduled-tasks.ts +42 -6
  165. package/src/workflows/video-generation.ts +36 -0
  166. package/src/workflows.ts +17 -1
  167. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  168. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -14,7 +14,7 @@
14
14
  // target lookup), and the selfhosted ControlRpc built over the events bus's NATS
15
15
  // request/reply connection.
16
16
 
17
- import { sandboxArchiveCaptureTimeoutMs, type Settings } from "@opengeni/config";
17
+ import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
18
18
  import {
19
19
  advanceWorkspaceGenerationForRetainedProcess,
20
20
  advanceWorkspaceGeneration,
@@ -45,7 +45,9 @@ import {
45
45
  RoutingBackendRecoveryRequiredError,
46
46
  RoutingSandboxSession,
47
47
  resolveModalCheckpointProviderBindingForSession,
48
+ sandboxProviderInstanceIdFromEnvelope,
48
49
  sandboxBackendForSdkBackendId,
50
+ selectBackend,
49
51
  verifySandboxExecReadiness,
50
52
  type ControlRpc,
51
53
  type EstablishedSandboxSession,
@@ -100,6 +102,8 @@ export type RoutingWiringServices = {
100
102
  established: EstablishedSandboxSession;
101
103
  leaseEpoch: number;
102
104
  }) => void;
105
+ /** Cancel bounded capture waits with the owning turn activity. */
106
+ waitSignal?: AbortSignal;
103
107
  };
104
108
 
105
109
  /** Worker-turn routing may final-ack streamed command output, so it must always
@@ -262,27 +266,7 @@ function homeRouteRecoveryError(
262
266
  export function providerIdentityFromResumeState(
263
267
  resumeState: Record<string, unknown>,
264
268
  ): string | null {
265
- const sessionState =
266
- resumeState.sessionState && typeof resumeState.sessionState === "object"
267
- ? (resumeState.sessionState as Record<string, unknown>)
268
- : null;
269
- const providerState =
270
- sessionState?.providerState && typeof sessionState.providerState === "object"
271
- ? (sessionState.providerState as Record<string, unknown>)
272
- : null;
273
- for (const field of [
274
- "sandboxId",
275
- "instanceId",
276
- "id",
277
- "hostId",
278
- "containerId",
279
- "workspaceRootPath",
280
- "agentId",
281
- ]) {
282
- const value = providerState?.[field];
283
- if (typeof value === "string" && value.length > 0) return value;
284
- }
285
- return null;
269
+ return sandboxProviderInstanceIdFromEnvelope(resumeState);
286
270
  }
287
271
 
288
272
  /**
@@ -448,7 +432,8 @@ function beforePersistableHomeMutation(
448
432
  expectedEpoch: backend.leaseEpoch,
449
433
  expectedInstanceId: backend.providerInstanceId,
450
434
  operation: op,
451
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
435
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
436
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
452
437
  });
453
438
  return { admission, providerBinding };
454
439
  };
@@ -583,7 +568,8 @@ function beforeRetainedProcessMutation(
583
568
  sessionId: ids.sessionId,
584
569
  processId: process.id,
585
570
  operation: op,
586
- captureWaitMs: sandboxArchiveCaptureTimeoutMs(services.settings),
571
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(services.settings),
572
+ ...(services.waitSignal ? { waitSignal: services.waitSignal } : {}),
587
573
  });
588
574
  }
589
575
 
@@ -884,8 +870,17 @@ export function wrapLazyTurnBoxWithRouting(
884
870
  const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
885
871
  const afterProcessMutation = afterRetainedProcessMutation(services, ids);
886
872
  const settleProcess = settleRetainedProcessForTurn(services, ids);
873
+ const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
874
+ const defaultSupportsPty = defaultBackend
875
+ ? selectBackend(defaultBackend).capabilities.Terminal.pty
876
+ : false;
887
877
  const syntheticSession: RoutableBackendSession = {
888
878
  state: { manifest: args.agentDefaultManifest },
879
+ // The SDK binds shell tools synchronously before lazy provisioning. Preserve
880
+ // the authoritative descriptor's PTY shape so a Modal home exposes
881
+ // write_stdin on its first turn operation; the async cancellation transport
882
+ // below still resolves the exact live route before executing a command.
883
+ supportsPty: () => defaultSupportsPty,
889
884
  };
890
885
  const routedResolver = makeActiveBackendResolver({
891
886
  workspaceId: ids.workspaceId,
@@ -20,10 +20,58 @@ export type WorkerRunTarget = {
20
20
  shutdown(): void;
21
21
  };
22
22
 
23
+ /** One process may poll several disjoint Temporal queues, but it still has one
24
+ * host lifecycle. Start and stop every poller as a unit; a failed poller drains
25
+ * its siblings and preserves the original failure. */
26
+ export function combineWorkerRunTargets(targets: readonly WorkerRunTarget[]): WorkerRunTarget {
27
+ if (targets.length === 0) {
28
+ throw new Error("a worker service requires at least one Temporal poller");
29
+ }
30
+ let runPromise: Promise<void> | undefined;
31
+ const shutdown = (): void => {
32
+ let firstError: unknown;
33
+ for (const target of targets) {
34
+ try {
35
+ target.shutdown();
36
+ } catch (error) {
37
+ firstError ??= error;
38
+ }
39
+ }
40
+ if (firstError !== undefined) throw firstError;
41
+ };
42
+ return {
43
+ run: () => {
44
+ runPromise ??= (async () => {
45
+ const runs = targets.map((target) => {
46
+ try {
47
+ return target.run();
48
+ } catch (error) {
49
+ return Promise.reject(error);
50
+ }
51
+ });
52
+ try {
53
+ await Promise.all(runs);
54
+ } catch (error) {
55
+ try {
56
+ shutdown();
57
+ } catch {
58
+ // Preserve the poller failure. Every sibling still received its
59
+ // shutdown request even when one request itself failed.
60
+ }
61
+ await Promise.allSettled(runs);
62
+ throw error;
63
+ }
64
+ })();
65
+ return runPromise;
66
+ },
67
+ shutdown,
68
+ };
69
+ }
70
+
23
71
  export type WorkerServiceLifecycle = {
24
72
  state(): WorkerLifecycleState;
25
73
  run(): Promise<void>;
26
- drain(reason?: string): void;
74
+ drain(reason?: string): boolean;
27
75
  close(): Promise<void>;
28
76
  };
29
77
 
@@ -74,27 +122,37 @@ export function createWorkerServiceLifecycle(input: {
74
122
  },
75
123
  drain: (_reason = "host request") => {
76
124
  if (state === "draining" || state === "stopped" || state === "failed") {
77
- return;
125
+ return true;
78
126
  }
127
+ const previousState = state;
128
+ safeLifecycleLog(() =>
129
+ input.observability.info("OpenGeni worker draining (graceful shutdown)", {
130
+ role: input.role,
131
+ errorClass: "WorkerLifecycleOperation",
132
+ errorCode: "worker_draining",
133
+ origin: "worker-lifecycle",
134
+ }),
135
+ );
79
136
  state = "draining";
80
- input.observability.info("OpenGeni worker draining (graceful shutdown)", {
81
- role: input.role,
82
- errorClass: "WorkerLifecycleOperation",
83
- errorCode: "worker_draining",
84
- origin: "worker-lifecycle",
85
- });
86
137
  try {
87
138
  input.worker.shutdown();
139
+ return true;
88
140
  } catch {
89
- input.observability.warn("worker shutdown request failed", {
90
- errorClass: "WorkerLifecycleOperationError",
91
- errorCode: "worker_shutdown_request_failed",
92
- origin: "worker-lifecycle",
93
- });
141
+ state = previousState;
142
+ safeLifecycleLog(() =>
143
+ input.observability.warn("worker shutdown request failed", {
144
+ errorClass: "WorkerLifecycleOperationError",
145
+ errorCode: "worker_shutdown_request_failed",
146
+ origin: "worker-lifecycle",
147
+ }),
148
+ );
149
+ return false;
94
150
  }
95
151
  },
96
152
  close: async () => {
97
- lifecycle.drain("service close");
153
+ if (!lifecycle.drain("service close")) {
154
+ throw new Error("worker shutdown request failed");
155
+ }
98
156
  if (runPromise) {
99
157
  await runPromise.catch(() => undefined);
100
158
  } else {
@@ -106,3 +164,11 @@ export function createWorkerServiceLifecycle(input: {
106
164
 
107
165
  return lifecycle;
108
166
  }
167
+
168
+ function safeLifecycleLog(log: () => void): void {
169
+ try {
170
+ log();
171
+ } catch {
172
+ // Telemetry cannot change worker lifecycle state or shutdown delivery.
173
+ }
174
+ }
@@ -91,11 +91,41 @@ export function turnActivityForTaskQueue(baseTaskQueue: string, receiptGatedCanc
91
91
  });
92
92
  }
93
93
 
94
+ export function videoGenerationActivityForTaskQueue(baseTaskQueue: string) {
95
+ return proxyActivities<Pick<typeof activities, "reconcileVideoGenerationOperation">>({
96
+ taskQueue: turnTaskQueue(baseTaskQueue),
97
+ startToCloseTimeout: "20 minutes",
98
+ retry: {
99
+ initialInterval: "1 second",
100
+ backoffCoefficient: 2,
101
+ maximumInterval: "30 seconds",
102
+ maximumAttempts: 3,
103
+ },
104
+ });
105
+ }
106
+
94
107
  export const documentActivity = proxyActivities<Pick<typeof activities, "indexDocument">>({
95
108
  startToCloseTimeout: "30 minutes",
96
109
  retry: { maximumAttempts: 1 },
97
110
  });
98
111
 
112
+ /** Connector inventory/download/index batches are provider-I/O activities. The
113
+ * implementation heartbeats between bounded items and owns its resumable
114
+ * checkpoint; Temporal retries therefore repeat only idempotent source/object
115
+ * operations and never invoke the agent runtime. */
116
+ export const knowledgeSourceSyncActivity = proxyActivities<
117
+ Pick<typeof activities, "runKnowledgeSourceSyncBatch">
118
+ >({
119
+ startToCloseTimeout: "1 hour",
120
+ heartbeatTimeout: "2 minutes",
121
+ retry: {
122
+ initialInterval: "2 seconds",
123
+ backoffCoefficient: 2,
124
+ maximumInterval: "1 minute",
125
+ maximumAttempts: 5,
126
+ },
127
+ });
128
+
99
129
  export function workflowFailureMessage(error: unknown): string {
100
130
  if (error instanceof Error) {
101
131
  return error.message;
@@ -2,7 +2,8 @@ import { proxyActivities } from "@temporalio/workflow";
2
2
  import type * as activities from "../activities";
3
3
 
4
4
  const reaperActivity = proxyActivities<
5
- Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots">
5
+ Pick<typeof activities, "reapExpiredFileUploads" | "maintainRetainedScreenshots"> &
6
+ Pick<typeof activities, "recoverVideoGenerationWorkflows">
6
7
  >({
7
8
  startToCloseTimeout: "5 minutes",
8
9
  retry: { maximumAttempts: 1 },
@@ -12,4 +13,5 @@ const reaperActivity = proxyActivities<
12
13
  export async function fileUploadReaperWorkflow(): Promise<void> {
13
14
  await reaperActivity.reapExpiredFileUploads();
14
15
  await reaperActivity.maintainRetainedScreenshots();
16
+ await reaperActivity.recoverVideoGenerationWorkflows();
15
17
  }
@@ -0,0 +1,83 @@
1
+ import { continueAsNew, defineSignal, setHandler, workflowInfo } from "@temporalio/workflow";
2
+ import type { RunKnowledgeSourceSyncBatchInput } from "../activities";
3
+ import { knowledgeSourceSyncActivity } from "./activities";
4
+
5
+ export type KnowledgeSourceSyncWorkflowInput = RunKnowledgeSourceSyncBatchInput;
6
+
7
+ export const knowledgeSourceSyncWake =
8
+ defineSignal<[KnowledgeSourceSyncWorkflowInput]>("knowledgeSourceSyncWake");
9
+
10
+ type KnowledgeSourceSyncWorkflowState = {
11
+ pending: KnowledgeSourceSyncWorkflowInput | null;
12
+ batchesThisRun: number;
13
+ };
14
+
15
+ const MAX_BATCHES_PER_WORKFLOW_RUN = 50;
16
+
17
+ /** One stable workflow ID serializes all fires for a source. Signals retain at
18
+ * most one newest pending wake; Postgres lease/checkpoint rows remain the
19
+ * authoritative overlap and resumability fence across workflow generations. */
20
+ export async function knowledgeSourceSyncWorkflow(
21
+ input: KnowledgeSourceSyncWorkflowInput,
22
+ carried?: KnowledgeSourceSyncWorkflowState,
23
+ ): Promise<void> {
24
+ let current = input;
25
+ let pending = carried?.pending ?? null;
26
+ let batchesThisRun = carried?.batchesThisRun ?? 0;
27
+
28
+ setHandler(knowledgeSourceSyncWake, (next) => {
29
+ if (
30
+ next.accountId !== input.accountId ||
31
+ next.workspaceId !== input.workspaceId ||
32
+ next.sourceId !== input.sourceId
33
+ ) {
34
+ return;
35
+ }
36
+ pending = next;
37
+ });
38
+
39
+ while (true) {
40
+ const result = await knowledgeSourceSyncActivity.runKnowledgeSourceSyncBatch(current);
41
+ batchesThisRun += 1;
42
+
43
+ if (result.action === "continue") {
44
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
45
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
46
+ pending,
47
+ batchesThisRun: 0,
48
+ });
49
+ }
50
+ continue;
51
+ }
52
+
53
+ if (
54
+ (result.action === "complete" || result.action === "failed") &&
55
+ result.bufferedScheduledTaskRunId
56
+ ) {
57
+ current = {
58
+ ...current,
59
+ scheduledTaskRunId: result.bufferedScheduledTaskRunId,
60
+ };
61
+ pending = null;
62
+ continue;
63
+ }
64
+
65
+ if (pending) {
66
+ current = pending;
67
+ pending = null;
68
+ if (workflowInfo().continueAsNewSuggested || batchesThisRun >= MAX_BATCHES_PER_WORKFLOW_RUN) {
69
+ await continueAsNew<typeof knowledgeSourceSyncWorkflow>(current, {
70
+ pending: null,
71
+ batchesThisRun: 0,
72
+ });
73
+ }
74
+ continue;
75
+ }
76
+
77
+ return;
78
+ }
79
+ }
80
+
81
+ export function knowledgeSourceSyncWorkflowId(sourceId: string): string {
82
+ return `knowledge-source-sync-${sourceId}`;
83
+ }
@@ -1,28 +1,190 @@
1
- // apps/worker/src/workflows/sandbox-reaper.ts — the ONE global reaper workflow.
2
- //
3
- // This is the action the single global reaper Temporal Schedule starts
4
- // (registered in apps/worker/src/index.ts, gated on sandboxOwnershipEnabled). It
5
- // runs on the global task queue — the SAME queue the worker pool listens on,
6
- // because the worker carries the runtime needed to resume-by-id and terminate a
7
- // provider box. There is NO per-session worker, NO per-session/per-group queue.
8
- //
9
- // The workflow is intentionally trivial: it delegates the whole sweep to the
10
- // stateless `reapSandboxLeases` activity (the sweep is DB + provider IO that
11
- // must run in an activity, not in workflow-deterministic code). The Schedule's
12
- // SKIP overlap policy means a slow sweep never overlaps itself; the activity is
13
- // idempotent regardless.
14
-
15
- import { proxyActivities } from "@temporalio/workflow";
1
+ // The global schedule performs only bounded inventory/maintenance work and
2
+ // starts one durable child per exact (workspace, group, epoch). Provider I/O is
3
+ // therefore parallel at the Temporal worker's activity-concurrency limit; one
4
+ // slow snapshot can delay only its own box, never the global reaper or a sibling.
5
+
6
+ import {
7
+ ParentClosePolicy,
8
+ WorkflowIdReusePolicy,
9
+ log,
10
+ proxyActivities,
11
+ startChild,
12
+ uuid4,
13
+ workflowInfo,
14
+ } from "@temporalio/workflow";
16
15
  import type * as activities from "../activities";
16
+ import {
17
+ SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
18
+ SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS,
19
+ SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
20
+ SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
21
+ SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
22
+ sandboxDrainWorkflowId,
23
+ sandboxLifecycleTaskQueue,
24
+ type SandboxDrainActivityInput,
25
+ type SandboxDrainWorkflowInput,
26
+ type SandboxLeaseSweepMaintenanceInput,
27
+ } from "../sandbox-reaper-contract";
28
+
29
+ function lifecycleTaskQueue(): string {
30
+ return sandboxLifecycleTaskQueue(workflowInfo().taskQueue);
31
+ }
32
+
33
+ function sweepActivities() {
34
+ return proxyActivities<Pick<typeof activities, "prepareSandboxLeaseSweep">>({
35
+ taskQueue: lifecycleTaskQueue(),
36
+ startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
37
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
38
+ // Inventory is the admission edge for every per-box child. A worker crash
39
+ // must not silently consume the only scheduled sweep.
40
+ retry: {
41
+ initialInterval: "1 second",
42
+ backoffCoefficient: 2,
43
+ maximumInterval: "30 seconds",
44
+ },
45
+ });
46
+ }
47
+
48
+ function maintenanceActivity() {
49
+ return proxyActivities<Pick<typeof activities, "maintainSandboxLeaseSweep">>({
50
+ taskQueue: lifecycleTaskQueue(),
51
+ startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
52
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
53
+ // Maintenance is ancillary and internally exact/idempotent. Never let one
54
+ // pathological pass keep the fixed-id V2 workflow open forever and thereby
55
+ // suppress all later drain inventories. The next Schedule tick retries it.
56
+ retry: { maximumAttempts: 1 },
57
+ });
58
+ }
59
+
60
+ function drainActivity(startToCloseTimeout: number) {
61
+ return proxyActivities<Pick<typeof activities, "drainSandboxLease">>({
62
+ taskQueue: lifecycleTaskQueue(),
63
+ startToCloseTimeout,
64
+ heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
65
+ // Each retry is a new exact DB capture attempt under one logical operation.
66
+ // Heartbeat timeout moves a crashed worker to a healthy poller promptly;
67
+ // exact-attempt CAS fencing makes late callbacks harmless.
68
+ retry: {
69
+ initialInterval: "1 second",
70
+ backoffCoefficient: 2,
71
+ maximumInterval: "30 seconds",
72
+ },
73
+ });
74
+ }
17
75
 
18
- // A short, bounded reaper activity. Unlike the 30-day agent turn, the sweep is a
19
- // fast DB pass + a handful of provider terminates; cap it well under the reaper
20
- // period so a wedged sweep is abandoned rather than blocking the next fire.
21
- const reaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
22
- startToCloseTimeout: "5 minutes",
76
+ // This proxy and workflow deliberately preserve the exact pre-refactor
77
+ // ScheduleActivity command. A patch marker would let a new binary replay old
78
+ // histories, but an old binary could not replay a history first executed by a
79
+ // new worker during a rolling deployment. Routing happens inside the activity,
80
+ // where changing implementation is safe and mixed old/new workers remain valid.
81
+ const legacyReaperActivity = proxyActivities<Pick<typeof activities, "reapSandboxLeases">>({
82
+ startToCloseTimeout: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
23
83
  retry: { maximumAttempts: 1 },
24
84
  });
25
85
 
86
+ export async function sandboxDrainWorkflow(input: SandboxDrainWorkflowInput): Promise<void> {
87
+ const activityInput: SandboxDrainActivityInput = {
88
+ ...input,
89
+ operationId: uuid4(),
90
+ };
91
+ if (input.timeoutClass === "fast") {
92
+ await drainActivity(SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS).drainSandboxLease(activityInput);
93
+ } else {
94
+ await drainActivity(SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS).drainSandboxLease(
95
+ activityInput,
96
+ );
97
+ }
98
+ }
99
+
100
+ /** Ancillary billing/reconciliation/GC is durable but never allowed to hold the
101
+ * fixed-ID inventory workflow open behind a provider backlog. One singleton
102
+ * maintenance child may queue on the lifecycle workers while every schedule
103
+ * tick remains free to inventory and dispatch newly-drainable boxes. */
104
+ export async function sandboxReaperMaintenanceWorkflow(
105
+ input: SandboxLeaseSweepMaintenanceInput,
106
+ ): Promise<void> {
107
+ try {
108
+ await maintenanceActivity().maintainSandboxLeaseSweep(input);
109
+ } catch (error) {
110
+ // The next Schedule tick starts another singleton after this run closes.
111
+ // Drain children were already made durable before this workflow was started.
112
+ log.warn("sandbox reaper maintenance deferred to a later sweep", {
113
+ error: error instanceof Error ? error.message : String(error),
114
+ });
115
+ }
116
+ }
117
+
118
+ function alreadyRunningChild(error: unknown): boolean {
119
+ let current: unknown = error;
120
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
121
+ const record = current as { name?: unknown; cause?: unknown };
122
+ if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
123
+ current = record.cause;
124
+ }
125
+ return false;
126
+ }
127
+
26
128
  export async function sandboxReaperWorkflow(): Promise<void> {
27
- await reaperActivity.reapSandboxLeases();
129
+ await legacyReaperActivity.reapSandboxLeases();
130
+ }
131
+
132
+ export async function sandboxReaperWorkflowV2(): Promise<void> {
133
+ const sweep = sweepActivities();
134
+ const plan = await sweep.prepareSandboxLeaseSweep();
135
+ const starts = await Promise.all(
136
+ plan.drainable.map(async (target) => {
137
+ try {
138
+ await startChild(sandboxDrainWorkflow, {
139
+ workflowId: sandboxDrainWorkflowId(target),
140
+ taskQueue: lifecycleTaskQueue(),
141
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
142
+ parentClosePolicy: ParentClosePolicy.ABANDON,
143
+ args: [
144
+ {
145
+ target,
146
+ timeoutClass: plan.timeoutClass,
147
+ snapshotTimeoutMs: plan.snapshotTimeoutMs,
148
+ captureTimeoutMs: plan.captureTimeoutMs,
149
+ },
150
+ ],
151
+ });
152
+ return "started" as const;
153
+ } catch (error) {
154
+ if (alreadyRunningChild(error)) return "already_running" as const;
155
+ log.error("sandbox reaper child start failed", {
156
+ workspaceId: target.workspaceId,
157
+ sandboxGroupId: target.sandboxGroupId,
158
+ leaseEpoch: target.leaseEpoch,
159
+ instanceId: target.instanceId,
160
+ error: error instanceof Error ? error.message : String(error),
161
+ });
162
+ return "failed" as const;
163
+ }
164
+ }),
165
+ );
166
+ try {
167
+ await startChild(sandboxReaperMaintenanceWorkflow, {
168
+ workflowId: SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
169
+ taskQueue: lifecycleTaskQueue(),
170
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
171
+ parentClosePolicy: ParentClosePolicy.ABANDON,
172
+ args: [
173
+ {
174
+ examined: plan.drainable.length,
175
+ started: starts.filter((outcome) => outcome === "started").length,
176
+ alreadyRunning: starts.filter((outcome) => outcome === "already_running").length,
177
+ startFailed: starts.filter((outcome) => outcome === "failed").length,
178
+ rotationsRequested: plan.rotationsRequested,
179
+ },
180
+ ],
181
+ });
182
+ } catch (error) {
183
+ if (alreadyRunningChild(error)) return;
184
+ // Every drain child is already durable and this inventory workflow still
185
+ // completes, so the next Schedule tick can retry maintenance independently.
186
+ log.warn("sandbox reaper maintenance child start deferred to next sweep", {
187
+ error: error instanceof Error ? error.message : String(error),
188
+ });
189
+ }
28
190
  }
@@ -1,6 +1,18 @@
1
- import { patched, workflowInfo } from "@temporalio/workflow";
1
+ import {
2
+ getExternalWorkflowHandle,
3
+ ParentClosePolicy,
4
+ patched,
5
+ startChild,
6
+ workflowInfo,
7
+ WorkflowIdReusePolicy,
8
+ } from "@temporalio/workflow";
2
9
  import type { TurnInitiator } from "@opengeni/contracts";
3
10
  import { scheduledTaskActivity } from "./activities";
11
+ import {
12
+ knowledgeSourceSyncWake,
13
+ knowledgeSourceSyncWorkflow,
14
+ knowledgeSourceSyncWorkflowId,
15
+ } from "./knowledge-source-sync";
4
16
 
5
17
  type ScheduledTaskFireWorkflowBase = {
6
18
  accountId: string;
@@ -16,7 +28,7 @@ export type ScheduledTaskFireWorkflowInput = ScheduledTaskFireWorkflowBase &
16
28
  initiator?: never;
17
29
  }
18
30
  | {
19
- triggerType: "manual";
31
+ triggerType: "manual" | "initial" | "provider_event" | "retry" | "repair";
20
32
  agentRunUsageIdempotencyKey: string;
21
33
  initiator: TurnInitiator;
22
34
  }
@@ -30,7 +42,7 @@ export async function scheduledTaskFireWorkflow(
30
42
  // the exact charging identity.
31
43
  if (
32
44
  patched("scheduled-task-manual-initiator-v1") &&
33
- input.triggerType === "manual" &&
45
+ input.triggerType !== "scheduled" &&
34
46
  (!input.agentRunUsageIdempotencyKey || !input.initiator)
35
47
  ) {
36
48
  return;
@@ -40,14 +52,38 @@ export async function scheduledTaskFireWorkflow(
40
52
  taskId: input.taskId,
41
53
  producerKey: workflowInfo().workflowId,
42
54
  };
43
- await scheduledTaskActivity.dispatchScheduledTaskRun(
44
- input.triggerType === "manual"
55
+ const result = await scheduledTaskActivity.dispatchScheduledTaskRun(
56
+ input.triggerType !== "scheduled"
45
57
  ? {
46
58
  ...base,
47
- triggerType: "manual",
59
+ triggerType: input.triggerType,
48
60
  agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
49
61
  initiator: input.initiator,
50
62
  }
51
63
  : { ...base, triggerType: "scheduled" },
52
64
  );
65
+ if (result.action !== "knowledge_source_sync") return;
66
+
67
+ const workflowId = knowledgeSourceSyncWorkflowId(result.sourceId);
68
+ try {
69
+ await startChild(knowledgeSourceSyncWorkflow, {
70
+ workflowId,
71
+ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
72
+ parentClosePolicy: ParentClosePolicy.ABANDON,
73
+ args: [result],
74
+ });
75
+ } catch (error) {
76
+ if (!alreadyRunningWorkflow(error)) throw error;
77
+ await getExternalWorkflowHandle(workflowId).signal(knowledgeSourceSyncWake, result);
78
+ }
79
+ }
80
+
81
+ function alreadyRunningWorkflow(error: unknown): boolean {
82
+ let current: unknown = error;
83
+ for (let depth = 0; depth < 4 && current && typeof current === "object"; depth += 1) {
84
+ const record = current as { name?: unknown; cause?: unknown };
85
+ if (record.name === "WorkflowExecutionAlreadyStartedError") return true;
86
+ current = record.cause;
87
+ }
88
+ return false;
53
89
  }
@@ -0,0 +1,36 @@
1
+ import { continueAsNew, sleep } from "@temporalio/workflow";
2
+ import { videoGenerationActivityForTaskQueue } from "./activities";
3
+
4
+ export type VideoGenerationWorkflowInput = {
5
+ accountId: string;
6
+ workspaceId: string;
7
+ operationId: string;
8
+ baseTaskQueue: string;
9
+ iterations?: number;
10
+ };
11
+
12
+ const CONTINUE_AS_NEW_AFTER = 100;
13
+
14
+ export function videoGenerationWorkflowId(operationId: string): string {
15
+ return `video-generation:${operationId}`;
16
+ }
17
+
18
+ /** Small deterministic orchestration; all private/provider state remains in Postgres. */
19
+ export async function videoGenerationWorkflow(input: VideoGenerationWorkflowInput): Promise<void> {
20
+ const activity = videoGenerationActivityForTaskQueue(input.baseTaskQueue);
21
+ let iterations = input.iterations ?? 0;
22
+ while (iterations < CONTINUE_AS_NEW_AFTER) {
23
+ const result = await activity.reconcileVideoGenerationOperation({
24
+ accountId: input.accountId,
25
+ workspaceId: input.workspaceId,
26
+ operationId: input.operationId,
27
+ });
28
+ if (result.action === "terminal") return;
29
+ await sleep(result.delayMs);
30
+ iterations += 1;
31
+ }
32
+ await continueAsNew<typeof videoGenerationWorkflow>({
33
+ ...input,
34
+ iterations: 0,
35
+ });
36
+ }