@opengeni/worker-bundle 0.16.21 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/dist/activities/agent-turn.d.ts +145 -32
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +3 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -7
  10. package/dist/activities/expiring-material-renewal.d.ts +5 -3
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +2 -2
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +32 -0
  25. package/dist/activities/retained-screenshots.d.ts +129 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +2 -2
  28. package/dist/activities/run-input.d.ts +5 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +11 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -147
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/{chunk-BJUW5LCM.js → chunk-RAMNYSAW.js} +15 -12
  64. package/dist/chunk-RAMNYSAW.js.map +1 -0
  65. package/dist/concurrency.d.ts +108 -5
  66. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  67. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  68. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  69. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  70. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  71. package/dist/editable-artifact-materializer.d.ts +262 -0
  72. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  73. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  74. package/dist/host-export.d.ts +1 -1
  75. package/dist/host-export.js +1 -1
  76. package/dist/index.d.ts +176 -10
  77. package/dist/index.js +675 -16460
  78. package/dist/index.js.map +1 -1
  79. package/dist/memory-pressure-guard.d.ts +57 -0
  80. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  81. package/dist/observability-metrics.d.ts +33 -0
  82. package/dist/sandbox-reaper-contract.d.ts +40 -0
  83. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  84. package/dist/sandbox-resume.d.ts +40 -7
  85. package/dist/sandbox-routing.d.ts +13 -6
  86. package/dist/worker-service-lifecycle.d.ts +6 -2
  87. package/dist/workflow-bundle.js +1044 -363
  88. package/dist/workflows/activities.d.ts +7 -1
  89. package/dist/workflows/document-index.d.ts +1 -1
  90. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  91. package/dist/workflows/rig-verification.d.ts +1 -1
  92. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  93. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  94. package/dist/workflows/session.d.ts +2 -2
  95. package/dist/workflows/video-generation.d.ts +10 -0
  96. package/dist/workflows/workflow-wake.d.ts +1 -1
  97. package/dist/workflows.d.ts +9 -7
  98. package/package.json +34 -15
  99. package/src/activities/agent-turn.ts +1905 -559
  100. package/src/activities/api-integrations.ts +138 -0
  101. package/src/activities/codemode-dispatcher.ts +335 -0
  102. package/src/activities/codemode-token-renewal.ts +64 -0
  103. package/src/activities/codex-capacity.ts +16 -7
  104. package/src/activities/codex-image-generation.ts +56 -0
  105. package/src/activities/codex-rotation.ts +19 -5
  106. package/src/activities/context-compaction.ts +47 -6
  107. package/src/activities/documents.ts +14 -4
  108. package/src/activities/environment.ts +52 -60
  109. package/src/activities/expiring-material-renewal.ts +16 -7
  110. package/src/activities/file-upload-reaper.ts +34 -3
  111. package/src/activities/gateway-image-generation.ts +146 -0
  112. package/src/activities/gateway-video-generation.ts +298 -0
  113. package/src/activities/generated-images.ts +860 -0
  114. package/src/activities/git-credential-renewal.ts +3 -3
  115. package/src/activities/goals.ts +6 -2
  116. package/src/activities/google-drive-changes.ts +454 -0
  117. package/src/activities/image-generation-operation.ts +333 -0
  118. package/src/activities/image-generation-references.ts +118 -0
  119. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  120. package/src/activities/knowledge-source-sync.ts +2840 -0
  121. package/src/activities/mcp-credentials.ts +1 -0
  122. package/src/activities/packs.ts +55 -55
  123. package/src/activities/recording.ts +4 -5
  124. package/src/activities/retained-screenshot-reaper.ts +202 -0
  125. package/src/activities/retained-screenshots.ts +1317 -0
  126. package/src/activities/rig-verification.ts +567 -77
  127. package/src/activities/run-credential-renewal.ts +9 -2
  128. package/src/activities/run-input.ts +40 -10
  129. package/src/activities/sandbox-images.ts +167 -0
  130. package/src/activities/sandbox-lease.ts +1065 -483
  131. package/src/activities/scheduled-tasks.ts +96 -7
  132. package/src/activities/session-state.ts +4 -4
  133. package/src/activities/streaming.ts +42 -3
  134. package/src/activities/types.ts +87 -5
  135. package/src/activities/video-generation-admission.ts +270 -0
  136. package/src/activities/video-generation-credential.ts +34 -0
  137. package/src/activities/video-generation-reconciliation.ts +578 -0
  138. package/src/activities/video-media-validation.ts +326 -0
  139. package/src/activities/video-output-retention.ts +314 -0
  140. package/src/activities/video-reference-staging.ts +263 -0
  141. package/src/activities/workflow-wake.ts +2 -2
  142. package/src/activities/workspace-capture.ts +77 -13
  143. package/src/activities-control.ts +54 -0
  144. package/src/activities-turn.ts +40 -0
  145. package/src/activities.ts +50 -118
  146. package/src/activity-services.ts +62 -0
  147. package/src/artifact-materializer-entry.ts +35 -0
  148. package/src/artifact-outbox-entry.ts +29 -0
  149. package/src/concurrency.ts +484 -6
  150. package/src/editable-artifact-hint-broker.ts +235 -0
  151. package/src/editable-artifact-materializer-launcher.ts +515 -0
  152. package/src/editable-artifact-materializer-service.ts +787 -0
  153. package/src/editable-artifact-materializer-storage.ts +413 -0
  154. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  155. package/src/editable-artifact-materializer.ts +1289 -0
  156. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  157. package/src/editable-artifact-outbox-service.ts +438 -0
  158. package/src/host-export-pump.ts +28 -11
  159. package/src/http.ts +3 -6
  160. package/src/index.ts +277 -72
  161. package/src/memory-pressure-guard.ts +489 -0
  162. package/src/model-checkpoint-memory-collector.ts +116 -0
  163. package/src/observability-metrics.ts +202 -7
  164. package/src/sandbox-reaper-contract.ts +73 -0
  165. package/src/sandbox-reaper-timeout.ts +79 -0
  166. package/src/sandbox-resume.ts +331 -157
  167. package/src/sandbox-routing.ts +49 -39
  168. package/src/worker-service-lifecycle.ts +82 -12
  169. package/src/workflows/activities.ts +30 -0
  170. package/src/workflows/file-upload-reaper.ts +6 -1
  171. package/src/workflows/knowledge-source-sync.ts +83 -0
  172. package/src/workflows/sandbox-reaper.ts +183 -21
  173. package/src/workflows/scheduled-tasks.ts +42 -6
  174. package/src/workflows/video-generation.ts +36 -0
  175. package/src/workflows.ts +17 -1
  176. package/dist/activities/redaction.d.ts +0 -1
  177. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  178. package/dist/chunk-BJUW5LCM.js.map +0 -1
  179. package/src/activities/redaction.ts +0 -9
  180. package/src/activities/toolspace-token-renewal.ts +0 -57
@@ -15,6 +15,23 @@ export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
15
15
  export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
16
16
  export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
17
17
  export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
18
+ export type TurnTaskQueueStats = {
19
+ /** Temporal activity tasks ready for a turn worker to accept. */
20
+ eligibleBacklog: number;
21
+ oldestBacklogAgeSeconds: number;
22
+ tasksAddRate: number;
23
+ tasksDispatchRate: number;
24
+ };
25
+
26
+ export type TemporalTurnTaskQueueStats = {
27
+ approximateBacklogCount?: unknown;
28
+ approximateBacklogAge?: {
29
+ seconds?: unknown;
30
+ nanos?: unknown;
31
+ } | null;
32
+ tasksAddRate?: unknown;
33
+ tasksDispatchRate?: unknown;
34
+ } | null;
18
35
 
19
36
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
20
37
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
@@ -37,11 +54,11 @@ function eventAttributes(attributes: Record<string, unknown> | undefined): Attri
37
54
  if (!attributes) {
38
55
  return undefined;
39
56
  }
40
- const sanitized: Attributes = {};
57
+ const projected: Attributes = {};
41
58
  for (const [key, value] of Object.entries(attributes)) {
42
- sanitized[key] = eventAttributeValue(value);
59
+ projected[key] = eventAttributeValue(value);
43
60
  }
44
- return sanitized;
61
+ return projected;
45
62
  }
46
63
 
47
64
  function eventAttributeValue(value: unknown): AttributeValue {
@@ -78,23 +95,24 @@ export function runtimeMetricsHooksForObservability(
78
95
  value: durationSeconds,
79
96
  });
80
97
  },
81
- onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
98
+ onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {
82
99
  observability.incrementCounter({
83
100
  name: "opengeni_sandbox_creates_total",
84
101
  help: "Total sandbox create attempts by backend and outcome.",
85
- labels: { backend, outcome },
102
+ labels: { backend, image_source: imageSource, outcome },
86
103
  });
87
104
  observability.observeHistogram({
88
105
  name: "opengeni_sandbox_create_duration_seconds",
89
106
  help: "Sandbox create duration in seconds by backend.",
90
- labels: { backend },
107
+ labels: { backend, image_source: imageSource },
91
108
  value: durationSeconds,
92
109
  });
93
110
  },
94
- onSandboxWarmingTimeout: () => {
111
+ onSandboxWarmingTimeout: ({ backend, stage }) => {
95
112
  observability.incrementCounter({
96
113
  name: "opengeni_sandbox_warming_timeouts_total",
97
114
  help: "Total sandbox warming timeouts.",
115
+ labels: { backend, stage },
98
116
  });
99
117
  },
100
118
  onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
@@ -330,6 +348,152 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
330
348
  });
331
349
  }
332
350
 
351
+ /**
352
+ * Record the authoritative turn activity queue rather than aggregate Postgres
353
+ * prompts. A paused human prompt is durable queue truth but is not runnable and
354
+ * never reaches this Temporal task queue. DescribeTaskQueue's approximate
355
+ * backlog count and age are explicitly documented by Temporal as autoscaling
356
+ * signals.
357
+ */
358
+ export function recordTurnTaskQueueStats(
359
+ observability: Observability,
360
+ stats: TurnTaskQueueStats,
361
+ ): void {
362
+ observability.setGauge({
363
+ name: "opengeni_turn_eligible_backlog",
364
+ help: "Temporal runAgentTurn activity tasks eligible for immediate worker admission.",
365
+ value: nonnegativeFinite(stats.eligibleBacklog),
366
+ });
367
+ observability.setGauge({
368
+ name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
369
+ help: "Approximate age of the oldest eligible runAgentTurn activity task.",
370
+ value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),
371
+ });
372
+ observability.setGauge({
373
+ name: "opengeni_turn_eligible_tasks_add_rate",
374
+ help: "Temporal runAgentTurn tasks added per second over its rolling window.",
375
+ value: nonnegativeFinite(stats.tasksAddRate),
376
+ });
377
+ observability.setGauge({
378
+ name: "opengeni_turn_eligible_tasks_dispatch_rate",
379
+ help: "Temporal runAgentTurn tasks dispatched per second over its rolling window.",
380
+ value: nonnegativeFinite(stats.tasksDispatchRate),
381
+ });
382
+ }
383
+
384
+ export function normalizeTurnTaskQueueStats(
385
+ stats: TemporalTurnTaskQueueStats | undefined,
386
+ ): TurnTaskQueueStats {
387
+ if (!stats) {
388
+ throw new Error("Temporal DescribeTaskQueue response omitted required stats");
389
+ }
390
+ const eligibleBacklog = requiredTemporalNumber(
391
+ stats.approximateBacklogCount,
392
+ "approximateBacklogCount",
393
+ { integer: true },
394
+ );
395
+ let oldestBacklogAgeSeconds = 0;
396
+ if (stats.approximateBacklogAge) {
397
+ const seconds = optionalTemporalNumber(
398
+ stats.approximateBacklogAge.seconds,
399
+ "approximateBacklogAge.seconds",
400
+ { integer: true },
401
+ );
402
+ const nanos = optionalTemporalNumber(
403
+ stats.approximateBacklogAge.nanos,
404
+ "approximateBacklogAge.nanos",
405
+ { integer: true },
406
+ );
407
+ if (nanos >= 1_000_000_000) {
408
+ throw new Error("Temporal approximateBacklogAge.nanos must be less than one second");
409
+ }
410
+ oldestBacklogAgeSeconds = seconds + nanos / 1_000_000_000;
411
+ } else if (eligibleBacklog > 0) {
412
+ throw new Error("Temporal stats omitted approximateBacklogAge for a nonzero backlog");
413
+ }
414
+ return {
415
+ eligibleBacklog,
416
+ oldestBacklogAgeSeconds,
417
+ tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
418
+ tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate"),
419
+ };
420
+ }
421
+
422
+ export function startTurnCapacityMonitor(input: {
423
+ observability: Observability;
424
+ read: () => Promise<TurnTaskQueueStats>;
425
+ intervalMs?: number;
426
+ now?: () => number;
427
+ }): { close: () => Promise<void> } {
428
+ const intervalMs = input.intervalMs ?? 15_000;
429
+ const now = input.now ?? Date.now;
430
+ const startedAt = now();
431
+ let lastSuccessAt: number | null = null;
432
+ let lastReadSucceeded = false;
433
+ let stopped = false;
434
+ let running: Promise<void> | null = null;
435
+ const recordStatus = () => {
436
+ const observedAt = now();
437
+ const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);
438
+ const set = (name: string, help: string, value: number) =>
439
+ input.observability.setGauge({ name, help, value });
440
+ set(
441
+ "opengeni_turn_capacity_monitor_last_read_success",
442
+ "Whether the latest Temporal turn-queue capacity read completed successfully.",
443
+ lastReadSucceeded ? 1 : 0,
444
+ );
445
+ set(
446
+ "opengeni_turn_capacity_monitor_last_success_timestamp_seconds",
447
+ "Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.",
448
+ lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,
449
+ );
450
+ set(
451
+ "opengeni_turn_capacity_monitor_last_success_age_seconds",
452
+ "Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.",
453
+ Math.max(0, successAgeMs / 1_000),
454
+ );
455
+ set(
456
+ "opengeni_turn_capacity_monitor_fresh",
457
+ "Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.",
458
+ lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
459
+ );
460
+ };
461
+ const refresh = () => {
462
+ // Advance freshness age even while a Temporal read is hung. Old backlog
463
+ // values remain observable for diagnosis but cease to be authoritative.
464
+ recordStatus();
465
+ if (stopped || running) return;
466
+ running = input
467
+ .read()
468
+ .then((stats) => {
469
+ recordTurnTaskQueueStats(input.observability, stats);
470
+ lastSuccessAt = now();
471
+ lastReadSucceeded = true;
472
+ recordStatus();
473
+ })
474
+ .catch((error) => {
475
+ lastReadSucceeded = false;
476
+ recordStatus();
477
+ input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
478
+ error: error instanceof Error ? error.message : String(error),
479
+ });
480
+ })
481
+ .finally(() => {
482
+ running = null;
483
+ });
484
+ };
485
+ refresh();
486
+ const timer = setInterval(refresh, intervalMs);
487
+ timer.unref?.();
488
+ return {
489
+ close: async () => {
490
+ stopped = true;
491
+ clearInterval(timer);
492
+ await running;
493
+ },
494
+ };
495
+ }
496
+
333
497
  export function recordSandboxLeaseGauges(
334
498
  observability: Observability,
335
499
  counts: Partial<Record<SandboxLeaseLiveness, number>>,
@@ -948,6 +1112,37 @@ function incrementBoundedModelCacheCounter(
948
1112
  totals.set(key, current + input.amount);
949
1113
  }
950
1114
 
1115
+ function nonnegativeFinite(value: number): number {
1116
+ return Number.isFinite(value) && value > 0 ? value : 0;
1117
+ }
1118
+
1119
+ function requiredTemporalNumber(
1120
+ value: unknown,
1121
+ field: string,
1122
+ options: { integer?: boolean } = {},
1123
+ ): number {
1124
+ if (value === null || value === undefined) {
1125
+ throw new Error(`Temporal stats omitted required ${field}`);
1126
+ }
1127
+ const parsed = Number(value);
1128
+ if (
1129
+ !Number.isFinite(parsed) ||
1130
+ parsed < 0 ||
1131
+ (options.integer && !Number.isSafeInteger(parsed))
1132
+ ) {
1133
+ throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);
1134
+ }
1135
+ return parsed;
1136
+ }
1137
+
1138
+ function optionalTemporalNumber(
1139
+ value: unknown,
1140
+ field: string,
1141
+ options: { integer?: boolean } = {},
1142
+ ): number {
1143
+ return value === null || value === undefined ? 0 : requiredTemporalNumber(value, field, options);
1144
+ }
1145
+
951
1146
  /**
952
1147
  * An opaque, stable, non-reversible tag for a codex credential (account) — the
953
1148
  * per-call log dimension the account-switch hypothesis correlates against. It is
@@ -0,0 +1,73 @@
1
+ export const SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
2
+ export const SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS = 5 * 60_000;
3
+ export const SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS = 65 * 60_000;
4
+ export const SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS = 60_000;
5
+ export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 20_000;
6
+ export const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS = 5_000;
7
+ // Match the DB inventory's fair 500-row bound so one sweep never discovers a
8
+ // due box and then needlessly defers it for another schedule period. Five
9
+ // hundred small StartChild commands remain far below Temporal history limits;
10
+ // actual provider I/O stays bounded by worker activity concurrency.
11
+ export const SANDBOX_REAPER_CHILD_DISPATCH_LIMIT = 500;
12
+ export const SANDBOX_REAPER_V2_WORKFLOW_ID = "opengeni-sandbox-lease-reaper-v2";
13
+ export const SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID = "opengeni-sandbox-lease-maintenance-v1";
14
+
15
+ // A protocol-specific queue is the rolling-deploy compatibility boundary for
16
+ // the split reaper activities. An old control worker polls only the base queue,
17
+ // so it can replay already-started legacy workflows but can never receive a new
18
+ // activity type that its binary does not register.
19
+ export const SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX = "-sandbox-lifecycle-v1";
20
+
21
+ export function sandboxLifecycleTaskQueue(baseTaskQueue: string): string {
22
+ return baseTaskQueue.endsWith(SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX)
23
+ ? baseTaskQueue
24
+ : `${baseTaskQueue}${SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX}`;
25
+ }
26
+
27
+ export type SandboxDrainTimeoutClass = "fast" | "extended";
28
+
29
+ export type SandboxDrainTarget = {
30
+ workspaceId: string;
31
+ sandboxGroupId: string;
32
+ instanceId: string | null;
33
+ leaseEpoch: number;
34
+ };
35
+
36
+ export type SandboxDrainActivityInput = {
37
+ target: SandboxDrainTarget;
38
+ timeoutClass: SandboxDrainTimeoutClass;
39
+ /** Provider operation budget frozen when the durable child is created. A
40
+ * rolling config change cannot make an existing fast child retry forever. */
41
+ snapshotTimeoutMs: number;
42
+ captureTimeoutMs: number;
43
+ /** Stable logical teardown operation; every Temporal activity retry keeps it
44
+ * while receiving a distinct exact capture-attempt id. */
45
+ operationId: string;
46
+ };
47
+
48
+ export type SandboxDrainWorkflowInput = Omit<SandboxDrainActivityInput, "operationId">;
49
+
50
+ export type SandboxLeaseSweepMaintenanceInput = {
51
+ examined: number;
52
+ started: number;
53
+ alreadyRunning: number;
54
+ startFailed: number;
55
+ rotationsRequested: number;
56
+ };
57
+
58
+ export function sandboxDrainTimeoutClass(captureTimeoutMs: number): SandboxDrainTimeoutClass {
59
+ return captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS <
60
+ SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
61
+ ? "fast"
62
+ : "extended";
63
+ }
64
+
65
+ export function sandboxDrainActivityTimeoutMs(timeoutClass: SandboxDrainTimeoutClass): number {
66
+ return timeoutClass === "fast"
67
+ ? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
68
+ : SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS;
69
+ }
70
+
71
+ export function sandboxDrainWorkflowId(target: SandboxDrainTarget): string {
72
+ return `sandbox-drain:${target.workspaceId}:${target.sandboxGroupId}:${target.leaseEpoch}`;
73
+ }
@@ -0,0 +1,79 @@
1
+ import {
2
+ SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
3
+ SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,
4
+ sandboxArchiveCaptureTimeoutMs,
5
+ type Settings,
6
+ } from "@opengeni/config";
7
+ import {
8
+ SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS,
9
+ sandboxDrainActivityTimeoutMs,
10
+ sandboxDrainTimeoutClass,
11
+ type SandboxDrainTimeoutClass,
12
+ } from "./sandbox-reaper-contract";
13
+
14
+ const STRICT_TIMEOUT_FENCE_MS = 1;
15
+
16
+ export function sandboxDrainTiming(settings: Pick<Settings, "sandboxSnapshotTimeoutMs">): {
17
+ snapshotTimeoutMs: number;
18
+ captureTimeoutMs: number;
19
+ timeoutClass: SandboxDrainTimeoutClass;
20
+ activityTimeoutMs: number;
21
+ } {
22
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
23
+ const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);
24
+ return {
25
+ snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,
26
+ captureTimeoutMs,
27
+ timeoutClass,
28
+ activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),
29
+ };
30
+ }
31
+
32
+ export function assertSandboxReaperActivityTimeout(
33
+ settings: Pick<Settings, "sandboxSnapshotTimeoutMs">,
34
+ ): void {
35
+ const timing = sandboxDrainTiming(settings);
36
+ if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {
37
+ throw new Error(
38
+ `Sandbox provider snapshot timeout (${timing.snapshotTimeoutMs}) must be strictly less than ` +
39
+ `its durable capture claim (${timing.captureTimeoutMs}).`,
40
+ );
41
+ }
42
+ if (
43
+ timing.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS + STRICT_TIMEOUT_FENCE_MS >=
44
+ timing.activityTimeoutMs
45
+ ) {
46
+ throw new Error(
47
+ `Sandbox ${timing.timeoutClass} drain activity timeout (${timing.activityTimeoutMs}) must ` +
48
+ `strictly exceed one durable archive capture and settlement margin ` +
49
+ `(${timing.captureTimeoutMs} + ${SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS} milliseconds).`,
50
+ );
51
+ }
52
+ }
53
+
54
+ export function assertSandboxDrainInputTiming(input: {
55
+ timeoutClass: SandboxDrainTimeoutClass;
56
+ snapshotTimeoutMs: number;
57
+ captureTimeoutMs: number;
58
+ }): void {
59
+ if (
60
+ !Number.isSafeInteger(input.snapshotTimeoutMs) ||
61
+ input.snapshotTimeoutMs <= 0 ||
62
+ input.snapshotTimeoutMs > SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS ||
63
+ !Number.isSafeInteger(input.captureTimeoutMs) ||
64
+ input.captureTimeoutMs <= 0 ||
65
+ input.captureTimeoutMs > SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS
66
+ ) {
67
+ throw new Error("Sandbox drain timeout values are invalid");
68
+ }
69
+ const inputActivityTimeoutMs = sandboxDrainActivityTimeoutMs(input.timeoutClass);
70
+ if (
71
+ sandboxDrainTimeoutClass(input.captureTimeoutMs) !== input.timeoutClass ||
72
+ input.snapshotTimeoutMs >= input.captureTimeoutMs ||
73
+ input.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS >= inputActivityTimeoutMs
74
+ ) {
75
+ throw new Error(
76
+ `Sandbox drain timing is inconsistent with the ${input.timeoutClass} activity budget`,
77
+ );
78
+ }
79
+ }