@opengeni/worker-bundle 0.16.29 → 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/activities/agent-turn.d.ts +85 -19
  2. package/dist/activities/api-integrations.d.ts +21 -0
  3. package/dist/activities/codemode-dispatcher.d.ts +35 -0
  4. package/dist/activities/codemode-token-renewal.d.ts +23 -0
  5. package/dist/activities/codex-capacity.d.ts +5 -5
  6. package/dist/activities/codex-image-generation.d.ts +21 -0
  7. package/dist/activities/context-compaction.d.ts +1 -0
  8. package/dist/activities/documents.d.ts +3 -2
  9. package/dist/activities/environment.d.ts +11 -8
  10. package/dist/activities/expiring-material-renewal.d.ts +1 -1
  11. package/dist/activities/file-upload-reaper.d.ts +7 -2
  12. package/dist/activities/gateway-image-generation.d.ts +39 -0
  13. package/dist/activities/gateway-video-generation.d.ts +53 -0
  14. package/dist/activities/generated-images.d.ts +112 -0
  15. package/dist/activities/git-credential-renewal.d.ts +1 -1
  16. package/dist/activities/goals.d.ts +2 -2
  17. package/dist/activities/google-drive-changes.d.ts +99 -0
  18. package/dist/activities/image-generation-operation.d.ts +74 -0
  19. package/dist/activities/image-generation-references.d.ts +25 -0
  20. package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
  21. package/dist/activities/knowledge-source-sync.d.ts +75 -0
  22. package/dist/activities/packs.d.ts +13 -26
  23. package/dist/activities/parent-wake.d.ts +1 -1
  24. package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
  25. package/dist/activities/retained-screenshots.d.ts +36 -0
  26. package/dist/activities/rig-verification.d.ts +82 -5
  27. package/dist/activities/run-credential-renewal.d.ts +1 -1
  28. package/dist/activities/run-input.d.ts +3 -1
  29. package/dist/activities/sandbox-images.d.ts +14 -0
  30. package/dist/activities/sandbox-lease.d.ts +47 -9
  31. package/dist/activities/scheduled-tasks.d.ts +2 -2
  32. package/dist/activities/session-state.d.ts +4 -4
  33. package/dist/activities/streaming.d.ts +1 -1
  34. package/dist/activities/types.d.ts +71 -5
  35. package/dist/activities/video-generation-admission.d.ts +38 -0
  36. package/dist/activities/video-generation-credential.d.ts +7 -0
  37. package/dist/activities/video-generation-reconciliation.d.ts +13 -0
  38. package/dist/activities/video-media-validation.d.ts +34 -0
  39. package/dist/activities/video-output-retention.d.ts +36 -0
  40. package/dist/activities/video-reference-staging.d.ts +49 -0
  41. package/dist/activities/workflow-wake.d.ts +2 -2
  42. package/dist/activities/workspace-capture.d.ts +10 -1
  43. package/dist/activities-control-44KBWLSM.js +6885 -0
  44. package/dist/activities-control-44KBWLSM.js.map +1 -0
  45. package/dist/activities-control.d.ts +289 -0
  46. package/dist/activities-turn-K4CEDEGJ.js +14675 -0
  47. package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
  48. package/dist/activities-turn.d.ts +17 -0
  49. package/dist/activities.d.ts +130 -150
  50. package/dist/activity-services.d.ts +10 -0
  51. package/dist/artifact-materializer-entry.d.ts +6 -0
  52. package/dist/artifact-materializer-entry.js +2662 -0
  53. package/dist/artifact-materializer-entry.js.map +1 -0
  54. package/dist/artifact-outbox-entry.d.ts +2 -0
  55. package/dist/artifact-outbox-entry.js +681 -0
  56. package/dist/artifact-outbox-entry.js.map +1 -0
  57. package/dist/chunk-E3RI3I33.js +828 -0
  58. package/dist/chunk-E3RI3I33.js.map +1 -0
  59. package/dist/chunk-GRFQTTSQ.js +448 -0
  60. package/dist/chunk-GRFQTTSQ.js.map +1 -0
  61. package/dist/chunk-KKCRGAGZ.js +3091 -0
  62. package/dist/chunk-KKCRGAGZ.js.map +1 -0
  63. package/dist/concurrency.d.ts +108 -5
  64. package/dist/editable-artifact-hint-broker.d.ts +49 -0
  65. package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
  66. package/dist/editable-artifact-materializer-service.d.ts +108 -0
  67. package/dist/editable-artifact-materializer-storage.d.ts +12 -0
  68. package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
  69. package/dist/editable-artifact-materializer.d.ts +262 -0
  70. package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
  71. package/dist/editable-artifact-outbox-service.d.ts +70 -0
  72. package/dist/host-export.d.ts +1 -1
  73. package/dist/index.d.ts +176 -10
  74. package/dist/index.js +662 -17374
  75. package/dist/index.js.map +1 -1
  76. package/dist/memory-pressure-guard.d.ts +57 -0
  77. package/dist/model-checkpoint-memory-collector.d.ts +22 -0
  78. package/dist/observability-metrics.d.ts +33 -0
  79. package/dist/sandbox-reaper-contract.d.ts +40 -0
  80. package/dist/sandbox-reaper-timeout.d.ts +14 -0
  81. package/dist/sandbox-resume.d.ts +34 -7
  82. package/dist/sandbox-routing.d.ts +2 -0
  83. package/dist/worker-service-lifecycle.d.ts +6 -2
  84. package/dist/workflow-bundle.js +1043 -363
  85. package/dist/workflows/activities.d.ts +7 -1
  86. package/dist/workflows/document-index.d.ts +1 -1
  87. package/dist/workflows/knowledge-source-sync.d.ts +13 -0
  88. package/dist/workflows/rig-verification.d.ts +1 -1
  89. package/dist/workflows/sandbox-reaper.d.ts +8 -0
  90. package/dist/workflows/scheduled-tasks.d.ts +1 -1
  91. package/dist/workflows/session.d.ts +2 -2
  92. package/dist/workflows/video-generation.d.ts +10 -0
  93. package/dist/workflows/workflow-wake.d.ts +1 -1
  94. package/dist/workflows.d.ts +9 -7
  95. package/package.json +34 -15
  96. package/src/activities/agent-turn.ts +1470 -246
  97. package/src/activities/api-integrations.ts +138 -0
  98. package/src/activities/codemode-dispatcher.ts +335 -0
  99. package/src/activities/codemode-token-renewal.ts +64 -0
  100. package/src/activities/codex-capacity.ts +6 -6
  101. package/src/activities/codex-image-generation.ts +56 -0
  102. package/src/activities/context-compaction.ts +38 -3
  103. package/src/activities/documents.ts +14 -4
  104. package/src/activities/environment.ts +45 -42
  105. package/src/activities/expiring-material-renewal.ts +1 -1
  106. package/src/activities/file-upload-reaper.ts +34 -3
  107. package/src/activities/gateway-image-generation.ts +146 -0
  108. package/src/activities/gateway-video-generation.ts +298 -0
  109. package/src/activities/generated-images.ts +860 -0
  110. package/src/activities/goals.ts +6 -2
  111. package/src/activities/google-drive-changes.ts +454 -0
  112. package/src/activities/image-generation-operation.ts +333 -0
  113. package/src/activities/image-generation-references.ts +118 -0
  114. package/src/activities/knowledge-source-sync-driver.ts +27 -0
  115. package/src/activities/knowledge-source-sync.ts +2840 -0
  116. package/src/activities/mcp-credentials.ts +1 -0
  117. package/src/activities/packs.ts +55 -55
  118. package/src/activities/retained-screenshot-reaper.ts +2 -2
  119. package/src/activities/retained-screenshots.ts +453 -28
  120. package/src/activities/rig-verification.ts +554 -43
  121. package/src/activities/run-input.ts +22 -6
  122. package/src/activities/sandbox-images.ts +167 -0
  123. package/src/activities/sandbox-lease.ts +1065 -483
  124. package/src/activities/scheduled-tasks.ts +77 -4
  125. package/src/activities/session-state.ts +4 -4
  126. package/src/activities/streaming.ts +42 -3
  127. package/src/activities/types.ts +87 -5
  128. package/src/activities/video-generation-admission.ts +270 -0
  129. package/src/activities/video-generation-credential.ts +34 -0
  130. package/src/activities/video-generation-reconciliation.ts +578 -0
  131. package/src/activities/video-media-validation.ts +326 -0
  132. package/src/activities/video-output-retention.ts +314 -0
  133. package/src/activities/video-reference-staging.ts +263 -0
  134. package/src/activities/workflow-wake.ts +2 -2
  135. package/src/activities/workspace-capture.ts +44 -10
  136. package/src/activities-control.ts +54 -0
  137. package/src/activities-turn.ts +40 -0
  138. package/src/activities.ts +48 -124
  139. package/src/activity-services.ts +62 -0
  140. package/src/artifact-materializer-entry.ts +35 -0
  141. package/src/artifact-outbox-entry.ts +29 -0
  142. package/src/concurrency.ts +484 -6
  143. package/src/editable-artifact-hint-broker.ts +235 -0
  144. package/src/editable-artifact-materializer-launcher.ts +515 -0
  145. package/src/editable-artifact-materializer-service.ts +787 -0
  146. package/src/editable-artifact-materializer-storage.ts +413 -0
  147. package/src/editable-artifact-materializer-subprocess.ts +956 -0
  148. package/src/editable-artifact-materializer.ts +1289 -0
  149. package/src/editable-artifact-outbox-dispatcher.ts +244 -0
  150. package/src/editable-artifact-outbox-service.ts +438 -0
  151. package/src/index.ts +277 -72
  152. package/src/memory-pressure-guard.ts +489 -0
  153. package/src/model-checkpoint-memory-collector.ts +116 -0
  154. package/src/observability-metrics.ts +199 -4
  155. package/src/sandbox-reaper-contract.ts +73 -0
  156. package/src/sandbox-reaper-timeout.ts +79 -0
  157. package/src/sandbox-resume.ts +294 -153
  158. package/src/sandbox-routing.ts +19 -24
  159. package/src/worker-service-lifecycle.ts +80 -14
  160. package/src/workflows/activities.ts +30 -0
  161. package/src/workflows/file-upload-reaper.ts +3 -1
  162. package/src/workflows/knowledge-source-sync.ts +83 -0
  163. package/src/workflows/sandbox-reaper.ts +183 -21
  164. package/src/workflows/scheduled-tasks.ts +42 -6
  165. package/src/workflows/video-generation.ts +36 -0
  166. package/src/workflows.ts +17 -1
  167. package/dist/activities/toolspace-token-renewal.d.ts +0 -23
  168. package/src/activities/toolspace-token-renewal.ts +0 -64
@@ -0,0 +1,62 @@
1
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
2
+ import { createDb } from "@opengeni/db";
3
+ import { createNatsEventBus } from "@opengeni/events";
4
+ import { createObservability } from "@opengeni/observability";
5
+ import { createObjectStorage } from "@opengeni/storage";
6
+ import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
7
+ import { observabilityEventLogger } from "./observability-metrics";
8
+
9
+ /**
10
+ * Build the dependency graph common to both worker roles exactly once.
11
+ *
12
+ * This module deliberately has no import edge to @opengeni/runtime or
13
+ * @opengeni/documents. Those graphs are added by the role-specific factories,
14
+ * preventing an idle process from retaining an unused model SDK or parser
15
+ * stack for its entire lifetime.
16
+ */
17
+ export function createSharedActivityServices(
18
+ dependencies: ActivityDependencies,
19
+ ): () => Promise<SharedActivityServices> {
20
+ let servicesPromise: Promise<SharedActivityServices> | null = null;
21
+
22
+ return async function services(): Promise<SharedActivityServices> {
23
+ servicesPromise ??= (async () => {
24
+ const settings = dependencies.settings ?? getSettings();
25
+ const observability =
26
+ dependencies.observability ?? createObservability(settings, { component: "worker" });
27
+ const searchPath = dbSearchPath(settings);
28
+ const dbClient = dependencies.db
29
+ ? null
30
+ : createDb(settings.databaseUrl, {
31
+ ...(searchPath ? { searchPath } : {}),
32
+ rlsStrategy: settings.rlsStrategy,
33
+ });
34
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
35
+
36
+ return {
37
+ settings,
38
+ db: dependencies.db ?? dbClient!.db,
39
+ bus:
40
+ dependencies.bus ??
41
+ (await createNatsEventBus(
42
+ settings.natsUrl,
43
+ controlPlaneAuth
44
+ ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
45
+ : undefined,
46
+ { logger: observabilityEventLogger(observability) },
47
+ )),
48
+ objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
49
+ observability,
50
+ wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
51
+ signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
52
+ inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
53
+ signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
54
+ startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
55
+ startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
56
+ entitlements: dependencies.entitlements ?? null,
57
+ connectionCredentials: dependencies.connectionCredentials ?? null,
58
+ };
59
+ })();
60
+ return await servicesPromise;
61
+ };
62
+ }
@@ -0,0 +1,35 @@
1
+ import { createMaterializerSidecarFromEnvironment } from "./editable-artifact-materializer-service";
2
+
3
+ /**
4
+ * Dedicated editable-artifact materializer process. This entry deliberately
5
+ * does not import/create a Temporal worker and is never called by control/turn
6
+ * startup. Deploy it as a separate sidecar/service with its own DSN.
7
+ */
8
+ export async function runEditableArtifactMaterializerSidecar(
9
+ environment: NodeJS.ProcessEnv = process.env,
10
+ ): Promise<void> {
11
+ const configured = await createMaterializerSidecarFromEnvironment(environment);
12
+ if (!configured) {
13
+ throw new Error("Editable artifact materializer sidecar is disabled");
14
+ }
15
+ const server = Bun.serve({
16
+ hostname: configured.httpHostname,
17
+ port: configured.httpPort,
18
+ fetch: configured.service.fetch,
19
+ });
20
+ const stop = () => configured.service.drain("process signal");
21
+ process.on("SIGTERM", stop);
22
+ process.on("SIGINT", stop);
23
+ try {
24
+ await configured.service.run();
25
+ } finally {
26
+ process.off("SIGTERM", stop);
27
+ process.off("SIGINT", stop);
28
+ server.stop(true);
29
+ await configured.service.close();
30
+ }
31
+ }
32
+
33
+ if (import.meta.main) {
34
+ await runEditableArtifactMaterializerSidecar();
35
+ }
@@ -0,0 +1,29 @@
1
+ import { createOutboxSidecarFromEnvironment } from "./editable-artifact-outbox-service";
2
+
3
+ /** Dedicated process: no Temporal/control worker is created or imported here. */
4
+ export async function runEditableArtifactOutboxSidecar(
5
+ environment: NodeJS.ProcessEnv = process.env,
6
+ ): Promise<void> {
7
+ const configured = await createOutboxSidecarFromEnvironment(environment);
8
+ if (!configured) throw new Error("Editable artifact outbox sidecar is disabled");
9
+ const server = Bun.serve({
10
+ hostname: "0.0.0.0",
11
+ port: configured.httpPort,
12
+ fetch: configured.service.fetch,
13
+ });
14
+ const stop = () => configured.service.drain("process signal");
15
+ process.on("SIGTERM", stop);
16
+ process.on("SIGINT", stop);
17
+ try {
18
+ await configured.service.run();
19
+ } finally {
20
+ process.off("SIGTERM", stop);
21
+ process.off("SIGINT", stop);
22
+ server.stop(true);
23
+ await configured.service.close();
24
+ }
25
+ }
26
+
27
+ if (import.meta.main) {
28
+ await runEditableArtifactOutboxSidecar();
29
+ }
@@ -1,5 +1,27 @@
1
1
  import type { Settings } from "@opengeni/config";
2
- import type { WorkerOptions } from "@temporalio/worker";
2
+ import type { Observability } from "@opengeni/observability";
3
+ import { readFileSync } from "node:fs";
4
+ import type {
5
+ ActivitySlotInfo,
6
+ CustomSlotSupplier,
7
+ SlotMarkUsedContext,
8
+ SlotPermit,
9
+ SlotReleaseContext,
10
+ SlotReserveContext,
11
+ WorkerOptions,
12
+ WorkerTuner,
13
+ } from "@temporalio/worker";
14
+
15
+ const MIB = 1024 * 1024;
16
+
17
+ /**
18
+ * Default fixed/HPA turn-worker admission contract. The configured fixed-mode
19
+ * ceiling may be lower, but every live Temporal permit reserves the complete
20
+ * hard budget until the runAgentTurn activity promise physically completes.
21
+ */
22
+ export const TURN_WORKER_MAX_CONCURRENT_TURNS = 16;
23
+ export const TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN = 100 * MIB;
24
+ export const TURN_WORKER_NATIVE_HEADROOM_BYTES = 512 * MIB;
3
25
 
4
26
  export type TurnWorkerConcurrencySettings = Pick<
5
27
  Settings,
@@ -15,10 +37,10 @@ export type TurnWorkerConcurrencyOptions = Pick<
15
37
  >;
16
38
 
17
39
  /**
18
- * Ordinary multi-worker deployments keep a fixed per-process ceiling so HPA or
19
- * an operator owns horizontal capacity. The single-machine profile selects the
20
- * resource tuner instead: Temporal admits activity slots while whole-system CPU
21
- * and memory remain below the configured targets, then stops polling more work.
40
+ * Fixed profiles keep a per-process ceiling. Resource-aware profiles retain a
41
+ * hard per-process maximum while Temporal meters new activity admission against
42
+ * cgroup/host CPU and memory. The ramp lets usage become visible before a burst
43
+ * can reserve the entire ceiling.
22
44
  */
23
45
  export function turnWorkerConcurrencyOptions(
24
46
  settings: TurnWorkerConcurrencySettings,
@@ -38,12 +60,46 @@ export function turnWorkerConcurrencyOptions(
38
60
  activityTaskSlotOptions: {
39
61
  minimumSlots: 1,
40
62
  maximumSlots: settings.turnWorkerMaxConcurrentTurns,
41
- rampThrottle: "50ms",
63
+ rampThrottle: "250ms",
42
64
  },
43
65
  },
44
66
  };
45
67
  }
46
68
 
69
+ export type TurnWorkerConcurrencyPlan = {
70
+ options: TurnWorkerConcurrencyOptions;
71
+ admission: MemoryAwareTurnSlotSupplier | null;
72
+ };
73
+
74
+ /**
75
+ * Build the runtime concurrency policy for a turn worker.
76
+ *
77
+ * Fixed/HPA deployments use the hard-reservation supplier below. The explicit
78
+ * non-HA single-machine profile keeps Temporal's native CPU/memory resource
79
+ * tuner because it controls whole-machine load rather than a pod cgroup; that
80
+ * profile documents that its configured maximum is not a per-turn reservation.
81
+ */
82
+ export function createTurnWorkerConcurrencyPlan(
83
+ settings: TurnWorkerConcurrencySettings,
84
+ options: Omit<TurnAdmissionOptions, "maximumTurns"> = {},
85
+ ): TurnWorkerConcurrencyPlan {
86
+ if (settings.turnWorkerConcurrencyMode === "resource-based") {
87
+ return {
88
+ options: turnWorkerConcurrencyOptions(settings),
89
+ admission: null,
90
+ };
91
+ }
92
+
93
+ const fixed = createTurnWorkerTuner({
94
+ ...options,
95
+ maximumTurns: settings.turnWorkerMaxConcurrentTurns,
96
+ });
97
+ return {
98
+ options: { tuner: fixed.tuner },
99
+ admission: fixed.admission,
100
+ };
101
+ }
102
+
47
103
  export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {
48
104
  concurrencyMode: "fixed" | "resource-based";
49
105
  maxConcurrentTurns: number;
@@ -61,3 +117,425 @@ export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySe
61
117
 
62
118
  export const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;
63
119
  export const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;
120
+
121
+ const CGROUP_MEMORY_FILES = [
122
+ {
123
+ limit: "/sys/fs/cgroup/memory.max",
124
+ current: "/sys/fs/cgroup/memory.current",
125
+ },
126
+ {
127
+ limit: "/sys/fs/cgroup/memory/memory.limit_in_bytes",
128
+ current: "/sys/fs/cgroup/memory/memory.usage_in_bytes",
129
+ },
130
+ ] as const;
131
+
132
+ // cgroup v1 uses enormous sentinels for "unlimited". Values this large are
133
+ // host capacity rather than a meaningful container admission boundary.
134
+ const UNLIMITED_CGROUP_BYTES = 1n << 60n;
135
+
136
+ export type CgroupMemorySnapshot = {
137
+ currentBytes: number;
138
+ limitBytes: number | null;
139
+ source: "cgroup-v1" | "cgroup-v2" | "process";
140
+ };
141
+
142
+ export type TurnAdmissionSnapshot = {
143
+ baselineBytes: number;
144
+ currentBytes: number;
145
+ limitBytes: number | null;
146
+ hardBytesPerTurn: number;
147
+ nativeHeadroomBytes: number;
148
+ maximumTurns: number;
149
+ memoryBoundCapacity: number;
150
+ reservedSlots: number;
151
+ usedSlots: number;
152
+ availableSlots: number;
153
+ };
154
+
155
+ export type TurnAdmissionOptions = {
156
+ maximumTurns?: number;
157
+ hardBytesPerTurn?: number;
158
+ nativeHeadroomBytes?: number;
159
+ memorySnapshot?: () => CgroupMemorySnapshot;
160
+ retryDelayMs?: number;
161
+ observability?: Observability;
162
+ };
163
+
164
+ type TurnSlotPermit = SlotPermit & {
165
+ readonly owner: symbol;
166
+ used: boolean;
167
+ };
168
+
169
+ /**
170
+ * Temporal custom activity-slot supplier for turn workers.
171
+ *
172
+ * A permit is required before Temporal polls a task. Reservation is allowed
173
+ * only when both independent safety checks pass:
174
+ *
175
+ * baseline + reserved-after × hard-turn-budget + native headroom <= limit
176
+ * current + reserved-after × hard-turn-budget + native headroom <= limit
177
+ *
178
+ * The first check enforces the release contract even when current RSS happens
179
+ * to be low. The second accounts for native/retained memory observed after
180
+ * startup and keeps the complete hard budget reserved for every live permit.
181
+ * A used permit is not evidence that its allocation is already visible in the
182
+ * cgroup, and unrelated native growth cannot safely be attributed to a turn.
183
+ * Kubernetes OOM is never an admission mechanism.
184
+ */
185
+ export class MemoryAwareTurnSlotSupplier implements CustomSlotSupplier<ActivitySlotInfo> {
186
+ readonly type = "custom" as const;
187
+
188
+ private readonly owner = Symbol("opengeni-turn-admission");
189
+ private readonly permits = new Set<TurnSlotPermit>();
190
+ private baselineBytes: number;
191
+ private readonly maximumTurns: number;
192
+ private readonly hardBytesPerTurn: number;
193
+ private readonly nativeHeadroomBytes: number;
194
+ private readonly memorySnapshot: () => CgroupMemorySnapshot;
195
+ private readonly retryDelayMs: number;
196
+ private readonly observability: Observability | undefined;
197
+
198
+ constructor(options: TurnAdmissionOptions = {}) {
199
+ this.maximumTurns = positiveInteger(
200
+ "maximumTurns",
201
+ options.maximumTurns ?? TURN_WORKER_MAX_CONCURRENT_TURNS,
202
+ );
203
+ this.hardBytesPerTurn = positiveInteger(
204
+ "hardBytesPerTurn",
205
+ options.hardBytesPerTurn ?? TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN,
206
+ );
207
+ this.nativeHeadroomBytes = nonnegativeInteger(
208
+ "nativeHeadroomBytes",
209
+ options.nativeHeadroomBytes ?? TURN_WORKER_NATIVE_HEADROOM_BYTES,
210
+ );
211
+ this.retryDelayMs = positiveInteger("retryDelayMs", options.retryDelayMs ?? 100);
212
+ this.memorySnapshot = options.memorySnapshot ?? readCgroupMemorySnapshot;
213
+ this.observability = options.observability;
214
+ const initial = this.memorySnapshot();
215
+ this.baselineBytes = initial.currentBytes;
216
+ if (initial.limitBytes !== null && this.memoryBoundCapacity(initial.limitBytes) < 1) {
217
+ throw new Error(
218
+ "Turn worker memory limit cannot safely admit one turn: " +
219
+ `baseline=${this.baselineBytes} hardTurn=${this.hardBytesPerTurn} ` +
220
+ `nativeHeadroom=${this.nativeHeadroomBytes} limit=${initial.limitBytes}`,
221
+ );
222
+ }
223
+ this.refreshMetrics(initial);
224
+ }
225
+
226
+ async reserveSlot(_ctx: SlotReserveContext, abortSignal: AbortSignal): Promise<SlotPermit> {
227
+ for (;;) {
228
+ if (abortSignal.aborted) throw abortError();
229
+ const permit = this.reserveIfSafe();
230
+ if (permit) return permit;
231
+ await abortableDelay(this.retryDelayMs, abortSignal);
232
+ }
233
+ }
234
+
235
+ tryReserveSlot(_ctx: SlotReserveContext): SlotPermit | null {
236
+ return this.reserveIfSafe();
237
+ }
238
+
239
+ markSlotUsed(ctx: SlotMarkUsedContext<ActivitySlotInfo>): void {
240
+ const permit = this.ownedPermit(ctx.permit);
241
+ permit.used = true;
242
+ this.refreshMetrics();
243
+ }
244
+
245
+ releaseSlot(ctx: SlotReleaseContext<ActivitySlotInfo>): void {
246
+ const permit = this.ownedPermit(ctx.permit);
247
+ this.permits.delete(permit);
248
+ this.refreshMetrics();
249
+ }
250
+
251
+ snapshot(memory = this.memorySnapshot()): TurnAdmissionSnapshot {
252
+ const usedSlots = this.usedSlots();
253
+ const reservedSlots = this.permits.size;
254
+ const availableSlots = this.availableSlots(memory);
255
+ // Never report less capacity than work already admitted. The dynamic
256
+ // denominator is exactly reserved + what a new poll could reserve now.
257
+ const memoryBoundCapacity = reservedSlots + availableSlots;
258
+ return {
259
+ baselineBytes: this.baselineBytes,
260
+ currentBytes: memory.currentBytes,
261
+ limitBytes: memory.limitBytes,
262
+ hardBytesPerTurn: this.hardBytesPerTurn,
263
+ nativeHeadroomBytes: this.nativeHeadroomBytes,
264
+ maximumTurns: this.maximumTurns,
265
+ memoryBoundCapacity,
266
+ reservedSlots,
267
+ usedSlots,
268
+ availableSlots,
269
+ };
270
+ }
271
+
272
+ /**
273
+ * Finalize the admission baseline after Temporal/native construction.
274
+ * Worker.create can retain memory after the constructor's provisional
275
+ * sample. That initialization delta must remain in the baseline for every
276
+ * later projection rather than competing with turn reservations.
277
+ */
278
+ finalizeStartupBaseline(): void {
279
+ const memory = this.memorySnapshot();
280
+ if (this.permits.size !== 0) {
281
+ throw new Error("Turn worker startup admission check ran after slot reservation");
282
+ }
283
+ const finalizedBaselineBytes = memory.currentBytes;
284
+ const finalizedCapacity =
285
+ memory.limitBytes === null
286
+ ? this.maximumTurns
287
+ : this.memoryBoundCapacity(memory.limitBytes, finalizedBaselineBytes);
288
+ if (memory.limitBytes !== null && finalizedCapacity < 1) {
289
+ throw new Error(
290
+ "Turn worker memory limit cannot safely admit one turn after worker initialization: " +
291
+ `provisionalBaseline=${this.baselineBytes} finalizedBaseline=${finalizedBaselineBytes} ` +
292
+ `hardTurn=${this.hardBytesPerTurn} nativeHeadroom=${this.nativeHeadroomBytes} ` +
293
+ `limit=${String(memory.limitBytes)}`,
294
+ );
295
+ }
296
+ this.baselineBytes = finalizedBaselineBytes;
297
+ this.refreshMetrics(memory);
298
+ }
299
+
300
+ private reserveIfSafe(): TurnSlotPermit | null {
301
+ const memory = this.memorySnapshot();
302
+ if (this.permits.size >= this.maximumTurns || !this.memoryAllowsAnother(memory)) {
303
+ this.refreshMetrics(memory);
304
+ return null;
305
+ }
306
+ const permit: TurnSlotPermit = { owner: this.owner, used: false };
307
+ this.permits.add(permit);
308
+ this.refreshMetrics(memory);
309
+ return permit;
310
+ }
311
+
312
+ private memoryAllowsAnother(memory: CgroupMemorySnapshot): boolean {
313
+ return this.availableSlots(memory) > 0;
314
+ }
315
+
316
+ private availableSlots(memory: CgroupMemorySnapshot): number {
317
+ const densityRemaining = this.maximumTurns - this.permits.size;
318
+ if (memory.limitBytes === null) return Math.max(0, densityRemaining);
319
+
320
+ const baselineRemaining = this.memoryBoundCapacity(memory.limitBytes) - this.permits.size;
321
+ const observedRemaining =
322
+ Math.floor(
323
+ (memory.limitBytes - memory.currentBytes - this.nativeHeadroomBytes) /
324
+ this.hardBytesPerTurn,
325
+ ) - this.permits.size;
326
+ return Math.max(0, Math.min(densityRemaining, baselineRemaining, observedRemaining));
327
+ }
328
+
329
+ private memoryBoundCapacity(limitBytes: number, baselineBytes = this.baselineBytes): number {
330
+ return Math.max(
331
+ 0,
332
+ Math.floor((limitBytes - baselineBytes - this.nativeHeadroomBytes) / this.hardBytesPerTurn),
333
+ );
334
+ }
335
+
336
+ private usedSlots(): number {
337
+ let used = 0;
338
+ for (const permit of this.permits) {
339
+ if (permit.used) used += 1;
340
+ }
341
+ return used;
342
+ }
343
+
344
+ private ownedPermit(permit: SlotPermit): TurnSlotPermit {
345
+ const candidate = permit as Partial<TurnSlotPermit>;
346
+ if (candidate.owner !== this.owner || !this.permits.has(permit as TurnSlotPermit)) {
347
+ throw new Error("Temporal returned a turn slot permit owned by another supplier");
348
+ }
349
+ return permit as TurnSlotPermit;
350
+ }
351
+
352
+ private refreshMetrics(memory = this.memorySnapshot()): void {
353
+ if (!this.observability) return;
354
+ const value = this.snapshot(memory);
355
+ const set = (name: string, help: string, metricValue: number) =>
356
+ this.observability?.setGauge({ name, help, value: metricValue });
357
+ set(
358
+ "opengeni_turn_slots_capacity",
359
+ "Turn slots this worker can safely support after the hard memory contract.",
360
+ value.memoryBoundCapacity,
361
+ );
362
+ set(
363
+ "opengeni_turn_slots_reserved",
364
+ "Temporal turn activity slots reserved for polling or execution in this worker.",
365
+ value.reservedSlots,
366
+ );
367
+ set(
368
+ "opengeni_turn_slots_used",
369
+ "Temporal turn activity slots currently executing in this worker.",
370
+ value.usedSlots,
371
+ );
372
+ set(
373
+ "opengeni_turn_slots_available",
374
+ "Memory-safe turn activity slots still available to reserve in this worker.",
375
+ value.availableSlots,
376
+ );
377
+ set(
378
+ "opengeni_turn_slot_saturation_ratio",
379
+ "Fraction of this worker's memory-safe turn capacity currently executing.",
380
+ value.memoryBoundCapacity === 0 ? 1 : value.usedSlots / value.memoryBoundCapacity,
381
+ );
382
+ set(
383
+ "opengeni_turn_admission_memory_baseline_bytes",
384
+ "Turn-worker cgroup memory after Temporal/native worker initialization.",
385
+ value.baselineBytes,
386
+ );
387
+ set(
388
+ "opengeni_turn_admission_memory_current_bytes",
389
+ "Current cgroup memory used for turn admission decisions.",
390
+ value.currentBytes,
391
+ );
392
+ set(
393
+ "opengeni_turn_admission_memory_limit_bytes",
394
+ "Finite cgroup memory limit used for turn admission, or zero when unlimited.",
395
+ value.limitBytes ?? 0,
396
+ );
397
+ set(
398
+ "opengeni_turn_admission_hard_bytes_per_turn",
399
+ "Hard incremental memory budget reserved for each admitted turn.",
400
+ value.hardBytesPerTurn,
401
+ );
402
+ set(
403
+ "opengeni_turn_admission_native_headroom_bytes",
404
+ "Cgroup memory retained for runtime, native, and GC headroom.",
405
+ value.nativeHeadroomBytes,
406
+ );
407
+ }
408
+ }
409
+
410
+ export function createTurnWorkerTuner(options: TurnAdmissionOptions = {}): {
411
+ tuner: WorkerTuner;
412
+ admission: MemoryAwareTurnSlotSupplier;
413
+ } {
414
+ const admission = new MemoryAwareTurnSlotSupplier(options);
415
+ return {
416
+ admission,
417
+ tuner: {
418
+ workflowTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
419
+ activityTaskSlotSupplier: admission,
420
+ localActivityTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
421
+ nexusTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
422
+ },
423
+ };
424
+ }
425
+
426
+ export function readCgroupMemorySnapshot(): CgroupMemorySnapshot {
427
+ for (const [index, files] of CGROUP_MEMORY_FILES.entries()) {
428
+ const limitRaw = readCgroupText(files.limit);
429
+ const currentRaw = readCgroupText(files.current);
430
+ if (limitRaw === null && currentRaw === null) continue;
431
+ if (limitRaw === null || currentRaw === null) {
432
+ throw new Error(
433
+ `Incomplete cgroup memory controller: limit=${files.limit} current=${files.current}`,
434
+ );
435
+ }
436
+ return {
437
+ currentBytes: parseCgroupCurrentValue(currentRaw),
438
+ limitBytes: parseCgroupLimitValue(limitRaw),
439
+ source: index === 0 ? "cgroup-v2" : "cgroup-v1",
440
+ };
441
+ }
442
+ return {
443
+ currentBytes: process.memoryUsage().rss,
444
+ limitBytes: null,
445
+ source: "process",
446
+ };
447
+ }
448
+
449
+ export function parseCgroupLimitValue(value: string): number | null {
450
+ if (value.trim() === "max") return null;
451
+ const parsed = parseCgroupBigInt(value);
452
+ if (parsed === null || parsed <= 0n) {
453
+ throw new Error(`Malformed finite cgroup memory limit: ${JSON.stringify(value.trim())}`);
454
+ }
455
+ if (parsed >= UNLIMITED_CGROUP_BYTES) return null;
456
+ const number = Number(parsed);
457
+ if (!Number.isSafeInteger(number)) {
458
+ throw new Error(
459
+ `Finite cgroup memory limit exceeds JavaScript's safe integer range: ${parsed}`,
460
+ );
461
+ }
462
+ return number;
463
+ }
464
+
465
+ export function parseCgroupCurrentValue(value: string): number {
466
+ const parsed = parseCgroupBigInt(value);
467
+ if (parsed === null || parsed < 0n) {
468
+ throw new Error(`Malformed cgroup current memory value: ${JSON.stringify(value.trim())}`);
469
+ }
470
+ const number = Number(parsed);
471
+ if (!Number.isSafeInteger(number)) {
472
+ throw new Error(`Cgroup current memory exceeds JavaScript's safe integer range: ${parsed}`);
473
+ }
474
+ return number;
475
+ }
476
+
477
+ function parseCgroupBigInt(value: string): bigint | null {
478
+ try {
479
+ return BigInt(value.trim());
480
+ } catch {
481
+ return null;
482
+ }
483
+ }
484
+
485
+ function readCgroupText(path: string): string | null {
486
+ try {
487
+ return readFileSync(path, "utf8");
488
+ } catch (error) {
489
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
490
+ throw new Error(
491
+ `Unable to read present cgroup memory controller file ${path}: ${
492
+ error instanceof Error ? error.message : String(error)
493
+ }`,
494
+ { cause: error },
495
+ );
496
+ }
497
+ }
498
+
499
+ function positiveInteger(name: string, value: number): number {
500
+ if (!Number.isSafeInteger(value) || value <= 0) {
501
+ throw new Error(`${name} must be a positive integer`);
502
+ }
503
+ return value;
504
+ }
505
+
506
+ function nonnegativeInteger(name: string, value: number): number {
507
+ if (!Number.isSafeInteger(value) || value < 0) {
508
+ throw new Error(`${name} must be a non-negative integer`);
509
+ }
510
+ return value;
511
+ }
512
+
513
+ function abortError(): DOMException {
514
+ return new DOMException("Turn slot reservation aborted", "AbortError");
515
+ }
516
+
517
+ function abortableDelay(milliseconds: number, signal: AbortSignal): Promise<void> {
518
+ return new Promise((resolve, reject) => {
519
+ if (signal.aborted) {
520
+ reject(abortError());
521
+ return;
522
+ }
523
+ const timer = setTimeout(() => {
524
+ signal.removeEventListener("abort", onAbort);
525
+ resolve();
526
+ }, milliseconds);
527
+ timer.unref?.();
528
+ const onAbort = () => {
529
+ clearTimeout(timer);
530
+ reject(abortError());
531
+ };
532
+ signal.addEventListener("abort", onAbort, { once: true });
533
+ });
534
+ }
535
+
536
+ // Session workflows finish at idle boundaries and replay from durable history.
537
+ // Bound sticky VM state explicitly: the SDK otherwise derives this from the
538
+ // host heap limit (roughly 600 cached workflows/GiB), which is far above this
539
+ // process's actual concurrent workflow-task capacity and permanently retains
540
+ // idle workflow state in every control replica.
541
+ export const CONTROL_WORKER_MAX_CACHED_WORKFLOWS = 64;