@opengeni/worker-bundle 0.16.29 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn.d.ts +85 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +5 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +11 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +13 -26
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +4 -4
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +71 -5
- package/dist/activities/video-generation-admission.d.ts +38 -0
- package/dist/activities/video-generation-credential.d.ts +7 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +49 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities-control-44KBWLSM.js +6885 -0
- package/dist/activities-control-44KBWLSM.js.map +1 -0
- package/dist/activities-control.d.ts +289 -0
- package/dist/activities-turn-K4CEDEGJ.js +14675 -0
- package/dist/activities-turn-K4CEDEGJ.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +130 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-E3RI3I33.js +828 -0
- package/dist/chunk-E3RI3I33.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-KKCRGAGZ.js +3091 -0
- package/dist/chunk-KKCRGAGZ.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +176 -10
- package/dist/index.js +662 -17374
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +22 -0
- package/dist/observability-metrics.d.ts +33 -0
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +2 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1043 -363
- package/dist/workflows/activities.d.ts +7 -1
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +9 -7
- package/package.json +34 -15
- package/src/activities/agent-turn.ts +1470 -246
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +6 -6
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +45 -42
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +6 -2
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +55 -55
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +77 -4
- package/src/activities/session-state.ts +4 -4
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +87 -5
- package/src/activities/video-generation-admission.ts +270 -0
- package/src/activities/video-generation-credential.ts +34 -0
- package/src/activities/video-generation-reconciliation.ts +578 -0
- package/src/activities/video-media-validation.ts +326 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +263 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities-control.ts +54 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +48 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +277 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +116 -0
- package/src/observability-metrics.ts +199 -4
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +19 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +30 -0
- package/src/workflows/file-upload-reaper.ts +3 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +17 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
2
|
+
import { createDb } from "@opengeni/db";
|
|
3
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
4
|
+
import { createObservability } from "@opengeni/observability";
|
|
5
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
6
|
+
import type { ActivityDependencies, SharedActivityServices } from "./activities/types";
|
|
7
|
+
import { observabilityEventLogger } from "./observability-metrics";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Build the dependency graph common to both worker roles exactly once.
|
|
11
|
+
*
|
|
12
|
+
* This module deliberately has no import edge to @opengeni/runtime or
|
|
13
|
+
* @opengeni/documents. Those graphs are added by the role-specific factories,
|
|
14
|
+
* preventing an idle process from retaining an unused model SDK or parser
|
|
15
|
+
* stack for its entire lifetime.
|
|
16
|
+
*/
|
|
17
|
+
export function createSharedActivityServices(
|
|
18
|
+
dependencies: ActivityDependencies,
|
|
19
|
+
): () => Promise<SharedActivityServices> {
|
|
20
|
+
let servicesPromise: Promise<SharedActivityServices> | null = null;
|
|
21
|
+
|
|
22
|
+
return async function services(): Promise<SharedActivityServices> {
|
|
23
|
+
servicesPromise ??= (async () => {
|
|
24
|
+
const settings = dependencies.settings ?? getSettings();
|
|
25
|
+
const observability =
|
|
26
|
+
dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
27
|
+
const searchPath = dbSearchPath(settings);
|
|
28
|
+
const dbClient = dependencies.db
|
|
29
|
+
? null
|
|
30
|
+
: createDb(settings.databaseUrl, {
|
|
31
|
+
...(searchPath ? { searchPath } : {}),
|
|
32
|
+
rlsStrategy: settings.rlsStrategy,
|
|
33
|
+
});
|
|
34
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
settings,
|
|
38
|
+
db: dependencies.db ?? dbClient!.db,
|
|
39
|
+
bus:
|
|
40
|
+
dependencies.bus ??
|
|
41
|
+
(await createNatsEventBus(
|
|
42
|
+
settings.natsUrl,
|
|
43
|
+
controlPlaneAuth
|
|
44
|
+
? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
|
|
45
|
+
: undefined,
|
|
46
|
+
{ logger: observabilityEventLogger(observability) },
|
|
47
|
+
)),
|
|
48
|
+
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
49
|
+
observability,
|
|
50
|
+
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
51
|
+
signalSessionAttemptQuiesced: dependencies.signalSessionAttemptQuiesced ?? null,
|
|
52
|
+
inspectSessionAttemptActivity: dependencies.inspectSessionAttemptActivity ?? null,
|
|
53
|
+
signalCodexCapacityWorkflow: dependencies.signalCodexCapacityWorkflow ?? null,
|
|
54
|
+
startSandboxReaperWorkflow: dependencies.startSandboxReaperWorkflow ?? null,
|
|
55
|
+
startVideoGenerationWorkflow: dependencies.startVideoGenerationWorkflow ?? null,
|
|
56
|
+
entitlements: dependencies.entitlements ?? null,
|
|
57
|
+
connectionCredentials: dependencies.connectionCredentials ?? null,
|
|
58
|
+
};
|
|
59
|
+
})();
|
|
60
|
+
return await servicesPromise;
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { createMaterializerSidecarFromEnvironment } from "./editable-artifact-materializer-service";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Dedicated editable-artifact materializer process. This entry deliberately
|
|
5
|
+
* does not import/create a Temporal worker and is never called by control/turn
|
|
6
|
+
* startup. Deploy it as a separate sidecar/service with its own DSN.
|
|
7
|
+
*/
|
|
8
|
+
export async function runEditableArtifactMaterializerSidecar(
|
|
9
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
10
|
+
): Promise<void> {
|
|
11
|
+
const configured = await createMaterializerSidecarFromEnvironment(environment);
|
|
12
|
+
if (!configured) {
|
|
13
|
+
throw new Error("Editable artifact materializer sidecar is disabled");
|
|
14
|
+
}
|
|
15
|
+
const server = Bun.serve({
|
|
16
|
+
hostname: configured.httpHostname,
|
|
17
|
+
port: configured.httpPort,
|
|
18
|
+
fetch: configured.service.fetch,
|
|
19
|
+
});
|
|
20
|
+
const stop = () => configured.service.drain("process signal");
|
|
21
|
+
process.on("SIGTERM", stop);
|
|
22
|
+
process.on("SIGINT", stop);
|
|
23
|
+
try {
|
|
24
|
+
await configured.service.run();
|
|
25
|
+
} finally {
|
|
26
|
+
process.off("SIGTERM", stop);
|
|
27
|
+
process.off("SIGINT", stop);
|
|
28
|
+
server.stop(true);
|
|
29
|
+
await configured.service.close();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (import.meta.main) {
|
|
34
|
+
await runEditableArtifactMaterializerSidecar();
|
|
35
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { createOutboxSidecarFromEnvironment } from "./editable-artifact-outbox-service";
|
|
2
|
+
|
|
3
|
+
/** Dedicated process: no Temporal/control worker is created or imported here. */
|
|
4
|
+
export async function runEditableArtifactOutboxSidecar(
|
|
5
|
+
environment: NodeJS.ProcessEnv = process.env,
|
|
6
|
+
): Promise<void> {
|
|
7
|
+
const configured = await createOutboxSidecarFromEnvironment(environment);
|
|
8
|
+
if (!configured) throw new Error("Editable artifact outbox sidecar is disabled");
|
|
9
|
+
const server = Bun.serve({
|
|
10
|
+
hostname: "0.0.0.0",
|
|
11
|
+
port: configured.httpPort,
|
|
12
|
+
fetch: configured.service.fetch,
|
|
13
|
+
});
|
|
14
|
+
const stop = () => configured.service.drain("process signal");
|
|
15
|
+
process.on("SIGTERM", stop);
|
|
16
|
+
process.on("SIGINT", stop);
|
|
17
|
+
try {
|
|
18
|
+
await configured.service.run();
|
|
19
|
+
} finally {
|
|
20
|
+
process.off("SIGTERM", stop);
|
|
21
|
+
process.off("SIGINT", stop);
|
|
22
|
+
server.stop(true);
|
|
23
|
+
await configured.service.close();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (import.meta.main) {
|
|
28
|
+
await runEditableArtifactOutboxSidecar();
|
|
29
|
+
}
|
package/src/concurrency.ts
CHANGED
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import type {
|
|
2
|
+
import type { Observability } from "@opengeni/observability";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import type {
|
|
5
|
+
ActivitySlotInfo,
|
|
6
|
+
CustomSlotSupplier,
|
|
7
|
+
SlotMarkUsedContext,
|
|
8
|
+
SlotPermit,
|
|
9
|
+
SlotReleaseContext,
|
|
10
|
+
SlotReserveContext,
|
|
11
|
+
WorkerOptions,
|
|
12
|
+
WorkerTuner,
|
|
13
|
+
} from "@temporalio/worker";
|
|
14
|
+
|
|
15
|
+
const MIB = 1024 * 1024;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Default fixed/HPA turn-worker admission contract. The configured fixed-mode
|
|
19
|
+
* ceiling may be lower, but every live Temporal permit reserves the complete
|
|
20
|
+
* hard budget until the runAgentTurn activity promise physically completes.
|
|
21
|
+
*/
|
|
22
|
+
export const TURN_WORKER_MAX_CONCURRENT_TURNS = 16;
|
|
23
|
+
export const TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN = 100 * MIB;
|
|
24
|
+
export const TURN_WORKER_NATIVE_HEADROOM_BYTES = 512 * MIB;
|
|
3
25
|
|
|
4
26
|
export type TurnWorkerConcurrencySettings = Pick<
|
|
5
27
|
Settings,
|
|
@@ -15,10 +37,10 @@ export type TurnWorkerConcurrencyOptions = Pick<
|
|
|
15
37
|
>;
|
|
16
38
|
|
|
17
39
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
40
|
+
* Fixed profiles keep a per-process ceiling. Resource-aware profiles retain a
|
|
41
|
+
* hard per-process maximum while Temporal meters new activity admission against
|
|
42
|
+
* cgroup/host CPU and memory. The ramp lets usage become visible before a burst
|
|
43
|
+
* can reserve the entire ceiling.
|
|
22
44
|
*/
|
|
23
45
|
export function turnWorkerConcurrencyOptions(
|
|
24
46
|
settings: TurnWorkerConcurrencySettings,
|
|
@@ -38,12 +60,46 @@ export function turnWorkerConcurrencyOptions(
|
|
|
38
60
|
activityTaskSlotOptions: {
|
|
39
61
|
minimumSlots: 1,
|
|
40
62
|
maximumSlots: settings.turnWorkerMaxConcurrentTurns,
|
|
41
|
-
rampThrottle: "
|
|
63
|
+
rampThrottle: "250ms",
|
|
42
64
|
},
|
|
43
65
|
},
|
|
44
66
|
};
|
|
45
67
|
}
|
|
46
68
|
|
|
69
|
+
export type TurnWorkerConcurrencyPlan = {
|
|
70
|
+
options: TurnWorkerConcurrencyOptions;
|
|
71
|
+
admission: MemoryAwareTurnSlotSupplier | null;
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Build the runtime concurrency policy for a turn worker.
|
|
76
|
+
*
|
|
77
|
+
* Fixed/HPA deployments use the hard-reservation supplier below. The explicit
|
|
78
|
+
* non-HA single-machine profile keeps Temporal's native CPU/memory resource
|
|
79
|
+
* tuner because it controls whole-machine load rather than a pod cgroup; that
|
|
80
|
+
* profile documents that its configured maximum is not a per-turn reservation.
|
|
81
|
+
*/
|
|
82
|
+
export function createTurnWorkerConcurrencyPlan(
|
|
83
|
+
settings: TurnWorkerConcurrencySettings,
|
|
84
|
+
options: Omit<TurnAdmissionOptions, "maximumTurns"> = {},
|
|
85
|
+
): TurnWorkerConcurrencyPlan {
|
|
86
|
+
if (settings.turnWorkerConcurrencyMode === "resource-based") {
|
|
87
|
+
return {
|
|
88
|
+
options: turnWorkerConcurrencyOptions(settings),
|
|
89
|
+
admission: null,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const fixed = createTurnWorkerTuner({
|
|
94
|
+
...options,
|
|
95
|
+
maximumTurns: settings.turnWorkerMaxConcurrentTurns,
|
|
96
|
+
});
|
|
97
|
+
return {
|
|
98
|
+
options: { tuner: fixed.tuner },
|
|
99
|
+
admission: fixed.admission,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
47
103
|
export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {
|
|
48
104
|
concurrencyMode: "fixed" | "resource-based";
|
|
49
105
|
maxConcurrentTurns: number;
|
|
@@ -61,3 +117,425 @@ export function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySe
|
|
|
61
117
|
|
|
62
118
|
export const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;
|
|
63
119
|
export const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;
|
|
120
|
+
|
|
121
|
+
const CGROUP_MEMORY_FILES = [
|
|
122
|
+
{
|
|
123
|
+
limit: "/sys/fs/cgroup/memory.max",
|
|
124
|
+
current: "/sys/fs/cgroup/memory.current",
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
limit: "/sys/fs/cgroup/memory/memory.limit_in_bytes",
|
|
128
|
+
current: "/sys/fs/cgroup/memory/memory.usage_in_bytes",
|
|
129
|
+
},
|
|
130
|
+
] as const;
|
|
131
|
+
|
|
132
|
+
// cgroup v1 uses enormous sentinels for "unlimited". Values this large are
|
|
133
|
+
// host capacity rather than a meaningful container admission boundary.
|
|
134
|
+
const UNLIMITED_CGROUP_BYTES = 1n << 60n;
|
|
135
|
+
|
|
136
|
+
export type CgroupMemorySnapshot = {
|
|
137
|
+
currentBytes: number;
|
|
138
|
+
limitBytes: number | null;
|
|
139
|
+
source: "cgroup-v1" | "cgroup-v2" | "process";
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
export type TurnAdmissionSnapshot = {
|
|
143
|
+
baselineBytes: number;
|
|
144
|
+
currentBytes: number;
|
|
145
|
+
limitBytes: number | null;
|
|
146
|
+
hardBytesPerTurn: number;
|
|
147
|
+
nativeHeadroomBytes: number;
|
|
148
|
+
maximumTurns: number;
|
|
149
|
+
memoryBoundCapacity: number;
|
|
150
|
+
reservedSlots: number;
|
|
151
|
+
usedSlots: number;
|
|
152
|
+
availableSlots: number;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export type TurnAdmissionOptions = {
|
|
156
|
+
maximumTurns?: number;
|
|
157
|
+
hardBytesPerTurn?: number;
|
|
158
|
+
nativeHeadroomBytes?: number;
|
|
159
|
+
memorySnapshot?: () => CgroupMemorySnapshot;
|
|
160
|
+
retryDelayMs?: number;
|
|
161
|
+
observability?: Observability;
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
type TurnSlotPermit = SlotPermit & {
|
|
165
|
+
readonly owner: symbol;
|
|
166
|
+
used: boolean;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Temporal custom activity-slot supplier for turn workers.
|
|
171
|
+
*
|
|
172
|
+
* A permit is required before Temporal polls a task. Reservation is allowed
|
|
173
|
+
* only when both independent safety checks pass:
|
|
174
|
+
*
|
|
175
|
+
* baseline + reserved-after × hard-turn-budget + native headroom <= limit
|
|
176
|
+
* current + reserved-after × hard-turn-budget + native headroom <= limit
|
|
177
|
+
*
|
|
178
|
+
* The first check enforces the release contract even when current RSS happens
|
|
179
|
+
* to be low. The second accounts for native/retained memory observed after
|
|
180
|
+
* startup and keeps the complete hard budget reserved for every live permit.
|
|
181
|
+
* A used permit is not evidence that its allocation is already visible in the
|
|
182
|
+
* cgroup, and unrelated native growth cannot safely be attributed to a turn.
|
|
183
|
+
* Kubernetes OOM is never an admission mechanism.
|
|
184
|
+
*/
|
|
185
|
+
export class MemoryAwareTurnSlotSupplier implements CustomSlotSupplier<ActivitySlotInfo> {
|
|
186
|
+
readonly type = "custom" as const;
|
|
187
|
+
|
|
188
|
+
private readonly owner = Symbol("opengeni-turn-admission");
|
|
189
|
+
private readonly permits = new Set<TurnSlotPermit>();
|
|
190
|
+
private baselineBytes: number;
|
|
191
|
+
private readonly maximumTurns: number;
|
|
192
|
+
private readonly hardBytesPerTurn: number;
|
|
193
|
+
private readonly nativeHeadroomBytes: number;
|
|
194
|
+
private readonly memorySnapshot: () => CgroupMemorySnapshot;
|
|
195
|
+
private readonly retryDelayMs: number;
|
|
196
|
+
private readonly observability: Observability | undefined;
|
|
197
|
+
|
|
198
|
+
constructor(options: TurnAdmissionOptions = {}) {
|
|
199
|
+
this.maximumTurns = positiveInteger(
|
|
200
|
+
"maximumTurns",
|
|
201
|
+
options.maximumTurns ?? TURN_WORKER_MAX_CONCURRENT_TURNS,
|
|
202
|
+
);
|
|
203
|
+
this.hardBytesPerTurn = positiveInteger(
|
|
204
|
+
"hardBytesPerTurn",
|
|
205
|
+
options.hardBytesPerTurn ?? TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN,
|
|
206
|
+
);
|
|
207
|
+
this.nativeHeadroomBytes = nonnegativeInteger(
|
|
208
|
+
"nativeHeadroomBytes",
|
|
209
|
+
options.nativeHeadroomBytes ?? TURN_WORKER_NATIVE_HEADROOM_BYTES,
|
|
210
|
+
);
|
|
211
|
+
this.retryDelayMs = positiveInteger("retryDelayMs", options.retryDelayMs ?? 100);
|
|
212
|
+
this.memorySnapshot = options.memorySnapshot ?? readCgroupMemorySnapshot;
|
|
213
|
+
this.observability = options.observability;
|
|
214
|
+
const initial = this.memorySnapshot();
|
|
215
|
+
this.baselineBytes = initial.currentBytes;
|
|
216
|
+
if (initial.limitBytes !== null && this.memoryBoundCapacity(initial.limitBytes) < 1) {
|
|
217
|
+
throw new Error(
|
|
218
|
+
"Turn worker memory limit cannot safely admit one turn: " +
|
|
219
|
+
`baseline=${this.baselineBytes} hardTurn=${this.hardBytesPerTurn} ` +
|
|
220
|
+
`nativeHeadroom=${this.nativeHeadroomBytes} limit=${initial.limitBytes}`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
this.refreshMetrics(initial);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async reserveSlot(_ctx: SlotReserveContext, abortSignal: AbortSignal): Promise<SlotPermit> {
|
|
227
|
+
for (;;) {
|
|
228
|
+
if (abortSignal.aborted) throw abortError();
|
|
229
|
+
const permit = this.reserveIfSafe();
|
|
230
|
+
if (permit) return permit;
|
|
231
|
+
await abortableDelay(this.retryDelayMs, abortSignal);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
tryReserveSlot(_ctx: SlotReserveContext): SlotPermit | null {
|
|
236
|
+
return this.reserveIfSafe();
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
markSlotUsed(ctx: SlotMarkUsedContext<ActivitySlotInfo>): void {
|
|
240
|
+
const permit = this.ownedPermit(ctx.permit);
|
|
241
|
+
permit.used = true;
|
|
242
|
+
this.refreshMetrics();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
releaseSlot(ctx: SlotReleaseContext<ActivitySlotInfo>): void {
|
|
246
|
+
const permit = this.ownedPermit(ctx.permit);
|
|
247
|
+
this.permits.delete(permit);
|
|
248
|
+
this.refreshMetrics();
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
snapshot(memory = this.memorySnapshot()): TurnAdmissionSnapshot {
|
|
252
|
+
const usedSlots = this.usedSlots();
|
|
253
|
+
const reservedSlots = this.permits.size;
|
|
254
|
+
const availableSlots = this.availableSlots(memory);
|
|
255
|
+
// Never report less capacity than work already admitted. The dynamic
|
|
256
|
+
// denominator is exactly reserved + what a new poll could reserve now.
|
|
257
|
+
const memoryBoundCapacity = reservedSlots + availableSlots;
|
|
258
|
+
return {
|
|
259
|
+
baselineBytes: this.baselineBytes,
|
|
260
|
+
currentBytes: memory.currentBytes,
|
|
261
|
+
limitBytes: memory.limitBytes,
|
|
262
|
+
hardBytesPerTurn: this.hardBytesPerTurn,
|
|
263
|
+
nativeHeadroomBytes: this.nativeHeadroomBytes,
|
|
264
|
+
maximumTurns: this.maximumTurns,
|
|
265
|
+
memoryBoundCapacity,
|
|
266
|
+
reservedSlots,
|
|
267
|
+
usedSlots,
|
|
268
|
+
availableSlots,
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Finalize the admission baseline after Temporal/native construction.
|
|
274
|
+
* Worker.create can retain memory after the constructor's provisional
|
|
275
|
+
* sample. That initialization delta must remain in the baseline for every
|
|
276
|
+
* later projection rather than competing with turn reservations.
|
|
277
|
+
*/
|
|
278
|
+
finalizeStartupBaseline(): void {
|
|
279
|
+
const memory = this.memorySnapshot();
|
|
280
|
+
if (this.permits.size !== 0) {
|
|
281
|
+
throw new Error("Turn worker startup admission check ran after slot reservation");
|
|
282
|
+
}
|
|
283
|
+
const finalizedBaselineBytes = memory.currentBytes;
|
|
284
|
+
const finalizedCapacity =
|
|
285
|
+
memory.limitBytes === null
|
|
286
|
+
? this.maximumTurns
|
|
287
|
+
: this.memoryBoundCapacity(memory.limitBytes, finalizedBaselineBytes);
|
|
288
|
+
if (memory.limitBytes !== null && finalizedCapacity < 1) {
|
|
289
|
+
throw new Error(
|
|
290
|
+
"Turn worker memory limit cannot safely admit one turn after worker initialization: " +
|
|
291
|
+
`provisionalBaseline=${this.baselineBytes} finalizedBaseline=${finalizedBaselineBytes} ` +
|
|
292
|
+
`hardTurn=${this.hardBytesPerTurn} nativeHeadroom=${this.nativeHeadroomBytes} ` +
|
|
293
|
+
`limit=${String(memory.limitBytes)}`,
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
this.baselineBytes = finalizedBaselineBytes;
|
|
297
|
+
this.refreshMetrics(memory);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
private reserveIfSafe(): TurnSlotPermit | null {
|
|
301
|
+
const memory = this.memorySnapshot();
|
|
302
|
+
if (this.permits.size >= this.maximumTurns || !this.memoryAllowsAnother(memory)) {
|
|
303
|
+
this.refreshMetrics(memory);
|
|
304
|
+
return null;
|
|
305
|
+
}
|
|
306
|
+
const permit: TurnSlotPermit = { owner: this.owner, used: false };
|
|
307
|
+
this.permits.add(permit);
|
|
308
|
+
this.refreshMetrics(memory);
|
|
309
|
+
return permit;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private memoryAllowsAnother(memory: CgroupMemorySnapshot): boolean {
|
|
313
|
+
return this.availableSlots(memory) > 0;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
private availableSlots(memory: CgroupMemorySnapshot): number {
|
|
317
|
+
const densityRemaining = this.maximumTurns - this.permits.size;
|
|
318
|
+
if (memory.limitBytes === null) return Math.max(0, densityRemaining);
|
|
319
|
+
|
|
320
|
+
const baselineRemaining = this.memoryBoundCapacity(memory.limitBytes) - this.permits.size;
|
|
321
|
+
const observedRemaining =
|
|
322
|
+
Math.floor(
|
|
323
|
+
(memory.limitBytes - memory.currentBytes - this.nativeHeadroomBytes) /
|
|
324
|
+
this.hardBytesPerTurn,
|
|
325
|
+
) - this.permits.size;
|
|
326
|
+
return Math.max(0, Math.min(densityRemaining, baselineRemaining, observedRemaining));
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
private memoryBoundCapacity(limitBytes: number, baselineBytes = this.baselineBytes): number {
|
|
330
|
+
return Math.max(
|
|
331
|
+
0,
|
|
332
|
+
Math.floor((limitBytes - baselineBytes - this.nativeHeadroomBytes) / this.hardBytesPerTurn),
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
private usedSlots(): number {
|
|
337
|
+
let used = 0;
|
|
338
|
+
for (const permit of this.permits) {
|
|
339
|
+
if (permit.used) used += 1;
|
|
340
|
+
}
|
|
341
|
+
return used;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
private ownedPermit(permit: SlotPermit): TurnSlotPermit {
|
|
345
|
+
const candidate = permit as Partial<TurnSlotPermit>;
|
|
346
|
+
if (candidate.owner !== this.owner || !this.permits.has(permit as TurnSlotPermit)) {
|
|
347
|
+
throw new Error("Temporal returned a turn slot permit owned by another supplier");
|
|
348
|
+
}
|
|
349
|
+
return permit as TurnSlotPermit;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
private refreshMetrics(memory = this.memorySnapshot()): void {
|
|
353
|
+
if (!this.observability) return;
|
|
354
|
+
const value = this.snapshot(memory);
|
|
355
|
+
const set = (name: string, help: string, metricValue: number) =>
|
|
356
|
+
this.observability?.setGauge({ name, help, value: metricValue });
|
|
357
|
+
set(
|
|
358
|
+
"opengeni_turn_slots_capacity",
|
|
359
|
+
"Turn slots this worker can safely support after the hard memory contract.",
|
|
360
|
+
value.memoryBoundCapacity,
|
|
361
|
+
);
|
|
362
|
+
set(
|
|
363
|
+
"opengeni_turn_slots_reserved",
|
|
364
|
+
"Temporal turn activity slots reserved for polling or execution in this worker.",
|
|
365
|
+
value.reservedSlots,
|
|
366
|
+
);
|
|
367
|
+
set(
|
|
368
|
+
"opengeni_turn_slots_used",
|
|
369
|
+
"Temporal turn activity slots currently executing in this worker.",
|
|
370
|
+
value.usedSlots,
|
|
371
|
+
);
|
|
372
|
+
set(
|
|
373
|
+
"opengeni_turn_slots_available",
|
|
374
|
+
"Memory-safe turn activity slots still available to reserve in this worker.",
|
|
375
|
+
value.availableSlots,
|
|
376
|
+
);
|
|
377
|
+
set(
|
|
378
|
+
"opengeni_turn_slot_saturation_ratio",
|
|
379
|
+
"Fraction of this worker's memory-safe turn capacity currently executing.",
|
|
380
|
+
value.memoryBoundCapacity === 0 ? 1 : value.usedSlots / value.memoryBoundCapacity,
|
|
381
|
+
);
|
|
382
|
+
set(
|
|
383
|
+
"opengeni_turn_admission_memory_baseline_bytes",
|
|
384
|
+
"Turn-worker cgroup memory after Temporal/native worker initialization.",
|
|
385
|
+
value.baselineBytes,
|
|
386
|
+
);
|
|
387
|
+
set(
|
|
388
|
+
"opengeni_turn_admission_memory_current_bytes",
|
|
389
|
+
"Current cgroup memory used for turn admission decisions.",
|
|
390
|
+
value.currentBytes,
|
|
391
|
+
);
|
|
392
|
+
set(
|
|
393
|
+
"opengeni_turn_admission_memory_limit_bytes",
|
|
394
|
+
"Finite cgroup memory limit used for turn admission, or zero when unlimited.",
|
|
395
|
+
value.limitBytes ?? 0,
|
|
396
|
+
);
|
|
397
|
+
set(
|
|
398
|
+
"opengeni_turn_admission_hard_bytes_per_turn",
|
|
399
|
+
"Hard incremental memory budget reserved for each admitted turn.",
|
|
400
|
+
value.hardBytesPerTurn,
|
|
401
|
+
);
|
|
402
|
+
set(
|
|
403
|
+
"opengeni_turn_admission_native_headroom_bytes",
|
|
404
|
+
"Cgroup memory retained for runtime, native, and GC headroom.",
|
|
405
|
+
value.nativeHeadroomBytes,
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export function createTurnWorkerTuner(options: TurnAdmissionOptions = {}): {
|
|
411
|
+
tuner: WorkerTuner;
|
|
412
|
+
admission: MemoryAwareTurnSlotSupplier;
|
|
413
|
+
} {
|
|
414
|
+
const admission = new MemoryAwareTurnSlotSupplier(options);
|
|
415
|
+
return {
|
|
416
|
+
admission,
|
|
417
|
+
tuner: {
|
|
418
|
+
workflowTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
|
|
419
|
+
activityTaskSlotSupplier: admission,
|
|
420
|
+
localActivityTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
|
|
421
|
+
nexusTaskSlotSupplier: { type: "fixed-size", numSlots: 1 },
|
|
422
|
+
},
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
export function readCgroupMemorySnapshot(): CgroupMemorySnapshot {
|
|
427
|
+
for (const [index, files] of CGROUP_MEMORY_FILES.entries()) {
|
|
428
|
+
const limitRaw = readCgroupText(files.limit);
|
|
429
|
+
const currentRaw = readCgroupText(files.current);
|
|
430
|
+
if (limitRaw === null && currentRaw === null) continue;
|
|
431
|
+
if (limitRaw === null || currentRaw === null) {
|
|
432
|
+
throw new Error(
|
|
433
|
+
`Incomplete cgroup memory controller: limit=${files.limit} current=${files.current}`,
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
return {
|
|
437
|
+
currentBytes: parseCgroupCurrentValue(currentRaw),
|
|
438
|
+
limitBytes: parseCgroupLimitValue(limitRaw),
|
|
439
|
+
source: index === 0 ? "cgroup-v2" : "cgroup-v1",
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
return {
|
|
443
|
+
currentBytes: process.memoryUsage().rss,
|
|
444
|
+
limitBytes: null,
|
|
445
|
+
source: "process",
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
export function parseCgroupLimitValue(value: string): number | null {
|
|
450
|
+
if (value.trim() === "max") return null;
|
|
451
|
+
const parsed = parseCgroupBigInt(value);
|
|
452
|
+
if (parsed === null || parsed <= 0n) {
|
|
453
|
+
throw new Error(`Malformed finite cgroup memory limit: ${JSON.stringify(value.trim())}`);
|
|
454
|
+
}
|
|
455
|
+
if (parsed >= UNLIMITED_CGROUP_BYTES) return null;
|
|
456
|
+
const number = Number(parsed);
|
|
457
|
+
if (!Number.isSafeInteger(number)) {
|
|
458
|
+
throw new Error(
|
|
459
|
+
`Finite cgroup memory limit exceeds JavaScript's safe integer range: ${parsed}`,
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
return number;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
export function parseCgroupCurrentValue(value: string): number {
|
|
466
|
+
const parsed = parseCgroupBigInt(value);
|
|
467
|
+
if (parsed === null || parsed < 0n) {
|
|
468
|
+
throw new Error(`Malformed cgroup current memory value: ${JSON.stringify(value.trim())}`);
|
|
469
|
+
}
|
|
470
|
+
const number = Number(parsed);
|
|
471
|
+
if (!Number.isSafeInteger(number)) {
|
|
472
|
+
throw new Error(`Cgroup current memory exceeds JavaScript's safe integer range: ${parsed}`);
|
|
473
|
+
}
|
|
474
|
+
return number;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function parseCgroupBigInt(value: string): bigint | null {
|
|
478
|
+
try {
|
|
479
|
+
return BigInt(value.trim());
|
|
480
|
+
} catch {
|
|
481
|
+
return null;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function readCgroupText(path: string): string | null {
|
|
486
|
+
try {
|
|
487
|
+
return readFileSync(path, "utf8");
|
|
488
|
+
} catch (error) {
|
|
489
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
490
|
+
throw new Error(
|
|
491
|
+
`Unable to read present cgroup memory controller file ${path}: ${
|
|
492
|
+
error instanceof Error ? error.message : String(error)
|
|
493
|
+
}`,
|
|
494
|
+
{ cause: error },
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
function positiveInteger(name: string, value: number): number {
|
|
500
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
501
|
+
throw new Error(`${name} must be a positive integer`);
|
|
502
|
+
}
|
|
503
|
+
return value;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
function nonnegativeInteger(name: string, value: number): number {
|
|
507
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
508
|
+
throw new Error(`${name} must be a non-negative integer`);
|
|
509
|
+
}
|
|
510
|
+
return value;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function abortError(): DOMException {
|
|
514
|
+
return new DOMException("Turn slot reservation aborted", "AbortError");
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function abortableDelay(milliseconds: number, signal: AbortSignal): Promise<void> {
|
|
518
|
+
return new Promise((resolve, reject) => {
|
|
519
|
+
if (signal.aborted) {
|
|
520
|
+
reject(abortError());
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
const timer = setTimeout(() => {
|
|
524
|
+
signal.removeEventListener("abort", onAbort);
|
|
525
|
+
resolve();
|
|
526
|
+
}, milliseconds);
|
|
527
|
+
timer.unref?.();
|
|
528
|
+
const onAbort = () => {
|
|
529
|
+
clearTimeout(timer);
|
|
530
|
+
reject(abortError());
|
|
531
|
+
};
|
|
532
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// Session workflows finish at idle boundaries and replay from durable history.
|
|
537
|
+
// Bound sticky VM state explicitly: the SDK otherwise derives this from the
|
|
538
|
+
// host heap limit (roughly 600 cached workflows/GiB), which is far above this
|
|
539
|
+
// process's actual concurrent workflow-task capacity and permanently retains
|
|
540
|
+
// idle workflow state in every control replica.
|
|
541
|
+
export const CONTROL_WORKER_MAX_CACHED_WORKFLOWS = 64;
|