@opengeni/worker-bundle 0.16.29 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +110 -19
- package/dist/activities/api-integrations.d.ts +21 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codemode-dispatcher.d.ts +35 -0
- package/dist/activities/codemode-token-renewal.d.ts +23 -0
- package/dist/activities/codex-capacity.d.ts +6 -5
- package/dist/activities/codex-image-generation.d.ts +21 -0
- package/dist/activities/context-compaction.d.ts +1 -0
- package/dist/activities/documents.d.ts +3 -2
- package/dist/activities/environment.d.ts +17 -8
- package/dist/activities/expiring-material-renewal.d.ts +1 -1
- package/dist/activities/file-upload-reaper.d.ts +7 -2
- package/dist/activities/gateway-image-generation.d.ts +39 -0
- package/dist/activities/gateway-video-generation.d.ts +53 -0
- package/dist/activities/generated-images.d.ts +112 -0
- package/dist/activities/git-credential-renewal.d.ts +1 -1
- package/dist/activities/goals.d.ts +2 -2
- package/dist/activities/google-drive-changes.d.ts +99 -0
- package/dist/activities/image-generation-operation.d.ts +74 -0
- package/dist/activities/image-generation-references.d.ts +25 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +23 -0
- package/dist/activities/knowledge-source-sync.d.ts +75 -0
- package/dist/activities/packs.d.ts +21 -36
- package/dist/activities/parent-wake.d.ts +1 -1
- package/dist/activities/retained-screenshot-reaper.d.ts +2 -2
- package/dist/activities/retained-screenshots.d.ts +36 -0
- package/dist/activities/rig-verification.d.ts +82 -5
- package/dist/activities/run-credential-renewal.d.ts +1 -1
- package/dist/activities/run-input.d.ts +3 -1
- package/dist/activities/sandbox-images.d.ts +14 -0
- package/dist/activities/sandbox-lease.d.ts +47 -9
- package/dist/activities/scheduled-tasks.d.ts +2 -2
- package/dist/activities/session-state.d.ts +7 -5
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/streaming.d.ts +1 -1
- package/dist/activities/types.d.ts +87 -5
- package/dist/activities/video-generation-admission.d.ts +43 -0
- package/dist/activities/video-generation-credential.d.ts +24 -0
- package/dist/activities/video-generation-reconciliation.d.ts +13 -0
- package/dist/activities/video-media-validation.d.ts +34 -0
- package/dist/activities/video-output-retention.d.ts +36 -0
- package/dist/activities/video-reference-staging.d.ts +55 -0
- package/dist/activities/workflow-wake.d.ts +2 -2
- package/dist/activities/workspace-capture.d.ts +10 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/activities-control-XURE4VKN.js +7221 -0
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +297 -0
- package/dist/activities-turn-6EBXGGLS.js +16036 -0
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities-turn.d.ts +17 -0
- package/dist/activities.d.ts +138 -150
- package/dist/activity-services.d.ts +10 -0
- package/dist/artifact-materializer-entry.d.ts +6 -0
- package/dist/artifact-materializer-entry.js +2662 -0
- package/dist/artifact-materializer-entry.js.map +1 -0
- package/dist/artifact-outbox-entry.d.ts +2 -0
- package/dist/artifact-outbox-entry.js +681 -0
- package/dist/artifact-outbox-entry.js.map +1 -0
- package/dist/chunk-5GRJIP5H.js +3092 -0
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/chunk-GRFQTTSQ.js +448 -0
- package/dist/chunk-GRFQTTSQ.js.map +1 -0
- package/dist/chunk-TYNPCG4U.js +906 -0
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/concurrency.d.ts +108 -5
- package/dist/editable-artifact-hint-broker.d.ts +49 -0
- package/dist/editable-artifact-materializer-launcher.d.ts +49 -0
- package/dist/editable-artifact-materializer-service.d.ts +108 -0
- package/dist/editable-artifact-materializer-storage.d.ts +12 -0
- package/dist/editable-artifact-materializer-subprocess.d.ts +50 -0
- package/dist/editable-artifact-materializer.d.ts +262 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +44 -0
- package/dist/editable-artifact-outbox-service.d.ts +70 -0
- package/dist/host-export.d.ts +1 -1
- package/dist/index.d.ts +189 -10
- package/dist/index.js +715 -17380
- package/dist/index.js.map +1 -1
- package/dist/memory-pressure-guard.d.ts +57 -0
- package/dist/model-checkpoint-memory-collector.d.ts +23 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +53 -1
- package/dist/sandbox-reaper-contract.d.ts +40 -0
- package/dist/sandbox-reaper-timeout.d.ts +14 -0
- package/dist/sandbox-resume.d.ts +34 -7
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/worker-service-lifecycle.d.ts +6 -2
- package/dist/workflow-bundle.js +1094 -370
- package/dist/workflows/activities.d.ts +8 -2
- package/dist/workflows/document-index.d.ts +1 -1
- package/dist/workflows/knowledge-source-sync.d.ts +13 -0
- package/dist/workflows/rig-verification.d.ts +1 -1
- package/dist/workflows/sandbox-reaper.d.ts +8 -0
- package/dist/workflows/scheduled-tasks.d.ts +1 -1
- package/dist/workflows/session.d.ts +2 -2
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows/video-generation.d.ts +10 -0
- package/dist/workflows/workflow-wake.d.ts +1 -1
- package/dist/workflows.d.ts +10 -7
- package/package.json +35 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +2600 -340
- package/src/activities/api-integrations.ts +138 -0
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codemode-dispatcher.ts +335 -0
- package/src/activities/codemode-token-renewal.ts +64 -0
- package/src/activities/codex-capacity.ts +54 -7
- package/src/activities/codex-image-generation.ts +56 -0
- package/src/activities/context-compaction.ts +38 -3
- package/src/activities/documents.ts +14 -4
- package/src/activities/environment.ts +54 -45
- package/src/activities/expiring-material-renewal.ts +1 -1
- package/src/activities/file-upload-reaper.ts +34 -3
- package/src/activities/gateway-image-generation.ts +146 -0
- package/src/activities/gateway-video-generation.ts +298 -0
- package/src/activities/generated-images.ts +860 -0
- package/src/activities/goals.ts +8 -3
- package/src/activities/google-drive-changes.ts +454 -0
- package/src/activities/image-generation-operation.ts +333 -0
- package/src/activities/image-generation-references.ts +118 -0
- package/src/activities/knowledge-source-sync-driver.ts +27 -0
- package/src/activities/knowledge-source-sync.ts +2840 -0
- package/src/activities/mcp-credentials.ts +1 -0
- package/src/activities/packs.ts +75 -152
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/retained-screenshot-reaper.ts +2 -2
- package/src/activities/retained-screenshots.ts +453 -28
- package/src/activities/rig-verification.ts +554 -43
- package/src/activities/run-input.ts +22 -6
- package/src/activities/sandbox-images.ts +167 -0
- package/src/activities/sandbox-lease.ts +1065 -483
- package/src/activities/scheduled-tasks.ts +104 -74
- package/src/activities/session-state.ts +22 -4
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/streaming.ts +42 -3
- package/src/activities/types.ts +104 -5
- package/src/activities/video-generation-admission.ts +303 -0
- package/src/activities/video-generation-credential.ts +112 -0
- package/src/activities/video-generation-reconciliation.ts +771 -0
- package/src/activities/video-media-validation.ts +332 -0
- package/src/activities/video-output-retention.ts +314 -0
- package/src/activities/video-reference-staging.ts +350 -0
- package/src/activities/workflow-wake.ts +2 -2
- package/src/activities/workspace-capture.ts +44 -10
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +58 -0
- package/src/activities-turn.ts +40 -0
- package/src/activities.ts +53 -124
- package/src/activity-services.ts +62 -0
- package/src/artifact-materializer-entry.ts +35 -0
- package/src/artifact-outbox-entry.ts +29 -0
- package/src/concurrency.ts +484 -6
- package/src/editable-artifact-hint-broker.ts +235 -0
- package/src/editable-artifact-materializer-launcher.ts +515 -0
- package/src/editable-artifact-materializer-service.ts +787 -0
- package/src/editable-artifact-materializer-storage.ts +413 -0
- package/src/editable-artifact-materializer-subprocess.ts +956 -0
- package/src/editable-artifact-materializer.ts +1289 -0
- package/src/editable-artifact-outbox-dispatcher.ts +244 -0
- package/src/editable-artifact-outbox-service.ts +438 -0
- package/src/index.ts +328 -72
- package/src/memory-pressure-guard.ts +489 -0
- package/src/model-checkpoint-memory-collector.ts +123 -0
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +297 -13
- package/src/sandbox-reaper-contract.ts +73 -0
- package/src/sandbox-reaper-timeout.ts +79 -0
- package/src/sandbox-resume.ts +294 -153
- package/src/sandbox-routing.ts +32 -24
- package/src/worker-service-lifecycle.ts +80 -14
- package/src/workflows/activities.ts +31 -0
- package/src/workflows/file-upload-reaper.ts +9 -1
- package/src/workflows/knowledge-source-sync.ts +83 -0
- package/src/workflows/sandbox-reaper.ts +183 -21
- package/src/workflows/scheduled-tasks.ts +42 -6
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows/video-generation.ts +36 -0
- package/src/workflows.ts +18 -1
- package/dist/activities/toolspace-token-renewal.d.ts +0 -23
- package/src/activities/toolspace-token-renewal.ts +0 -64
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { freemem, totalmem } from "node:os";
|
|
3
|
+
import { posix } from "node:path";
|
|
4
|
+
import type { Settings } from "@opengeni/config";
|
|
5
|
+
import type { Observability } from "@opengeni/observability";
|
|
6
|
+
|
|
7
|
+
export type TurnWorkerMemoryScope = "host" | "cgroup";
|
|
8
|
+
|
|
9
|
+
export type TurnWorkerMemoryScopeSnapshot = {
|
|
10
|
+
scope: TurnWorkerMemoryScope;
|
|
11
|
+
usedBytes: number;
|
|
12
|
+
limitBytes: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type TurnWorkerMemoryPressureSnapshot = {
|
|
16
|
+
processRssBytes: number;
|
|
17
|
+
processHeapUsedBytes: number;
|
|
18
|
+
processExternalBytes: number;
|
|
19
|
+
processArrayBuffersBytes: number;
|
|
20
|
+
scopes: TurnWorkerMemoryScopeSnapshot[];
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export type TurnWorkerMemoryPressureGuard = {
|
|
24
|
+
sampleNow(): void;
|
|
25
|
+
close(): void;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
type GuardSettings = Pick<
|
|
29
|
+
Settings,
|
|
30
|
+
| "turnWorkerConcurrencyMode"
|
|
31
|
+
| "turnWorkerTargetMemoryUsage"
|
|
32
|
+
| "turnWorkerEmergencyMemoryUsage"
|
|
33
|
+
| "turnWorkerMemoryGuardIntervalMs"
|
|
34
|
+
| "turnWorkerMemoryGuardSustainMs"
|
|
35
|
+
>;
|
|
36
|
+
|
|
37
|
+
type GuardDependencies = {
|
|
38
|
+
now?: () => number;
|
|
39
|
+
sample?: () => TurnWorkerMemoryPressureSnapshot;
|
|
40
|
+
setInterval?: typeof setInterval;
|
|
41
|
+
clearInterval?: typeof clearInterval;
|
|
42
|
+
gc?: () => void;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
type SnapshotDependencies = {
|
|
46
|
+
readText?: (path: string) => string | null;
|
|
47
|
+
hostTotalBytes?: () => number;
|
|
48
|
+
hostAvailableBytes?: () => number;
|
|
49
|
+
processMemoryUsage?: () => Pick<
|
|
50
|
+
NodeJS.MemoryUsage,
|
|
51
|
+
"rss" | "heapUsed" | "external" | "arrayBuffers"
|
|
52
|
+
>;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
export type ProcessCgroupMembership = {
|
|
56
|
+
v2Path: string | null;
|
|
57
|
+
v1MemoryPath: string | null;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type CgroupMemoryFilePair = {
|
|
61
|
+
currentPath: string;
|
|
62
|
+
maxPath: string;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
const CGROUP_V2_MOUNT = "/sys/fs/cgroup";
|
|
66
|
+
const CGROUP_V1_MEMORY_MOUNT = "/sys/fs/cgroup/memory";
|
|
67
|
+
|
|
68
|
+
export function parseLinuxMeminfo(
|
|
69
|
+
value: string,
|
|
70
|
+
): { totalBytes: number; availableBytes: number } | null {
|
|
71
|
+
const fields = new Map<string, number>();
|
|
72
|
+
for (const line of value.split("\n")) {
|
|
73
|
+
const match = /^([A-Za-z_()]+):\s+(\d+)\s+kB$/.exec(line.trim());
|
|
74
|
+
if (!match) continue;
|
|
75
|
+
fields.set(match[1] as string, Number(match[2]) * 1024);
|
|
76
|
+
}
|
|
77
|
+
const totalBytes = fields.get("MemTotal");
|
|
78
|
+
const availableBytes = fields.get("MemAvailable");
|
|
79
|
+
if (
|
|
80
|
+
totalBytes === undefined ||
|
|
81
|
+
availableBytes === undefined ||
|
|
82
|
+
!Number.isFinite(totalBytes) ||
|
|
83
|
+
!Number.isFinite(availableBytes) ||
|
|
84
|
+
totalBytes <= 0 ||
|
|
85
|
+
availableBytes < 0
|
|
86
|
+
) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
return { totalBytes, availableBytes: Math.min(totalBytes, availableBytes) };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function parseFiniteCgroupMemoryLimit(value: string): number | null {
|
|
93
|
+
const trimmed = value.trim();
|
|
94
|
+
if (!trimmed || trimmed === "max") return null;
|
|
95
|
+
const parsed = Number(trimmed);
|
|
96
|
+
// cgroup v1 represents "unlimited" with a near-int64 sentinel. Treat only
|
|
97
|
+
// realistic finite limits as authority; host headroom remains the fallback.
|
|
98
|
+
if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed >= 2 ** 60) return null;
|
|
99
|
+
return parsed;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function parseProcessCgroupMembership(value: string): ProcessCgroupMembership {
|
|
103
|
+
let v2Path: string | null = null;
|
|
104
|
+
let v1MemoryPath: string | null = null;
|
|
105
|
+
for (const line of value.split("\n")) {
|
|
106
|
+
const firstSeparator = line.indexOf(":");
|
|
107
|
+
const secondSeparator = line.indexOf(":", firstSeparator + 1);
|
|
108
|
+
if (firstSeparator <= 0 || secondSeparator < 0) continue;
|
|
109
|
+
const hierarchyId = line.slice(0, firstSeparator);
|
|
110
|
+
const controllers = line.slice(firstSeparator + 1, secondSeparator);
|
|
111
|
+
const path = normalizeCgroupPath(line.slice(secondSeparator + 1));
|
|
112
|
+
if (!path) continue;
|
|
113
|
+
if (hierarchyId === "0" && controllers === "") {
|
|
114
|
+
v2Path = path;
|
|
115
|
+
}
|
|
116
|
+
if (controllers.split(",").includes("memory")) {
|
|
117
|
+
v1MemoryPath = path;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return { v2Path, v1MemoryPath };
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function cgroupMemoryFilePairs(membership: ProcessCgroupMembership): CgroupMemoryFilePair[] {
|
|
124
|
+
const pairs: CgroupMemoryFilePair[] = [];
|
|
125
|
+
const seen = new Set<string>();
|
|
126
|
+
const appendHierarchy = (
|
|
127
|
+
mount: string,
|
|
128
|
+
path: string | null,
|
|
129
|
+
currentFilename: string,
|
|
130
|
+
maxFilename: string,
|
|
131
|
+
) => {
|
|
132
|
+
for (const candidate of cgroupAncestorPaths(path ?? "/")) {
|
|
133
|
+
const relative = candidate === "/" ? "" : candidate.slice(1);
|
|
134
|
+
const currentPath = posix.join(mount, relative, currentFilename);
|
|
135
|
+
const maxPath = posix.join(mount, relative, maxFilename);
|
|
136
|
+
const key = `${currentPath}\0${maxPath}`;
|
|
137
|
+
if (seen.has(key)) continue;
|
|
138
|
+
seen.add(key);
|
|
139
|
+
pairs.push({ currentPath, maxPath });
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
appendHierarchy(CGROUP_V2_MOUNT, membership.v2Path, "memory.current", "memory.max");
|
|
144
|
+
appendHierarchy(
|
|
145
|
+
CGROUP_V1_MEMORY_MOUNT,
|
|
146
|
+
membership.v1MemoryPath,
|
|
147
|
+
"memory.usage_in_bytes",
|
|
148
|
+
"memory.limit_in_bytes",
|
|
149
|
+
);
|
|
150
|
+
return pairs;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function readTurnWorkerMemoryPressureSnapshot(
|
|
154
|
+
dependencies: SnapshotDependencies = {},
|
|
155
|
+
): TurnWorkerMemoryPressureSnapshot {
|
|
156
|
+
const readText = dependencies.readText ?? readTextOrNull;
|
|
157
|
+
const linux = parseLinuxMeminfo(readText("/proc/meminfo") ?? "");
|
|
158
|
+
const hostTotalBytes = linux?.totalBytes ?? (dependencies.hostTotalBytes ?? totalmem)();
|
|
159
|
+
const hostAvailableBytes =
|
|
160
|
+
linux?.availableBytes ?? (dependencies.hostAvailableBytes ?? freemem)();
|
|
161
|
+
const host = {
|
|
162
|
+
scope: "host" as const,
|
|
163
|
+
usedBytes: Math.max(0, hostTotalBytes - Math.min(hostTotalBytes, hostAvailableBytes)),
|
|
164
|
+
limitBytes: hostTotalBytes,
|
|
165
|
+
};
|
|
166
|
+
const cgroup = readCgroupMemoryScope(readText);
|
|
167
|
+
// A finite process cgroup is the pod's actual admission boundary. Host
|
|
168
|
+
// pressure includes unrelated workloads and must not make every replica
|
|
169
|
+
// drain simultaneously; use it only when no finite cgroup authority exists.
|
|
170
|
+
const processMemory = (dependencies.processMemoryUsage ?? process.memoryUsage)();
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
processRssBytes: processMemory.rss,
|
|
174
|
+
processHeapUsedBytes: processMemory.heapUsed,
|
|
175
|
+
processExternalBytes: processMemory.external,
|
|
176
|
+
processArrayBuffersBytes: processMemory.arrayBuffers,
|
|
177
|
+
scopes: [cgroup ?? host],
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export function turnWorkerMemoryPressureGuardEnabled(
|
|
182
|
+
role: "control" | "turn",
|
|
183
|
+
settings: Pick<Settings, "turnWorkerConcurrencyMode">,
|
|
184
|
+
): boolean {
|
|
185
|
+
return role === "turn" && settings.turnWorkerConcurrencyMode === "resource-based";
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function createTurnWorkerMemoryPressureGuard(input: {
|
|
189
|
+
settings: GuardSettings;
|
|
190
|
+
observability: Observability;
|
|
191
|
+
drain: () => void;
|
|
192
|
+
dependencies?: GuardDependencies;
|
|
193
|
+
}): TurnWorkerMemoryPressureGuard {
|
|
194
|
+
const now = input.dependencies?.now ?? Date.now;
|
|
195
|
+
const sample = input.dependencies?.sample ?? readTurnWorkerMemoryPressureSnapshot;
|
|
196
|
+
const schedule = input.dependencies?.setInterval ?? setInterval;
|
|
197
|
+
const cancel = input.dependencies?.clearInterval ?? clearInterval;
|
|
198
|
+
const collectGarbage = input.dependencies?.gc ?? (() => Bun.gc(false));
|
|
199
|
+
const breachStartedAt = new Map<TurnWorkerMemoryScope, number>();
|
|
200
|
+
let lastGcAt = Number.NEGATIVE_INFINITY;
|
|
201
|
+
let drainRequested = false;
|
|
202
|
+
let closed = false;
|
|
203
|
+
|
|
204
|
+
const sampleNow = () => {
|
|
205
|
+
if (closed) return;
|
|
206
|
+
const observedAt = now();
|
|
207
|
+
let snapshot: TurnWorkerMemoryPressureSnapshot;
|
|
208
|
+
try {
|
|
209
|
+
snapshot = sample();
|
|
210
|
+
} catch {
|
|
211
|
+
breachStartedAt.clear();
|
|
212
|
+
safeIncrementCounter(input.observability, {
|
|
213
|
+
name: "opengeni_turn_worker_memory_guard_sample_failures_total",
|
|
214
|
+
help: "Turn-worker memory pressure guard samples that could not be read.",
|
|
215
|
+
});
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const validScopes = snapshot.scopes.filter(
|
|
220
|
+
({ usedBytes, limitBytes }) =>
|
|
221
|
+
Number.isFinite(usedBytes) &&
|
|
222
|
+
Number.isFinite(limitBytes) &&
|
|
223
|
+
usedBytes >= 0 &&
|
|
224
|
+
limitBytes > 0,
|
|
225
|
+
);
|
|
226
|
+
if (validScopes.length === 0) {
|
|
227
|
+
breachStartedAt.clear();
|
|
228
|
+
safeIncrementCounter(input.observability, {
|
|
229
|
+
name: "opengeni_turn_worker_memory_guard_sample_failures_total",
|
|
230
|
+
help: "Turn-worker memory pressure guard samples that could not be read.",
|
|
231
|
+
});
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const sustainedBreaches: Array<{
|
|
236
|
+
scope: TurnWorkerMemoryScope;
|
|
237
|
+
utilizationRatio: number;
|
|
238
|
+
breachSeconds: number;
|
|
239
|
+
}> = [];
|
|
240
|
+
for (const [kind, value] of [
|
|
241
|
+
["rss", snapshot.processRssBytes],
|
|
242
|
+
["heap_used", snapshot.processHeapUsedBytes],
|
|
243
|
+
["external", snapshot.processExternalBytes],
|
|
244
|
+
["array_buffers", snapshot.processArrayBuffersBytes],
|
|
245
|
+
] as const) {
|
|
246
|
+
safeSetGauge(input.observability, {
|
|
247
|
+
name: "opengeni_turn_worker_process_memory_bytes",
|
|
248
|
+
help: "Turn-worker process memory by runtime-reported category.",
|
|
249
|
+
labels: { kind },
|
|
250
|
+
value: Number.isFinite(value) && value >= 0 ? value : 0,
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
let shouldCollectGarbage = false;
|
|
254
|
+
for (const scope of validScopes) {
|
|
255
|
+
const utilizationRatio = scope.usedBytes / scope.limitBytes;
|
|
256
|
+
const processRssRatio = snapshot.processRssBytes / scope.limitBytes;
|
|
257
|
+
safeSetGauge(input.observability, {
|
|
258
|
+
name: "opengeni_turn_worker_memory_guard_utilization_ratio",
|
|
259
|
+
help: "Observed memory utilization ratio used by the turn-worker pressure guard.",
|
|
260
|
+
labels: { scope: scope.scope },
|
|
261
|
+
value: utilizationRatio,
|
|
262
|
+
});
|
|
263
|
+
safeSetGauge(input.observability, {
|
|
264
|
+
name: "opengeni_turn_worker_memory_guard_target_ratio",
|
|
265
|
+
help: "Memory admission target enforced by the Temporal resource tuner.",
|
|
266
|
+
labels: { scope: scope.scope },
|
|
267
|
+
value: input.settings.turnWorkerTargetMemoryUsage,
|
|
268
|
+
});
|
|
269
|
+
safeSetGauge(input.observability, {
|
|
270
|
+
name: "opengeni_turn_worker_memory_guard_emergency_ratio",
|
|
271
|
+
help: "Emergency memory threshold enforced by the turn-worker pressure guard.",
|
|
272
|
+
labels: { scope: scope.scope },
|
|
273
|
+
value: input.settings.turnWorkerEmergencyMemoryUsage,
|
|
274
|
+
});
|
|
275
|
+
safeSetGauge(input.observability, {
|
|
276
|
+
name: "opengeni_turn_worker_memory_guard_available_bytes",
|
|
277
|
+
help: "Memory headroom remaining in the turn-worker pressure guard scope.",
|
|
278
|
+
labels: { scope: scope.scope },
|
|
279
|
+
value: Math.max(0, scope.limitBytes - scope.usedBytes),
|
|
280
|
+
});
|
|
281
|
+
safeSetGauge(input.observability, {
|
|
282
|
+
name: "opengeni_turn_worker_memory_guard_process_rss_ratio",
|
|
283
|
+
help: "Turn-worker RSS as a ratio of the observed host or finite cgroup memory scope.",
|
|
284
|
+
labels: { scope: scope.scope },
|
|
285
|
+
value: processRssRatio,
|
|
286
|
+
});
|
|
287
|
+
shouldCollectGarbage ||= processRssRatio >= input.settings.turnWorkerTargetMemoryUsage;
|
|
288
|
+
|
|
289
|
+
if (utilizationRatio < input.settings.turnWorkerEmergencyMemoryUsage) {
|
|
290
|
+
breachStartedAt.delete(scope.scope);
|
|
291
|
+
safeSetGauge(input.observability, {
|
|
292
|
+
name: "opengeni_turn_worker_memory_guard_breach_seconds",
|
|
293
|
+
help: "Seconds the turn-worker emergency threshold has remained continuously breached.",
|
|
294
|
+
labels: { scope: scope.scope },
|
|
295
|
+
value: 0,
|
|
296
|
+
});
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const startedAt = breachStartedAt.get(scope.scope) ?? observedAt;
|
|
301
|
+
breachStartedAt.set(scope.scope, startedAt);
|
|
302
|
+
const breachMs = Math.max(0, observedAt - startedAt);
|
|
303
|
+
const breachSeconds = breachMs / 1_000;
|
|
304
|
+
safeSetGauge(input.observability, {
|
|
305
|
+
name: "opengeni_turn_worker_memory_guard_breach_seconds",
|
|
306
|
+
help: "Seconds the turn-worker emergency threshold has remained continuously breached.",
|
|
307
|
+
labels: { scope: scope.scope },
|
|
308
|
+
value: breachSeconds,
|
|
309
|
+
});
|
|
310
|
+
if (breachMs >= input.settings.turnWorkerMemoryGuardSustainMs) {
|
|
311
|
+
sustainedBreaches.push({ scope: scope.scope, utilizationRatio, breachSeconds });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Large request buffers can become unreachable well before JSC's ordinary
|
|
316
|
+
// heap trigger notices native/external pressure. Nudge an asynchronous GC
|
|
317
|
+
// at most once per sustain window before escalating to a worker drain.
|
|
318
|
+
const sustainedBreachesHadGcOpportunity =
|
|
319
|
+
sustainedBreaches.length > 0 &&
|
|
320
|
+
sustainedBreaches.every(({ scope }) => {
|
|
321
|
+
const breachStarted = breachStartedAt.get(scope);
|
|
322
|
+
return breachStarted !== undefined && lastGcAt >= breachStarted && lastGcAt < observedAt;
|
|
323
|
+
});
|
|
324
|
+
let collectedGarbageThisSample = false;
|
|
325
|
+
if (
|
|
326
|
+
shouldCollectGarbage &&
|
|
327
|
+
observedAt - lastGcAt >= input.settings.turnWorkerMemoryGuardSustainMs &&
|
|
328
|
+
(sustainedBreaches.length === 0 || !sustainedBreachesHadGcOpportunity)
|
|
329
|
+
) {
|
|
330
|
+
lastGcAt = observedAt;
|
|
331
|
+
try {
|
|
332
|
+
collectGarbage();
|
|
333
|
+
collectedGarbageThisSample = true;
|
|
334
|
+
safeIncrementCounter(input.observability, {
|
|
335
|
+
name: "opengeni_turn_worker_memory_guard_gc_total",
|
|
336
|
+
help: "Proactive garbage-collection nudges under process memory pressure.",
|
|
337
|
+
});
|
|
338
|
+
} catch {
|
|
339
|
+
safeIncrementCounter(input.observability, {
|
|
340
|
+
name: "opengeni_turn_worker_memory_guard_gc_failures_total",
|
|
341
|
+
help: "Proactive memory-pressure garbage-collection nudges that failed.",
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// The nudge is asynchronous. Give it one sampling interval to make the
|
|
347
|
+
// reclaimed RSS visible before deciding that process recycling is needed.
|
|
348
|
+
if (
|
|
349
|
+
(collectedGarbageThisSample && !sustainedBreachesHadGcOpportunity) ||
|
|
350
|
+
drainRequested ||
|
|
351
|
+
sustainedBreaches.length === 0
|
|
352
|
+
) {
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const primaryBreach = sustainedBreaches.sort(
|
|
357
|
+
(left, right) => right.utilizationRatio - left.utilizationRatio,
|
|
358
|
+
)[0];
|
|
359
|
+
if (!primaryBreach) return;
|
|
360
|
+
|
|
361
|
+
drainRequested = true;
|
|
362
|
+
safeIncrementCounter(input.observability, {
|
|
363
|
+
name: "opengeni_turn_worker_memory_guard_drains_total",
|
|
364
|
+
help: "Graceful turn-worker drains requested after sustained memory pressure.",
|
|
365
|
+
labels: { scope: primaryBreach.scope },
|
|
366
|
+
});
|
|
367
|
+
safeWarn(input.observability, "turn worker memory guard requested graceful drain", {
|
|
368
|
+
errorClass: "WorkerLifecycleOperation",
|
|
369
|
+
errorCode: "worker_draining",
|
|
370
|
+
origin: "worker-lifecycle",
|
|
371
|
+
reason: `memory pressure (${primaryBreach.scope}, ${boundedRatio(primaryBreach.utilizationRatio)} >= ${input.settings.turnWorkerEmergencyMemoryUsage}, ${primaryBreach.breachSeconds}s)`,
|
|
372
|
+
});
|
|
373
|
+
try {
|
|
374
|
+
input.drain();
|
|
375
|
+
} catch {
|
|
376
|
+
drainRequested = false;
|
|
377
|
+
safeIncrementCounter(input.observability, {
|
|
378
|
+
name: "opengeni_turn_worker_memory_guard_drain_failures_total",
|
|
379
|
+
help: "Turn-worker memory pressure guard drain requests that failed synchronously.",
|
|
380
|
+
labels: { scope: primaryBreach.scope },
|
|
381
|
+
});
|
|
382
|
+
safeWarn(input.observability, "turn worker memory guard drain request failed", {
|
|
383
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
384
|
+
errorCode: "worker_shutdown_request_failed",
|
|
385
|
+
origin: "worker-lifecycle",
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
const timer = schedule(sampleNow, input.settings.turnWorkerMemoryGuardIntervalMs);
|
|
391
|
+
timer.unref?.();
|
|
392
|
+
sampleNow();
|
|
393
|
+
|
|
394
|
+
return {
|
|
395
|
+
sampleNow,
|
|
396
|
+
close: () => {
|
|
397
|
+
if (closed) return;
|
|
398
|
+
closed = true;
|
|
399
|
+
cancel(timer);
|
|
400
|
+
},
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function readCgroupMemoryScope(
|
|
405
|
+
readText: (path: string) => string | null,
|
|
406
|
+
): TurnWorkerMemoryScopeSnapshot | null {
|
|
407
|
+
const membership = parseProcessCgroupMembership(readText("/proc/self/cgroup") ?? "");
|
|
408
|
+
let mostPressured: TurnWorkerMemoryScopeSnapshot | null = null;
|
|
409
|
+
for (const { currentPath, maxPath } of cgroupMemoryFilePairs(membership)) {
|
|
410
|
+
const currentRaw = readText(currentPath);
|
|
411
|
+
const maxRaw = readText(maxPath);
|
|
412
|
+
if (currentRaw === null || maxRaw === null) continue;
|
|
413
|
+
const current = Number(currentRaw.trim());
|
|
414
|
+
const limit = parseFiniteCgroupMemoryLimit(maxRaw);
|
|
415
|
+
if (!Number.isFinite(current) || current < 0 || limit === null) continue;
|
|
416
|
+
const candidate = { scope: "cgroup" as const, usedBytes: current, limitBytes: limit };
|
|
417
|
+
if (
|
|
418
|
+
!mostPressured ||
|
|
419
|
+
candidate.usedBytes / candidate.limitBytes >
|
|
420
|
+
mostPressured.usedBytes / mostPressured.limitBytes
|
|
421
|
+
) {
|
|
422
|
+
mostPressured = candidate;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return mostPressured;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function readTextOrNull(path: string): string | null {
|
|
429
|
+
try {
|
|
430
|
+
return readFileSync(path, "utf8");
|
|
431
|
+
} catch {
|
|
432
|
+
return null;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function boundedRatio(value: number): number {
|
|
437
|
+
return Math.round(Math.max(0, Math.min(value, 10)) * 1_000_000) / 1_000_000;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
function normalizeCgroupPath(value: string): string | null {
|
|
441
|
+
const trimmed = value.trim();
|
|
442
|
+
if (!trimmed.startsWith("/") || trimmed.includes("\0")) return null;
|
|
443
|
+
if (trimmed.split("/").includes("..")) return null;
|
|
444
|
+
return posix.normalize(trimmed);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function cgroupAncestorPaths(path: string): string[] {
|
|
448
|
+
const ancestors: string[] = [];
|
|
449
|
+
let current = path;
|
|
450
|
+
while (true) {
|
|
451
|
+
ancestors.push(current);
|
|
452
|
+
if (current === "/") return ancestors;
|
|
453
|
+
current = posix.dirname(current);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function safeSetGauge(
|
|
458
|
+
observability: Observability,
|
|
459
|
+
input: Parameters<Observability["setGauge"]>[0],
|
|
460
|
+
): void {
|
|
461
|
+
try {
|
|
462
|
+
observability.setGauge(input);
|
|
463
|
+
} catch {
|
|
464
|
+
// Telemetry cannot disable the memory-safety action.
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function safeIncrementCounter(
|
|
469
|
+
observability: Observability,
|
|
470
|
+
input: Parameters<Observability["incrementCounter"]>[0],
|
|
471
|
+
): void {
|
|
472
|
+
try {
|
|
473
|
+
observability.incrementCounter(input);
|
|
474
|
+
} catch {
|
|
475
|
+
// Telemetry cannot disable the memory-safety action.
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function safeWarn(
|
|
480
|
+
observability: Observability,
|
|
481
|
+
message: string,
|
|
482
|
+
attributes: Parameters<Observability["warn"]>[1],
|
|
483
|
+
): void {
|
|
484
|
+
try {
|
|
485
|
+
observability.warn(message, attributes);
|
|
486
|
+
} catch {
|
|
487
|
+
// Telemetry cannot disable the memory-safety action.
|
|
488
|
+
}
|
|
489
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
|
|
3
|
+
export const MODEL_CHECKPOINT_GC_PRESSURE_BYTES = 512 * 1024 * 1024;
|
|
4
|
+
export const MODEL_CHECKPOINT_GC_COOLDOWN_MS = 30_000;
|
|
5
|
+
|
|
6
|
+
type ProcessMemory = Pick<NodeJS.MemoryUsage, "heapUsed" | "external">;
|
|
7
|
+
|
|
8
|
+
type ModelCheckpointMemoryCollectorDependencies = {
|
|
9
|
+
now?: () => number;
|
|
10
|
+
memoryUsage?: () => ProcessMemory;
|
|
11
|
+
gc?: () => void;
|
|
12
|
+
setTimeout?: typeof setTimeout;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type ModelCheckpointMemoryCollector = {
|
|
16
|
+
schedule(observability: Observability): void;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export function modelCheckpointPressureBytes(memory: ProcessMemory): number {
|
|
20
|
+
// `external` includes ArrayBuffer storage in Bun/Node's memoryUsage contract.
|
|
21
|
+
// Heap and external allocations are independent, so max(heap, external)
|
|
22
|
+
// undercounts the actual reclaimable pressure whenever both are large.
|
|
23
|
+
return memory.heapUsed + memory.external;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Reclaim request serialization and stream buffers only after one model
|
|
28
|
+
* response has crossed the durable conversation checkpoint. The callback runs
|
|
29
|
+
* on a later task so the completed response's stack can unwind first. Active
|
|
30
|
+
* turns and their exact model context remain strongly reachable and are never
|
|
31
|
+
* shortened, restarted, or rewritten by this process-local collection.
|
|
32
|
+
*/
|
|
33
|
+
export function createModelCheckpointMemoryCollector(
|
|
34
|
+
dependencies: ModelCheckpointMemoryCollectorDependencies = {},
|
|
35
|
+
): ModelCheckpointMemoryCollector {
|
|
36
|
+
const now = dependencies.now ?? Date.now;
|
|
37
|
+
const memoryUsage = dependencies.memoryUsage ?? process.memoryUsage;
|
|
38
|
+
const collectGarbage = dependencies.gc ?? (() => Bun.gc(true));
|
|
39
|
+
const scheduleTask = dependencies.setTimeout ?? setTimeout;
|
|
40
|
+
let scheduled = false;
|
|
41
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
schedule(observability) {
|
|
45
|
+
if (scheduled) return;
|
|
46
|
+
scheduled = true;
|
|
47
|
+
try {
|
|
48
|
+
const timer = scheduleTask(() => {
|
|
49
|
+
scheduled = false;
|
|
50
|
+
let memory: ProcessMemory;
|
|
51
|
+
try {
|
|
52
|
+
memory = memoryUsage();
|
|
53
|
+
} catch {
|
|
54
|
+
safeIncrementCounter(observability, {
|
|
55
|
+
name: "opengeni_turn_worker_model_checkpoint_gc_failures_total",
|
|
56
|
+
help: "Model-checkpoint garbage collections that could not sample or collect memory.",
|
|
57
|
+
});
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const pressureBytes = modelCheckpointPressureBytes(memory);
|
|
62
|
+
const observedAt = now();
|
|
63
|
+
if (
|
|
64
|
+
!Number.isFinite(pressureBytes) ||
|
|
65
|
+
pressureBytes < MODEL_CHECKPOINT_GC_PRESSURE_BYTES ||
|
|
66
|
+
observedAt - lastCollectionAt < MODEL_CHECKPOINT_GC_COOLDOWN_MS
|
|
67
|
+
) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
lastCollectionAt = observedAt;
|
|
72
|
+
const startedAt = performance.now();
|
|
73
|
+
try {
|
|
74
|
+
collectGarbage();
|
|
75
|
+
safeIncrementCounter(observability, {
|
|
76
|
+
name: "opengeni_turn_worker_model_checkpoint_gc_total",
|
|
77
|
+
help: "Forced garbage collections after durable model-response checkpoints.",
|
|
78
|
+
});
|
|
79
|
+
safeObserveHistogram(observability, {
|
|
80
|
+
name: "opengeni_turn_worker_model_checkpoint_gc_duration_seconds",
|
|
81
|
+
help: "Forced garbage-collection duration after durable model-response checkpoints.",
|
|
82
|
+
value: Math.max(0, performance.now() - startedAt) / 1_000,
|
|
83
|
+
});
|
|
84
|
+
} catch {
|
|
85
|
+
safeIncrementCounter(observability, {
|
|
86
|
+
name: "opengeni_turn_worker_model_checkpoint_gc_failures_total",
|
|
87
|
+
help: "Model-checkpoint garbage collections that could not sample or collect memory.",
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}, 0);
|
|
91
|
+
try {
|
|
92
|
+
timer.unref?.();
|
|
93
|
+
} catch {
|
|
94
|
+
// A timer implementation without a usable unref is still safe.
|
|
95
|
+
}
|
|
96
|
+
} catch {
|
|
97
|
+
scheduled = false;
|
|
98
|
+
}
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function safeIncrementCounter(
|
|
104
|
+
observability: Observability,
|
|
105
|
+
input: Parameters<Observability["incrementCounter"]>[0],
|
|
106
|
+
): void {
|
|
107
|
+
try {
|
|
108
|
+
observability.incrementCounter(input);
|
|
109
|
+
} catch {
|
|
110
|
+
// Telemetry cannot disable collection at a durable model checkpoint.
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function safeObserveHistogram(
|
|
115
|
+
observability: Observability,
|
|
116
|
+
input: Parameters<Observability["observeHistogram"]>[0],
|
|
117
|
+
): void {
|
|
118
|
+
try {
|
|
119
|
+
observability.observeHistogram(input);
|
|
120
|
+
} catch {
|
|
121
|
+
// Telemetry cannot disable collection at a durable model checkpoint.
|
|
122
|
+
}
|
|
123
|
+
}
|