@ricsam/r5d-worker 0.0.111 → 0.0.113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -6
- package/dist/cjs/command-launcher.cjs +345 -0
- package/dist/cjs/inline-output.cjs +62 -0
- package/dist/cjs/main.cjs +149 -102
- package/dist/cjs/package.json +1 -1
- package/dist/cjs/supervisor.cjs +1 -1
- package/dist/mjs/command-launcher.mjs +316 -0
- package/dist/mjs/inline-output.mjs +37 -0
- package/dist/mjs/main.mjs +151 -103
- package/dist/mjs/package.json +1 -1
- package/dist/mjs/supervisor.mjs +1 -1
- package/dist/types/command-launcher.d.ts +61 -0
- package/dist/types/inline-output.d.ts +3 -0
- package/dist/types/supervisor-launcher-cleanup-fixture.d.ts +1 -0
- package/package.json +1 -1
- package/dist/cjs/job-resources.cjs +0 -335
- package/dist/mjs/job-resources.mjs +0 -293
- package/dist/types/job-resources.d.ts +0 -61
|
@@ -1,335 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
|
-
var __defProp = Object.defineProperty;
|
|
4
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
-
var __export = (target, all) => {
|
|
9
|
-
for (var name in all)
|
|
10
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
-
};
|
|
12
|
-
var __copyProps = (to, from, except, desc) => {
|
|
13
|
-
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
-
for (let key of __getOwnPropNames(from))
|
|
15
|
-
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
-
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
-
}
|
|
18
|
-
return to;
|
|
19
|
-
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
|
-
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
-
var job_resources_exports = {};
|
|
30
|
-
__export(job_resources_exports, {
|
|
31
|
-
WORKER_INLINE_OUTPUT_MAX_BYTES: () => WORKER_INLINE_OUTPUT_MAX_BYTES,
|
|
32
|
-
WorkerJobCancelledError: () => WorkerJobCancelledError,
|
|
33
|
-
WorkerJobCapacityError: () => WorkerJobCapacityError,
|
|
34
|
-
WorkerJobResourceController: () => WorkerJobResourceController,
|
|
35
|
-
collectWorkerInlineOutput: () => collectWorkerInlineOutput,
|
|
36
|
-
createWorkerJobResources: () => createWorkerJobResources,
|
|
37
|
-
effectiveWorkerMemory: () => effectiveWorkerMemory,
|
|
38
|
-
workerDelegatedCgroupRoot: () => workerDelegatedCgroupRoot,
|
|
39
|
-
workerJobBudget: () => workerJobBudget
|
|
40
|
-
});
|
|
41
|
-
module.exports = __toCommonJS(job_resources_exports);
|
|
42
|
-
var import_node_fs = __toESM(require("node:fs"), 1);
|
|
43
|
-
var import_node_os = __toESM(require("node:os"), 1);
|
|
44
|
-
var import_node_path = __toESM(require("node:path"), 1);
|
|
45
|
-
const MIB = 1024 * 1024;
|
|
46
|
-
const WORKER_INLINE_OUTPUT_MAX_BYTES = 8 * MIB;
|
|
47
|
-
async function collectWorkerInlineOutput(stream, maxBytes = WORKER_INLINE_OUTPUT_MAX_BYTES, signal) {
|
|
48
|
-
if (!stream) return "";
|
|
49
|
-
signal?.throwIfAborted();
|
|
50
|
-
const reader = stream.getReader();
|
|
51
|
-
const abort = () => {
|
|
52
|
-
void reader.cancel().catch(() => {
|
|
53
|
-
});
|
|
54
|
-
};
|
|
55
|
-
signal?.addEventListener("abort", abort, { once: true });
|
|
56
|
-
const decoder = new TextDecoder();
|
|
57
|
-
const parts = [];
|
|
58
|
-
let bytes = 0;
|
|
59
|
-
try {
|
|
60
|
-
while (true) {
|
|
61
|
-
const next = await reader.read();
|
|
62
|
-
signal?.throwIfAborted();
|
|
63
|
-
if (next.done) break;
|
|
64
|
-
bytes += next.value.byteLength;
|
|
65
|
-
if (bytes > maxBytes) {
|
|
66
|
-
void reader.cancel().catch(() => {
|
|
67
|
-
});
|
|
68
|
-
throw new Error(`Inline command output exceeded ${maxBytes} bytes. Use streaming execution for large output.`);
|
|
69
|
-
}
|
|
70
|
-
parts.push(decoder.decode(next.value, { stream: true }));
|
|
71
|
-
}
|
|
72
|
-
parts.push(decoder.decode());
|
|
73
|
-
return parts.join("");
|
|
74
|
-
} finally {
|
|
75
|
-
signal?.removeEventListener("abort", abort);
|
|
76
|
-
reader.releaseLock();
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
function positiveInteger(value, fallback, name) {
|
|
80
|
-
const result = value === void 0 ? fallback : Number(value);
|
|
81
|
-
if (!Number.isSafeInteger(result) || result <= 0) throw new Error(`${name} must be a positive integer`);
|
|
82
|
-
return result;
|
|
83
|
-
}
|
|
84
|
-
function workerJobBudget(effectiveMemoryBytes, env = process.env) {
|
|
85
|
-
const generalSlots = positiveInteger(env.R5D_WORKER_JOB_CONCURRENCY, 2, "R5D_WORKER_JOB_CONCURRENCY");
|
|
86
|
-
const utilitySlots = positiveInteger(env.R5D_WORKER_UTILITY_CONCURRENCY, 2, "R5D_WORKER_UTILITY_CONCURRENCY");
|
|
87
|
-
const memoryPercent = positiveInteger(env.R5D_WORKER_JOBS_MEMORY_PERCENT, 70, "R5D_WORKER_JOBS_MEMORY_PERCENT");
|
|
88
|
-
if (memoryPercent > 70) throw new Error("Worker jobs may reserve at most 70% of effective memory");
|
|
89
|
-
const aggregateBytes = positiveInteger(
|
|
90
|
-
env.R5D_WORKER_JOBS_MEMORY_BYTES,
|
|
91
|
-
Math.floor(effectiveMemoryBytes * memoryPercent / 100),
|
|
92
|
-
"R5D_WORKER_JOBS_MEMORY_BYTES"
|
|
93
|
-
);
|
|
94
|
-
const utilityBytes = positiveInteger(
|
|
95
|
-
env.R5D_WORKER_UTILITY_MEMORY_BYTES,
|
|
96
|
-
Math.min(256 * MIB, Math.floor(aggregateBytes / 10)),
|
|
97
|
-
"R5D_WORKER_UTILITY_MEMORY_BYTES"
|
|
98
|
-
);
|
|
99
|
-
const generalBytes = positiveInteger(
|
|
100
|
-
env.R5D_WORKER_JOB_MEMORY_BYTES,
|
|
101
|
-
Math.floor((aggregateBytes - utilityBytes * utilitySlots) / generalSlots),
|
|
102
|
-
"R5D_WORKER_JOB_MEMORY_BYTES"
|
|
103
|
-
);
|
|
104
|
-
if (aggregateBytes > Math.floor(effectiveMemoryBytes * 0.7)) throw new Error("Worker jobs may reserve at most 70% of effective memory");
|
|
105
|
-
if (generalBytes * generalSlots + utilityBytes * utilitySlots > aggregateBytes)
|
|
106
|
-
throw new Error("Worker job limits exceed the aggregate memory budget");
|
|
107
|
-
const queueTimeoutMs = positiveInteger(env.R5D_WORKER_JOB_QUEUE_TIMEOUT_MS, 6e4, "R5D_WORKER_JOB_QUEUE_TIMEOUT_MS");
|
|
108
|
-
if (queueTimeoutMs > 6e4) throw new Error("Worker job queue timeout cannot exceed 60000ms");
|
|
109
|
-
return {
|
|
110
|
-
aggregateBytes,
|
|
111
|
-
generalBytes,
|
|
112
|
-
utilityBytes,
|
|
113
|
-
generalSlots,
|
|
114
|
-
utilitySlots,
|
|
115
|
-
queueTimeoutMs,
|
|
116
|
-
maxQueued: positiveInteger(env.R5D_WORKER_JOB_QUEUE_LIMIT, 128, "R5D_WORKER_JOB_QUEUE_LIMIT"),
|
|
117
|
-
pidsPerJob: positiveInteger(env.R5D_WORKER_JOB_PIDS, 512, "R5D_WORKER_JOB_PIDS")
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
function workerDelegatedCgroupRoot(cgroupText, mount = "/sys/fs/cgroup") {
|
|
121
|
-
const relative = cgroupText.split("\n").find((line) => line.startsWith("0::/"))?.slice(3);
|
|
122
|
-
if (!relative) return void 0;
|
|
123
|
-
const segments = relative.split("/").filter(Boolean);
|
|
124
|
-
const index = segments.indexOf("r5d-worker.service");
|
|
125
|
-
if (index < 0 || segments[index + 1] !== "control") return void 0;
|
|
126
|
-
return import_node_path.default.join(mount, ...segments.slice(0, index + 1));
|
|
127
|
-
}
|
|
128
|
-
function effectiveWorkerMemory(root, hostBytes = import_node_os.default.totalmem(), mount = "/sys/fs/cgroup") {
|
|
129
|
-
let result = hostBytes;
|
|
130
|
-
if (!root) return result;
|
|
131
|
-
let current = root;
|
|
132
|
-
while (current === mount || current.startsWith(`${mount}/`)) {
|
|
133
|
-
try {
|
|
134
|
-
const limit = Number(import_node_fs.default.readFileSync(import_node_path.default.join(current, "memory.max"), "utf8").trim());
|
|
135
|
-
if (Number.isSafeInteger(limit) && limit > 0) result = Math.min(result, limit);
|
|
136
|
-
} catch {
|
|
137
|
-
}
|
|
138
|
-
if (current === mount) break;
|
|
139
|
-
current = import_node_path.default.dirname(current);
|
|
140
|
-
}
|
|
141
|
-
return result;
|
|
142
|
-
}
|
|
143
|
-
class WorkerJobCapacityError extends Error {
|
|
144
|
-
constructor() {
|
|
145
|
-
super("Worker command capacity is busy. No command was started; retry later or stop another command or terminal.");
|
|
146
|
-
this.name = "WorkerJobCapacityError";
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
class WorkerJobCancelledError extends Error {
|
|
150
|
-
constructor() {
|
|
151
|
-
super("Command cancelled before it started");
|
|
152
|
-
this.name = "WorkerJobCancelledError";
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
class WorkerJobResourceController {
|
|
156
|
-
constructor(budget, delegatedRoot) {
|
|
157
|
-
this.budget = budget;
|
|
158
|
-
this.isolation = delegatedRoot ? "cgroup-v2" : "concurrency-only";
|
|
159
|
-
if (delegatedRoot) {
|
|
160
|
-
if (import_node_fs.default.readFileSync(import_node_path.default.join(delegatedRoot, "cgroup.procs"), "utf8").trim())
|
|
161
|
-
throw new Error("Worker cgroup delegation root must be empty; configure DelegateSubgroup=control");
|
|
162
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(delegatedRoot, "cgroup.subtree_control"), "+memory +pids");
|
|
163
|
-
this.jobsRoot = import_node_path.default.join(delegatedRoot, "jobs");
|
|
164
|
-
import_node_fs.default.mkdirSync(this.jobsRoot, { recursive: true });
|
|
165
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.max"), String(budget.aggregateBytes));
|
|
166
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.swap.max"), "0");
|
|
167
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "memory.oom.group"), "0");
|
|
168
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(this.jobsRoot, "cgroup.subtree_control"), "+memory +pids");
|
|
169
|
-
for (const name of import_node_fs.default.readdirSync(this.jobsRoot)) {
|
|
170
|
-
if (!/^job-[0-9a-f-]+$/.test(name)) continue;
|
|
171
|
-
const old = import_node_path.default.join(this.jobsRoot, name);
|
|
172
|
-
if (!import_node_fs.default.statSync(old).isDirectory()) continue;
|
|
173
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(old, "cgroup.kill"), "1");
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
budget;
|
|
178
|
-
isolation;
|
|
179
|
-
jobsRoot;
|
|
180
|
-
active = /* @__PURE__ */ new Map();
|
|
181
|
-
waiters = [];
|
|
182
|
-
timer;
|
|
183
|
-
get status() {
|
|
184
|
-
return { isolation: this.isolation, active: this.active.size, queued: this.waiters.length };
|
|
185
|
-
}
|
|
186
|
-
acquire(input) {
|
|
187
|
-
try {
|
|
188
|
-
input.assertAdmission();
|
|
189
|
-
if (this.active.has(input.id) || this.waiters.some((entry) => entry.id === input.id))
|
|
190
|
-
throw new Error(`Duplicate resource admission: ${input.id}`);
|
|
191
|
-
if (this.available(input.lane)) return Promise.resolve(this.start(input));
|
|
192
|
-
if (this.waiters.length >= this.budget.maxQueued) throw new WorkerJobCapacityError();
|
|
193
|
-
return new Promise((resolve, reject) => {
|
|
194
|
-
this.waiters.push({ ...input, resolve, reject, deadline: Date.now() + this.budget.queueTimeoutMs });
|
|
195
|
-
this.timer ??= setInterval(() => this.drain(), Math.min(100, this.budget.queueTimeoutMs));
|
|
196
|
-
});
|
|
197
|
-
} catch (error) {
|
|
198
|
-
return Promise.reject(error);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
cancel(id) {
|
|
202
|
-
for (let index = this.waiters.length - 1; index >= 0; index--) {
|
|
203
|
-
if (this.waiters[index].id !== id) continue;
|
|
204
|
-
this.waiters.splice(index, 1)[0].reject(new WorkerJobCancelledError());
|
|
205
|
-
}
|
|
206
|
-
this.active.get(id)?.lease.kill();
|
|
207
|
-
this.drain();
|
|
208
|
-
}
|
|
209
|
-
cancelSession(sessionId) {
|
|
210
|
-
this.cancelMatching((_id, ownerSessionId) => ownerSessionId === sessionId);
|
|
211
|
-
}
|
|
212
|
-
cancelMatching(matches) {
|
|
213
|
-
const ids = /* @__PURE__ */ new Set([
|
|
214
|
-
...this.waiters.filter((entry) => matches(entry.id, entry.sessionId)).map((entry) => entry.id),
|
|
215
|
-
...[...this.active].filter(([id, entry]) => matches(id, entry.sessionId)).map(([id]) => id)
|
|
216
|
-
]);
|
|
217
|
-
const errors = [];
|
|
218
|
-
for (const id of ids) {
|
|
219
|
-
try {
|
|
220
|
-
this.cancel(id);
|
|
221
|
-
} catch (error) {
|
|
222
|
-
errors.push(error);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
if (errors.length) throw new AggregateError(errors, "Worker job cancellation was incomplete");
|
|
226
|
-
}
|
|
227
|
-
available(lane) {
|
|
228
|
-
return [...this.active.values()].filter((entry) => entry.lane === lane).length < (lane === "general" ? this.budget.generalSlots : this.budget.utilitySlots);
|
|
229
|
-
}
|
|
230
|
-
drain() {
|
|
231
|
-
for (let index = 0; index < this.waiters.length; ) {
|
|
232
|
-
const waiter = this.waiters[index];
|
|
233
|
-
try {
|
|
234
|
-
waiter.assertAdmission();
|
|
235
|
-
if (Date.now() >= waiter.deadline) throw new WorkerJobCapacityError();
|
|
236
|
-
if (!this.available(waiter.lane)) {
|
|
237
|
-
index++;
|
|
238
|
-
continue;
|
|
239
|
-
}
|
|
240
|
-
this.waiters.splice(index, 1);
|
|
241
|
-
waiter.resolve(this.start(waiter));
|
|
242
|
-
} catch (error) {
|
|
243
|
-
if (this.waiters[index] === waiter) this.waiters.splice(index, 1);
|
|
244
|
-
waiter.reject(error);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
if (this.waiters.length === 0 && this.timer) {
|
|
248
|
-
clearInterval(this.timer);
|
|
249
|
-
this.timer = void 0;
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
start(input) {
|
|
253
|
-
input.assertAdmission();
|
|
254
|
-
const memoryBytes = input.lane === "general" ? this.budget.generalBytes : this.budget.utilityBytes;
|
|
255
|
-
const cgroupPath = this.jobsRoot ? import_node_path.default.join(this.jobsRoot, `job-${crypto.randomUUID()}`) : void 0;
|
|
256
|
-
if (cgroupPath) {
|
|
257
|
-
import_node_fs.default.mkdirSync(cgroupPath);
|
|
258
|
-
try {
|
|
259
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.max"), String(memoryBytes));
|
|
260
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.swap.max"), "0");
|
|
261
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "memory.oom.group"), "1");
|
|
262
|
-
import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "pids.max"), String(this.budget.pidsPerJob));
|
|
263
|
-
} catch (error) {
|
|
264
|
-
import_node_fs.default.rmdirSync(cgroupPath);
|
|
265
|
-
throw error;
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
let released = false;
|
|
269
|
-
let revoked = false;
|
|
270
|
-
let releasePromise;
|
|
271
|
-
const kill = () => {
|
|
272
|
-
revoked = true;
|
|
273
|
-
if (!released && cgroupPath) import_node_fs.default.writeFileSync(import_node_path.default.join(cgroupPath, "cgroup.kill"), "1");
|
|
274
|
-
};
|
|
275
|
-
const lease = {
|
|
276
|
-
cgroupPath,
|
|
277
|
-
memoryBytes,
|
|
278
|
-
wrap: (argv) => {
|
|
279
|
-
if (released) throw new Error("Worker job resource lease was already released");
|
|
280
|
-
if (revoked) throw new WorkerJobCancelledError();
|
|
281
|
-
if (!argv.length) throw new Error("Command argv must not be empty");
|
|
282
|
-
return cgroupPath ? [
|
|
283
|
-
"/bin/sh",
|
|
284
|
-
"-c",
|
|
285
|
-
'set -e; printf "%s" "$$" > "$1/cgroup.procs"; read -r oom < /proc/self/oom_score_adj; if [ "$oom" -lt 0 ]; then printf 0 > /proc/self/oom_score_adj; fi; shift; exec "$@"',
|
|
286
|
-
"r5d-job",
|
|
287
|
-
cgroupPath,
|
|
288
|
-
...argv
|
|
289
|
-
] : argv;
|
|
290
|
-
},
|
|
291
|
-
kill,
|
|
292
|
-
memoryLimitExceeded: () => {
|
|
293
|
-
if (!cgroupPath || released) return false;
|
|
294
|
-
return /^oom_kill [1-9][0-9]*$/m.test(import_node_fs.default.readFileSync(import_node_path.default.join(cgroupPath, "memory.events"), "utf8"));
|
|
295
|
-
},
|
|
296
|
-
release: () => releasePromise ??= (async () => {
|
|
297
|
-
kill();
|
|
298
|
-
if (cgroupPath) {
|
|
299
|
-
const deadline = Date.now() + 2e3;
|
|
300
|
-
while (/^populated 1$/m.test(import_node_fs.default.readFileSync(import_node_path.default.join(cgroupPath, "cgroup.events"), "utf8"))) {
|
|
301
|
-
if (Date.now() >= deadline) throw new Error(`Worker job cgroup remained populated: ${cgroupPath}`);
|
|
302
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
303
|
-
}
|
|
304
|
-
import_node_fs.default.rmdirSync(cgroupPath);
|
|
305
|
-
}
|
|
306
|
-
released = true;
|
|
307
|
-
this.active.delete(input.id);
|
|
308
|
-
this.drain();
|
|
309
|
-
})()
|
|
310
|
-
};
|
|
311
|
-
this.active.set(input.id, { lane: input.lane, sessionId: input.sessionId, lease });
|
|
312
|
-
return lease;
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
function createWorkerJobResources(env = process.env) {
|
|
316
|
-
const mode = env.R5D_WORKER_RESOURCE_ISOLATION ?? "auto";
|
|
317
|
-
if (mode !== "auto" && mode !== "required") throw new Error("R5D_WORKER_RESOURCE_ISOLATION must be auto or required");
|
|
318
|
-
const root = process.platform === "linux" ? workerDelegatedCgroupRoot(import_node_fs.default.readFileSync("/proc/self/cgroup", "utf8")) : void 0;
|
|
319
|
-
if (mode === "required" && !root)
|
|
320
|
-
throw new Error("Worker requires delegated cgroup v2 resource isolation; configure Delegate=memory pids and DelegateSubgroup=control");
|
|
321
|
-
const budget = workerJobBudget(effectiveWorkerMemory(root), env);
|
|
322
|
-
return new WorkerJobResourceController(budget, root);
|
|
323
|
-
}
|
|
324
|
-
// Annotate the CommonJS export names for ESM import in node:
|
|
325
|
-
0 && (module.exports = {
|
|
326
|
-
WORKER_INLINE_OUTPUT_MAX_BYTES,
|
|
327
|
-
WorkerJobCancelledError,
|
|
328
|
-
WorkerJobCapacityError,
|
|
329
|
-
WorkerJobResourceController,
|
|
330
|
-
collectWorkerInlineOutput,
|
|
331
|
-
createWorkerJobResources,
|
|
332
|
-
effectiveWorkerMemory,
|
|
333
|
-
workerDelegatedCgroupRoot,
|
|
334
|
-
workerJobBudget
|
|
335
|
-
});
|
|
@@ -1,293 +0,0 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
|
-
import os from "node:os";
|
|
3
|
-
import path from "node:path";
|
|
4
|
-
const MIB = 1024 * 1024;
|
|
5
|
-
const WORKER_INLINE_OUTPUT_MAX_BYTES = 8 * MIB;
|
|
6
|
-
async function collectWorkerInlineOutput(stream, maxBytes = WORKER_INLINE_OUTPUT_MAX_BYTES, signal) {
|
|
7
|
-
if (!stream) return "";
|
|
8
|
-
signal?.throwIfAborted();
|
|
9
|
-
const reader = stream.getReader();
|
|
10
|
-
const abort = () => {
|
|
11
|
-
void reader.cancel().catch(() => {
|
|
12
|
-
});
|
|
13
|
-
};
|
|
14
|
-
signal?.addEventListener("abort", abort, { once: true });
|
|
15
|
-
const decoder = new TextDecoder();
|
|
16
|
-
const parts = [];
|
|
17
|
-
let bytes = 0;
|
|
18
|
-
try {
|
|
19
|
-
while (true) {
|
|
20
|
-
const next = await reader.read();
|
|
21
|
-
signal?.throwIfAborted();
|
|
22
|
-
if (next.done) break;
|
|
23
|
-
bytes += next.value.byteLength;
|
|
24
|
-
if (bytes > maxBytes) {
|
|
25
|
-
void reader.cancel().catch(() => {
|
|
26
|
-
});
|
|
27
|
-
throw new Error(`Inline command output exceeded ${maxBytes} bytes. Use streaming execution for large output.`);
|
|
28
|
-
}
|
|
29
|
-
parts.push(decoder.decode(next.value, { stream: true }));
|
|
30
|
-
}
|
|
31
|
-
parts.push(decoder.decode());
|
|
32
|
-
return parts.join("");
|
|
33
|
-
} finally {
|
|
34
|
-
signal?.removeEventListener("abort", abort);
|
|
35
|
-
reader.releaseLock();
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
function positiveInteger(value, fallback, name) {
|
|
39
|
-
const result = value === void 0 ? fallback : Number(value);
|
|
40
|
-
if (!Number.isSafeInteger(result) || result <= 0) throw new Error(`${name} must be a positive integer`);
|
|
41
|
-
return result;
|
|
42
|
-
}
|
|
43
|
-
function workerJobBudget(effectiveMemoryBytes, env = process.env) {
|
|
44
|
-
const generalSlots = positiveInteger(env.R5D_WORKER_JOB_CONCURRENCY, 2, "R5D_WORKER_JOB_CONCURRENCY");
|
|
45
|
-
const utilitySlots = positiveInteger(env.R5D_WORKER_UTILITY_CONCURRENCY, 2, "R5D_WORKER_UTILITY_CONCURRENCY");
|
|
46
|
-
const memoryPercent = positiveInteger(env.R5D_WORKER_JOBS_MEMORY_PERCENT, 70, "R5D_WORKER_JOBS_MEMORY_PERCENT");
|
|
47
|
-
if (memoryPercent > 70) throw new Error("Worker jobs may reserve at most 70% of effective memory");
|
|
48
|
-
const aggregateBytes = positiveInteger(
|
|
49
|
-
env.R5D_WORKER_JOBS_MEMORY_BYTES,
|
|
50
|
-
Math.floor(effectiveMemoryBytes * memoryPercent / 100),
|
|
51
|
-
"R5D_WORKER_JOBS_MEMORY_BYTES"
|
|
52
|
-
);
|
|
53
|
-
const utilityBytes = positiveInteger(
|
|
54
|
-
env.R5D_WORKER_UTILITY_MEMORY_BYTES,
|
|
55
|
-
Math.min(256 * MIB, Math.floor(aggregateBytes / 10)),
|
|
56
|
-
"R5D_WORKER_UTILITY_MEMORY_BYTES"
|
|
57
|
-
);
|
|
58
|
-
const generalBytes = positiveInteger(
|
|
59
|
-
env.R5D_WORKER_JOB_MEMORY_BYTES,
|
|
60
|
-
Math.floor((aggregateBytes - utilityBytes * utilitySlots) / generalSlots),
|
|
61
|
-
"R5D_WORKER_JOB_MEMORY_BYTES"
|
|
62
|
-
);
|
|
63
|
-
if (aggregateBytes > Math.floor(effectiveMemoryBytes * 0.7)) throw new Error("Worker jobs may reserve at most 70% of effective memory");
|
|
64
|
-
if (generalBytes * generalSlots + utilityBytes * utilitySlots > aggregateBytes)
|
|
65
|
-
throw new Error("Worker job limits exceed the aggregate memory budget");
|
|
66
|
-
const queueTimeoutMs = positiveInteger(env.R5D_WORKER_JOB_QUEUE_TIMEOUT_MS, 6e4, "R5D_WORKER_JOB_QUEUE_TIMEOUT_MS");
|
|
67
|
-
if (queueTimeoutMs > 6e4) throw new Error("Worker job queue timeout cannot exceed 60000ms");
|
|
68
|
-
return {
|
|
69
|
-
aggregateBytes,
|
|
70
|
-
generalBytes,
|
|
71
|
-
utilityBytes,
|
|
72
|
-
generalSlots,
|
|
73
|
-
utilitySlots,
|
|
74
|
-
queueTimeoutMs,
|
|
75
|
-
maxQueued: positiveInteger(env.R5D_WORKER_JOB_QUEUE_LIMIT, 128, "R5D_WORKER_JOB_QUEUE_LIMIT"),
|
|
76
|
-
pidsPerJob: positiveInteger(env.R5D_WORKER_JOB_PIDS, 512, "R5D_WORKER_JOB_PIDS")
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
function workerDelegatedCgroupRoot(cgroupText, mount = "/sys/fs/cgroup") {
|
|
80
|
-
const relative = cgroupText.split("\n").find((line) => line.startsWith("0::/"))?.slice(3);
|
|
81
|
-
if (!relative) return void 0;
|
|
82
|
-
const segments = relative.split("/").filter(Boolean);
|
|
83
|
-
const index = segments.indexOf("r5d-worker.service");
|
|
84
|
-
if (index < 0 || segments[index + 1] !== "control") return void 0;
|
|
85
|
-
return path.join(mount, ...segments.slice(0, index + 1));
|
|
86
|
-
}
|
|
87
|
-
function effectiveWorkerMemory(root, hostBytes = os.totalmem(), mount = "/sys/fs/cgroup") {
|
|
88
|
-
let result = hostBytes;
|
|
89
|
-
if (!root) return result;
|
|
90
|
-
let current = root;
|
|
91
|
-
while (current === mount || current.startsWith(`${mount}/`)) {
|
|
92
|
-
try {
|
|
93
|
-
const limit = Number(fs.readFileSync(path.join(current, "memory.max"), "utf8").trim());
|
|
94
|
-
if (Number.isSafeInteger(limit) && limit > 0) result = Math.min(result, limit);
|
|
95
|
-
} catch {
|
|
96
|
-
}
|
|
97
|
-
if (current === mount) break;
|
|
98
|
-
current = path.dirname(current);
|
|
99
|
-
}
|
|
100
|
-
return result;
|
|
101
|
-
}
|
|
102
|
-
class WorkerJobCapacityError extends Error {
|
|
103
|
-
constructor() {
|
|
104
|
-
super("Worker command capacity is busy. No command was started; retry later or stop another command or terminal.");
|
|
105
|
-
this.name = "WorkerJobCapacityError";
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
class WorkerJobCancelledError extends Error {
|
|
109
|
-
constructor() {
|
|
110
|
-
super("Command cancelled before it started");
|
|
111
|
-
this.name = "WorkerJobCancelledError";
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
class WorkerJobResourceController {
|
|
115
|
-
constructor(budget, delegatedRoot) {
|
|
116
|
-
this.budget = budget;
|
|
117
|
-
this.isolation = delegatedRoot ? "cgroup-v2" : "concurrency-only";
|
|
118
|
-
if (delegatedRoot) {
|
|
119
|
-
if (fs.readFileSync(path.join(delegatedRoot, "cgroup.procs"), "utf8").trim())
|
|
120
|
-
throw new Error("Worker cgroup delegation root must be empty; configure DelegateSubgroup=control");
|
|
121
|
-
fs.writeFileSync(path.join(delegatedRoot, "cgroup.subtree_control"), "+memory +pids");
|
|
122
|
-
this.jobsRoot = path.join(delegatedRoot, "jobs");
|
|
123
|
-
fs.mkdirSync(this.jobsRoot, { recursive: true });
|
|
124
|
-
fs.writeFileSync(path.join(this.jobsRoot, "memory.max"), String(budget.aggregateBytes));
|
|
125
|
-
fs.writeFileSync(path.join(this.jobsRoot, "memory.swap.max"), "0");
|
|
126
|
-
fs.writeFileSync(path.join(this.jobsRoot, "memory.oom.group"), "0");
|
|
127
|
-
fs.writeFileSync(path.join(this.jobsRoot, "cgroup.subtree_control"), "+memory +pids");
|
|
128
|
-
for (const name of fs.readdirSync(this.jobsRoot)) {
|
|
129
|
-
if (!/^job-[0-9a-f-]+$/.test(name)) continue;
|
|
130
|
-
const old = path.join(this.jobsRoot, name);
|
|
131
|
-
if (!fs.statSync(old).isDirectory()) continue;
|
|
132
|
-
fs.writeFileSync(path.join(old, "cgroup.kill"), "1");
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
budget;
|
|
137
|
-
isolation;
|
|
138
|
-
jobsRoot;
|
|
139
|
-
active = /* @__PURE__ */ new Map();
|
|
140
|
-
waiters = [];
|
|
141
|
-
timer;
|
|
142
|
-
get status() {
|
|
143
|
-
return { isolation: this.isolation, active: this.active.size, queued: this.waiters.length };
|
|
144
|
-
}
|
|
145
|
-
acquire(input) {
|
|
146
|
-
try {
|
|
147
|
-
input.assertAdmission();
|
|
148
|
-
if (this.active.has(input.id) || this.waiters.some((entry) => entry.id === input.id))
|
|
149
|
-
throw new Error(`Duplicate resource admission: ${input.id}`);
|
|
150
|
-
if (this.available(input.lane)) return Promise.resolve(this.start(input));
|
|
151
|
-
if (this.waiters.length >= this.budget.maxQueued) throw new WorkerJobCapacityError();
|
|
152
|
-
return new Promise((resolve, reject) => {
|
|
153
|
-
this.waiters.push({ ...input, resolve, reject, deadline: Date.now() + this.budget.queueTimeoutMs });
|
|
154
|
-
this.timer ??= setInterval(() => this.drain(), Math.min(100, this.budget.queueTimeoutMs));
|
|
155
|
-
});
|
|
156
|
-
} catch (error) {
|
|
157
|
-
return Promise.reject(error);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
cancel(id) {
|
|
161
|
-
for (let index = this.waiters.length - 1; index >= 0; index--) {
|
|
162
|
-
if (this.waiters[index].id !== id) continue;
|
|
163
|
-
this.waiters.splice(index, 1)[0].reject(new WorkerJobCancelledError());
|
|
164
|
-
}
|
|
165
|
-
this.active.get(id)?.lease.kill();
|
|
166
|
-
this.drain();
|
|
167
|
-
}
|
|
168
|
-
cancelSession(sessionId) {
|
|
169
|
-
this.cancelMatching((_id, ownerSessionId) => ownerSessionId === sessionId);
|
|
170
|
-
}
|
|
171
|
-
cancelMatching(matches) {
|
|
172
|
-
const ids = /* @__PURE__ */ new Set([
|
|
173
|
-
...this.waiters.filter((entry) => matches(entry.id, entry.sessionId)).map((entry) => entry.id),
|
|
174
|
-
...[...this.active].filter(([id, entry]) => matches(id, entry.sessionId)).map(([id]) => id)
|
|
175
|
-
]);
|
|
176
|
-
const errors = [];
|
|
177
|
-
for (const id of ids) {
|
|
178
|
-
try {
|
|
179
|
-
this.cancel(id);
|
|
180
|
-
} catch (error) {
|
|
181
|
-
errors.push(error);
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
if (errors.length) throw new AggregateError(errors, "Worker job cancellation was incomplete");
|
|
185
|
-
}
|
|
186
|
-
available(lane) {
|
|
187
|
-
return [...this.active.values()].filter((entry) => entry.lane === lane).length < (lane === "general" ? this.budget.generalSlots : this.budget.utilitySlots);
|
|
188
|
-
}
|
|
189
|
-
drain() {
|
|
190
|
-
for (let index = 0; index < this.waiters.length; ) {
|
|
191
|
-
const waiter = this.waiters[index];
|
|
192
|
-
try {
|
|
193
|
-
waiter.assertAdmission();
|
|
194
|
-
if (Date.now() >= waiter.deadline) throw new WorkerJobCapacityError();
|
|
195
|
-
if (!this.available(waiter.lane)) {
|
|
196
|
-
index++;
|
|
197
|
-
continue;
|
|
198
|
-
}
|
|
199
|
-
this.waiters.splice(index, 1);
|
|
200
|
-
waiter.resolve(this.start(waiter));
|
|
201
|
-
} catch (error) {
|
|
202
|
-
if (this.waiters[index] === waiter) this.waiters.splice(index, 1);
|
|
203
|
-
waiter.reject(error);
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
if (this.waiters.length === 0 && this.timer) {
|
|
207
|
-
clearInterval(this.timer);
|
|
208
|
-
this.timer = void 0;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
start(input) {
|
|
212
|
-
input.assertAdmission();
|
|
213
|
-
const memoryBytes = input.lane === "general" ? this.budget.generalBytes : this.budget.utilityBytes;
|
|
214
|
-
const cgroupPath = this.jobsRoot ? path.join(this.jobsRoot, `job-${crypto.randomUUID()}`) : void 0;
|
|
215
|
-
if (cgroupPath) {
|
|
216
|
-
fs.mkdirSync(cgroupPath);
|
|
217
|
-
try {
|
|
218
|
-
fs.writeFileSync(path.join(cgroupPath, "memory.max"), String(memoryBytes));
|
|
219
|
-
fs.writeFileSync(path.join(cgroupPath, "memory.swap.max"), "0");
|
|
220
|
-
fs.writeFileSync(path.join(cgroupPath, "memory.oom.group"), "1");
|
|
221
|
-
fs.writeFileSync(path.join(cgroupPath, "pids.max"), String(this.budget.pidsPerJob));
|
|
222
|
-
} catch (error) {
|
|
223
|
-
fs.rmdirSync(cgroupPath);
|
|
224
|
-
throw error;
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
let released = false;
|
|
228
|
-
let revoked = false;
|
|
229
|
-
let releasePromise;
|
|
230
|
-
const kill = () => {
|
|
231
|
-
revoked = true;
|
|
232
|
-
if (!released && cgroupPath) fs.writeFileSync(path.join(cgroupPath, "cgroup.kill"), "1");
|
|
233
|
-
};
|
|
234
|
-
const lease = {
|
|
235
|
-
cgroupPath,
|
|
236
|
-
memoryBytes,
|
|
237
|
-
wrap: (argv) => {
|
|
238
|
-
if (released) throw new Error("Worker job resource lease was already released");
|
|
239
|
-
if (revoked) throw new WorkerJobCancelledError();
|
|
240
|
-
if (!argv.length) throw new Error("Command argv must not be empty");
|
|
241
|
-
return cgroupPath ? [
|
|
242
|
-
"/bin/sh",
|
|
243
|
-
"-c",
|
|
244
|
-
'set -e; printf "%s" "$$" > "$1/cgroup.procs"; read -r oom < /proc/self/oom_score_adj; if [ "$oom" -lt 0 ]; then printf 0 > /proc/self/oom_score_adj; fi; shift; exec "$@"',
|
|
245
|
-
"r5d-job",
|
|
246
|
-
cgroupPath,
|
|
247
|
-
...argv
|
|
248
|
-
] : argv;
|
|
249
|
-
},
|
|
250
|
-
kill,
|
|
251
|
-
memoryLimitExceeded: () => {
|
|
252
|
-
if (!cgroupPath || released) return false;
|
|
253
|
-
return /^oom_kill [1-9][0-9]*$/m.test(fs.readFileSync(path.join(cgroupPath, "memory.events"), "utf8"));
|
|
254
|
-
},
|
|
255
|
-
release: () => releasePromise ??= (async () => {
|
|
256
|
-
kill();
|
|
257
|
-
if (cgroupPath) {
|
|
258
|
-
const deadline = Date.now() + 2e3;
|
|
259
|
-
while (/^populated 1$/m.test(fs.readFileSync(path.join(cgroupPath, "cgroup.events"), "utf8"))) {
|
|
260
|
-
if (Date.now() >= deadline) throw new Error(`Worker job cgroup remained populated: ${cgroupPath}`);
|
|
261
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
262
|
-
}
|
|
263
|
-
fs.rmdirSync(cgroupPath);
|
|
264
|
-
}
|
|
265
|
-
released = true;
|
|
266
|
-
this.active.delete(input.id);
|
|
267
|
-
this.drain();
|
|
268
|
-
})()
|
|
269
|
-
};
|
|
270
|
-
this.active.set(input.id, { lane: input.lane, sessionId: input.sessionId, lease });
|
|
271
|
-
return lease;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
function createWorkerJobResources(env = process.env) {
|
|
275
|
-
const mode = env.R5D_WORKER_RESOURCE_ISOLATION ?? "auto";
|
|
276
|
-
if (mode !== "auto" && mode !== "required") throw new Error("R5D_WORKER_RESOURCE_ISOLATION must be auto or required");
|
|
277
|
-
const root = process.platform === "linux" ? workerDelegatedCgroupRoot(fs.readFileSync("/proc/self/cgroup", "utf8")) : void 0;
|
|
278
|
-
if (mode === "required" && !root)
|
|
279
|
-
throw new Error("Worker requires delegated cgroup v2 resource isolation; configure Delegate=memory pids and DelegateSubgroup=control");
|
|
280
|
-
const budget = workerJobBudget(effectiveWorkerMemory(root), env);
|
|
281
|
-
return new WorkerJobResourceController(budget, root);
|
|
282
|
-
}
|
|
283
|
-
export {
|
|
284
|
-
WORKER_INLINE_OUTPUT_MAX_BYTES,
|
|
285
|
-
WorkerJobCancelledError,
|
|
286
|
-
WorkerJobCapacityError,
|
|
287
|
-
WorkerJobResourceController,
|
|
288
|
-
collectWorkerInlineOutput,
|
|
289
|
-
createWorkerJobResources,
|
|
290
|
-
effectiveWorkerMemory,
|
|
291
|
-
workerDelegatedCgroupRoot,
|
|
292
|
-
workerJobBudget
|
|
293
|
-
};
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
export declare const WORKER_INLINE_OUTPUT_MAX_BYTES: number;
|
|
2
|
-
/** Inline RPC output lives in the control process, outside the child's cgroup. */
|
|
3
|
-
export declare function collectWorkerInlineOutput(stream: ReadableStream<Uint8Array> | null, maxBytes?: number, signal?: AbortSignal): Promise<string>;
|
|
4
|
-
export type WorkerJobLane = "general" | "utility";
|
|
5
|
-
export type WorkerJobBudget = {
|
|
6
|
-
aggregateBytes: number;
|
|
7
|
-
generalBytes: number;
|
|
8
|
-
utilityBytes: number;
|
|
9
|
-
generalSlots: number;
|
|
10
|
-
utilitySlots: number;
|
|
11
|
-
queueTimeoutMs: number;
|
|
12
|
-
maxQueued: number;
|
|
13
|
-
pidsPerJob: number;
|
|
14
|
-
};
|
|
15
|
-
export declare function workerJobBudget(effectiveMemoryBytes: number, env?: NodeJS.ProcessEnv): WorkerJobBudget;
|
|
16
|
-
export declare function workerDelegatedCgroupRoot(cgroupText: string, mount?: string): string | undefined;
|
|
17
|
-
export declare function effectiveWorkerMemory(root: string | undefined, hostBytes?: number, mount?: string): number;
|
|
18
|
-
export declare class WorkerJobCapacityError extends Error {
|
|
19
|
-
constructor();
|
|
20
|
-
}
|
|
21
|
-
export declare class WorkerJobCancelledError extends Error {
|
|
22
|
-
constructor();
|
|
23
|
-
}
|
|
24
|
-
type Acquisition = {
|
|
25
|
-
id: string;
|
|
26
|
-
lane: WorkerJobLane;
|
|
27
|
-
sessionId?: string;
|
|
28
|
-
assertAdmission: () => void;
|
|
29
|
-
};
|
|
30
|
-
export type WorkerJobLease = {
|
|
31
|
-
readonly cgroupPath?: string;
|
|
32
|
-
readonly memoryBytes: number;
|
|
33
|
-
wrap(argv: string[]): string[];
|
|
34
|
-
kill(): void;
|
|
35
|
-
memoryLimitExceeded(): boolean;
|
|
36
|
-
release(): Promise<void>;
|
|
37
|
-
};
|
|
38
|
-
/** One controller per runtime, shared by every project and socket generation. */
|
|
39
|
-
export declare class WorkerJobResourceController {
|
|
40
|
-
readonly budget: WorkerJobBudget;
|
|
41
|
-
readonly isolation: "cgroup-v2" | "concurrency-only";
|
|
42
|
-
private readonly jobsRoot?;
|
|
43
|
-
private readonly active;
|
|
44
|
-
private readonly waiters;
|
|
45
|
-
private timer?;
|
|
46
|
-
constructor(budget: WorkerJobBudget, delegatedRoot?: string);
|
|
47
|
-
get status(): {
|
|
48
|
-
isolation: string;
|
|
49
|
-
active: number;
|
|
50
|
-
queued: number;
|
|
51
|
-
};
|
|
52
|
-
acquire(input: Acquisition): Promise<WorkerJobLease>;
|
|
53
|
-
cancel(id: string): void;
|
|
54
|
-
cancelSession(sessionId: string): void;
|
|
55
|
-
cancelMatching(matches: (id: string, sessionId?: string) => boolean): void;
|
|
56
|
-
private available;
|
|
57
|
-
private drain;
|
|
58
|
-
private start;
|
|
59
|
-
}
|
|
60
|
-
export declare function createWorkerJobResources(env?: NodeJS.ProcessEnv): WorkerJobResourceController;
|
|
61
|
-
export {};
|