@shogo-ai/worker 2.3.12 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +25 -25
- package/package.json +2 -1
- package/src/lib/__tests__/helpers/fake-cloud.ts +139 -0
- package/src/lib/__tests__/resource-limits-exec-budget.test.ts +116 -0
- package/src/lib/__tests__/runtime-manager-exec-budget-env.test.ts +87 -0
- package/src/lib/__tests__/tunnel-worker-block.test.ts +243 -0
- package/src/lib/resource-limits.ts +153 -10
- package/src/lib/runtime-manager.ts +78 -3
- package/src/lib/tunnel-host.ts +253 -0
- package/src/lib/tunnel-worker.ts +223 -0
- package/src/lib/tunnel.ts +37 -2
- package/src/lib/web-worker.d.ts +23 -0
|
@@ -26,8 +26,13 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
26
26
|
import { join } from 'node:path';
|
|
27
27
|
|
|
28
28
|
export interface RuntimeResourceLimits {
|
|
29
|
-
/** Per-project RAM ceiling in MB
|
|
29
|
+
/** Per-project RAM ceiling in MB for the runtime's OWN processes. Always > 0
|
|
30
|
+
* when limits are active. Not the budget for the commands an agent runs. */
|
|
30
31
|
memoryMB: number;
|
|
32
|
+
/** Hard OS cap for the whole process tree in MB: {@link memoryMB} plus the
|
|
33
|
+
* agent-command budget. The exec guard (which writes a note) must trip before
|
|
34
|
+
* the kernel (which writes nothing), so the kernel cap has to sit above it. */
|
|
35
|
+
osMemoryMB: number;
|
|
31
36
|
/** CPU ceiling as percent of ONE core (100 = one full core). 0 = no cap. */
|
|
32
37
|
cpuPercent: number;
|
|
33
38
|
/** Max tasks/pids in the group (cgroup pids.max). */
|
|
@@ -39,10 +44,47 @@ type Logger = Pick<Console, 'log' | 'warn' | 'error'>;
|
|
|
39
44
|
/** cgroup CPU period in microseconds (100ms), the systemd/cgroup default. */
|
|
40
45
|
const CPU_PERIOD_US = 100_000;
|
|
41
46
|
|
|
47
|
+
/** `host-limits.json`, as published by the desktop app (SHO-203). */
|
|
48
|
+
export interface HostLimitsFile {
|
|
49
|
+
/** MB the agent's `exec` commands may use. */
|
|
50
|
+
execMemoryMB?: number;
|
|
51
|
+
/** Total physical RAM of the machine, for messages. */
|
|
52
|
+
totalMemMB?: number;
|
|
53
|
+
source?: 'auto' | 'settings';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Parse `host-limits.json`. Missing or malformed reads as `null`. */
|
|
57
|
+
export function readHostLimitsFile(path: string): HostLimitsFile | null {
|
|
58
|
+
try {
|
|
59
|
+
const parsed = JSON.parse(readFileSync(path, 'utf-8')) as HostLimitsFile;
|
|
60
|
+
return parsed && typeof parsed === 'object' ? parsed : null;
|
|
61
|
+
} catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The command budget from the environment: an explicit `SHOGO_EXEC_MEMORY_MB`
|
|
68
|
+
* (headless worker, tests) wins, otherwise the watched `host-limits.json`.
|
|
69
|
+
* 0 = no budget known.
|
|
70
|
+
*/
|
|
71
|
+
export function resolveExecMemoryMB(env: NodeJS.ProcessEnv): number {
|
|
72
|
+
const explicit = parseInt(env.SHOGO_EXEC_MEMORY_MB || '', 10);
|
|
73
|
+
if (Number.isFinite(explicit) && explicit > 0) return explicit;
|
|
74
|
+
if (!env.SHOGO_HOST_LIMITS_FILE) return 0;
|
|
75
|
+
const mb = readHostLimitsFile(env.SHOGO_HOST_LIMITS_FILE)?.execMemoryMB;
|
|
76
|
+
return typeof mb === 'number' && Number.isFinite(mb) && mb > 0 ? Math.floor(mb) : 0;
|
|
77
|
+
}
|
|
78
|
+
|
|
42
79
|
/**
|
|
43
80
|
* Resolve the active resource limits from the environment. Returns `null` when
|
|
44
81
|
* no ceiling is configured (`RUNTIME_MEMORY_MB` unset or <= 0), which disables
|
|
45
82
|
* all enforcement for backwards compatibility.
|
|
83
|
+
*
|
|
84
|
+
* `memoryMB` stays the runtime's own leak ceiling (the RSS watchdog and the heap
|
|
85
|
+
* cap use it). `osMemoryMB` is the kernel-level cap for the whole tree, raised
|
|
86
|
+
* by the agent-command budget so a big `exec` command is stopped by the guard —
|
|
87
|
+
* which writes an actionable note — instead of silently OOM-killed (SHO-203).
|
|
46
88
|
*/
|
|
47
89
|
export function resolveResourceLimits(env: NodeJS.ProcessEnv): RuntimeResourceLimits | null {
|
|
48
90
|
const mb = parseInt(env.RUNTIME_MEMORY_MB || '', 10);
|
|
@@ -54,7 +96,7 @@ export function resolveResourceLimits(env: NodeJS.ProcessEnv): RuntimeResourceLi
|
|
|
54
96
|
const tasksRaw = parseInt(env.RUNTIME_TASKS_MAX || '', 10);
|
|
55
97
|
const tasksMax = Number.isFinite(tasksRaw) && tasksRaw > 0 ? tasksRaw : 4096;
|
|
56
98
|
|
|
57
|
-
return { memoryMB: mb, cpuPercent, tasksMax };
|
|
99
|
+
return { memoryMB: mb, osMemoryMB: mb + resolveExecMemoryMB(env), cpuPercent, tasksMax };
|
|
58
100
|
}
|
|
59
101
|
|
|
60
102
|
/**
|
|
@@ -98,6 +140,11 @@ export function hasSystemdRun(): boolean {
|
|
|
98
140
|
* command untouched (`wrapped: false`) on non-Linux or when systemd-run is
|
|
99
141
|
* absent — the caller then falls back to {@link attachDirectCgroup} or the
|
|
100
142
|
* watchdog.
|
|
143
|
+
*
|
|
144
|
+
* The cap is `limits.osMemoryMB` (runtime ceiling + agent-command budget): the
|
|
145
|
+
* scope holds the runtime's own processes *and* the `exec` shells it spawns, so
|
|
146
|
+
* sized to the runtime ceiling alone it killed a 22.5 GB command on a 128 GB
|
|
147
|
+
* machine with no message (SHO-203).
|
|
101
148
|
*/
|
|
102
149
|
export function wrapSpawnForCgroup(opts: {
|
|
103
150
|
command: string;
|
|
@@ -112,10 +159,10 @@ export function wrapSpawnForCgroup(opts: {
|
|
|
112
159
|
}
|
|
113
160
|
|
|
114
161
|
const props = [
|
|
115
|
-
'-p', `MemoryMax=${limits.
|
|
162
|
+
'-p', `MemoryMax=${limits.osMemoryMB}M`,
|
|
116
163
|
// MemoryHigh throttles + reclaims before the hard OOM at MemoryMax, so a
|
|
117
164
|
// spiky build is slowed rather than instantly killed.
|
|
118
|
-
'-p', `MemoryHigh=${
|
|
165
|
+
'-p', `MemoryHigh=${memoryHighMB(limits.osMemoryMB)}M`,
|
|
119
166
|
'-p', `TasksMax=${limits.tasksMax}`,
|
|
120
167
|
];
|
|
121
168
|
if (limits.cpuPercent > 0) props.push('-p', `CPUQuota=${limits.cpuPercent}%`);
|
|
@@ -131,10 +178,20 @@ export function wrapSpawnForCgroup(opts: {
|
|
|
131
178
|
command,
|
|
132
179
|
...args,
|
|
133
180
|
];
|
|
134
|
-
logger?.log(`[resource-limits] wrapping spawn in systemd scope ${scopeName} (MemoryMax=${limits.
|
|
181
|
+
logger?.log(`[resource-limits] wrapping spawn in systemd scope ${scopeName} (MemoryMax=${limits.osMemoryMB}M)`);
|
|
135
182
|
return { command: 'systemd-run', args: wrappedArgs, wrapped: true };
|
|
136
183
|
}
|
|
137
184
|
|
|
185
|
+
/** Soft cap for a hard cap: 10% below, so reclaim starts before the OOM. */
|
|
186
|
+
function memoryHighMB(memoryMB: number): number {
|
|
187
|
+
return Math.max(128, Math.floor(memoryMB * 0.9));
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** The cgroup v2 directory {@link attachDirectCgroup} puts a runtime in. */
|
|
191
|
+
export function directCgroupDir(pid: number): string {
|
|
192
|
+
return join('/sys/fs/cgroup', 'shogo', `rt-${pid}`);
|
|
193
|
+
}
|
|
194
|
+
|
|
138
195
|
/**
|
|
139
196
|
* Fallback for Linux hosts without systemd: write a cgroup v2 directly and move
|
|
140
197
|
* `pid` (plus its future children) into it. Best-effort — returns `false` when
|
|
@@ -153,10 +210,10 @@ export function attachDirectCgroup(opts: {
|
|
|
153
210
|
// cgroup v2 unified hierarchy exposes cgroup.controllers at the root.
|
|
154
211
|
if (!existsSync(join(root, 'cgroup.controllers'))) return false;
|
|
155
212
|
|
|
156
|
-
const dir =
|
|
213
|
+
const dir = directCgroupDir(pid);
|
|
157
214
|
try {
|
|
158
215
|
mkdirSync(dir, { recursive: true });
|
|
159
|
-
writeFileSync(join(dir, 'memory.max'), String(limits.
|
|
216
|
+
writeFileSync(join(dir, 'memory.max'), String(limits.osMemoryMB * 1024 * 1024));
|
|
160
217
|
if (limits.cpuPercent > 0) {
|
|
161
218
|
const quota = Math.floor((limits.cpuPercent / 100) * CPU_PERIOD_US);
|
|
162
219
|
writeFileSync(join(dir, 'cpu.max'), `${quota} ${CPU_PERIOD_US}`);
|
|
@@ -164,7 +221,7 @@ export function attachDirectCgroup(opts: {
|
|
|
164
221
|
try { writeFileSync(join(dir, 'pids.max'), String(limits.tasksMax)); } catch { /* controller may be absent */ }
|
|
165
222
|
// Moving the pid in also captures children forked after this point.
|
|
166
223
|
writeFileSync(join(dir, 'cgroup.procs'), String(pid));
|
|
167
|
-
logger?.log(`[resource-limits] attached pid ${pid} to cgroup ${dir} (memory.max=${limits.
|
|
224
|
+
logger?.log(`[resource-limits] attached pid ${pid} to cgroup ${dir} (memory.max=${limits.osMemoryMB}M)`);
|
|
168
225
|
return true;
|
|
169
226
|
} catch (err: any) {
|
|
170
227
|
logger?.warn(`[resource-limits] direct cgroup attach failed for pid ${pid}: ${err?.message ?? err}`);
|
|
@@ -172,6 +229,92 @@ export function attachDirectCgroup(opts: {
|
|
|
172
229
|
}
|
|
173
230
|
}
|
|
174
231
|
|
|
232
|
+
// ─── Live re-apply (SHO-203) ─────────────────────────────────────────
|
|
233
|
+
|
|
234
|
+
export interface LiveMemoryCapOptions {
|
|
235
|
+
/** Runtime process group id, used for a direct cgroup. */
|
|
236
|
+
pid?: number | null;
|
|
237
|
+
/** `systemd-run --scope` unit name, when the spawn was cgroup-wrapped. */
|
|
238
|
+
scopeName?: string | null;
|
|
239
|
+
limits: RuntimeResourceLimits;
|
|
240
|
+
logger?: Logger;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Swappable effects, so the generated `systemctl` args / file write are testable. */
|
|
244
|
+
export interface LiveMemoryCapDeps {
|
|
245
|
+
platform?: NodeJS.Platform;
|
|
246
|
+
run?: (command: string, args: string[]) => void;
|
|
247
|
+
writeFile?: (path: string, data: string) => void;
|
|
248
|
+
exists?: (path: string) => boolean;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Re-apply the memory cap to a runtime that is already running, after the
|
|
253
|
+
* agent-command budget changed. Without this a raised budget would only land
|
|
254
|
+
* on the next spawn — an app restart — which is exactly the complaint in
|
|
255
|
+
* SHO-203.
|
|
256
|
+
*
|
|
257
|
+
* - Linux + systemd scope: `systemctl --user set-property --runtime`.
|
|
258
|
+
* - Linux + direct cgroup: rewrite `memory.max` (and `memory.high`).
|
|
259
|
+
* - Windows: the Job Object's limit is fixed at assign time; there is no API
|
|
260
|
+
* to change it in place, so it lands on the next spawn (the RSS watchdog
|
|
261
|
+
* covers the gap).
|
|
262
|
+
* - macOS: the watchdog reads the budget from the limits file itself, so
|
|
263
|
+
* nothing to do here.
|
|
264
|
+
*/
|
|
265
|
+
export function applyLiveMemoryCap(
|
|
266
|
+
opts: LiveMemoryCapOptions,
|
|
267
|
+
deps: LiveMemoryCapDeps = {},
|
|
268
|
+
): boolean {
|
|
269
|
+
const platform = deps.platform ?? process.platform;
|
|
270
|
+
const { limits, logger } = opts;
|
|
271
|
+
if (platform === 'win32') {
|
|
272
|
+
logger?.log('[resource-limits] Job Object memory limit applies on the next runtime spawn');
|
|
273
|
+
return false;
|
|
274
|
+
}
|
|
275
|
+
if (platform !== 'linux') return false;
|
|
276
|
+
|
|
277
|
+
if (opts.scopeName) {
|
|
278
|
+
const args = [
|
|
279
|
+
'--user',
|
|
280
|
+
'set-property',
|
|
281
|
+
'--runtime',
|
|
282
|
+
opts.scopeName,
|
|
283
|
+
`MemoryMax=${limits.osMemoryMB}M`,
|
|
284
|
+
`MemoryHigh=${memoryHighMB(limits.osMemoryMB)}M`,
|
|
285
|
+
];
|
|
286
|
+
if (deps.run) {
|
|
287
|
+
deps.run('systemctl', args);
|
|
288
|
+
return true;
|
|
289
|
+
}
|
|
290
|
+
execFile('systemctl', args, { timeout: 5000 }, (err) => {
|
|
291
|
+
if (err) {
|
|
292
|
+
logger?.warn(`[resource-limits] set-property ${opts.scopeName} failed: ${err.message}`);
|
|
293
|
+
} else {
|
|
294
|
+
logger?.log(`[resource-limits] ${opts.scopeName} MemoryMax=${limits.osMemoryMB}M`);
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
return true;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
if (!opts.pid) return false;
|
|
301
|
+
const dir = directCgroupDir(opts.pid);
|
|
302
|
+
const exists = deps.exists ?? existsSync;
|
|
303
|
+
const write = deps.writeFile ?? writeFileSync;
|
|
304
|
+
try {
|
|
305
|
+
if (!exists(join(dir, 'memory.max'))) return false;
|
|
306
|
+
write(join(dir, 'memory.max'), String(limits.osMemoryMB * 1024 * 1024));
|
|
307
|
+
if (exists(join(dir, 'memory.high'))) {
|
|
308
|
+
write(join(dir, 'memory.high'), String(memoryHighMB(limits.osMemoryMB) * 1024 * 1024));
|
|
309
|
+
}
|
|
310
|
+
logger?.log(`[resource-limits] raised cgroup ${dir} memory.max to ${limits.osMemoryMB}M`);
|
|
311
|
+
return true;
|
|
312
|
+
} catch (err: any) {
|
|
313
|
+
logger?.warn(`[resource-limits] live cgroup update failed for ${dir}: ${err?.message ?? err}`);
|
|
314
|
+
return false;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
175
318
|
// ─── Windows: Job Object (optional native helper) ────────────────────
|
|
176
319
|
|
|
177
320
|
/**
|
|
@@ -203,11 +346,11 @@ export function tryAttachJobObject(opts: {
|
|
|
203
346
|
};
|
|
204
347
|
if (typeof helper.limitProcess !== 'function') return false;
|
|
205
348
|
const ok = helper.limitProcess(pid, {
|
|
206
|
-
memoryBytes: limits.
|
|
349
|
+
memoryBytes: limits.osMemoryMB * 1024 * 1024,
|
|
207
350
|
killOnClose: true,
|
|
208
351
|
});
|
|
209
352
|
if (ok) {
|
|
210
|
-
logger?.log(`[resource-limits] attached pid ${pid} to Job Object (${limits.
|
|
353
|
+
logger?.log(`[resource-limits] attached pid ${pid} to Job Object (${limits.osMemoryMB}M)`);
|
|
211
354
|
}
|
|
212
355
|
return ok;
|
|
213
356
|
} catch {
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
*/
|
|
34
34
|
import { type ChildProcess, spawn, spawnSync } from './child-process.ts';
|
|
35
35
|
import { createHmac, randomBytes } from 'node:crypto';
|
|
36
|
-
import { existsSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
|
|
36
|
+
import { existsSync, mkdirSync, readdirSync, rmSync, unwatchFile, watchFile } from 'node:fs';
|
|
37
37
|
import { createConnection } from 'node:net';
|
|
38
38
|
import { tmpdir } from 'node:os';
|
|
39
39
|
import { dirname, join } from 'node:path';
|
|
@@ -41,6 +41,7 @@ import { resolveRuntime, type ResolvedRuntime } from './runtime-resolver.ts';
|
|
|
41
41
|
import {
|
|
42
42
|
RssWatchdog,
|
|
43
43
|
applyHeapEnvCap,
|
|
44
|
+
applyLiveMemoryCap,
|
|
44
45
|
attachDirectCgroup,
|
|
45
46
|
readExecPidsFile,
|
|
46
47
|
resolveResourceLimits,
|
|
@@ -538,6 +539,9 @@ interface InternalRuntime {
|
|
|
538
539
|
* or when no limit is configured. Stopped on exit/stop.
|
|
539
540
|
*/
|
|
540
541
|
limitWatchdog: RssWatchdog | null;
|
|
542
|
+
/** `systemd-run --scope` unit name when the spawn was cgroup-wrapped, so a
|
|
543
|
+
* later budget change can be re-applied to the live scope (SHO-203). */
|
|
544
|
+
cgroupScope?: string | null;
|
|
541
545
|
lastError?: string;
|
|
542
546
|
spawnConfig: ProjectSpawnConfig;
|
|
543
547
|
/**
|
|
@@ -688,6 +692,9 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
688
692
|
/** Timer that refreshes every running runtime's cached activity sample. */
|
|
689
693
|
private activityTimer: ReturnType<typeof setInterval> | null = null;
|
|
690
694
|
|
|
695
|
+
/** `host-limits.json` this manager watches for live limit changes (SHO-203). */
|
|
696
|
+
private hostLimitsFile: string | null = null;
|
|
697
|
+
|
|
691
698
|
constructor(opts: WorkerRuntimeManagerOptions = {}) {
|
|
692
699
|
this.opts = opts;
|
|
693
700
|
this.log = opts.logger ?? console;
|
|
@@ -697,6 +704,7 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
697
704
|
// time — see runtime-activity.ts. Unref'd so it never holds the process
|
|
698
705
|
// open.
|
|
699
706
|
this.ensureActivitySampler();
|
|
707
|
+
this.watchHostLimits();
|
|
700
708
|
}
|
|
701
709
|
|
|
702
710
|
/**
|
|
@@ -1339,6 +1347,7 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1339
1347
|
|
|
1340
1348
|
async stopAll(signal: NodeJS.Signals = 'SIGTERM'): Promise<void> {
|
|
1341
1349
|
this.stopped = true;
|
|
1350
|
+
this.unwatchHostLimits();
|
|
1342
1351
|
if (this.activityTimer) {
|
|
1343
1352
|
clearInterval(this.activityTimer);
|
|
1344
1353
|
this.activityTimer = null;
|
|
@@ -1420,18 +1429,22 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1420
1429
|
let command = baseSpawn.command;
|
|
1421
1430
|
let args = baseSpawn.args;
|
|
1422
1431
|
let cgroupWrapped = false;
|
|
1432
|
+
let cgroupScope: string | null = null;
|
|
1423
1433
|
if (limits) {
|
|
1434
|
+
const scopeName = `shogo-rt-${slot.projectId.slice(0, 8)}-${Date.now().toString(36)}`;
|
|
1424
1435
|
const wrapped = wrapSpawnForCgroup({
|
|
1425
1436
|
command: baseSpawn.command,
|
|
1426
1437
|
args: baseSpawn.args,
|
|
1427
1438
|
limits,
|
|
1428
|
-
scopeName
|
|
1439
|
+
scopeName,
|
|
1429
1440
|
logger: this.log,
|
|
1430
1441
|
});
|
|
1431
1442
|
command = wrapped.command;
|
|
1432
1443
|
args = wrapped.args;
|
|
1433
1444
|
cgroupWrapped = wrapped.wrapped;
|
|
1445
|
+
cgroupScope = wrapped.wrapped ? scopeName : null;
|
|
1434
1446
|
}
|
|
1447
|
+
slot.cgroupScope = cgroupScope;
|
|
1435
1448
|
|
|
1436
1449
|
this.log.log(
|
|
1437
1450
|
`[WorkerRuntimeManager] Spawning agent-runtime for ${slot.projectId} ` +
|
|
@@ -1605,6 +1618,60 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1605
1618
|
return this.resolvedLimits;
|
|
1606
1619
|
}
|
|
1607
1620
|
|
|
1621
|
+
/**
|
|
1622
|
+
* Watch `host-limits.json` so a Settings change to the agent-command budget
|
|
1623
|
+
* lands on the app that is already running (SHO-203). An env var can only
|
|
1624
|
+
* change at spawn; a file can change now. `fs.watchFile` polls, which is why
|
|
1625
|
+
* it also picks the file up on a fresh install — no IPC plumbing needed.
|
|
1626
|
+
*/
|
|
1627
|
+
private watchHostLimits(): void {
|
|
1628
|
+
const file = (this.opts.env ?? process.env).SHOGO_HOST_LIMITS_FILE;
|
|
1629
|
+
if (!file || this.hostLimitsFile) return;
|
|
1630
|
+
this.hostLimitsFile = file;
|
|
1631
|
+
watchFile(file, { interval: 2000 }, () => this.applyHostLimitsChange());
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
/** Re-resolve the limits and push the new OS cap onto every live runtime. */
|
|
1635
|
+
private applyHostLimitsChange(): void {
|
|
1636
|
+
const env = this.opts.env ?? process.env;
|
|
1637
|
+
let limits: RuntimeResourceLimits | null;
|
|
1638
|
+
try {
|
|
1639
|
+
limits = resolveResourceLimits(env);
|
|
1640
|
+
} catch (err: any) {
|
|
1641
|
+
this.log.warn(`[WorkerRuntimeManager] host limits re-read failed: ${err?.message ?? err}`);
|
|
1642
|
+
return;
|
|
1643
|
+
}
|
|
1644
|
+
// Future spawns read the memo, so keep it in step with the file.
|
|
1645
|
+
this.resolvedLimits = limits;
|
|
1646
|
+
if (!limits) return;
|
|
1647
|
+
const live = [...this.runtimes.values()].filter((slot) => slot.pid && slot.status !== 'stopped');
|
|
1648
|
+
if (live.length === 0) return;
|
|
1649
|
+
this.log.log(
|
|
1650
|
+
`[WorkerRuntimeManager] agent-command budget changed — memory cap now ${limits.osMemoryMB}M ` +
|
|
1651
|
+
`(${limits.memoryMB}M runtime + ${limits.osMemoryMB - limits.memoryMB}M commands)`,
|
|
1652
|
+
);
|
|
1653
|
+
for (const slot of live) {
|
|
1654
|
+
try {
|
|
1655
|
+
applyLiveMemoryCap({
|
|
1656
|
+
pid: slot.pid,
|
|
1657
|
+
scopeName: slot.cgroupScope ?? null,
|
|
1658
|
+
limits,
|
|
1659
|
+
logger: this.log,
|
|
1660
|
+
});
|
|
1661
|
+
} catch (err: any) {
|
|
1662
|
+
this.log.warn(
|
|
1663
|
+
`[WorkerRuntimeManager] live memory cap for ${slot.projectId} failed: ${err?.message ?? err}`,
|
|
1664
|
+
);
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
private unwatchHostLimits(): void {
|
|
1670
|
+
if (!this.hostLimitsFile) return;
|
|
1671
|
+
unwatchFile(this.hostLimitsFile);
|
|
1672
|
+
this.hostLimitsFile = null;
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1608
1675
|
/**
|
|
1609
1676
|
* Apply the strongest available OS-level cap to a freshly-spawned runtime,
|
|
1610
1677
|
* or start the RSS watchdog as a fallback. Called right after spawn so most
|
|
@@ -1803,7 +1870,15 @@ export class WorkerRuntimeManager implements RuntimeResolver {
|
|
|
1803
1870
|
// the runtime publishes their PIDs so the watchdog leaves them out, and
|
|
1804
1871
|
// budgets them itself, stopping the largest command instead of the turn.
|
|
1805
1872
|
env.SHOGO_EXEC_PIDS_FILE = this.execPidsFile(slot);
|
|
1806
|
-
env
|
|
1873
|
+
const limitSource = this.opts.env ?? process.env;
|
|
1874
|
+
if (limitSource.SHOGO_HOST_LIMITS_FILE) {
|
|
1875
|
+
// Desktop: the budget is the user's setting, published in a file the
|
|
1876
|
+
// runtime watches, so it changes on a running app (SHO-203). Never
|
|
1877
|
+
// default it to the runtime ceiling — that is the bug this fixed.
|
|
1878
|
+
env.SHOGO_HOST_LIMITS_FILE = limitSource.SHOGO_HOST_LIMITS_FILE;
|
|
1879
|
+
} else {
|
|
1880
|
+
env.SHOGO_EXEC_MEMORY_MB ??= String(limits.memoryMB);
|
|
1881
|
+
}
|
|
1807
1882
|
}
|
|
1808
1883
|
|
|
1809
1884
|
return env;
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
// Copyright (C) 2026 Shogo Technologies, Inc.
|
|
3
|
+
/**
|
|
4
|
+
* Main-thread side of the tunnel Worker (SHO-185 §5)
|
|
5
|
+
*
|
|
6
|
+
* `startTunnelInWorker()` puts `WorkerTunnel` on its own thread and answers the
|
|
7
|
+
* questions it cannot answer itself: which local URL a path maps to, the runtime
|
|
8
|
+
* token for a project, and — once per heartbeat — which runtimes are up. The
|
|
9
|
+
* socket, its timers and its local `fetch`es never touch this thread, so a busy
|
|
10
|
+
* local API cannot cost the machine its connection any more.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { existsSync } from 'node:fs'
|
|
14
|
+
import { dirname, join } from 'node:path'
|
|
15
|
+
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
16
|
+
import type { ResolveRejection, RuntimeResolver } from './tunnel'
|
|
17
|
+
import type { HostProjectStatus, HostReply, WorkerEvent, WorkerRequest } from './tunnel-worker'
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* How long the worker gets to answer `ready` before it is written off.
|
|
21
|
+
*
|
|
22
|
+
* The worker posts `ready` from its `init` handler, before it touches the
|
|
23
|
+
* network, so this is only ever spent on a worker that never came up at all
|
|
24
|
+
* (crashed on load, wrong entry file, a runtime that quietly refuses to spawn
|
|
25
|
+
* one). A tunnel that is slow to connect is not slow to be ready.
|
|
26
|
+
*/
|
|
27
|
+
const READY_TIMEOUT_MS = 10_000
|
|
28
|
+
|
|
29
|
+
/** This module's directory: `resources/bundle/` once packaged, `src/lib/` in a checkout. */
|
|
30
|
+
const HOST_DIR = dirname(fileURLToPath(import.meta.url))
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The Worker entry file that sits next to this module.
|
|
34
|
+
*
|
|
35
|
+
* A source checkout has only `tunnel-worker.ts`; the desktop ships the compiled
|
|
36
|
+
* `resources/bundle/tunnel-worker.js` (see `apps/desktop/scripts/bundle-api.mjs`)
|
|
37
|
+
* and no source tree at all. Relative to `import.meta.url` — which inside
|
|
38
|
+
* `resources/bundle/api.js` is the bundle itself — `new URL('./tunnel-worker.ts')`
|
|
39
|
+
* could only ever name the checkout, so the packaged app asked for a `.ts` file
|
|
40
|
+
* that was never shipped, the worker never started, and the tunnel vanished.
|
|
41
|
+
*
|
|
42
|
+
* `SHOGO_TUNNEL_WORKER_PATH` overrides the search for packagers that relocate
|
|
43
|
+
* the bundle.
|
|
44
|
+
*/
|
|
45
|
+
export function resolveTunnelWorkerEntry(env: NodeJS.ProcessEnv = process.env): string {
|
|
46
|
+
const override = env.SHOGO_TUNNEL_WORKER_PATH?.trim()
|
|
47
|
+
if (override) return override
|
|
48
|
+
|
|
49
|
+
const candidates = [join(HOST_DIR, 'tunnel-worker.js'), join(HOST_DIR, 'tunnel-worker.ts')]
|
|
50
|
+
const found = candidates.find((candidate) => existsSync(candidate))
|
|
51
|
+
if (found) return found
|
|
52
|
+
|
|
53
|
+
throw new Error(
|
|
54
|
+
`no tunnel worker entry next to ${HOST_DIR} (looked for ${candidates.join(', ')}); ` +
|
|
55
|
+
`packaging must ship the tunnel-worker build output beside the API bundle`,
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface TunnelHostOptions {
|
|
60
|
+
apiKey: string
|
|
61
|
+
cloudUrl: string
|
|
62
|
+
name?: string
|
|
63
|
+
kind?: string
|
|
64
|
+
wsUrlOverride?: string
|
|
65
|
+
/** Everything the tunnel asks about the local machine. */
|
|
66
|
+
resolver: RuntimeResolver
|
|
67
|
+
logger?: Pick<Console, 'log' | 'warn' | 'error'>
|
|
68
|
+
onAuthRevoked?: (reason: string) => void
|
|
69
|
+
/** Called when the cloud tells us this machine's instance id (or forgets it). */
|
|
70
|
+
onInstanceId?: (instanceId: string | null) => void
|
|
71
|
+
/** Called when the socket's connected state changes. */
|
|
72
|
+
onConnection?: (connected: boolean) => void
|
|
73
|
+
/** The worker died (on load or later). The host is not usable after this. */
|
|
74
|
+
onWorkerError?: (error: Error) => void
|
|
75
|
+
/** Overrides the readiness deadline. Tests only. */
|
|
76
|
+
readyTimeoutMs?: number
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface TunnelHost {
|
|
80
|
+
stop(): void
|
|
81
|
+
instanceId(): string | null
|
|
82
|
+
isConnected(): boolean
|
|
83
|
+
/** Resolves once the worker has the tunnel running. */
|
|
84
|
+
ready(): Promise<void>
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Start the tunnel on its own thread.
|
|
89
|
+
*
|
|
90
|
+
* Throws if the worker cannot be created (a bundle without the entry file, a
|
|
91
|
+
* runtime that will not spawn one), which is the caller's cue to fall back to an
|
|
92
|
+
* in-process tunnel rather than lose remote control entirely. A worker that is
|
|
93
|
+
* created but never answers `ready` rejects `ready()` for the same reason — the
|
|
94
|
+
* throw alone is not enough, because a missing entry file does not always fail
|
|
95
|
+
* synchronously.
|
|
96
|
+
*/
|
|
97
|
+
export function startTunnelInWorker(options: TunnelHostOptions): TunnelHost {
|
|
98
|
+
const log = options.logger ?? console
|
|
99
|
+
const entry = resolveTunnelWorkerEntry()
|
|
100
|
+
// The web Worker API, which is what Bun implements (and what the worker file
|
|
101
|
+
// uses on its side with `self.postMessage`). `node:worker_threads` needs
|
|
102
|
+
// `parentPort` instead, and mixing the two silently never delivers a message.
|
|
103
|
+
const worker = new Worker(pathToFileURL(entry))
|
|
104
|
+
|
|
105
|
+
const readyTimeoutMs = options.readyTimeoutMs ?? READY_TIMEOUT_MS
|
|
106
|
+
let instanceId: string | null = null
|
|
107
|
+
let connected = false
|
|
108
|
+
let stopped = false
|
|
109
|
+
let readySettled = false
|
|
110
|
+
let settleReady: () => void = () => {}
|
|
111
|
+
let failReady: (error: Error) => void = () => {}
|
|
112
|
+
const ready = new Promise<void>((resolve, reject) => {
|
|
113
|
+
settleReady = resolve
|
|
114
|
+
failReady = reject
|
|
115
|
+
})
|
|
116
|
+
// Nobody is obliged to await `ready()`; an unobserved rejection would take the
|
|
117
|
+
// process down instead of letting the caller fall back.
|
|
118
|
+
void ready.catch(() => {})
|
|
119
|
+
|
|
120
|
+
const readyTimer = setTimeout(() => {
|
|
121
|
+
failReady(new Error(`tunnel worker (${entry}) did not answer ready within ${readyTimeoutMs}ms`))
|
|
122
|
+
}, readyTimeoutMs)
|
|
123
|
+
void ready.then(
|
|
124
|
+
() => {
|
|
125
|
+
readySettled = true
|
|
126
|
+
clearTimeout(readyTimer)
|
|
127
|
+
},
|
|
128
|
+
() => {
|
|
129
|
+
readySettled = true
|
|
130
|
+
clearTimeout(readyTimer)
|
|
131
|
+
},
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
/** What the last `resolve` decided, so a rejection needs no extra round trip. */
|
|
135
|
+
const rejections = new Map<string, ResolveRejection>()
|
|
136
|
+
|
|
137
|
+
worker.onmessage = (event: MessageEvent) => {
|
|
138
|
+
const data = event.data as any
|
|
139
|
+
if (!data || typeof data !== 'object') return
|
|
140
|
+
|
|
141
|
+
if (data.type === 'ready') {
|
|
142
|
+
settleReady()
|
|
143
|
+
return
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (data.type === 'log') {
|
|
147
|
+
const sink = data.level === 'error' ? log.error : data.level === 'warn' ? log.warn : log.log
|
|
148
|
+
sink.call(log, data.message)
|
|
149
|
+
return
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
if (data.type === 'event') {
|
|
153
|
+
if (data.name === 'instance') {
|
|
154
|
+
instanceId = data.instanceId ?? null
|
|
155
|
+
options.onInstanceId?.(instanceId)
|
|
156
|
+
} else if (data.name === 'connection') {
|
|
157
|
+
connected = Boolean(data.connected)
|
|
158
|
+
options.onConnection?.(connected)
|
|
159
|
+
} else if (data.name === 'auth-revoked') {
|
|
160
|
+
options.onAuthRevoked?.(String(data.reason ?? 'auth revoked'))
|
|
161
|
+
}
|
|
162
|
+
return
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
void answer(data as WorkerRequest)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
worker.onerror = (event: any) => {
|
|
169
|
+
const message = event?.message ?? event
|
|
170
|
+
log.error(`[TunnelHost] worker error: ${message}`)
|
|
171
|
+
const error = new Error(`tunnel worker failed: ${message}`)
|
|
172
|
+
// A worker that dies on load never posts `ready`. Reject now so the caller
|
|
173
|
+
// can fall back instead of waiting out the whole deadline.
|
|
174
|
+
if (!readySettled) failReady(error)
|
|
175
|
+
// A worker that dies later leaves a socket nobody is watching, which looks
|
|
176
|
+
// exactly like an offline machine. Same answer: hand it back to the caller.
|
|
177
|
+
options.onWorkerError?.(error)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function answer(request: WorkerRequest): Promise<void> {
|
|
181
|
+
const resolver = options.resolver
|
|
182
|
+
try {
|
|
183
|
+
if (request.type === 'resolve') {
|
|
184
|
+
const url = await resolver.resolveLocalUrl(request.path, request.projectId)
|
|
185
|
+
const rejection =
|
|
186
|
+
url === null
|
|
187
|
+
? await Promise.resolve(resolver.describeRejection?.(request.path, request.projectId))
|
|
188
|
+
: undefined
|
|
189
|
+
if (rejection) rejections.set(`${request.path}|${request.projectId ?? ''}`, rejection)
|
|
190
|
+
reply({ type: 'resolved', id: request.id, url, rejection })
|
|
191
|
+
return
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (request.type === 'token') {
|
|
195
|
+
const token = await resolver.deriveRuntimeToken(request.projectId)
|
|
196
|
+
reply({ type: 'token-result', id: request.id, token })
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (request.type === 'refresh') {
|
|
201
|
+
const projects: HostProjectStatus[] = resolver.getActiveProjects().map((projectId) => {
|
|
202
|
+
const status = resolver.status(projectId)
|
|
203
|
+
return { projectId, status: status?.status ?? 'unknown', agentPort: status?.agentPort }
|
|
204
|
+
})
|
|
205
|
+
reply({
|
|
206
|
+
type: 'refresh-result',
|
|
207
|
+
id: request.id,
|
|
208
|
+
projects,
|
|
209
|
+
rejections: Object.fromEntries(rejections),
|
|
210
|
+
})
|
|
211
|
+
return
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (request.type === 'machineSnapshot') {
|
|
215
|
+
const snapshot = (await resolver.machineSnapshot?.()) ?? null
|
|
216
|
+
reply({ type: 'machine-snapshot-result', id: request.id, snapshot })
|
|
217
|
+
}
|
|
218
|
+
} catch (error: any) {
|
|
219
|
+
log.warn(`[TunnelHost] ${request.type} failed: ${error?.message ?? error}`)
|
|
220
|
+
if (request.type === 'resolve') reply({ type: 'resolved', id: request.id, url: null })
|
|
221
|
+
else if (request.type === 'token') reply({ type: 'token-result', id: request.id, token: null })
|
|
222
|
+
else if (request.type === 'refresh') reply({ type: 'refresh-result', id: request.id, projects: [] })
|
|
223
|
+
else reply({ type: 'machine-snapshot-result', id: request.id, snapshot: null })
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function reply(message: HostReply): void {
|
|
228
|
+
if (stopped) return
|
|
229
|
+
worker.postMessage(message)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
worker.postMessage({
|
|
233
|
+
type: 'init',
|
|
234
|
+
config: {
|
|
235
|
+
apiKey: options.apiKey,
|
|
236
|
+
cloudUrl: options.cloudUrl,
|
|
237
|
+
name: options.name,
|
|
238
|
+
kind: options.kind,
|
|
239
|
+
wsUrlOverride: options.wsUrlOverride,
|
|
240
|
+
},
|
|
241
|
+
})
|
|
242
|
+
|
|
243
|
+
return {
|
|
244
|
+
stop() {
|
|
245
|
+
stopped = true
|
|
246
|
+
clearTimeout(readyTimer)
|
|
247
|
+
void worker.terminate()
|
|
248
|
+
},
|
|
249
|
+
instanceId: () => instanceId,
|
|
250
|
+
isConnected: () => connected,
|
|
251
|
+
ready: () => ready,
|
|
252
|
+
}
|
|
253
|
+
}
|