c8ctl-plugin-nano 1.20.0 → 1.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/c8ctl-plugin.js +1238 -6
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
openSync,
|
|
36
36
|
readFileSync,
|
|
37
37
|
writeFileSync,
|
|
38
|
+
appendFileSync,
|
|
38
39
|
rmSync,
|
|
39
40
|
readdirSync,
|
|
40
41
|
chmodSync,
|
|
@@ -43,15 +44,18 @@ import {
|
|
|
43
44
|
statfsSync,
|
|
44
45
|
lstatSync,
|
|
45
46
|
mkdtempSync,
|
|
47
|
+
closeSync,
|
|
46
48
|
watchFile,
|
|
47
49
|
unwatchFile,
|
|
48
50
|
} from 'node:fs';
|
|
49
|
-
import {
|
|
50
|
-
import {
|
|
51
|
+
import { createConnection, createServer } from 'node:net';
|
|
52
|
+
import { randomUUID, createHash, randomBytes } from 'node:crypto';
|
|
53
|
+
import { homedir, platform as osPlatform, devNull, tmpdir, hostname } from 'node:os';
|
|
51
54
|
import { join, isAbsolute, resolve as resolvePath, dirname, basename, sep } from 'node:path';
|
|
52
55
|
import { createRequire } from 'node:module';
|
|
53
56
|
import { fileURLToPath } from 'node:url';
|
|
54
57
|
import { createInterface } from 'node:readline/promises';
|
|
58
|
+
import { createInterface as createReadline } from 'node:readline';
|
|
55
59
|
import { platformForHost } from './platforms.mjs';
|
|
56
60
|
|
|
57
61
|
const requireFromHere = createRequire(import.meta.url);
|
|
@@ -119,6 +123,7 @@ const READINESS_POLL_MS = 500;
|
|
|
119
123
|
const HEALTH_TIMEOUT_MS = 1_500;
|
|
120
124
|
const STOP_GRACE_MS = 8_000;
|
|
121
125
|
const PROCESSOS_STATE_FILE = 'processos.json';
|
|
126
|
+
const SUPERVISOR_STATE_FILE = 'supervisor.json';
|
|
122
127
|
const PROCESSOS_DEFAULT_PORT = 8090;
|
|
123
128
|
const DEFAULT_NANO_URL = 'http://localhost:8080';
|
|
124
129
|
|
|
@@ -193,6 +198,34 @@ function getLogDir() {
|
|
|
193
198
|
return join(getStateHome(), 'logs');
|
|
194
199
|
}
|
|
195
200
|
|
|
201
|
+
// ---------------------------------------------------------------------------
|
|
202
|
+
// Worker supervisor paths (see the `supervisor` command). The supervisor is a
|
|
203
|
+
// detached daemon that manages a fleet of `nano work` child processes; it keeps
|
|
204
|
+
// its own state file, a control socket, and per-worker + daemon log files.
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
function getSupervisorStateFile() {
|
|
208
|
+
return join(getStateHome(), SUPERVISOR_STATE_FILE);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function getSupervisorLogDir() {
|
|
212
|
+
return join(getLogDir(), 'supervisor');
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Deterministic control-socket path shared by the daemon and every client.
|
|
217
|
+
* Derived from a hash of the (possibly overridden) state home so distinct
|
|
218
|
+
* C8CTL_NANO_HOME instances get distinct sockets, and kept SHORT to stay under
|
|
219
|
+
* the ~104-byte AF_UNIX `sun_path` limit on macOS regardless of username. On
|
|
220
|
+
* Windows a named pipe is used instead. The chosen path is also recorded in the
|
|
221
|
+
* state file so clients can prefer the daemon's own reported path.
|
|
222
|
+
*/
|
|
223
|
+
function getSupervisorSocketPath() {
|
|
224
|
+
const hash = createHash('sha1').update(getStateHome()).digest('hex').slice(0, 8);
|
|
225
|
+
if (osPlatform() === 'win32') return `\\\\.\\pipe\\c8ctl-nano-sup-${hash}`;
|
|
226
|
+
return join(tmpdir(), `c8ctl-nano-sup-${hash}.sock`);
|
|
227
|
+
}
|
|
228
|
+
|
|
196
229
|
// ---------------------------------------------------------------------------
|
|
197
230
|
// Persistent plugin config (config.json) — user settings that survive across
|
|
198
231
|
// clusters: the binary path and the workspace (models/workers) location.
|
|
@@ -389,7 +422,7 @@ function launcherEnvMarkers(resolved) {
|
|
|
389
422
|
// Argument parsing
|
|
390
423
|
// ---------------------------------------------------------------------------
|
|
391
424
|
|
|
392
|
-
const VALID_SUBCOMMANDS = ['start', 'stop', 'status', 'logs', 'log', 'restart', 'pause', 'resume', 'clean', 'set', 'config', 'update', 'hire', 'assign', 'work'];
|
|
425
|
+
const VALID_SUBCOMMANDS = ['start', 'stop', 'status', 'logs', 'log', 'restart', 'pause', 'resume', 'clean', 'set', 'config', 'update', 'hire', 'assign', 'work', 'supervisor'];
|
|
393
426
|
|
|
394
427
|
/**
|
|
395
428
|
* Parse positional args + flags into a normalized request.
|
|
@@ -2846,7 +2879,10 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult
|
|
|
2846
2879
|
*/
|
|
2847
2880
|
async function workAgent(req, flags) {
|
|
2848
2881
|
const logger = getLogger();
|
|
2849
|
-
|
|
2882
|
+
// The hire to run always comes from the positional profile. `--name` no longer
|
|
2883
|
+
// selects the hire (that was a footgun: `work reviewer --name coder` silently
|
|
2884
|
+
// ran `coder`); it now names THIS worker instance (see `workerName` below).
|
|
2885
|
+
const name = req.positional[0];
|
|
2850
2886
|
|
|
2851
2887
|
if (!name) {
|
|
2852
2888
|
const hires = readHires();
|
|
@@ -2869,6 +2905,19 @@ async function workAgent(req, flags) {
|
|
|
2869
2905
|
}
|
|
2870
2906
|
const profile = normalized.profile;
|
|
2871
2907
|
|
|
2908
|
+
// This worker's identity, surfaced to the broker as the `workerName` on every
|
|
2909
|
+
// activateJobs call (`‹workerName›:‹jobType›`). An explicit `--name` wins;
|
|
2910
|
+
// otherwise auto-generate `‹host›-‹profile›-‹random›` so two workers of the
|
|
2911
|
+
// same profile (e.g. launched by the supervisor) stay distinct at the broker
|
|
2912
|
+
// and in logs. A blank/whitespace `--name` falls back to auto (mirrors the
|
|
2913
|
+
// supervisor path); a non-blank one must be a safe worker-name token.
|
|
2914
|
+
const explicitName = flags?.name ? String(flags.name).trim() : '';
|
|
2915
|
+
if (explicitName !== '' && !isValidWorkerName(explicitName)) {
|
|
2916
|
+
logger.error(`Invalid --name "${flags.name}": use only letters, digits, and . _ -`);
|
|
2917
|
+
process.exit(1);
|
|
2918
|
+
}
|
|
2919
|
+
const workerName = explicitName !== '' ? explicitName : autoWorkerName(name);
|
|
2920
|
+
|
|
2872
2921
|
if (!globalThis.c8ctl || typeof globalThis.c8ctl.createClient !== 'function') {
|
|
2873
2922
|
logger.error('work requires the c8ctl runtime (createClient). Run it via the c8ctl CLI.');
|
|
2874
2923
|
process.exit(1);
|
|
@@ -3012,6 +3061,7 @@ async function workAgent(req, flags) {
|
|
|
3012
3061
|
const camunda = globalThis.c8ctl.createClient();
|
|
3013
3062
|
|
|
3014
3063
|
logger.info(`Putting "${name}" [${profile.rank}] to work → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
|
|
3064
|
+
logger.info(` worker: ${workerName}`);
|
|
3015
3065
|
logger.info(` model: ${profile.model || '(none)'}; capabilities: ${profile.capabilities.join(', ') || '(none)'}`);
|
|
3016
3066
|
logger.info(` sandbox: ${sandbox}${isContainer ? ` (image ${image})` : ''}`);
|
|
3017
3067
|
const profileEnvKeys = Object.keys(profileEnv);
|
|
@@ -3027,7 +3077,7 @@ async function workAgent(req, flags) {
|
|
|
3027
3077
|
const makeWorker = (jobType) =>
|
|
3028
3078
|
camunda.createJobWorker({
|
|
3029
3079
|
jobType,
|
|
3030
|
-
workerName: `${
|
|
3080
|
+
workerName: `${workerName}:${jobType}`,
|
|
3031
3081
|
maxParallelJobs,
|
|
3032
3082
|
jobTimeoutMs: jobLockMs,
|
|
3033
3083
|
pollTimeoutMs,
|
|
@@ -3381,6 +3431,1150 @@ async function workAgent(req, flags) {
|
|
|
3381
3431
|
});
|
|
3382
3432
|
}
|
|
3383
3433
|
|
|
3434
|
+
// ---------------------------------------------------------------------------
|
|
3435
|
+
// supervisor — run & manage a fleet of `nano work` children from one terminal.
|
|
3436
|
+
//
|
|
3437
|
+
// `nano work` needs the c8ctl host runtime (createClient), so worker loops
|
|
3438
|
+
// cannot run inside a bare detached process. The supervisor is therefore a
|
|
3439
|
+
// process *manager*: a detached daemon spawns one `c8ctl nano work <profile>`
|
|
3440
|
+
// child per worker, restarts crashed children with capped backoff, and serves a
|
|
3441
|
+
// control socket (newline-delimited JSON) used by both the management
|
|
3442
|
+
// subcommands (status/add/remove/restart/stop/logs — no interactive surface
|
|
3443
|
+
// needed) and the interactive `attach` console, which can be detached from
|
|
3444
|
+
// (leaving the daemon running) or used to `stop` the whole fleet.
|
|
3445
|
+
// ---------------------------------------------------------------------------
|
|
3446
|
+
|
|
3447
|
+
const SUPERVISOR_BACKOFF_BASE_MS = 1_000;
|
|
3448
|
+
const SUPERVISOR_BACKOFF_MAX_MS = 30_000;
|
|
3449
|
+
// A child that stayed up at least this long before exiting is not crash-looping,
|
|
3450
|
+
// so its restart backoff is reset to zero.
|
|
3451
|
+
const SUPERVISOR_HEALTHY_UPTIME_MS = 60_000;
|
|
3452
|
+
const SUPERVISOR_CONNECT_TIMEOUT_MS = 6_000;
|
|
3453
|
+
// End-to-end deadline for a single request: once connected, a wedged/incompatible
|
|
3454
|
+
// daemon that accepts but never sends a `final` frame must not hang the client.
|
|
3455
|
+
const SUPERVISOR_RESPONSE_TIMEOUT_MS = 15_000;
|
|
3456
|
+
// Tighter end-to-end deadline for quick liveness probes (status checks used by
|
|
3457
|
+
// liveSupervisor/ensureSupervisor). Without this, a daemon that accepts the
|
|
3458
|
+
// connection but never returns a `final` frame would still block the "fast"
|
|
3459
|
+
// probe for the full SUPERVISOR_RESPONSE_TIMEOUT_MS, hanging stop/remove/restart.
|
|
3460
|
+
const SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS = 2_000;
|
|
3461
|
+
// Hard cap on a single connection's inbound buffer, so a misbehaving client
|
|
3462
|
+
// can't grow the daemon's memory without bound with a newline-free frame.
|
|
3463
|
+
const SUPERVISOR_MAX_FRAME_BYTES = 1 << 20; // 1 MiB
|
|
3464
|
+
|
|
3465
|
+
// The `nano work` flags forwarded verbatim to each spawned child.
|
|
3466
|
+
// kind: 'value' → `--flag v`; 'boolean' → `--flag`; 'list' → repeated `--flag v`.
|
|
3467
|
+
const WORK_FORWARD_FLAGS = {
|
|
3468
|
+
'max-parallel': 'value',
|
|
3469
|
+
'job-timeout': 'value',
|
|
3470
|
+
'lock-grace': 'value',
|
|
3471
|
+
'poll-timeout': 'value',
|
|
3472
|
+
sandbox: 'value',
|
|
3473
|
+
image: 'value',
|
|
3474
|
+
'secret-resolver': 'value',
|
|
3475
|
+
'reap-age': 'value',
|
|
3476
|
+
'reap-interval': 'value',
|
|
3477
|
+
'min-free-mb': 'value',
|
|
3478
|
+
'clone-timeout': 'value',
|
|
3479
|
+
'keep-runs': 'boolean',
|
|
3480
|
+
stream: 'boolean',
|
|
3481
|
+
arg: 'list',
|
|
3482
|
+
env: 'list',
|
|
3483
|
+
'job-type': 'list',
|
|
3484
|
+
};
|
|
3485
|
+
|
|
3486
|
+
/**
|
|
3487
|
+
* Reconstruct the `work` argv tail from a parsed flags object, so `supervisor
|
|
3488
|
+
* add <profile> [work flags]` forwards those flags to the spawned child. Pure.
|
|
3489
|
+
*/
|
|
3490
|
+
function reconstructWorkArgs(flags) {
|
|
3491
|
+
const out = [];
|
|
3492
|
+
if (!flags || typeof flags !== 'object') return out;
|
|
3493
|
+
for (const [name, kind] of Object.entries(WORK_FORWARD_FLAGS)) {
|
|
3494
|
+
const v = flags[name];
|
|
3495
|
+
if (v === undefined || v === null) continue;
|
|
3496
|
+
if (kind === 'boolean') {
|
|
3497
|
+
if (v === true || v === 'true') out.push(`--${name}`);
|
|
3498
|
+
} else if (kind === 'list') {
|
|
3499
|
+
const items = Array.isArray(v) ? v : [v];
|
|
3500
|
+
for (const item of items) {
|
|
3501
|
+
if (item === undefined || item === null) continue;
|
|
3502
|
+
out.push(`--${name}`, String(item));
|
|
3503
|
+
}
|
|
3504
|
+
} else if (v !== '') {
|
|
3505
|
+
out.push(`--${name}`, String(v));
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
return out;
|
|
3509
|
+
}
|
|
3510
|
+
|
|
3511
|
+
/**
|
|
3512
|
+
* Sanitize one token for use inside a worker name: keep `[A-Za-z0-9._-]`,
|
|
3513
|
+
* collapse every other run to a single `-`, and trim leading/trailing
|
|
3514
|
+
* separators. Returns `fallback` when nothing survives (e.g. an all-symbol
|
|
3515
|
+
* input). Pure.
|
|
3516
|
+
*/
|
|
3517
|
+
function sanitizeNameToken(raw, fallback = 'x') {
|
|
3518
|
+
const s = String(raw ?? '')
|
|
3519
|
+
.trim()
|
|
3520
|
+
.replace(/[^A-Za-z0-9._-]+/g, '-')
|
|
3521
|
+
.replace(/^[-._]+|[-._]+$/g, '');
|
|
3522
|
+
return s || fallback;
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
/**
|
|
3526
|
+
* An explicit worker name (`--name`) is valid iff — after trimming — it is a
|
|
3527
|
+
* non-empty run of `[A-Za-z0-9._-]`. That charset is the intersection of what
|
|
3528
|
+
* is safe in a broker `workerName` (no `:` to corrupt the `‹name›:‹jobType›`
|
|
3529
|
+
* form) and what survives `supervisorWorkerLogFile`'s filename sanitization
|
|
3530
|
+
* unchanged (so distinct ids can never collapse onto the same `worker-‹id›.log`
|
|
3531
|
+
* or escape the log dir). Auto-generated names are already in this shape;
|
|
3532
|
+
* operator-supplied names are validated against it so both invariants hold.
|
|
3533
|
+
* Pure.
|
|
3534
|
+
*/
|
|
3535
|
+
function isValidWorkerName(name) {
|
|
3536
|
+
const s = typeof name === 'string' ? name.trim() : '';
|
|
3537
|
+
return s !== '' && /^[A-Za-z0-9._-]+$/.test(s);
|
|
3538
|
+
}
|
|
3539
|
+
|
|
3540
|
+
/** A short, lowercase, collision-resistant suffix for auto worker names. */
|
|
3541
|
+
function randomNameSuffix(bytes = 4) {
|
|
3542
|
+
return randomBytes(bytes).toString('hex');
|
|
3543
|
+
}
|
|
3544
|
+
|
|
3545
|
+
/**
|
|
3546
|
+
* Auto-generated worker name: `‹short-hostname›-‹profile›-‹random›`. The host
|
|
3547
|
+
* defaults to this machine's short hostname (the first dot-label, lowercased);
|
|
3548
|
+
* the random suffix keeps two same-profile workers on the same host distinct.
|
|
3549
|
+
* `host`/`rand` are injectable so tests can assert a deterministic shape. Pure
|
|
3550
|
+
* given its options.
|
|
3551
|
+
*/
|
|
3552
|
+
function autoWorkerName(profile, { host = hostname(), rand = randomNameSuffix } = {}) {
|
|
3553
|
+
const shortHost = sanitizeNameToken(String(host || '').split('.')[0].toLowerCase(), 'host');
|
|
3554
|
+
const prof = sanitizeNameToken(profile, 'worker');
|
|
3555
|
+
const suffix = sanitizeNameToken(typeof rand === 'function' ? rand() : rand, '0');
|
|
3556
|
+
return `${shortHost}-${prof}-${suffix}`;
|
|
3557
|
+
}
|
|
3558
|
+
|
|
3559
|
+
/**
|
|
3560
|
+
* Split a supervised-worker name (`--name X`, `--name=X`, `-n X`) out of a raw
|
|
3561
|
+
* token list, returning `{ name, rest }` where `rest` is the remaining work
|
|
3562
|
+
* flags. Used by the interactive console's `add`, whose tokens aren't parsed by
|
|
3563
|
+
* the CLI flag layer. Last occurrence wins; a trailing `--name` with no value
|
|
3564
|
+
* yields `name: undefined`. Pure.
|
|
3565
|
+
*/
|
|
3566
|
+
function extractNameFlag(parts) {
|
|
3567
|
+
const rest = [];
|
|
3568
|
+
let name;
|
|
3569
|
+
const list = Array.isArray(parts) ? parts : [];
|
|
3570
|
+
for (let i = 0; i < list.length; i++) {
|
|
3571
|
+
const tok = String(list[i]);
|
|
3572
|
+
const eq = /^(?:--name|-n)=(.*)$/.exec(tok);
|
|
3573
|
+
if (eq) { name = eq[1]; continue; }
|
|
3574
|
+
if (tok === '--name' || tok === '-n') {
|
|
3575
|
+
if (i + 1 < list.length) { name = String(list[i + 1]); i++; }
|
|
3576
|
+
continue;
|
|
3577
|
+
}
|
|
3578
|
+
rest.push(tok);
|
|
3579
|
+
}
|
|
3580
|
+
return { name: name != null && name.trim() !== '' ? name.trim() : undefined, rest };
|
|
3581
|
+
}
|
|
3582
|
+
|
|
3583
|
+
/** Assign a unique, stable worker id from a profile name (pure). */
|
|
3584
|
+
function supervisorWorkerId(profile, taken) {
|
|
3585
|
+
const base = String(profile || '').trim() || 'worker';
|
|
3586
|
+
const set = taken instanceof Set ? taken : new Set(taken || []);
|
|
3587
|
+
if (!set.has(base)) return base;
|
|
3588
|
+
for (let i = 2; ; i++) {
|
|
3589
|
+
const candidate = `${base}#${i}`;
|
|
3590
|
+
if (!set.has(candidate)) return candidate;
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3593
|
+
|
|
3594
|
+
/**
|
|
3595
|
+
* Redact sensitive values from a reconstructed `work` argv before logging, so
|
|
3596
|
+
* supervisor logs never capture secrets. Both `--env NAME=VALUE` and the
|
|
3597
|
+
* inline `--env=NAME=VALUE` form become `NAME=***` (the value passed to
|
|
3598
|
+
* `nano work` is untouched). Pure.
|
|
3599
|
+
*/
|
|
3600
|
+
function redactWorkArgs(args) {
|
|
3601
|
+
const out = [];
|
|
3602
|
+
const list = Array.isArray(args) ? args : [];
|
|
3603
|
+
const redactPair = (pair) => {
|
|
3604
|
+
const eq = pair.indexOf('=');
|
|
3605
|
+
return eq === -1 ? '***' : `${pair.slice(0, eq)}=***`;
|
|
3606
|
+
};
|
|
3607
|
+
for (let i = 0; i < list.length; i++) {
|
|
3608
|
+
const tok = String(list[i]);
|
|
3609
|
+
if (tok === '--env' && i + 1 < list.length) {
|
|
3610
|
+
out.push(tok, redactPair(String(list[i + 1])));
|
|
3611
|
+
i++;
|
|
3612
|
+
} else if (tok.startsWith('--env=')) {
|
|
3613
|
+
out.push(`--env=${redactPair(tok.slice('--env='.length))}`);
|
|
3614
|
+
} else {
|
|
3615
|
+
out.push(tok);
|
|
3616
|
+
}
|
|
3617
|
+
}
|
|
3618
|
+
return out;
|
|
3619
|
+
}
|
|
3620
|
+
|
|
3621
|
+
/** Capped exponential restart backoff for a crash-looping child (pure). */
|
|
3622
|
+
function supervisorBackoffMs(restarts, base = SUPERVISOR_BACKOFF_BASE_MS, max = SUPERVISOR_BACKOFF_MAX_MS) {
|
|
3623
|
+
const n = Math.max(0, Number(restarts) || 0);
|
|
3624
|
+
return Math.min(max, base * 2 ** Math.min(n, 20));
|
|
3625
|
+
}
|
|
3626
|
+
|
|
3627
|
+
/** Newline-delimited JSON framing for the control socket (pure). */
|
|
3628
|
+
function encodeFrame(obj) {
|
|
3629
|
+
return JSON.stringify(obj) + '\n';
|
|
3630
|
+
}
|
|
3631
|
+
|
|
3632
|
+
/** Split a buffered string into complete JSON frames + a remainder (pure). */
|
|
3633
|
+
function decodeFrames(buffer) {
|
|
3634
|
+
const frames = [];
|
|
3635
|
+
let rest = String(buffer ?? '');
|
|
3636
|
+
let idx;
|
|
3637
|
+
while ((idx = rest.indexOf('\n')) >= 0) {
|
|
3638
|
+
const line = rest.slice(0, idx).trim();
|
|
3639
|
+
rest = rest.slice(idx + 1);
|
|
3640
|
+
if (!line) continue;
|
|
3641
|
+
try { frames.push(JSON.parse(line)); } catch { /* skip malformed frame */ }
|
|
3642
|
+
}
|
|
3643
|
+
return { frames, rest };
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3646
|
+
/** Humanise a millisecond duration compactly (pure). */
|
|
3647
|
+
function formatDuration(ms) {
|
|
3648
|
+
const s = Math.floor((Number(ms) || 0) / 1000);
|
|
3649
|
+
if (s < 60) return `${s}s`;
|
|
3650
|
+
const m = Math.floor(s / 60);
|
|
3651
|
+
if (m < 60) return `${m}m${s % 60}s`;
|
|
3652
|
+
const h = Math.floor(m / 60);
|
|
3653
|
+
if (h < 24) return `${h}h${m % 60}m`;
|
|
3654
|
+
const d = Math.floor(h / 24);
|
|
3655
|
+
return `${d}d${h % 24}h`;
|
|
3656
|
+
}
|
|
3657
|
+
|
|
3658
|
+
/** Project a live/stored worker record to a status row (pure w.r.t. `now`). */
|
|
3659
|
+
function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
3660
|
+
const alive = isPidAlive(w.pid);
|
|
3661
|
+
const uptimeMs = alive && w.startedAt ? Math.max(0, now - new Date(w.startedAt).getTime()) : 0;
|
|
3662
|
+
return {
|
|
3663
|
+
id: w.id,
|
|
3664
|
+
profile: w.profile,
|
|
3665
|
+
pid: alive ? w.pid : null,
|
|
3666
|
+
state: w.stopping ? 'stopping' : alive ? 'running' : 'down',
|
|
3667
|
+
restarts: Number(w.restarts) || 0,
|
|
3668
|
+
uptimeMs,
|
|
3669
|
+
lastExit: w.lastExit ?? null,
|
|
3670
|
+
args: Array.isArray(w.args) ? w.args : [],
|
|
3671
|
+
};
|
|
3672
|
+
}
|
|
3673
|
+
|
|
3674
|
+
/** Render a supervisor status object as an aligned text table. */
|
|
3675
|
+
function formatSupervisorStatus(status) {
|
|
3676
|
+
const lines = [];
|
|
3677
|
+
const d = status.daemon || {};
|
|
3678
|
+
const alive = d.pid ? isPidAlive(d.pid) : false;
|
|
3679
|
+
lines.push('Supervisor:');
|
|
3680
|
+
lines.push(` daemon pid: ${d.pid ?? '-'} ${alive ? '(alive)' : '(dead — stale state)'}`);
|
|
3681
|
+
if (d.startedAt) lines.push(` started: ${d.startedAt}`);
|
|
3682
|
+
if (d.socket) lines.push(` control: ${d.socket}`);
|
|
3683
|
+
const workers = Array.isArray(status.workers) ? status.workers : [];
|
|
3684
|
+
lines.push('');
|
|
3685
|
+
if (workers.length === 0) {
|
|
3686
|
+
lines.push(' No workers. Add one with: c8ctl nano supervisor add <profile>');
|
|
3687
|
+
return lines.join('\n');
|
|
3688
|
+
}
|
|
3689
|
+
const rows = workers.map((w) => ({
|
|
3690
|
+
id: String(w.id),
|
|
3691
|
+
profile: String(w.profile),
|
|
3692
|
+
state: String(w.state),
|
|
3693
|
+
pid: w.pid ? String(w.pid) : '-',
|
|
3694
|
+
restarts: String(w.restarts),
|
|
3695
|
+
uptime: w.state === 'running' ? formatDuration(w.uptimeMs) : '-',
|
|
3696
|
+
last: w.lastExit ? String(w.lastExit) : '-',
|
|
3697
|
+
}));
|
|
3698
|
+
const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
|
|
3699
|
+
const cols = ['id', 'profile', 'state', 'pid', 'restarts', 'uptime', 'last'];
|
|
3700
|
+
const width = {};
|
|
3701
|
+
for (const c of cols) width[c] = Math.max(head[c].length, ...rows.map((r) => r[c].length));
|
|
3702
|
+
const fmt = (r) => ' ' + cols.map((c) => r[c].padEnd(width[c])).join(' ');
|
|
3703
|
+
lines.push(fmt(head));
|
|
3704
|
+
for (const r of rows) lines.push(fmt(r));
|
|
3705
|
+
return lines.join('\n');
|
|
3706
|
+
}
|
|
3707
|
+
|
|
3708
|
+
function readSupervisorState() {
|
|
3709
|
+
const file = getSupervisorStateFile();
|
|
3710
|
+
if (!existsSync(file)) return null;
|
|
3711
|
+
try {
|
|
3712
|
+
return JSON.parse(readFileSync(file, 'utf-8'));
|
|
3713
|
+
} catch {
|
|
3714
|
+
return null;
|
|
3715
|
+
}
|
|
3716
|
+
}
|
|
3717
|
+
|
|
3718
|
+
function writeSupervisorState(state) {
|
|
3719
|
+
mkdirSync(getStateHome(), { recursive: true });
|
|
3720
|
+
// Atomic + owner-only: write to a same-dir temp file (mode 0600) then rename
|
|
3721
|
+
// over the target, so a concurrent reader never sees a torn file and the
|
|
3722
|
+
// state (which records worker argv) isn't world-readable.
|
|
3723
|
+
const target = getSupervisorStateFile();
|
|
3724
|
+
const tmp = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
3725
|
+
writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
|
|
3726
|
+
try { renameSync(tmp, target); }
|
|
3727
|
+
catch (err) { try { rmSync(tmp, { force: true }); } catch { /* best effort */ } throw err; }
|
|
3728
|
+
}
|
|
3729
|
+
|
|
3730
|
+
function clearSupervisorState() {
|
|
3731
|
+
const file = getSupervisorStateFile();
|
|
3732
|
+
try { if (existsSync(file)) rmSync(file); } catch { /* best effort */ }
|
|
3733
|
+
}
|
|
3734
|
+
|
|
3735
|
+
/** Running daemon state (pid alive) or null. */
|
|
3736
|
+
function runningSupervisor() {
|
|
3737
|
+
const state = readSupervisorState();
|
|
3738
|
+
return state && isPidAlive(state.pid) ? state : null;
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
/** Synthesize a state-file-shaped object from a live `status` response. */
|
|
3742
|
+
function stateFromStatus(res, socketPath) {
|
|
3743
|
+
return {
|
|
3744
|
+
pid: res.daemon?.pid,
|
|
3745
|
+
startedAt: res.daemon?.startedAt,
|
|
3746
|
+
socket: res.daemon?.socket || socketPath,
|
|
3747
|
+
logFile: res.daemon?.logFile,
|
|
3748
|
+
workers: res.workers || [],
|
|
3749
|
+
};
|
|
3750
|
+
}
|
|
3751
|
+
|
|
3752
|
+
/**
|
|
3753
|
+
* Resolve a live supervisor, healing a missing/stale state file. Returns the
|
|
3754
|
+
* running state (pid alive) if present; otherwise probes the deterministic
|
|
3755
|
+
* control socket and, if a daemon answers, re-persists and returns its state so
|
|
3756
|
+
* management commands still work when supervisor.json was deleted/cleaned.
|
|
3757
|
+
* Returns null when nothing is listening.
|
|
3758
|
+
*/
|
|
3759
|
+
async function liveSupervisor() {
|
|
3760
|
+
const running = runningSupervisor();
|
|
3761
|
+
if (running) return running;
|
|
3762
|
+
try {
|
|
3763
|
+
const socketPath = getSupervisorSocketPath();
|
|
3764
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
3765
|
+
if (res && res.ok) {
|
|
3766
|
+
const state = stateFromStatus(res, socketPath);
|
|
3767
|
+
try { writeSupervisorState(state); } catch { /* best effort */ }
|
|
3768
|
+
return state;
|
|
3769
|
+
}
|
|
3770
|
+
} catch { /* no live daemon on the socket */ }
|
|
3771
|
+
return null;
|
|
3772
|
+
}
|
|
3773
|
+
|
|
3774
|
+
/** How to re-invoke the c8ctl CLI to spawn the daemon + `work` children. */
|
|
3775
|
+
function c8ctlInvocation() {
|
|
3776
|
+
const entry = process.env.C8CTL_NANO_ENTRY || process.argv[1];
|
|
3777
|
+
return { exec: process.execPath, entry };
|
|
3778
|
+
}
|
|
3779
|
+
|
|
3780
|
+
function supervisorDaemonLogFile() {
|
|
3781
|
+
return join(getSupervisorLogDir(), 'daemon.log');
|
|
3782
|
+
}
|
|
3783
|
+
|
|
3784
|
+
function supervisorWorkerLogFile(id) {
|
|
3785
|
+
return join(getSupervisorLogDir(), `worker-${String(id).replace(/[^\w.#-]/g, '_')}.log`);
|
|
3786
|
+
}
|
|
3787
|
+
|
|
3788
|
+
function waitForChildExit(child, timeoutMs) {
|
|
3789
|
+
return new Promise((resolve) => {
|
|
3790
|
+
if (!child || child.exitCode !== null || child.signalCode !== null) return resolve();
|
|
3791
|
+
let done = false;
|
|
3792
|
+
const finish = () => { if (done) return; done = true; clearTimeout(t); resolve(); };
|
|
3793
|
+
const t = setTimeout(finish, timeoutMs);
|
|
3794
|
+
child.once('exit', finish);
|
|
3795
|
+
});
|
|
3796
|
+
}
|
|
3797
|
+
|
|
3798
|
+
// --- Daemon ----------------------------------------------------------------
|
|
3799
|
+
|
|
3800
|
+
/**
|
|
3801
|
+
* The supervisor daemon body. Runs under `c8ctl nano supervisor __daemon`,
|
|
3802
|
+
* spawned detached by `startSupervisorDaemon`. Never returns — it runs until a
|
|
3803
|
+
* stop request or SIGTERM, then drains children and exits.
|
|
3804
|
+
*/
|
|
3805
|
+
async function runSupervisorDaemon() {
|
|
3806
|
+
const startedAt = new Date().toISOString();
|
|
3807
|
+
const { exec, entry } = c8ctlInvocation();
|
|
3808
|
+
const socketPath = getSupervisorSocketPath();
|
|
3809
|
+
const daemonLogFile = supervisorDaemonLogFile();
|
|
3810
|
+
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
3811
|
+
|
|
3812
|
+
const workers = new Map();
|
|
3813
|
+
const attachClients = new Set();
|
|
3814
|
+
let shuttingDown = false;
|
|
3815
|
+
|
|
3816
|
+
// Daemon-wide mutation serialization: `add`/`remove`/`restart` must not
|
|
3817
|
+
// interleave, or two clients racing the same worker could each spawn an
|
|
3818
|
+
// untracked child. Every mutation runs to completion before the next starts.
|
|
3819
|
+
let opQueue = Promise.resolve();
|
|
3820
|
+
const serializeOp = (fn) => {
|
|
3821
|
+
const run = opQueue.then(fn, fn);
|
|
3822
|
+
opQueue = run.then(() => {}, () => {});
|
|
3823
|
+
return run;
|
|
3824
|
+
};
|
|
3825
|
+
|
|
3826
|
+
const dlog = (msg) => {
|
|
3827
|
+
try { appendFileSync(daemonLogFile, `[${new Date().toISOString()}] ${msg}\n`); } catch { /* best effort */ }
|
|
3828
|
+
};
|
|
3829
|
+
|
|
3830
|
+
const workerPublic = (w) => summarizeSupervisorWorker(w);
|
|
3831
|
+
|
|
3832
|
+
const persist = () => {
|
|
3833
|
+
try {
|
|
3834
|
+
writeSupervisorState({
|
|
3835
|
+
pid: process.pid,
|
|
3836
|
+
startedAt,
|
|
3837
|
+
socket: socketPath,
|
|
3838
|
+
logFile: daemonLogFile,
|
|
3839
|
+
workers: [...workers.values()].map((w) => ({
|
|
3840
|
+
id: w.id, profile: w.profile, args: w.args, pid: isPidAlive(w.pid) ? w.pid : null,
|
|
3841
|
+
startedAt: w.startedAt || null, restarts: w.restarts, lastExit: w.lastExit ?? null,
|
|
3842
|
+
stopping: !!w.stopping, logFile: w.logFile,
|
|
3843
|
+
})),
|
|
3844
|
+
});
|
|
3845
|
+
} catch { /* best effort */ }
|
|
3846
|
+
};
|
|
3847
|
+
|
|
3848
|
+
const broadcast = (frame) => {
|
|
3849
|
+
const data = encodeFrame(frame);
|
|
3850
|
+
for (const sock of attachClients) {
|
|
3851
|
+
try { sock.write(data); } catch { /* client gone */ }
|
|
3852
|
+
}
|
|
3853
|
+
};
|
|
3854
|
+
|
|
3855
|
+
const startWorker = (w) => {
|
|
3856
|
+
let fd;
|
|
3857
|
+
try { fd = openSync(w.logFile, 'a'); } catch { fd = 'ignore'; }
|
|
3858
|
+
// `--name w.id` makes the child's broker workerName match this worker's
|
|
3859
|
+
// supervisor id, so the same profile launched twice is distinct end-to-end.
|
|
3860
|
+
const child = spawn(exec, [entry, 'nano', 'work', w.profile, '--name', w.id, ...w.args], {
|
|
3861
|
+
env: process.env,
|
|
3862
|
+
stdio: ['ignore', fd, fd],
|
|
3863
|
+
});
|
|
3864
|
+
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* dup'd into child */ } }
|
|
3865
|
+
w.child = child;
|
|
3866
|
+
w.pid = child.pid || null;
|
|
3867
|
+
w.startedAt = new Date().toISOString();
|
|
3868
|
+
w.spawnedAt = Date.now();
|
|
3869
|
+
dlog(`worker '${w.id}' (profile ${w.profile}) started pid ${w.pid}: work ${[w.profile, ...redactWorkArgs(w.args)].join(' ')}`);
|
|
3870
|
+
broadcast({ type: 'event', event: 'worker-start', worker: workerPublic(w) });
|
|
3871
|
+
|
|
3872
|
+
// A spawn failure (ENOENT/EMFILE/…) emits only 'error' with no 'exit', so
|
|
3873
|
+
// both paths funnel through one death handler that schedules a restart.
|
|
3874
|
+
// `settled` guards the error+exit double-fire; the `w.child !== child` check
|
|
3875
|
+
// ignores a stale child's late exit after `restart` swapped in a new one
|
|
3876
|
+
// (which would otherwise clobber the live pid and leak a duplicate worker).
|
|
3877
|
+
let settled = false;
|
|
3878
|
+
const handleDeath = (reason) => {
|
|
3879
|
+
if (w.child !== child || settled) return;
|
|
3880
|
+
settled = true;
|
|
3881
|
+
w.pid = null;
|
|
3882
|
+
w.lastExit = reason;
|
|
3883
|
+
const ranMs = Date.now() - (w.spawnedAt || Date.now());
|
|
3884
|
+
if (ranMs >= SUPERVISOR_HEALTHY_UPTIME_MS) w.restarts = 0;
|
|
3885
|
+
if (w.stopping || shuttingDown || !workers.has(w.id)) { persist(); return; }
|
|
3886
|
+
const delay = supervisorBackoffMs(w.restarts);
|
|
3887
|
+
w.restarts += 1;
|
|
3888
|
+
dlog(`worker '${w.id}' down (${reason}); restarting in ${delay}ms (restart #${w.restarts})`);
|
|
3889
|
+
broadcast({ type: 'event', event: 'worker-exit', worker: workerPublic(w), restartInMs: delay });
|
|
3890
|
+
w.restartTimer = setTimeout(() => {
|
|
3891
|
+
w.restartTimer = null;
|
|
3892
|
+
if (!w.stopping && !shuttingDown && workers.has(w.id)) startWorker(w);
|
|
3893
|
+
}, delay);
|
|
3894
|
+
if (typeof w.restartTimer.unref === 'function') w.restartTimer.unref();
|
|
3895
|
+
persist();
|
|
3896
|
+
};
|
|
3897
|
+
child.on('error', (err) => handleDeath(`spawn error: ${err.message}`));
|
|
3898
|
+
child.on('exit', (code, signal) => handleDeath(signal ? `signal ${signal}` : `code ${code}`));
|
|
3899
|
+
persist();
|
|
3900
|
+
};
|
|
3901
|
+
|
|
3902
|
+
const addWorker = (profile, args, name) => {
|
|
3903
|
+
const taken = new Set(workers.keys());
|
|
3904
|
+
let id;
|
|
3905
|
+
if (name != null && String(name).trim() !== '') {
|
|
3906
|
+
id = String(name).trim();
|
|
3907
|
+
if (!isValidWorkerName(id)) throw new Error(`invalid worker name "${id}": use only letters, digits, and . _ -`);
|
|
3908
|
+
if (taken.has(id)) throw new Error(`a worker named "${id}" already exists`);
|
|
3909
|
+
} else {
|
|
3910
|
+
// No explicit name → auto ‹host›-‹profile›-‹random›. The random suffix is
|
|
3911
|
+
// collision-resistant, but never hand back a duplicate id.
|
|
3912
|
+
id = autoWorkerName(profile);
|
|
3913
|
+
while (taken.has(id)) id = autoWorkerName(profile);
|
|
3914
|
+
}
|
|
3915
|
+
const w = {
|
|
3916
|
+
id, profile: String(profile), args: Array.isArray(args) ? args.map(String) : [],
|
|
3917
|
+
restarts: 0, stopping: false, lastExit: null, logFile: supervisorWorkerLogFile(id),
|
|
3918
|
+
};
|
|
3919
|
+
workers.set(id, w);
|
|
3920
|
+
startWorker(w);
|
|
3921
|
+
return w;
|
|
3922
|
+
};
|
|
3923
|
+
|
|
3924
|
+
const stopWorker = async (id) => {
|
|
3925
|
+
const w = workers.get(id);
|
|
3926
|
+
if (!w) return false;
|
|
3927
|
+
w.stopping = true;
|
|
3928
|
+
if (w.restartTimer) { clearTimeout(w.restartTimer); w.restartTimer = null; }
|
|
3929
|
+
const pid = w.pid;
|
|
3930
|
+
if (w.child && pid) {
|
|
3931
|
+
try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
|
|
3932
|
+
await waitForChildExit(w.child, STOP_GRACE_MS);
|
|
3933
|
+
if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
|
|
3934
|
+
}
|
|
3935
|
+
return true;
|
|
3936
|
+
};
|
|
3937
|
+
|
|
3938
|
+
const removeWorker = async (id) => {
|
|
3939
|
+
if (!workers.has(id)) return false;
|
|
3940
|
+
await stopWorker(id);
|
|
3941
|
+
workers.delete(id);
|
|
3942
|
+
dlog(`worker '${id}' removed`);
|
|
3943
|
+
broadcast({ type: 'event', event: 'worker-remove', id });
|
|
3944
|
+
persist();
|
|
3945
|
+
return true;
|
|
3946
|
+
};
|
|
3947
|
+
|
|
3948
|
+
const restartWorker = async (id) => {
|
|
3949
|
+
const w = workers.get(id);
|
|
3950
|
+
if (!w) return false;
|
|
3951
|
+
await stopWorker(id);
|
|
3952
|
+
w.stopping = false;
|
|
3953
|
+
w.restarts = 0;
|
|
3954
|
+
startWorker(w);
|
|
3955
|
+
dlog(`worker '${id}' restarted`);
|
|
3956
|
+
return true;
|
|
3957
|
+
};
|
|
3958
|
+
|
|
3959
|
+
// Resolve a target token to worker ids: exact id, else all with that profile.
|
|
3960
|
+
const resolveTargets = (target) => {
|
|
3961
|
+
const t = String(target || '').trim();
|
|
3962
|
+
if (!t) return [];
|
|
3963
|
+
if (t === 'all' || t === '*') return [...workers.keys()];
|
|
3964
|
+
if (workers.has(t)) return [t];
|
|
3965
|
+
return [...workers.values()].filter((w) => w.profile === t).map((w) => w.id);
|
|
3966
|
+
};
|
|
3967
|
+
|
|
3968
|
+
const statusFrame = (final) => ({
|
|
3969
|
+
ok: true,
|
|
3970
|
+
type: 'status',
|
|
3971
|
+
daemon: { pid: process.pid, startedAt, socket: socketPath, logFile: daemonLogFile },
|
|
3972
|
+
workers: [...workers.values()].map(workerPublic),
|
|
3973
|
+
...(final ? { final: true } : {}),
|
|
3974
|
+
});
|
|
3975
|
+
|
|
3976
|
+
const shutdown = async (signal) => {
|
|
3977
|
+
if (shuttingDown) return;
|
|
3978
|
+
shuttingDown = true;
|
|
3979
|
+
// Let any in-flight mutation finish before we snapshot the worker set, so
|
|
3980
|
+
// an add/restart racing the shutdown can't leave an orphaned child behind.
|
|
3981
|
+
try { await opQueue; } catch { /* mutation already logged */ }
|
|
3982
|
+
dlog(`received ${signal || 'stop'} — stopping ${workers.size} worker(s)`);
|
|
3983
|
+
await Promise.all([...workers.keys()].map((id) => stopWorker(id)));
|
|
3984
|
+
broadcast({ type: 'event', event: 'daemon-stop' });
|
|
3985
|
+
try { server.close(); } catch { /* ignore */ }
|
|
3986
|
+
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
3987
|
+
clearSupervisorState();
|
|
3988
|
+
process.exit(0);
|
|
3989
|
+
};
|
|
3990
|
+
|
|
3991
|
+
const handleRequest = async (req, sock) => {
|
|
3992
|
+
const op = req && req.op;
|
|
3993
|
+
try {
|
|
3994
|
+
switch (op) {
|
|
3995
|
+
case 'status':
|
|
3996
|
+
sock.write(encodeFrame(statusFrame(true)));
|
|
3997
|
+
break;
|
|
3998
|
+
case 'add': {
|
|
3999
|
+
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
4000
|
+
if (!req.profile) { sock.write(encodeFrame({ ok: false, error: 'add requires a profile', final: true })); break; }
|
|
4001
|
+
// A supervised worker's name is supplied out-of-band as `req.name`
|
|
4002
|
+
// (which the daemon forwards to the child as `nano work … --name`).
|
|
4003
|
+
// A bare `--name` inside the forwarded work args is therefore ambiguous
|
|
4004
|
+
// — it would fight the supervisor-assigned id — so reject it here and
|
|
4005
|
+
// steer the operator to the dedicated flag.
|
|
4006
|
+
// `req.args` comes from untrusted JSON and may be non-array (e.g. a
|
|
4007
|
+
// string or object). Coerce to an array of string tokens before
|
|
4008
|
+
// scanning/forwarding so a malformed payload yields a clean rejection
|
|
4009
|
+
// instead of throwing a generic request error.
|
|
4010
|
+
const args = Array.isArray(req.args) ? req.args.filter((a) => typeof a === 'string') : [];
|
|
4011
|
+
const badName = args.find((a) => a === '--name' || a === '-n' || /^--name=/.test(a) || /^-n=/.test(a));
|
|
4012
|
+
if (badName) { sock.write(encodeFrame({ ok: false, error: 'name a supervised worker with `--name` on `supervisor add`, not inside its work flags', final: true })); break; }
|
|
4013
|
+
const stored = readHires()[String(req.profile)];
|
|
4014
|
+
if (!stored) { sock.write(encodeFrame({ ok: false, error: `no hire named "${req.profile}"`, final: true })); break; }
|
|
4015
|
+
const name = typeof req.name === 'string' ? req.name.trim() : '';
|
|
4016
|
+
const w = await serializeOp(() => addWorker(req.profile, args, name));
|
|
4017
|
+
sock.write(encodeFrame({ ok: true, type: 'added', worker: workerPublic(w), final: true }));
|
|
4018
|
+
break;
|
|
4019
|
+
}
|
|
4020
|
+
case 'remove': {
|
|
4021
|
+
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
4022
|
+
const removed = await serializeOp(async () => {
|
|
4023
|
+
const ids = resolveTargets(req.target);
|
|
4024
|
+
for (const id of ids) await removeWorker(id);
|
|
4025
|
+
return ids;
|
|
4026
|
+
});
|
|
4027
|
+
sock.write(encodeFrame({ ok: true, type: 'removed', removed, final: true }));
|
|
4028
|
+
break;
|
|
4029
|
+
}
|
|
4030
|
+
case 'restart': {
|
|
4031
|
+
if (shuttingDown) { sock.write(encodeFrame({ ok: false, error: 'supervisor is shutting down', final: true })); break; }
|
|
4032
|
+
const restarted = await serializeOp(async () => {
|
|
4033
|
+
const ids = resolveTargets(req.target);
|
|
4034
|
+
for (const id of ids) await restartWorker(id);
|
|
4035
|
+
return ids;
|
|
4036
|
+
});
|
|
4037
|
+
sock.write(encodeFrame({ ok: true, type: 'restarted', restarted, final: true }));
|
|
4038
|
+
break;
|
|
4039
|
+
}
|
|
4040
|
+
case 'attach':
|
|
4041
|
+
attachClients.add(sock);
|
|
4042
|
+
sock.write(encodeFrame(statusFrame(false)));
|
|
4043
|
+
break;
|
|
4044
|
+
case 'stop':
|
|
4045
|
+
sock.write(encodeFrame({ ok: true, type: 'stopping', final: true }));
|
|
4046
|
+
setTimeout(() => shutdown('stop'), 50);
|
|
4047
|
+
break;
|
|
4048
|
+
default:
|
|
4049
|
+
sock.write(encodeFrame({ ok: false, error: `unknown op "${op}"`, final: true }));
|
|
4050
|
+
}
|
|
4051
|
+
} catch (err) {
|
|
4052
|
+
try { sock.write(encodeFrame({ ok: false, error: String(err && err.message || err), final: true })); } catch { /* ignore */ }
|
|
4053
|
+
}
|
|
4054
|
+
};
|
|
4055
|
+
|
|
4056
|
+
// Bind the control socket. A stale unix socket file from a crashed daemon
|
|
4057
|
+
// would make listen() fail with EADDRINUSE even though nobody is listening;
|
|
4058
|
+
// remove it first (we already know no live daemon owns our state).
|
|
4059
|
+
if (osPlatform() !== 'win32') { try { rmSync(socketPath, { force: true }); } catch { /* ignore */ } }
|
|
4060
|
+
|
|
4061
|
+
const server = createServer((sock) => {
|
|
4062
|
+
sock.setEncoding('utf8');
|
|
4063
|
+
let buf = '';
|
|
4064
|
+
// Serialize requests per connection: handleRequest is async and mutates the
|
|
4065
|
+
// shared workers map, so a second 'data' event arriving mid-await must not
|
|
4066
|
+
// interleave add/remove/restart. Chain each frame onto a per-socket queue.
|
|
4067
|
+
let queue = Promise.resolve();
|
|
4068
|
+
sock.on('data', (chunk) => {
|
|
4069
|
+
buf += chunk;
|
|
4070
|
+
// Cap by UTF-8 byte length, not string length: buf is a decoded string
|
|
4071
|
+
// whose .length counts UTF-16 code units, so multibyte input could hold
|
|
4072
|
+
// far more than SUPERVISOR_MAX_FRAME_BYTES in memory before being dropped.
|
|
4073
|
+
if (Buffer.byteLength(buf, 'utf8') > SUPERVISOR_MAX_FRAME_BYTES) {
|
|
4074
|
+
dlog(`control connection exceeded ${SUPERVISOR_MAX_FRAME_BYTES} bytes without a complete frame — dropping`);
|
|
4075
|
+
try { sock.destroy(); } catch { /* ignore */ }
|
|
4076
|
+
buf = '';
|
|
4077
|
+
return;
|
|
4078
|
+
}
|
|
4079
|
+
const { frames, rest } = decodeFrames(buf);
|
|
4080
|
+
buf = rest;
|
|
4081
|
+
for (const req of frames) {
|
|
4082
|
+
queue = queue.then(() => handleRequest(req, sock)).catch((err) => dlog(`request error: ${err?.message || err}`));
|
|
4083
|
+
}
|
|
4084
|
+
});
|
|
4085
|
+
sock.on('close', () => attachClients.delete(sock));
|
|
4086
|
+
sock.on('error', () => attachClients.delete(sock));
|
|
4087
|
+
});
|
|
4088
|
+
|
|
4089
|
+
// Create the control socket owner-only from the start. The socket file lives
|
|
4090
|
+
// in shared tmpdir(); libuv binds it synchronously inside listen(), so a
|
|
4091
|
+
// restrictive umask around that call closes the TOCTOU window where another
|
|
4092
|
+
// local user could connect before the chmod below lands. Restore the previous
|
|
4093
|
+
// umask immediately after — the listen() bind is synchronous, so no unrelated
|
|
4094
|
+
// file creation can interleave. Unix only; on Windows umask/mode are no-ops.
|
|
4095
|
+
const isWin = osPlatform() === 'win32';
|
|
4096
|
+
const prevUmask = isWin ? null : process.umask(0o177);
|
|
4097
|
+
try {
|
|
4098
|
+
await new Promise((resolve, reject) => {
|
|
4099
|
+
server.once('error', reject);
|
|
4100
|
+
server.listen(socketPath, resolve);
|
|
4101
|
+
});
|
|
4102
|
+
} catch (err) {
|
|
4103
|
+
dlog(`failed to bind control socket ${socketPath}: ${err.message}`);
|
|
4104
|
+
process.exit(1);
|
|
4105
|
+
} finally {
|
|
4106
|
+
if (!isWin) { try { process.umask(prevUmask); } catch { /* ignore */ } }
|
|
4107
|
+
}
|
|
4108
|
+
|
|
4109
|
+
// Lock the control socket to the owner so another local user can't drive the
|
|
4110
|
+
// supervisor (stop/add/remove). Unix only — Windows named pipes are secured
|
|
4111
|
+
// by their own ACLs, not filesystem mode bits. This chmod is now a backstop
|
|
4112
|
+
// for the owner-only umask applied around listen() above.
|
|
4113
|
+
if (!isWin) {
|
|
4114
|
+
try { chmodSync(socketPath, 0o600); } catch (err) { dlog(`could not chmod control socket: ${err.message}`); }
|
|
4115
|
+
}
|
|
4116
|
+
|
|
4117
|
+
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
|
4118
|
+
process.once('SIGINT', () => shutdown('SIGINT'));
|
|
4119
|
+
dlog(`supervisor daemon up (pid ${process.pid}) — control ${socketPath}`);
|
|
4120
|
+
persist();
|
|
4121
|
+
|
|
4122
|
+
// Keep the event loop alive indefinitely; the server holds it, but add an
|
|
4123
|
+
// explicit never-resolving guard so a transient server close can't exit us.
|
|
4124
|
+
await new Promise(() => {});
|
|
4125
|
+
}
|
|
4126
|
+
|
|
4127
|
+
// --- Client (management subcommands + attach) ------------------------------
|
|
4128
|
+
|
|
4129
|
+
/** Connect to the control socket, resolving with the socket once connected. */
|
|
4130
|
+
function supervisorConnect(socketPath, { timeoutMs = SUPERVISOR_CONNECT_TIMEOUT_MS } = {}) {
|
|
4131
|
+
return new Promise((resolve, reject) => {
|
|
4132
|
+
const sock = createConnection(socketPath);
|
|
4133
|
+
let settled = false;
|
|
4134
|
+
const timer = setTimeout(() => {
|
|
4135
|
+
if (settled) return;
|
|
4136
|
+
settled = true;
|
|
4137
|
+
sock.destroy();
|
|
4138
|
+
reject(new Error(`timed out connecting to supervisor at ${socketPath}`));
|
|
4139
|
+
}, timeoutMs);
|
|
4140
|
+
sock.once('connect', () => {
|
|
4141
|
+
if (settled) return;
|
|
4142
|
+
settled = true;
|
|
4143
|
+
clearTimeout(timer);
|
|
4144
|
+
sock.setEncoding('utf8');
|
|
4145
|
+
resolve(sock);
|
|
4146
|
+
});
|
|
4147
|
+
sock.once('error', (err) => {
|
|
4148
|
+
if (settled) return;
|
|
4149
|
+
settled = true;
|
|
4150
|
+
clearTimeout(timer);
|
|
4151
|
+
reject(err);
|
|
4152
|
+
});
|
|
4153
|
+
});
|
|
4154
|
+
}
|
|
4155
|
+
|
|
4156
|
+
/** Send one request and collect frames until a `final:true` frame arrives. */
|
|
4157
|
+
function supervisorRequest(req, { socketPath, timeoutMs, responseTimeoutMs = SUPERVISOR_RESPONSE_TIMEOUT_MS } = {}) {
|
|
4158
|
+
const path = socketPath || (readSupervisorState()?.socket) || getSupervisorSocketPath();
|
|
4159
|
+
return new Promise((resolve, reject) => {
|
|
4160
|
+
supervisorConnect(path, { timeoutMs }).then((sock) => {
|
|
4161
|
+
let buf = '';
|
|
4162
|
+
let settled = false;
|
|
4163
|
+
const finish = (fn, arg) => { if (settled) return; settled = true; clearTimeout(timer); try { sock.end(); } catch { /* ignore */ } fn(arg); };
|
|
4164
|
+
const done = (result) => finish(resolve, result);
|
|
4165
|
+
const fail = (err) => finish(reject, err);
|
|
4166
|
+
// End-to-end response deadline: a daemon that accepts the connection but
|
|
4167
|
+
// never sends a `final` frame must not hang the caller forever.
|
|
4168
|
+
const timer = setTimeout(() => {
|
|
4169
|
+
if (settled) return;
|
|
4170
|
+
settled = true;
|
|
4171
|
+
try { sock.destroy(); } catch { /* ignore */ }
|
|
4172
|
+
reject(new Error(`timed out waiting for supervisor response from ${path}`));
|
|
4173
|
+
}, responseTimeoutMs);
|
|
4174
|
+
sock.on('data', (chunk) => {
|
|
4175
|
+
buf += chunk;
|
|
4176
|
+
const { frames, rest } = decodeFrames(buf);
|
|
4177
|
+
buf = rest;
|
|
4178
|
+
for (const frame of frames) {
|
|
4179
|
+
if (frame.final) return done(frame);
|
|
4180
|
+
}
|
|
4181
|
+
});
|
|
4182
|
+
sock.on('error', fail);
|
|
4183
|
+
sock.on('close', () => done({ ok: false, error: 'connection closed before response' }));
|
|
4184
|
+
sock.write(encodeFrame(req));
|
|
4185
|
+
}).catch(reject);
|
|
4186
|
+
});
|
|
4187
|
+
}
|
|
4188
|
+
|
|
4189
|
+
/**
|
|
4190
|
+
* Ensure a daemon is running, spawning it detached if not, and return its
|
|
4191
|
+
* running state. Polls the control socket until it answers a status request.
|
|
4192
|
+
*/
|
|
4193
|
+
async function startSupervisorDaemon() {
|
|
4194
|
+
const existing = runningSupervisor();
|
|
4195
|
+
if (existing) return existing;
|
|
4196
|
+
|
|
4197
|
+
const socketPath = getSupervisorSocketPath();
|
|
4198
|
+
// The state file may be missing (deleted, cleaned up, or not yet written)
|
|
4199
|
+
// while a daemon is still listening on the deterministic socket. Adopt that
|
|
4200
|
+
// live daemon instead of spawning a second one that would orphan the
|
|
4201
|
+
// original and its workers.
|
|
4202
|
+
try {
|
|
4203
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
4204
|
+
if (res && res.ok) {
|
|
4205
|
+
// Re-persist the adopted daemon's state so subsequent pid-based checks
|
|
4206
|
+
// (runningSupervisor()) work immediately, instead of staying broken until
|
|
4207
|
+
// some later command happens to heal supervisor.json.
|
|
4208
|
+
const adopted = runningSupervisor() || stateFromStatus(res, socketPath);
|
|
4209
|
+
try { writeSupervisorState(adopted); } catch { /* best effort */ }
|
|
4210
|
+
return adopted;
|
|
4211
|
+
}
|
|
4212
|
+
} catch { /* no live daemon on the socket — safe to (re)spawn */ }
|
|
4213
|
+
|
|
4214
|
+
clearSupervisorState(); // clear any stale marker from a dead daemon
|
|
4215
|
+
|
|
4216
|
+
const { exec, entry } = c8ctlInvocation();
|
|
4217
|
+
mkdirSync(getSupervisorLogDir(), { recursive: true });
|
|
4218
|
+
const logFile = supervisorDaemonLogFile();
|
|
4219
|
+
let fd;
|
|
4220
|
+
try { fd = openSync(logFile, 'a'); } catch { fd = 'ignore'; }
|
|
4221
|
+
const child = spawn(exec, [entry, 'nano', 'supervisor', '__daemon'], {
|
|
4222
|
+
env: process.env,
|
|
4223
|
+
detached: true,
|
|
4224
|
+
stdio: ['ignore', fd, fd],
|
|
4225
|
+
});
|
|
4226
|
+
child.unref();
|
|
4227
|
+
if (typeof fd === 'number') { try { closeSync(fd); } catch { /* ignore */ } }
|
|
4228
|
+
if (typeof child.pid !== 'number') throw new Error('failed to spawn supervisor daemon');
|
|
4229
|
+
|
|
4230
|
+
|
|
4231
|
+
const deadline = Date.now() + SUPERVISOR_CONNECT_TIMEOUT_MS;
|
|
4232
|
+
while (Date.now() < deadline) {
|
|
4233
|
+
try {
|
|
4234
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath, timeoutMs: 750, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
4235
|
+
if (res && res.ok) {
|
|
4236
|
+
// The daemon can answer `status` on the socket a beat before it has
|
|
4237
|
+
// written supervisor.json. Fall back to the live status response so
|
|
4238
|
+
// callers always get a state object with a usable pid.
|
|
4239
|
+
return runningSupervisor() || readSupervisorState() || stateFromStatus(res, socketPath);
|
|
4240
|
+
}
|
|
4241
|
+
} catch { /* not up yet */ }
|
|
4242
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
4243
|
+
}
|
|
4244
|
+
throw new Error(`supervisor daemon did not become ready (see ${logFile})`);
|
|
4245
|
+
}
|
|
4246
|
+
|
|
4247
|
+
async function supervisorStartCmd(req, flags) {
|
|
4248
|
+
const logger = getLogger();
|
|
4249
|
+
const state = await startSupervisorDaemon();
|
|
4250
|
+
logger.info(`Supervisor daemon running (pid ${state.pid}).`);
|
|
4251
|
+
|
|
4252
|
+
const specs = normalizeArgList(flags?.worker);
|
|
4253
|
+
const workArgs = reconstructWorkArgs(flags);
|
|
4254
|
+
// `--name` names a single launched worker. With several `--worker` specs a lone
|
|
4255
|
+
// name can't apply to all of them, so honour it only for a single spec and let
|
|
4256
|
+
// the rest auto-name; warn so the intent isn't silently dropped.
|
|
4257
|
+
const explicitName = flags?.name ? String(flags.name).trim() : undefined;
|
|
4258
|
+
if (explicitName && specs.length > 1) {
|
|
4259
|
+
logger.warn('--name is ignored when starting multiple --worker specs; each is auto-named.');
|
|
4260
|
+
}
|
|
4261
|
+
const nameFor = (i) => (explicitName && specs.length === 1 ? explicitName : undefined);
|
|
4262
|
+
for (let i = 0; i < specs.length; i++) {
|
|
4263
|
+
const profile = specs[i];
|
|
4264
|
+
const res = await supervisorRequest({ op: 'add', profile, name: nameFor(i), args: workArgs });
|
|
4265
|
+
if (res.ok) logger.info(` + worker "${res.worker.id}" (profile ${profile})`);
|
|
4266
|
+
else logger.error(` ! could not add "${profile}": ${res.error}`);
|
|
4267
|
+
}
|
|
4268
|
+
|
|
4269
|
+
if (coerceBool(flags?.attach, false)) {
|
|
4270
|
+
await attachSupervisorConsole(runningSupervisor() || state);
|
|
4271
|
+
return;
|
|
4272
|
+
}
|
|
4273
|
+
await supervisorStatusCmd();
|
|
4274
|
+
logger.info('');
|
|
4275
|
+
logger.info('Attach an interactive console with: c8ctl nano supervisor');
|
|
4276
|
+
logger.info('Manage without it: c8ctl nano supervisor add|remove|restart|status|stop');
|
|
4277
|
+
}
|
|
4278
|
+
|
|
4279
|
+
async function supervisorStatusCmd() {
|
|
4280
|
+
const logger = getLogger();
|
|
4281
|
+
const running = runningSupervisor();
|
|
4282
|
+
if (!running) {
|
|
4283
|
+
// The state file may be missing (deleted/cleaned) while a daemon is still
|
|
4284
|
+
// listening on the deterministic socket — same case startSupervisorDaemon
|
|
4285
|
+
// adopts. Probe it before declaring the supervisor down, and re-persist so
|
|
4286
|
+
// the state file is healed for later pid-based checks.
|
|
4287
|
+
try {
|
|
4288
|
+
const res = await supervisorRequest({ op: 'status' }, { socketPath: getSupervisorSocketPath(), timeoutMs: 500, responseTimeoutMs: SUPERVISOR_PROBE_RESPONSE_TIMEOUT_MS });
|
|
4289
|
+
if (res && res.ok) {
|
|
4290
|
+
try { writeSupervisorState(stateFromStatus(res, getSupervisorSocketPath())); } catch { /* best effort */ }
|
|
4291
|
+
logger.info(formatSupervisorStatus(res));
|
|
4292
|
+
return;
|
|
4293
|
+
}
|
|
4294
|
+
} catch { /* no live daemon on the socket — genuinely down */ }
|
|
4295
|
+
const stale = readSupervisorState();
|
|
4296
|
+
if (stale) {
|
|
4297
|
+
logger.info('Supervisor: not running (stale state — daemon pid is dead).');
|
|
4298
|
+
logger.info(' Start it with: c8ctl nano supervisor start');
|
|
4299
|
+
} else {
|
|
4300
|
+
logger.info('Supervisor: not running.');
|
|
4301
|
+
logger.info(' Start it with: c8ctl nano supervisor start (or attach: c8ctl nano supervisor)');
|
|
4302
|
+
}
|
|
4303
|
+
return;
|
|
4304
|
+
}
|
|
4305
|
+
try {
|
|
4306
|
+
const res = await supervisorRequest({ op: 'status' });
|
|
4307
|
+
if (res.ok) { logger.info(formatSupervisorStatus(res)); return; }
|
|
4308
|
+
} catch { /* fall back to state file below */ }
|
|
4309
|
+
// Socket unreachable but pid alive — render from the last persisted state.
|
|
4310
|
+
logger.info(formatSupervisorStatus({
|
|
4311
|
+
daemon: { pid: running.pid, startedAt: running.startedAt, socket: running.socket },
|
|
4312
|
+
workers: (running.workers || []).map((w) => summarizeSupervisorWorker(w)),
|
|
4313
|
+
}));
|
|
4314
|
+
}
|
|
4315
|
+
|
|
4316
|
+
async function supervisorAddCmd(req, flags) {
|
|
4317
|
+
const logger = getLogger();
|
|
4318
|
+
// The positional profile is what runs; `--name` names this worker instance
|
|
4319
|
+
// (forwarded to the child as `nano work … --name`, and used as its supervisor
|
|
4320
|
+
// id). Omit `--name` to auto-generate ‹host›-‹profile›-‹random›.
|
|
4321
|
+
const profile = req.positional[1];
|
|
4322
|
+
if (!profile) { logger.error('Usage: c8ctl nano supervisor add <profile> [--name <worker>] [work flags]'); process.exit(1); }
|
|
4323
|
+
const name = flags?.name ? String(flags.name).trim() : undefined;
|
|
4324
|
+
await startSupervisorDaemon();
|
|
4325
|
+
const res = await supervisorRequest({ op: 'add', profile, name, args: reconstructWorkArgs(flags) });
|
|
4326
|
+
if (res.ok) logger.info(`Added worker "${res.worker.id}" (profile ${profile}); pid ${res.worker.pid ?? 'starting'}.`);
|
|
4327
|
+
else { logger.error(`Could not add "${profile}": ${res.error}`); process.exit(1); }
|
|
4328
|
+
}
|
|
4329
|
+
|
|
4330
|
+
async function supervisorRemoveCmd(req) {
|
|
4331
|
+
const logger = getLogger();
|
|
4332
|
+
const target = req.positional[1];
|
|
4333
|
+
if (!target) { logger.error('Usage: c8ctl nano supervisor remove <id|profile|all>'); process.exit(1); }
|
|
4334
|
+
if (!await liveSupervisor()) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
4335
|
+
const res = await supervisorRequest({ op: 'remove', target });
|
|
4336
|
+
if (res.ok && res.removed.length > 0) logger.info(`Removed worker(s): ${res.removed.join(', ')}.`);
|
|
4337
|
+
else if (res.ok) { logger.warn(`No worker matched "${target}".`); }
|
|
4338
|
+
else { logger.error(res.error); process.exit(1); }
|
|
4339
|
+
}
|
|
4340
|
+
|
|
4341
|
+
async function supervisorRestartCmd(req) {
|
|
4342
|
+
const logger = getLogger();
|
|
4343
|
+
const target = req.positional[1];
|
|
4344
|
+
if (!target) { logger.error('Usage: c8ctl nano supervisor restart <id|profile|all>'); process.exit(1); }
|
|
4345
|
+
if (!await liveSupervisor()) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
4346
|
+
const res = await supervisorRequest({ op: 'restart', target });
|
|
4347
|
+
if (res.ok && res.restarted.length > 0) logger.info(`Restarted worker(s): ${res.restarted.join(', ')}.`);
|
|
4348
|
+
else if (res.ok) { logger.warn(`No worker matched "${target}".`); }
|
|
4349
|
+
else { logger.error(res.error); process.exit(1); }
|
|
4350
|
+
}
|
|
4351
|
+
|
|
4352
|
+
async function supervisorStopCmd() {
|
|
4353
|
+
const logger = getLogger();
|
|
4354
|
+
const running = await liveSupervisor();
|
|
4355
|
+
if (!running) {
|
|
4356
|
+
if (readSupervisorState()) { clearSupervisorState(); logger.info('Cleared stale supervisor state.'); }
|
|
4357
|
+
else logger.warn('Supervisor is not running — nothing to stop.');
|
|
4358
|
+
return;
|
|
4359
|
+
}
|
|
4360
|
+
try {
|
|
4361
|
+
await supervisorRequest({ op: 'stop' });
|
|
4362
|
+
} catch {
|
|
4363
|
+
// Socket unreachable — fall back to signalling the daemon pid directly.
|
|
4364
|
+
try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ }
|
|
4365
|
+
}
|
|
4366
|
+
// Gate the wait loop and the SIGKILL fallback on the daemon pid we captured,
|
|
4367
|
+
// not on runningSupervisor()/the state file: the daemon clears its state file
|
|
4368
|
+
// as part of shutting down (and liveSupervisor()/external cleanup can remove
|
|
4369
|
+
// it too), so a state-file check can report "gone" while the process is still
|
|
4370
|
+
// alive — which would break the loop early and skip the SIGKILL fallback,
|
|
4371
|
+
// leaving a wedged daemon and its worker process group running.
|
|
4372
|
+
const deadline = Date.now() + STOP_GRACE_MS + 2_000;
|
|
4373
|
+
while (Date.now() < deadline) {
|
|
4374
|
+
if (!isPidAlive(running.pid)) break;
|
|
4375
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
4376
|
+
}
|
|
4377
|
+
if (isPidAlive(running.pid)) {
|
|
4378
|
+
logger.warn(`Supervisor (pid ${running.pid}) did not stop gracefully — sending SIGKILL.`);
|
|
4379
|
+
// The daemon is spawned detached (a process-group leader) and its workers
|
|
4380
|
+
// are children in that group, so SIGKILL the whole group to avoid orphaning
|
|
4381
|
+
// `nano work` processes. Fall back to the bare pid (e.g. on Windows, or if
|
|
4382
|
+
// the daemon isn't a group leader).
|
|
4383
|
+
let killedGroup = false;
|
|
4384
|
+
if (osPlatform() !== 'win32') {
|
|
4385
|
+
try { process.kill(-running.pid, 'SIGKILL'); killedGroup = true; } catch { /* fall back below */ }
|
|
4386
|
+
}
|
|
4387
|
+
if (!killedGroup) { try { process.kill(running.pid, 'SIGKILL'); } catch { /* ignore */ } }
|
|
4388
|
+
clearSupervisorState();
|
|
4389
|
+
}
|
|
4390
|
+
logger.info('Supervisor stopped.');
|
|
4391
|
+
}
|
|
4392
|
+
|
|
4393
|
+
function supervisorLogsCmd(req) {
|
|
4394
|
+
const logger = getLogger();
|
|
4395
|
+
const id = req.positional[1];
|
|
4396
|
+
const file = id ? supervisorWorkerLogFile(id) : supervisorDaemonLogFile();
|
|
4397
|
+
if (!existsSync(file)) {
|
|
4398
|
+
logger.error(`No log file at ${file}.${id ? ` (unknown worker "${id}"?)` : ''}`);
|
|
4399
|
+
process.exit(1);
|
|
4400
|
+
}
|
|
4401
|
+
const follow = Boolean(req.follow);
|
|
4402
|
+
const tailArgs = follow ? ['-n', '200', '-F', file] : ['-n', '200', file];
|
|
4403
|
+
const proc = spawn('tail', tailArgs, { stdio: ['ignore', 'inherit', 'inherit'] });
|
|
4404
|
+
proc.on('error', () => {
|
|
4405
|
+
// tail unavailable (e.g. Windows): print the tail ourselves, no follow.
|
|
4406
|
+
if (follow) logger.warn('`--follow` is not supported without `tail` on this platform; printing the current tail only.');
|
|
4407
|
+
try {
|
|
4408
|
+
const lines = readFileSync(file, 'utf-8').split('\n');
|
|
4409
|
+
logger.info(lines.slice(-200).join('\n'));
|
|
4410
|
+
} catch (err) { logger.error(`Could not read ${file}: ${err.message}`); }
|
|
4411
|
+
});
|
|
4412
|
+
}
|
|
4413
|
+
|
|
4414
|
+
/**
|
|
4415
|
+
* Interactive attach console. Streams live events from the daemon and accepts
|
|
4416
|
+
* line commands. `detach` (or Ctrl-D) disconnects but leaves the daemon
|
|
4417
|
+
* running; `stop` tears the fleet down.
|
|
4418
|
+
*/
|
|
4419
|
+
async function attachSupervisorConsole(state) {
|
|
4420
|
+
const logger = getLogger();
|
|
4421
|
+
const socketPath = state?.socket || getSupervisorSocketPath();
|
|
4422
|
+
let sock;
|
|
4423
|
+
try {
|
|
4424
|
+
sock = await supervisorConnect(socketPath);
|
|
4425
|
+
} catch (err) {
|
|
4426
|
+
logger.error(`Could not attach to supervisor: ${err.message}`);
|
|
4427
|
+
process.exit(1);
|
|
4428
|
+
}
|
|
4429
|
+
|
|
4430
|
+
const out = (s) => process.stdout.write(s + '\n');
|
|
4431
|
+
out('Attached to nano worker supervisor. Type "help" for commands.');
|
|
4432
|
+
out('Detach (leave it running) with "detach" or Ctrl-D; tear it down with "stop".');
|
|
4433
|
+
sock.write(encodeFrame({ op: 'attach' }));
|
|
4434
|
+
|
|
4435
|
+
let buf = '';
|
|
4436
|
+
sock.on('data', (chunk) => {
|
|
4437
|
+
buf += chunk;
|
|
4438
|
+
const { frames, rest } = decodeFrames(buf);
|
|
4439
|
+
buf = rest;
|
|
4440
|
+
for (const frame of frames) {
|
|
4441
|
+
if (frame.type === 'status') {
|
|
4442
|
+
out('');
|
|
4443
|
+
out(formatSupervisorStatus(frame));
|
|
4444
|
+
} else if (frame.type === 'event') {
|
|
4445
|
+
const w = frame.worker;
|
|
4446
|
+
if (frame.event === 'worker-start') out(`• worker ${w.id} started (pid ${w.pid}).`);
|
|
4447
|
+
else if (frame.event === 'worker-exit') out(`• worker ${w.id} exited (${w.lastExit}); restarting in ${formatDuration(frame.restartInMs)}.`);
|
|
4448
|
+
else if (frame.event === 'worker-remove') out(`• worker ${frame.id} removed.`);
|
|
4449
|
+
else if (frame.event === 'daemon-stop') out('• supervisor stopping.');
|
|
4450
|
+
} else if (frame.type === 'added') {
|
|
4451
|
+
out(`• added worker ${frame.worker.id}.`);
|
|
4452
|
+
} else if (frame.type === 'removed') {
|
|
4453
|
+
out(`• removed: ${frame.removed.join(', ') || '(none matched)'}.`);
|
|
4454
|
+
} else if (frame.type === 'restarted') {
|
|
4455
|
+
out(`• restarted: ${frame.restarted.join(', ') || '(none matched)'}.`);
|
|
4456
|
+
} else if (frame.ok === false) {
|
|
4457
|
+
out(`! ${frame.error}`);
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
});
|
|
4461
|
+
|
|
4462
|
+
const rl = createReadline({ input: process.stdin, output: process.stdout, prompt: 'supervisor> ' });
|
|
4463
|
+
rl.prompt();
|
|
4464
|
+
|
|
4465
|
+
await new Promise((resolve) => {
|
|
4466
|
+
let stopping = false;
|
|
4467
|
+
const finish = () => { try { rl.close(); } catch { /* ignore */ } try { sock.end(); } catch { /* ignore */ } resolve(); };
|
|
4468
|
+
|
|
4469
|
+
sock.on('close', () => { if (!stopping) out('\nSupervisor connection closed.'); finish(); });
|
|
4470
|
+
|
|
4471
|
+
rl.on('line', (line) => {
|
|
4472
|
+
const parts = String(line).trim().split(/\s+/).filter(Boolean);
|
|
4473
|
+
const cmd = (parts.shift() || '').toLowerCase();
|
|
4474
|
+
switch (cmd) {
|
|
4475
|
+
case '': break;
|
|
4476
|
+
case 'help':
|
|
4477
|
+
out('Commands: status | add <profile> [--name <worker>] [work flags] |');
|
|
4478
|
+
out(' remove <id|profile|all> | restart <id|profile|all> |');
|
|
4479
|
+
out(' logs [id] | detach | stop | help');
|
|
4480
|
+
break;
|
|
4481
|
+
case 'status': sock.write(encodeFrame({ op: 'status' })); break;
|
|
4482
|
+
case 'add': {
|
|
4483
|
+
const profile = parts.shift();
|
|
4484
|
+
if (!profile) { out('usage: add <profile> [--name <worker>] [work flags]'); break; }
|
|
4485
|
+
const { name, rest } = extractNameFlag(parts);
|
|
4486
|
+
sock.write(encodeFrame({ op: 'add', profile, name, args: rest }));
|
|
4487
|
+
break;
|
|
4488
|
+
}
|
|
4489
|
+
case 'remove': case 'rm': {
|
|
4490
|
+
const target = parts.shift();
|
|
4491
|
+
if (!target) { out('usage: remove <id|profile|all>'); break; }
|
|
4492
|
+
sock.write(encodeFrame({ op: 'remove', target }));
|
|
4493
|
+
break;
|
|
4494
|
+
}
|
|
4495
|
+
case 'restart': {
|
|
4496
|
+
const target = parts.shift();
|
|
4497
|
+
if (!target) { out('usage: restart <id|profile|all>'); break; }
|
|
4498
|
+
sock.write(encodeFrame({ op: 'restart', target }));
|
|
4499
|
+
break;
|
|
4500
|
+
}
|
|
4501
|
+
case 'logs': case 'log': {
|
|
4502
|
+
const file = parts[0] ? supervisorWorkerLogFile(parts[0]) : supervisorDaemonLogFile();
|
|
4503
|
+
try {
|
|
4504
|
+
const lines = readFileSync(file, 'utf-8').split('\n');
|
|
4505
|
+
out(lines.slice(-30).join('\n'));
|
|
4506
|
+
} catch { out(`no log at ${file}`); }
|
|
4507
|
+
break;
|
|
4508
|
+
}
|
|
4509
|
+
case 'detach': case 'quit': case 'exit':
|
|
4510
|
+
out('Detaching — supervisor keeps running. Reattach with: c8ctl nano supervisor');
|
|
4511
|
+
finish();
|
|
4512
|
+
return;
|
|
4513
|
+
case 'stop':
|
|
4514
|
+
stopping = true;
|
|
4515
|
+
out('Stopping supervisor…');
|
|
4516
|
+
sock.write(encodeFrame({ op: 'stop' }));
|
|
4517
|
+
setTimeout(finish, 500);
|
|
4518
|
+
return;
|
|
4519
|
+
default:
|
|
4520
|
+
out(`unknown command "${cmd}" — type "help"`);
|
|
4521
|
+
}
|
|
4522
|
+
rl.prompt();
|
|
4523
|
+
});
|
|
4524
|
+
|
|
4525
|
+
// Ctrl-D (EOF) detaches, leaving the daemon running.
|
|
4526
|
+
rl.on('close', () => {
|
|
4527
|
+
if (stopping) return;
|
|
4528
|
+
out('\nDetaching — supervisor keeps running. Reattach with: c8ctl nano supervisor');
|
|
4529
|
+
finish();
|
|
4530
|
+
});
|
|
4531
|
+
});
|
|
4532
|
+
}
|
|
4533
|
+
|
|
4534
|
+
/** Dispatch the `supervisor` subcommand's action. */
|
|
4535
|
+
async function supervisorCommand(req, flags) {
|
|
4536
|
+
const action = (req.positional[0] || '').toLowerCase();
|
|
4537
|
+
switch (action) {
|
|
4538
|
+
case '__daemon':
|
|
4539
|
+
await runSupervisorDaemon();
|
|
4540
|
+
return;
|
|
4541
|
+
case '':
|
|
4542
|
+
case 'attach': {
|
|
4543
|
+
const state = await startSupervisorDaemon();
|
|
4544
|
+
await attachSupervisorConsole(runningSupervisor() || state);
|
|
4545
|
+
return;
|
|
4546
|
+
}
|
|
4547
|
+
case 'start':
|
|
4548
|
+
await supervisorStartCmd(req, flags);
|
|
4549
|
+
return;
|
|
4550
|
+
case 'status':
|
|
4551
|
+
case 'list':
|
|
4552
|
+
case 'ls':
|
|
4553
|
+
await supervisorStatusCmd();
|
|
4554
|
+
return;
|
|
4555
|
+
case 'add':
|
|
4556
|
+
await supervisorAddCmd(req, flags);
|
|
4557
|
+
return;
|
|
4558
|
+
case 'remove':
|
|
4559
|
+
case 'rm':
|
|
4560
|
+
await supervisorRemoveCmd(req);
|
|
4561
|
+
return;
|
|
4562
|
+
case 'restart':
|
|
4563
|
+
await supervisorRestartCmd(req);
|
|
4564
|
+
return;
|
|
4565
|
+
case 'stop':
|
|
4566
|
+
await supervisorStopCmd();
|
|
4567
|
+
return;
|
|
4568
|
+
case 'logs':
|
|
4569
|
+
case 'log':
|
|
4570
|
+
supervisorLogsCmd(req);
|
|
4571
|
+
return;
|
|
4572
|
+
default:
|
|
4573
|
+
getLogger().error(`Unknown supervisor action "${action}". Use: start|status|add|remove|restart|stop|logs|attach`);
|
|
4574
|
+
process.exit(1);
|
|
4575
|
+
}
|
|
4576
|
+
}
|
|
4577
|
+
|
|
3384
4578
|
// ---------------------------------------------------------------------------
|
|
3385
4579
|
// update — pull a new nanobpmn release onto a machine with an existing install.
|
|
3386
4580
|
// The plugin (and the bundled server binary, shipped via the matching platform
|
|
@@ -4743,6 +5937,31 @@ export {
|
|
|
4743
5937
|
RESERVED_RESULT_KEYS,
|
|
4744
5938
|
SANDBOXES,
|
|
4745
5939
|
};
|
|
5940
|
+
export {
|
|
5941
|
+
reconstructWorkArgs,
|
|
5942
|
+
supervisorWorkerId,
|
|
5943
|
+
autoWorkerName,
|
|
5944
|
+
sanitizeNameToken,
|
|
5945
|
+
isValidWorkerName,
|
|
5946
|
+
randomNameSuffix,
|
|
5947
|
+
extractNameFlag,
|
|
5948
|
+
redactWorkArgs,
|
|
5949
|
+
supervisorBackoffMs,
|
|
5950
|
+
encodeFrame,
|
|
5951
|
+
decodeFrames,
|
|
5952
|
+
formatDuration,
|
|
5953
|
+
summarizeSupervisorWorker,
|
|
5954
|
+
formatSupervisorStatus,
|
|
5955
|
+
WORK_FORWARD_FLAGS,
|
|
5956
|
+
runSupervisorDaemon,
|
|
5957
|
+
startSupervisorDaemon,
|
|
5958
|
+
supervisorRequest,
|
|
5959
|
+
runningSupervisor,
|
|
5960
|
+
readSupervisorState,
|
|
5961
|
+
clearSupervisorState,
|
|
5962
|
+
getSupervisorSocketPath,
|
|
5963
|
+
getSupervisorStateFile,
|
|
5964
|
+
};
|
|
4746
5965
|
|
|
4747
5966
|
export const metadata = {
|
|
4748
5967
|
name: 'c8ctl-plugin-nano',
|
|
@@ -4782,6 +6001,12 @@ export const metadata = {
|
|
|
4782
6001
|
{ command: 'c8ctl nano hire --name coder --rank senior --command "agent-harness" --sandbox docker --image ghcr.io/acme/agent:1', description: 'Create a profile that runs each job in a throwaway Docker container' },
|
|
4783
6002
|
{ command: 'c8ctl nano work reviewer', description: 'Spawn Nano job workers for the "reviewer" profile and poll for work' },
|
|
4784
6003
|
{ command: 'c8ctl nano work coder --sandbox docker --image ghcr.io/acme/agent:1', description: 'Run jobs in isolated containers with disk-hygiene reaping' },
|
|
6004
|
+
{ command: 'c8ctl nano supervisor start --worker reviewer --worker coder', description: 'Start a detached supervisor managing several workers from one terminal' },
|
|
6005
|
+
{ command: 'c8ctl nano supervisor', description: 'Attach an interactive console to the supervisor (detach with Ctrl-D, leaving it running)' },
|
|
6006
|
+
{ command: 'c8ctl nano supervisor status', description: 'List supervised workers (pid, state, restarts, uptime) without the console' },
|
|
6007
|
+
{ command: 'c8ctl nano supervisor add decider --max-parallel 2', description: 'Add a supervised worker (forwarding work flags) to the running supervisor' },
|
|
6008
|
+
{ command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
|
|
6009
|
+
{ command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
|
|
4785
6010
|
],
|
|
4786
6011
|
},
|
|
4787
6012
|
processos: {
|
|
@@ -4822,7 +6047,7 @@ export const commands = {
|
|
|
4822
6047
|
workspace: { type: 'boolean', description: 'clean: also delete the workspace (models + workers)' },
|
|
4823
6048
|
check: { type: 'boolean', description: 'update: only report whether a new release is available; do not install' },
|
|
4824
6049
|
binary: { type: 'string', description: 'Path to the nanobpmn server binary' },
|
|
4825
|
-
name: { type: 'string', description: '
|
|
6050
|
+
name: { type: 'string', description: 'work/supervisor add: worker name (auto ‹host›-‹profile›-‹random› if omitted); hire/assign: agent profile name' },
|
|
4826
6051
|
rank: { type: 'string', description: 'hire: agent rank (principal|senior|junior|decider)' },
|
|
4827
6052
|
command: { type: 'string', description: 'hire: CLI command that runs the agent harness (e.g. copilot, claude, pi)' },
|
|
4828
6053
|
arg: { type: 'string', multiple: true, description: 'hire/work: command-line switch/arg appended to the harness command (repeatable), e.g. --arg --allow-all. Persisted on hire; work appends more.' },
|
|
@@ -4844,6 +6069,8 @@ export const commands = {
|
|
|
4844
6069
|
'lock-grace': { type: 'string', description: 'work: extra ms added to --job-timeout to derive the broker activation lock, so the worker reports before the lock lapses (default 120000)' },
|
|
4845
6070
|
'poll-timeout': { type: 'string', description: 'work: broker long-poll window in ms each activateJobs request is held open (fewer reconnects → fewer transient connect errors); default 30000, 0 = broker default, negative = return immediately' },
|
|
4846
6071
|
'job-type': { type: 'string', multiple: true, description: 'work: extra job type to service alongside the rank×capability matrix (repeatable)' },
|
|
6072
|
+
worker: { type: 'string', multiple: true, description: 'supervisor start: profile to launch as a supervised worker (repeatable)' },
|
|
6073
|
+
attach: { type: 'boolean', description: 'supervisor start: attach the interactive console after starting the daemon' },
|
|
4847
6074
|
},
|
|
4848
6075
|
handler: async (args, flags) => {
|
|
4849
6076
|
const logger = getLogger();
|
|
@@ -4901,6 +6128,9 @@ export const commands = {
|
|
|
4901
6128
|
case 'work':
|
|
4902
6129
|
await workAgent(req, flags);
|
|
4903
6130
|
break;
|
|
6131
|
+
case 'supervisor':
|
|
6132
|
+
await supervisorCommand(req, flags);
|
|
6133
|
+
break;
|
|
4904
6134
|
}
|
|
4905
6135
|
} catch (error) {
|
|
4906
6136
|
logger.error(`nano ${req.subcommand} failed: ${error instanceof Error ? error.message : error}`);
|
|
@@ -4992,6 +6222,7 @@ function printUsage() {
|
|
|
4992
6222
|
console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--list]');
|
|
4993
6223
|
console.log(' c8ctl nano assign <profileName> [<capability> ...] [--name <n>] [--capabilities <a,b>]');
|
|
4994
6224
|
console.log(' c8ctl nano work <profileName> [--arg <switch> ...] [--max-parallel <n>] [--job-timeout <ms>] [--lock-grace <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
|
|
6225
|
+
console.log(' c8ctl nano supervisor [start|status|add|remove|restart|stop|logs|attach] ... (manage many workers from one terminal)');
|
|
4995
6226
|
console.log('');
|
|
4996
6227
|
console.log('Subcommands:');
|
|
4997
6228
|
console.log(' start Spawn an N-node local cluster wired to talk to each other on localhost');
|
|
@@ -5008,6 +6239,7 @@ function printUsage() {
|
|
|
5008
6239
|
console.log(' hire Create a CLI agent worker profile (rank + capabilities → job-type matrix)');
|
|
5009
6240
|
console.log(' assign Grant new capabilities (roles) to an existing hire (additive)');
|
|
5010
6241
|
console.log(' work Run a hired profile as Nano job workers, polling for work until Ctrl-C');
|
|
6242
|
+
console.log(' supervisor Run/manage a fleet of workers from one terminal (detachable console + non-interactive control)');
|
|
5011
6243
|
console.log('');
|
|
5012
6244
|
console.log('Options:');
|
|
5013
6245
|
console.log(' <nodes> Number of nodes to start (default 1)');
|