c8ctl-plugin-nano 1.50.0 → 1.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/c8ctl-plugin.js +178 -388
- package/package.json +11 -10
- package/supervisor-engine.mjs +76 -1
- package/supervisor.dist.js +6 -6
package/c8ctl-plugin.js
CHANGED
|
@@ -136,11 +136,6 @@ const READINESS_TIMEOUT_MS = 60_000;
|
|
|
136
136
|
const READINESS_POLL_MS = 500;
|
|
137
137
|
const HEALTH_TIMEOUT_MS = 1_500;
|
|
138
138
|
const STOP_GRACE_MS = 8_000;
|
|
139
|
-
// Backoff applied when a poller fails a lease fast because the worker is already
|
|
140
|
-
// running another job (issue #142 single-flight). Long enough that the deferred
|
|
141
|
-
// job doesn't tight-loop re-activating while the first runs, short enough that it
|
|
142
|
-
// is picked up promptly once the worker frees up.
|
|
143
|
-
const WORKER_BUSY_RETRY_BACKOFF_MS = 5_000;
|
|
144
139
|
// Upper bound on one `--auto` engine-read reconcile (enumerate deployed
|
|
145
140
|
// definitions + fetch each BPMN). A read that stalls past this is treated as a
|
|
146
141
|
// transient failure so the running poller set is KEPT and, crucially, shutdown
|
|
@@ -3288,14 +3283,12 @@ async function createAgenticEndpoint(opts) {
|
|
|
3288
3283
|
//
|
|
3289
3284
|
// Returns everything a JS caller needs to run `makeSupervisor` — the assembled
|
|
3290
3285
|
// `deps`, the seeded `registry` (so workers can be add/remove'd live), and the
|
|
3291
|
-
// `makeSupervisor` + `Effect` handles from the bundle. NOTE (
|
|
3292
|
-
//
|
|
3293
|
-
//
|
|
3294
|
-
//
|
|
3295
|
-
//
|
|
3296
|
-
//
|
|
3297
|
-
// this seam unblocks. Constructing deps is side-effect-free (no socket opens, no
|
|
3298
|
-
// activation) until the caller forks `supervisor.run`.
|
|
3286
|
+
// `makeSupervisor` + `Effect` handles from the bundle. NOTE (issue #172): the
|
|
3287
|
+
// hot-path flip this seam unblocks is now DONE — `workAgent` runs
|
|
3288
|
+
// `makeSupervisor(deps).run` as the single per-host owner, having retired the
|
|
3289
|
+
// per-type SDK pollers, the process-wide `singleFlight`, the per-process reconcile
|
|
3290
|
+
// crawl, and the per-job lock extender. Constructing deps stays side-effect-free
|
|
3291
|
+
// (no socket opens, no activation) until the caller forks `supervisor.run`.
|
|
3299
3292
|
//
|
|
3300
3293
|
// @param {object} opts
|
|
3301
3294
|
// @param {{ run(job): Promise<void> }} opts.runner raw job runner (required to run)
|
|
@@ -3312,7 +3305,7 @@ async function createAgenticEndpoint(opts) {
|
|
|
3312
3305
|
// @param {{ searchProcessDefinitionKeys: Function, getProcessDefinitionXml: Function }} [opts.reconcileReader] raw reconcile reader overriding the default keep-alive `defaultC8RestReader`
|
|
3313
3306
|
// @param {typeof fetch} [opts.fetchImpl] injected fetch (tests)
|
|
3314
3307
|
// @param {NodeJS.ProcessEnv} [opts.env]
|
|
3315
|
-
// @returns {Promise<{ deps: object, registry: object, makeSupervisor: Function, Effect: object, Fiber: object }>}
|
|
3308
|
+
// @returns {Promise<{ deps: object, registry: object, settle: { complete: Function, fail: Function }, makeSupervisor: Function, Effect: object, Fiber: object }>}
|
|
3316
3309
|
async function createSupervisorDeps(opts = {}) {
|
|
3317
3310
|
const {
|
|
3318
3311
|
runner,
|
|
@@ -3373,9 +3366,8 @@ async function createSupervisorDeps(opts = {}) {
|
|
|
3373
3366
|
};
|
|
3374
3367
|
}
|
|
3375
3368
|
|
|
3376
|
-
const
|
|
3377
|
-
|
|
3378
|
-
);
|
|
3369
|
+
const rawEngine = createRawEngineClient({ baseUrl: rc.baseUrl, token: rc.token, authHeaders: resolvedAuthHeaders, worker, fetchImpl });
|
|
3370
|
+
const engine = rt.makeEngineClient(rawEngine);
|
|
3379
3371
|
// `reconcileReader` (a raw `{ searchProcessDefinitionKeys, getProcessDefinitionXml }`)
|
|
3380
3372
|
// may be injected to override the default keep-alive `httpC8RestReader` — the
|
|
3381
3373
|
// caller may already hold one, and a test drives the crawl without a socket.
|
|
@@ -3402,7 +3394,24 @@ async function createSupervisorDeps(opts = {}) {
|
|
|
3402
3394
|
config: scope ? { ...config, scope } : config,
|
|
3403
3395
|
});
|
|
3404
3396
|
|
|
3405
|
-
|
|
3397
|
+
// The `settle` seam (issue #156, escalation answer (a)): the runner settles a
|
|
3398
|
+
// finished job through the SAME engine client the supervisor activates/extends
|
|
3399
|
+
// with — one base URL, one (rotating) auth resolver — so a `complete`/`fail`
|
|
3400
|
+
// can never drift from the activation's credentials. A plain `ActivatedJob`
|
|
3401
|
+
// carries no `job.complete()`/`job.fail()` (unlike the SDK job object), so the
|
|
3402
|
+
// supervisor path settles via these `{ complete, fail }` calls. This is the
|
|
3403
|
+
// direct analogue of returning `registry` for live worker add/remove: the
|
|
3404
|
+
// caller wires it into `runAgentJob`'s completion, replacing the SDK job object.
|
|
3405
|
+
// Route through the LIFTED `engine` port (not `rawEngine`) + `Effect.runPromise`
|
|
3406
|
+
// so a settle rejection surfaces as the normalized `SupervisorError` the rest of
|
|
3407
|
+
// the runtime uses (activate/extendLock), and any future port instrumentation
|
|
3408
|
+
// covers the settle path too.
|
|
3409
|
+
const settle = {
|
|
3410
|
+
complete: (jobKey, variables) => rt.Effect.runPromise(engine.complete(jobKey, variables)),
|
|
3411
|
+
fail: (jobKey, opts2) => rt.Effect.runPromise(engine.fail(jobKey, opts2)),
|
|
3412
|
+
};
|
|
3413
|
+
|
|
3414
|
+
return { deps, registry, settle, makeSupervisor: rt.makeSupervisor, Effect: rt.Effect, Fiber: rt.Fiber };
|
|
3406
3415
|
}
|
|
3407
3416
|
|
|
3408
3417
|
// Build the content endpoint. Per issue #63 / nano-bpm #759 the non-binary
|
|
@@ -5513,80 +5522,12 @@ function baseAgentEnv(profile, job) {
|
|
|
5513
5522
|
};
|
|
5514
5523
|
}
|
|
5515
5524
|
|
|
5516
|
-
|
|
5517
|
-
|
|
5518
|
-
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
* serving N job types could lease and run up to N jobs at once — each holding its
|
|
5523
|
-
* own PTY + git workspace + broker lock-extender — the exact failure the
|
|
5524
|
-
* "one job per worker" invariant exists to prevent.
|
|
5525
|
-
*
|
|
5526
|
-
* This is a capacity-1, non-blocking mutex shared by EVERY per-type poller: the
|
|
5527
|
-
* first poller to `tryAcquire()` runs its job to completion (releasing in a
|
|
5528
|
-
* `finally`); any other poller that finds the permit already held must NOT begin
|
|
5529
|
-
* a second job (the caller fails the lease fast so the broker re-queues it rather
|
|
5530
|
-
* than leaving it "claimed but idle"). `tryAcquire`/`release` are synchronous
|
|
5531
|
-
* check-and-set, so the single-threaded event loop makes them race-free across
|
|
5532
|
-
* the concurrently-invoked async job handlers.
|
|
5533
|
-
*/
|
|
5534
|
-
function createSingleFlight() {
|
|
5535
|
-
let held = false;
|
|
5536
|
-
return {
|
|
5537
|
-
/** Take the permit if free; returns false when a job is already in flight. */
|
|
5538
|
-
tryAcquire() {
|
|
5539
|
-
if (held) return false;
|
|
5540
|
-
held = true;
|
|
5541
|
-
return true;
|
|
5542
|
-
},
|
|
5543
|
-
/** Release the permit. Idempotent: redundant calls are safe no-ops, though the normal path releases once per acquire (in a `finally`). */
|
|
5544
|
-
release() {
|
|
5545
|
-
held = false;
|
|
5546
|
-
},
|
|
5547
|
-
/** True while a job holds the permit. */
|
|
5548
|
-
get busy() {
|
|
5549
|
-
return held;
|
|
5550
|
-
},
|
|
5551
|
-
};
|
|
5552
|
-
}
|
|
5553
|
-
|
|
5554
|
-
/**
|
|
5555
|
-
* Keep a leased job's broker activation lock ahead of *now* while the harness is
|
|
5556
|
-
* running, so a long agent run never has its lock lapse and get re-activated (a
|
|
5557
|
-
* second worker starting → the classic stale complete/fail 409). The lock is NOT
|
|
5558
|
-
* hardcoded up front: we refresh it to `windowMs` — a duration-from-now, per the
|
|
5559
|
-
* UpdateJobTimeout contract ("the duration of the new timeout in ms, starting
|
|
5560
|
-
* from the current moment"), so calls SET rather than accumulate — every
|
|
5561
|
-
* `intervalMs`. The deadline therefore stays a bounded `windowMs` ahead of now.
|
|
5562
|
-
* The instant we stop refreshing (harness exit / idle-kill / hard cap) the lock
|
|
5563
|
-
* lapses within `windowMs` and the broker reclaims the job — fast node-loss
|
|
5564
|
-
* recovery. Because the harness is always killed locally before we stop, the lock
|
|
5565
|
-
* strictly outlives our local run, so a reclaim never races a still-running agent.
|
|
5566
|
-
*
|
|
5567
|
-
* Returns a stop() to call once the run settles. Extension failures are logged
|
|
5568
|
-
* and swallowed — a transient network blip must not crash the job handler. Older
|
|
5569
|
-
* SDKs without `modifyJobTimeout` degrade to the fixed initial lock (a no-op stop).
|
|
5570
|
-
*/
|
|
5571
|
-
function startLockExtender(job, windowMs, intervalMs, tag, logger) {
|
|
5572
|
-
if (!(windowMs > 0) || !(intervalMs > 0)) {
|
|
5573
|
-
return () => {};
|
|
5574
|
-
}
|
|
5575
|
-
if (typeof job?.modifyJobTimeout !== 'function') {
|
|
5576
|
-
logger?.warn?.(`${tag}: job.modifyJobTimeout unavailable — activation lock will NOT be auto-extended; a run longer than ${windowMs}ms risks being reclaimed and executed twice`);
|
|
5577
|
-
return () => {};
|
|
5578
|
-
}
|
|
5579
|
-
const extend = () => Promise.resolve()
|
|
5580
|
-
.then(() => job.modifyJobTimeout({ newTimeoutMs: windowMs }))
|
|
5581
|
-
.catch((err) => logger?.warn?.(`${tag}: lock extend failed — ${err?.message ?? err}`));
|
|
5582
|
-
// Renew immediately so the harness starts with a full, fresh window no matter
|
|
5583
|
-
// how much of the initial activation lease provisioning (clone/checkout) ate.
|
|
5584
|
-
extend();
|
|
5585
|
-
const timer = setInterval(extend, intervalMs);
|
|
5586
|
-
// Never let the heartbeat keep the process alive on shutdown.
|
|
5587
|
-
if (typeof timer.unref === 'function') timer.unref();
|
|
5588
|
-
return () => clearInterval(timer);
|
|
5589
|
-
}
|
|
5525
|
+
// Issue #172 retired `createSingleFlight` (the process-wide capacity-1 mutex the
|
|
5526
|
+
// per-type SDK pollers shared) and `startLockExtender` (the per-job broker-lock
|
|
5527
|
+
// heartbeat). Both are now owned by the single-owner supervisor runtime: race-free
|
|
5528
|
+
// per-type slot accounting lives in `supervisor/src/registry.ts` (this worker
|
|
5529
|
+
// registers with capacity 1), and the lock lifecycle — extend-winner-before-start
|
|
5530
|
+
// plus a `Schedule`-driven heartbeat — lives in `supervisor/src/dispatch.ts`.
|
|
5590
5531
|
|
|
5591
5532
|
/**
|
|
5592
5533
|
* Run a single activated job through the profile's CLI command (one-shot),
|
|
@@ -7016,26 +6957,21 @@ async function workAgent(req, flags) {
|
|
|
7016
6957
|
};
|
|
7017
6958
|
// One job per worker, hard-wired (there is deliberately no --max-parallel
|
|
7018
6959
|
// flag): an agent harness holds a PTY + a git workspace for the whole life of
|
|
7019
|
-
// a job, so a worker must never lease a second job concurrently.
|
|
7020
|
-
//
|
|
7021
|
-
//
|
|
7022
|
-
|
|
7023
|
-
//
|
|
7024
|
-
// only serializes ONE job-type poller, but this process runs one poller per
|
|
7025
|
-
// job type, so nothing stops N pollers from each leasing + running a job
|
|
7026
|
-
// concurrently. This capacity-1 mutex, shared by every poller's jobHandler,
|
|
7027
|
-
// enforces the real "one job per worker" invariant: while any job is in flight
|
|
7028
|
-
// on any job type, no other poller starts a second one.
|
|
7029
|
-
const singleFlight = createSingleFlight();
|
|
6960
|
+
// a job, so a worker must never lease a second job concurrently. Issue #172:
|
|
6961
|
+
// this is now enforced by registering this worker with capacity 1 in the
|
|
6962
|
+
// single-owner runtime's shared registry (one worker, capacity 1, across all
|
|
6963
|
+
// job types) — retiring the process-wide capacity-1 single-flight the per-type
|
|
6964
|
+
// SDK pollers needed. Race-free slot accounting lives in the runtime registry.
|
|
7030
6965
|
// The broker job-activation lock is NOT hardcoded up front. A fixed timeout is
|
|
7031
6966
|
// impossible to size for an agent: too short reclaims a still-working job (a
|
|
7032
6967
|
// second agent starts + the stale complete/fail is rejected 409), too long
|
|
7033
|
-
// strands a dead worker's job. Instead the
|
|
7034
|
-
// `recovery-window` ahead of *now* while the harness runs
|
|
7035
|
-
//
|
|
7036
|
-
//
|
|
7037
|
-
// `idle-timeout` (max silence before
|
|
7038
|
-
// lock is held only while the agent is
|
|
6968
|
+
// strands a dead worker's job. Instead the runtime's dispatch lifecycle keeps
|
|
6969
|
+
// the lock a bounded `recovery-window` ahead of *now* while the harness runs
|
|
6970
|
+
// (extend-winner-before-start + a Schedule heartbeat in dispatch.ts), so long
|
|
6971
|
+
// runs never lose their lock, and a dead/killed worker's job is reclaimed
|
|
6972
|
+
// within one window. Liveness is enforced by `idle-timeout` (max silence before
|
|
6973
|
+
// the harness is killed as wedged), so the lock is held only while the agent is
|
|
6974
|
+
// alive AND producing output.
|
|
7039
6975
|
const recoveryWindowMs = intFlag(flags?.['recovery-window'], 5 * 60_000);
|
|
7040
6976
|
const idleTimeoutMs = intFlag(flags?.['idle-timeout'], 5 * 60_000);
|
|
7041
6977
|
// `--job-timeout` is now an OPTIONAL absolute hard cap on total harness runtime
|
|
@@ -7234,7 +7170,7 @@ async function workAgent(req, flags) {
|
|
|
7234
7170
|
}
|
|
7235
7171
|
const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
|
|
7236
7172
|
logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
|
|
7237
|
-
logger.info(` one job per worker (
|
|
7173
|
+
logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
|
|
7238
7174
|
// Warm the gh-token cache now, off the job-handling path: githubCloneToken()
|
|
7239
7175
|
// may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
|
|
7240
7176
|
// credential fallback, and doing that inside a job handler would block the
|
|
@@ -7630,46 +7566,20 @@ async function workAgent(req, flags) {
|
|
|
7630
7566
|
const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
|
|
7631
7567
|
const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
|
|
7632
7568
|
|
|
7633
|
-
//
|
|
7634
|
-
//
|
|
7635
|
-
//
|
|
7636
|
-
|
|
7637
|
-
|
|
7638
|
-
|
|
7639
|
-
|
|
7640
|
-
|
|
7641
|
-
|
|
7642
|
-
|
|
7643
|
-
|
|
7644
|
-
|
|
7645
|
-
|
|
7646
|
-
|
|
7647
|
-
// anything — so the broker re-queues it (retries preserved) instead of it
|
|
7648
|
-
// sitting "claimed but idle" while the first job runs. Gating here, at the
|
|
7649
|
-
// point activation surfaces as a handler call, is the cross-poller gate
|
|
7650
|
-
// the per-type maxParallelJobs cannot provide.
|
|
7651
|
-
if (!singleFlight.tryAcquire()) {
|
|
7652
|
-
// Not a failure — preserve the broker-provided retries verbatim so
|
|
7653
|
-
// re-dispatch doesn't decrement (or resurrect) the job. Keep a real 0
|
|
7654
|
-
// as 0 (an already-incidentable job must stay that way); only default
|
|
7655
|
-
// to 1 when the count is missing/invalid.
|
|
7656
|
-
const rawRetries = Number(job.retries);
|
|
7657
|
-
const retries = Number.isInteger(rawRetries) && rawRetries >= 0 ? rawRetries : 1;
|
|
7658
|
-
logger.info(`[${jobType}] job ${job.jobKey} deferred — worker already running another job; releasing lease for re-dispatch.`);
|
|
7659
|
-
return job.fail({
|
|
7660
|
-
errorMessage: 'worker busy: one job per worker (single-flight across all job types)',
|
|
7661
|
-
retries,
|
|
7662
|
-
retryBackOff: WORKER_BUSY_RETRY_BACKOFF_MS,
|
|
7663
|
-
});
|
|
7664
|
-
}
|
|
7665
|
-
recordJobStart(job, jobType);
|
|
7666
|
-
// Auto-extend the broker lock for the whole life of this job (harness run
|
|
7667
|
-
// + git finalize + complete/fail), stopped in the outer finally. The lock
|
|
7668
|
-
// is held only while the harness stays alive and productive — a silent
|
|
7669
|
-
// hang is killed by the idle-timeout, which resolves runAgentJob and stops
|
|
7670
|
-
// the extension, so the broker can reclaim the job.
|
|
7671
|
-
let stopLockExtender = () => {};
|
|
7672
|
-
try {
|
|
7569
|
+
// The agent job runner (issue #172 hot-path flip). The single-owner supervisor
|
|
7570
|
+
// runtime dispatches each activated job to this `run(job)`; it executes the
|
|
7571
|
+
// harness exactly as the retired per-type SDK jobHandler did, but SETTLES via the
|
|
7572
|
+
// injected `settle` seam (engine complete/fail) because the plain ActivatedJob
|
|
7573
|
+
// carries no SDK `job.complete()/job.fail()`. Capacity (one job per worker), the
|
|
7574
|
+
// activation long-polls, the lock lifecycle, and reconcile are all owned by the
|
|
7575
|
+
// runtime now — retiring the per-type pollers, the process-wide single-flight,
|
|
7576
|
+
// the per-process 1+N reconcile crawl, and the per-job lock extender.
|
|
7577
|
+
let settle;
|
|
7578
|
+
const runner = {
|
|
7579
|
+
run: async (job) => {
|
|
7580
|
+
const jobType = job.type;
|
|
7581
|
+
recordJobStart(job, jobType);
|
|
7582
|
+
try {
|
|
7673
7583
|
logger.info(`[${jobType}] job ${job.jobKey} (instance ${job.processInstanceKey ?? '-'}) → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
|
|
7674
7584
|
|
|
7675
7585
|
// Disk-budget admission shed: if the engine data root is below the free
|
|
@@ -7681,7 +7591,7 @@ async function workAgent(req, flags) {
|
|
|
7681
7591
|
const freeMb = budget.free != null ? Math.round(budget.free / 1_048_576) : '?';
|
|
7682
7592
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7683
7593
|
logger.warn(`[${jobType}] job ${job.jobKey} shed — low disk (${freeMb}MB free); retries left ${retries}`);
|
|
7684
|
-
return
|
|
7594
|
+
return settle.fail(job.jobKey, { errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
|
|
7685
7595
|
}
|
|
7686
7596
|
}
|
|
7687
7597
|
|
|
@@ -7713,7 +7623,7 @@ async function workAgent(req, flags) {
|
|
|
7713
7623
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7714
7624
|
const msg = err instanceof ProvisionError ? err.message : `prompt resource fetch failed: ${err.message}`;
|
|
7715
7625
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7716
|
-
return
|
|
7626
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7717
7627
|
}
|
|
7718
7628
|
|
|
7719
7629
|
// Assemble + normalize the task envelope from headers (defaults) and
|
|
@@ -7724,27 +7634,24 @@ async function workAgent(req, flags) {
|
|
|
7724
7634
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7725
7635
|
const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
|
|
7726
7636
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7727
|
-
return
|
|
7637
|
+
return settle.fail(job.jobKey, { errorMessage: msg, retries });
|
|
7728
7638
|
}
|
|
7729
7639
|
|
|
7730
7640
|
// #130: per-task liveness overrides. Precedence: envelope override →
|
|
7731
7641
|
// worker-flag default → built-in default, each clamped to a sane max so
|
|
7732
7642
|
// a task can't request an unbounded window. Absent envelope fields leave
|
|
7733
|
-
// the worker-flag behaviour unchanged. These drive
|
|
7734
|
-
// liveness (idle/recovery)
|
|
7735
|
-
//
|
|
7643
|
+
// the worker-flag behaviour unchanged. These drive the harness idle
|
|
7644
|
+
// liveness (idle/recovery/hard-cap) so a JVM-heavy task can widen its own
|
|
7645
|
+
// window without a global flag change. NOTE: after the hot-path flip
|
|
7646
|
+
// (#172) the broker lock cadence/window is owned by the supervisor
|
|
7647
|
+
// runtime dispatch config at the worker level (the `dispatch` config's
|
|
7648
|
+
// `recoveryWindowMs` / `extendIntervalMs`), so a per-task override no
|
|
7649
|
+
// longer widens the broker lock window — only the harness liveness.
|
|
7736
7650
|
const {
|
|
7737
7651
|
idleTimeoutMs: effectiveIdleTimeoutMs,
|
|
7738
7652
|
recoveryWindowMs: effectiveRecoveryWindowMs,
|
|
7739
7653
|
hardCapMs: effectiveHardCapMs,
|
|
7740
7654
|
} = resolveLivenessOverrides(envelope.task, { idleTimeoutMs, recoveryWindowMs, hardCapMs });
|
|
7741
|
-
// Recompute the lock-extend cadence from the effective recovery window
|
|
7742
|
-
// (same ~1/3-of-window rule as at startup) so a widened window still
|
|
7743
|
-
// renews comfortably before it lapses.
|
|
7744
|
-
const effectiveLockExtendIntervalMs = Math.min(
|
|
7745
|
-
Math.max(5_000, Math.floor(effectiveRecoveryWindowMs / 3)),
|
|
7746
|
-
Math.max(1, Math.floor(effectiveRecoveryWindowMs * 0.75)),
|
|
7747
|
-
);
|
|
7748
7655
|
if (
|
|
7749
7656
|
effectiveIdleTimeoutMs !== idleTimeoutMs ||
|
|
7750
7657
|
effectiveRecoveryWindowMs !== recoveryWindowMs ||
|
|
@@ -7774,7 +7681,7 @@ async function workAgent(req, flags) {
|
|
|
7774
7681
|
: 'repository.url is missing';
|
|
7775
7682
|
const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
|
|
7776
7683
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7777
|
-
return
|
|
7684
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7778
7685
|
}
|
|
7779
7686
|
}
|
|
7780
7687
|
|
|
@@ -7784,12 +7691,10 @@ async function workAgent(req, flags) {
|
|
|
7784
7691
|
const hasRepo = !isContainer && !!envelope.repository?.url;
|
|
7785
7692
|
let runDir = null;
|
|
7786
7693
|
let provisioned = null;
|
|
7787
|
-
//
|
|
7788
|
-
//
|
|
7789
|
-
//
|
|
7790
|
-
//
|
|
7791
|
-
// duplicate-activation / stale-409 race. The `finally` below stops it.
|
|
7792
|
-
stopLockExtender = startLockExtender(job, effectiveRecoveryWindowMs, effectiveLockExtendIntervalMs, `[${jobType}] job ${job.jobKey}`, logger);
|
|
7694
|
+
// The broker activation lock is owned by the single-owner runtime's dispatch
|
|
7695
|
+
// lifecycle (supervisor/src/dispatch.ts): it extends the winner to the
|
|
7696
|
+
// recovery window BEFORE this runner starts and heartbeats it on a Schedule
|
|
7697
|
+
// for the whole run, so the retired per-job startLockExtender is gone here.
|
|
7793
7698
|
let cwd;
|
|
7794
7699
|
let extraEnv;
|
|
7795
7700
|
let repoToken = null;
|
|
@@ -7831,7 +7736,7 @@ async function workAgent(req, flags) {
|
|
|
7831
7736
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7832
7737
|
const msg = err instanceof ProvisionError ? err.message : `provisioning error: ${err.message}`;
|
|
7833
7738
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7834
|
-
return
|
|
7739
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7835
7740
|
}
|
|
7836
7741
|
} else if (!isContainer) {
|
|
7837
7742
|
// Repo-less host job (issue #129, hardening 1): nothing is provisioned,
|
|
@@ -7857,7 +7762,7 @@ async function workAgent(req, flags) {
|
|
|
7857
7762
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7858
7763
|
const msg = `could not create a temp workspace under the runs root: ${err.message}`;
|
|
7859
7764
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7860
|
-
return
|
|
7765
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7861
7766
|
}
|
|
7862
7767
|
}
|
|
7863
7768
|
|
|
@@ -8013,7 +7918,7 @@ async function workAgent(req, flags) {
|
|
|
8013
7918
|
const resultKeys = Object.keys(resultVars);
|
|
8014
7919
|
if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
|
|
8015
7920
|
else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
|
|
8016
|
-
return await
|
|
7921
|
+
return await settle.complete(job.jobKey, {
|
|
8017
7922
|
...resultVars,
|
|
8018
7923
|
[AGENT_RESULT_KEY]: resultEnvelope,
|
|
8019
7924
|
output: result.stdout,
|
|
@@ -8030,249 +7935,136 @@ async function workAgent(req, flags) {
|
|
|
8030
7935
|
|| (result.stderr || '').trim() + (result.stderrTruncated && (result.stderr || '').trim() ? ' [stderr truncated]' : '')
|
|
8031
7936
|
|| (result.signal ? `terminated by signal ${result.signal}` : `exit code ${result.exitCode}`);
|
|
8032
7937
|
logger.warn(`[${jobType}] job ${job.jobKey} failed (${detail}); retries left ${retries}`);
|
|
8033
|
-
return await
|
|
7938
|
+
return await settle.fail(job.jobKey, {
|
|
8034
7939
|
errorMessage: `agent "${profile.name}" failed: ${detail}`.slice(0, 2000),
|
|
8035
7940
|
retries,
|
|
8036
7941
|
variables: { [AGENT_RESULT_KEY]: resultEnvelope },
|
|
8037
7942
|
});
|
|
8038
|
-
|
|
8039
|
-
|
|
8040
|
-
recordJobEnd(job);
|
|
8041
|
-
// Release the process-wide single-flight permit LAST, once this job's
|
|
8042
|
-
// lock-extender is stopped and its bookkeeping cleared, so another
|
|
8043
|
-
// poller can only begin after this job is fully settled.
|
|
8044
|
-
singleFlight.release();
|
|
8045
|
-
}
|
|
8046
|
-
},
|
|
8047
|
-
});
|
|
8048
|
-
|
|
8049
|
-
// Live worker registry keyed by job type, so the profile watcher can add or
|
|
8050
|
-
// drain individual pollers without disturbing the others. `draining` is the
|
|
8051
|
-
// shutdown latch (shared with the watcher so a reconcile can't race a stop).
|
|
8052
|
-
const workers = new Map();
|
|
8053
|
-
let draining = false;
|
|
8054
|
-
|
|
8055
|
-
const drainWorker = async (w) => {
|
|
8056
|
-
try {
|
|
8057
|
-
if (typeof w.stopGracefully === 'function') {
|
|
8058
|
-
await w.stopGracefully({ waitUpToMs: STOP_GRACE_MS });
|
|
8059
|
-
} else if (typeof w.stop === 'function') {
|
|
8060
|
-
await w.stop();
|
|
7943
|
+
} finally {
|
|
7944
|
+
recordJobEnd(job);
|
|
8061
7945
|
}
|
|
8062
|
-
|
|
8063
|
-
} catch {
|
|
8064
|
-
return false; // best-effort: never let one worker's stop failure hang us
|
|
8065
|
-
}
|
|
8066
|
-
};
|
|
8067
|
-
|
|
8068
|
-
const spawnJobType = (jobType) => {
|
|
8069
|
-
if (workers.has(jobType)) return false;
|
|
8070
|
-
workers.set(jobType, makeWorker(jobType));
|
|
8071
|
-
return true;
|
|
7946
|
+
},
|
|
8072
7947
|
};
|
|
8073
7948
|
|
|
8074
|
-
|
|
8075
|
-
|
|
8076
|
-
//
|
|
8077
|
-
//
|
|
8078
|
-
|
|
8079
|
-
|
|
8080
|
-
|
|
8081
|
-
|
|
8082
|
-
|
|
8083
|
-
|
|
7949
|
+
// Compose the runtime deps over the plugin's real edges (issue #156 seam) and
|
|
7950
|
+
// register THIS worker (capacity 1) into the shared registry. In --auto the
|
|
7951
|
+
// runtime's reconcile loop rewrites this worker's serviceable types from the
|
|
7952
|
+
// engine read (`autoWorkerId`); otherwise the profile watch below does.
|
|
7953
|
+
const composed = await createSupervisorDeps({
|
|
7954
|
+
runner,
|
|
7955
|
+
camunda,
|
|
7956
|
+
restConfig,
|
|
7957
|
+
worker: workerName,
|
|
7958
|
+
workers: [{ id: workerName, types: jobTypes, capacity: 1 }],
|
|
7959
|
+
autoWorkerId: autoMode ? workerName : undefined,
|
|
7960
|
+
scope: autoScope,
|
|
7961
|
+
config: {
|
|
7962
|
+
activation: { requestTimeoutMs: pollTimeoutMs },
|
|
7963
|
+
dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
|
|
7964
|
+
},
|
|
7965
|
+
});
|
|
7966
|
+
settle = composed.settle;
|
|
7967
|
+
const {
|
|
7968
|
+
deps: supervisorDeps,
|
|
7969
|
+
registry: workerRegistry,
|
|
7970
|
+
makeSupervisor: makeSupervisorRuntime,
|
|
7971
|
+
Effect: SupervisorEffect,
|
|
7972
|
+
Fiber: SupervisorFiber,
|
|
7973
|
+
} = composed;
|
|
7974
|
+
|
|
7975
|
+
// Run the single per-host owner. `Effect.runFork` keeps the process alive on the
|
|
7976
|
+
// runtime's own Schedule cadences (reconcile + activation/idle loop), so even an
|
|
7977
|
+
// --auto worker that starts with zero types stays up and fills them in on the
|
|
7978
|
+
// next reconcile — the same liveness the retired ref'd auto-poll timer provided.
|
|
7979
|
+
const supervisor = await SupervisorEffect.runPromise(makeSupervisorRuntime(supervisorDeps));
|
|
7980
|
+
const supervisorFiber = SupervisorEffect.runFork(supervisor.run);
|
|
7981
|
+
|
|
7982
|
+
// Non-auto live retype: the runtime's reconcile loop only rewrites the --auto
|
|
7983
|
+
// worker's types (from the engine read), so keep watching the profile file to
|
|
7984
|
+
// honour `nano assign` — a profile edit rewrites THIS worker's serviceable types
|
|
7985
|
+
// in the shared registry without a restart (a single `registry.setTypes` call,
|
|
7986
|
+
// the direct analogue of the retired per-process reconcile). In --auto the
|
|
7987
|
+
// runtime owns the desired set, so no profile watch is installed.
|
|
8084
7988
|
const configFile = getConfigFile();
|
|
8085
7989
|
const WATCH_INTERVAL_MS = 1500;
|
|
8086
|
-
|
|
8087
|
-
|
|
8088
|
-
|
|
8089
|
-
|
|
8090
|
-
|
|
8091
|
-
|
|
8092
|
-
|
|
8093
|
-
// the update until the next change fires.
|
|
8094
|
-
let reconcileRequested = false;
|
|
8095
|
-
// Handle to the in-flight reconcile so shutdown can wait for it to finish
|
|
8096
|
-
// before snapshotting `workers` (avoids double-stops / missed drains).
|
|
8097
|
-
let inFlightReconcile = null;
|
|
8098
|
-
|
|
8099
|
-
// Desired job types. In `--auto` this is the engine's deployed agent job types
|
|
8100
|
-
// (∪ --job-type extras), read fresh each pass; a transient engine-read failure
|
|
8101
|
-
// returns { skip } so the running set is KEPT, never torn down. Otherwise it is
|
|
8102
|
-
// the CURRENT on-disk profile's matrix (∪ extras), with { skip } for a
|
|
8103
|
-
// transient/torn read, a vanished profile, or an invalid edit — callers must
|
|
8104
|
-
// then KEEP the running set, never tear down.
|
|
8105
|
-
const desiredJobTypes = async () => {
|
|
8106
|
-
if (autoMode) {
|
|
8107
|
-
try {
|
|
8108
|
-
const autoTypes = await resolveAutoJobTypes({ restConfig, scope: autoScope });
|
|
8109
|
-
return { jobTypes: [...new Set([...autoTypes, ...extraJobTypes])] };
|
|
8110
|
-
} catch (err) {
|
|
8111
|
-
return { skip: `engine read failed: ${err?.message || err}` };
|
|
8112
|
-
}
|
|
8113
|
-
}
|
|
8114
|
-
let stored;
|
|
8115
|
-
try {
|
|
8116
|
-
stored = readHiresStrict()[name];
|
|
8117
|
-
} catch {
|
|
8118
|
-
// config.json exists but doesn't parse (e.g. a torn write): the profile is
|
|
8119
|
-
// NOT necessarily gone, so don't claim it was deleted — skip this pass.
|
|
8120
|
-
return { skip: 'config unreadable' };
|
|
8121
|
-
}
|
|
8122
|
-
if (!stored) return { skip: 'deleted' };
|
|
8123
|
-
const norm = normalizeStoredProfile(name, stored);
|
|
8124
|
-
if (norm.error) return { skip: norm.error };
|
|
8125
|
-
const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
|
|
8126
|
-
return { jobTypes: [...new Set([...m, ...extraJobTypes])] };
|
|
8127
|
-
};
|
|
8128
|
-
|
|
8129
|
-
const reconcile = () => {
|
|
8130
|
-
if (draining) return inFlightReconcile || Promise.resolve();
|
|
8131
|
-
if (reconciling) {
|
|
8132
|
-
// A change landed mid-reconcile — remember it so the current pass loops
|
|
8133
|
-
// once more rather than leaving the worker set stale until the next edit.
|
|
8134
|
-
// Return the ACTUAL in-flight promise (not a fresh short-lived one) so a
|
|
8135
|
-
// caller — including shutdown — waits for the real reconcile to finish.
|
|
8136
|
-
reconcileRequested = true;
|
|
8137
|
-
return inFlightReconcile || Promise.resolve();
|
|
8138
|
-
}
|
|
8139
|
-
reconciling = true;
|
|
8140
|
-
reconcileRequested = false;
|
|
8141
|
-
inFlightReconcile = (async () => {
|
|
8142
|
-
try {
|
|
8143
|
-
do {
|
|
8144
|
-
reconcileRequested = false;
|
|
8145
|
-
await runReconcilePass();
|
|
8146
|
-
} while (reconcileRequested && !draining);
|
|
8147
|
-
} finally {
|
|
8148
|
-
reconciling = false;
|
|
8149
|
-
inFlightReconcile = null;
|
|
8150
|
-
}
|
|
8151
|
-
})();
|
|
8152
|
-
return inFlightReconcile;
|
|
8153
|
-
};
|
|
8154
|
-
|
|
8155
|
-
const runReconcilePass = async () => {
|
|
8156
|
-
const desired = await desiredJobTypes();
|
|
8157
|
-
if (desired.skip) {
|
|
8158
|
-
if (autoMode) {
|
|
8159
|
-
logger.warn(`--auto reconcile skipped — ${desired.skip}; keeping the current ${workers.size} worker(s) running.`);
|
|
8160
|
-
} else if (desired.skip === 'deleted') {
|
|
8161
|
-
logger.warn(`Profile "${name}" is gone from config — keeping the current ${workers.size} worker(s) running.`);
|
|
8162
|
-
} else {
|
|
8163
|
-
logger.warn(`Profile "${name}" reload skipped — ${desired.skip}; keeping current workers.`);
|
|
8164
|
-
}
|
|
8165
|
-
return;
|
|
8166
|
-
}
|
|
8167
|
-
const { added, removed } = diffJobTypes([...workers.keys()], desired.jobTypes);
|
|
8168
|
-
if (added.length === 0 && removed.length === 0) return;
|
|
8169
|
-
const source = autoMode ? 'engine deployed set' : `Profile "${name}"`;
|
|
8170
|
-
logger.info(`${source} changed — reconciling job types (+${added.length} / -${removed.length}).`);
|
|
8171
|
-
for (const jt of added) {
|
|
8172
|
-
spawnJobType(jt);
|
|
8173
|
-
logger.info(` + now listening on ${jt}`);
|
|
8174
|
-
}
|
|
8175
|
-
await Promise.all(
|
|
8176
|
-
removed.map(async (jt) => {
|
|
8177
|
-
const w = workers.get(jt);
|
|
8178
|
-
logger.info(` - draining ${jt} …`);
|
|
8179
|
-
const ok = await drainWorker(w);
|
|
8180
|
-
if (ok) {
|
|
8181
|
-
// Only drop it from the registry once it has actually stopped, so a
|
|
8182
|
-
// failed drain stays tracked and gets retried on the next reconcile
|
|
8183
|
-
// pass (or on shutdown) instead of leaking an untracked poller.
|
|
8184
|
-
workers.delete(jt);
|
|
8185
|
-
logger.info(` - stopped ${jt}`);
|
|
8186
|
-
} else {
|
|
8187
|
-
logger.warn(` - ${jt} did not stop cleanly; keeping it tracked so it is retried on the next reconcile or shutdown.`);
|
|
8188
|
-
}
|
|
8189
|
-
}),
|
|
8190
|
-
);
|
|
8191
|
-
logger.info(` now listening on ${workers.size} job type(s): ${[...workers.keys()].join(' ')}`);
|
|
8192
|
-
};
|
|
8193
|
-
|
|
8194
|
-
// Reconcile trigger. In `--auto` a periodic engine poll re-reads the deployed
|
|
8195
|
-
// agent job types; otherwise a profile-file watch fires on profile edits.
|
|
8196
|
-
let autoPollTimer = null;
|
|
8197
|
-
if (autoMode) {
|
|
8198
|
-
// Self-standing interval poll (not watchFile) since the desired set is
|
|
8199
|
-
// derived from the engine, not the on-disk profile. Skip a tick while a
|
|
8200
|
-
// reconcile is already in flight: calling reconcile() then would set
|
|
8201
|
-
// reconcileRequested and make the in-flight pass loop back-to-back, so an
|
|
8202
|
-
// engine read that consistently outlasts AUTO_POLL_INTERVAL_MS would run
|
|
8203
|
-
// reconciles as fast as the read completes and hammer the broker. Skipping
|
|
8204
|
-
// keeps polling rate-limited to the configured interval regardless of
|
|
8205
|
-
// engine-read latency; the next tick re-reads the latest engine state.
|
|
8206
|
-
autoPollTimer = setInterval(() => {
|
|
8207
|
-
if (inFlightReconcile) return;
|
|
8208
|
-
reconcile().catch((err) => logger.warn(`--auto reconcile failed: ${err?.message || err}`));
|
|
8209
|
-
}, AUTO_POLL_INTERVAL_MS);
|
|
8210
|
-
// Deliberately REF'd (unlike the reaper/run-dir hygiene timers, which are
|
|
8211
|
-
// unref'd): in `--auto` this poll IS the retry loop, and it must keep the
|
|
8212
|
-
// process alive even with zero pollers. When the INITIAL engine read fails
|
|
8213
|
-
// (transient miss, or the engine isn't up yet) the worker registers 0
|
|
8214
|
-
// pollers; nothing else holds the event loop open (the SDK client with no
|
|
8215
|
-
// job workers doesn't, and the hygiene timers are unref'd), so an unref'd
|
|
8216
|
-
// poll timer would let the process exit 0 — the observed crash-loop under a
|
|
8217
|
-
// supervisor (jwulf/c8ctl-plugin-nano#93). Keeping it ref'd makes the worker
|
|
8218
|
-
// stay up and re-read on the next poll, exactly as the initial-read warning
|
|
8219
|
-
// promises. Shutdown clears it (clearInterval), so Ctrl-C/SIGTERM still exit.
|
|
8220
|
-
} else {
|
|
8221
|
-
// `watchFile` (polling stat) is deliberate over `fs.watch`: it survives the
|
|
8222
|
-
// atomic temp+rename that `writeConfig` does (fs.watch would rebind to the old
|
|
8223
|
-
// inode and go silent), and it's uniform across platforms. Profile edits are
|
|
8224
|
-
// rare + manual, so a ~1.5s poll latency is fine.
|
|
7990
|
+
let draining = false;
|
|
7991
|
+
if (!autoMode) {
|
|
7992
|
+
// Serialize reloads on a chain (never overlap a `setTypes` write) and coalesce
|
|
7993
|
+
// with a monotonic generation guard, so a slow older reload can never apply a
|
|
7994
|
+
// stale job-type set after a newer edit has already superseded it.
|
|
7995
|
+
let reloadSeq = 0;
|
|
7996
|
+
let reloadChain = Promise.resolve();
|
|
8225
7997
|
watchFile(configFile, { interval: WATCH_INTERVAL_MS }, (curr, prev) => {
|
|
7998
|
+
// A callback can already be queued when teardown flips `draining`; bail so we
|
|
7999
|
+
// never write to the shared registry (or race its teardown) during shutdown.
|
|
8000
|
+
if (draining) return;
|
|
8226
8001
|
// Fires each interval; act only on real changes. Compare mtime, ctime and
|
|
8227
|
-
// size, not mtime alone:
|
|
8228
|
-
//
|
|
8229
|
-
// differ, and an mtime-only guard would skip a genuine profile update.
|
|
8002
|
+
// size, not mtime alone: coarse-mtime filesystems (or two edits in one tick)
|
|
8003
|
+
// can leave mtimeMs unchanged while size/ctimeMs differ.
|
|
8230
8004
|
if (
|
|
8231
8005
|
curr.mtimeMs === prev.mtimeMs &&
|
|
8232
8006
|
curr.ctimeMs === prev.ctimeMs &&
|
|
8233
8007
|
curr.size === prev.size
|
|
8234
8008
|
) return;
|
|
8235
|
-
|
|
8236
|
-
|
|
8237
|
-
|
|
8238
|
-
|
|
8009
|
+
const mySeq = ++reloadSeq;
|
|
8010
|
+
reloadChain = reloadChain.then(async () => {
|
|
8011
|
+
if (draining) return;
|
|
8012
|
+
// A newer edit already landed while we were queued — skip this stale
|
|
8013
|
+
// reload so its (older) job-type set never lands after the newer one.
|
|
8014
|
+
if (mySeq !== reloadSeq) return;
|
|
8015
|
+
let stored;
|
|
8016
|
+
try {
|
|
8017
|
+
stored = readHiresStrict()[name];
|
|
8018
|
+
} catch (err) {
|
|
8019
|
+
// config exists but doesn't parse (torn write) — keep current types
|
|
8020
|
+
logger.warn(`Profile "${name}" reload skipped — ${configFile} unreadable/unparseable (keeping current job types): ${err?.message || err}`);
|
|
8021
|
+
return;
|
|
8022
|
+
}
|
|
8023
|
+
if (!stored) {
|
|
8024
|
+
// profile vanished — keep serving the current types
|
|
8025
|
+
logger.warn(`Profile "${name}" reload skipped — profile no longer present in ${configFile} (keeping current job types)`);
|
|
8026
|
+
return;
|
|
8027
|
+
}
|
|
8028
|
+
const norm = normalizeStoredProfile(name, stored);
|
|
8029
|
+
if (norm.error) {
|
|
8030
|
+
// invalid edit — keep current types
|
|
8031
|
+
logger.warn(`Profile "${name}" reload skipped — invalid profile edit (keeping current job types): ${norm.error}`);
|
|
8032
|
+
return;
|
|
8033
|
+
}
|
|
8034
|
+
const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
|
|
8035
|
+
const desired = [...new Set([...m, ...extraJobTypes])];
|
|
8036
|
+
if (draining) return; // teardown began while we were reading — don't write
|
|
8037
|
+
if (mySeq !== reloadSeq) return; // superseded during the async read — skip
|
|
8038
|
+
await SupervisorEffect.runPromise(workerRegistry.setTypes(workerName, desired));
|
|
8039
|
+
logger.info(`Profile "${name}" changed — now servicing ${desired.length} job type(s): ${desired.join(' ')}`);
|
|
8040
|
+
}).catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
|
|
8239
8041
|
});
|
|
8240
8042
|
}
|
|
8241
8043
|
|
|
8242
|
-
// Keep the process alive until a stop signal, then
|
|
8044
|
+
// Keep the process alive until a stop signal, then interrupt the runtime loop
|
|
8045
|
+
// and tear down visibility. Interrupting the supervisor fiber runs the runtime's
|
|
8046
|
+
// bracketed teardown (release slots, stop the heartbeat + agentic scope).
|
|
8243
8047
|
await new Promise((resolve) => {
|
|
8244
8048
|
const stop = async (signal) => {
|
|
8245
8049
|
if (draining) return;
|
|
8246
8050
|
draining = true;
|
|
8247
|
-
|
|
8248
|
-
|
|
8249
|
-
// `workers` — this prevents double-stops, missed drains, or a wrong worker
|
|
8250
|
-
// count on exit.
|
|
8251
|
-
if (autoPollTimer) clearInterval(autoPollTimer);
|
|
8252
|
-
else unwatchFile(configFile);
|
|
8253
|
-
if (inFlightReconcile) {
|
|
8254
|
-
logger.info('Waiting for in-flight reconcile to finish before shutdown…');
|
|
8255
|
-
await inFlightReconcile;
|
|
8256
|
-
}
|
|
8257
|
-
const list = [...workers.values()];
|
|
8258
|
-
logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
|
|
8051
|
+
if (!autoMode) unwatchFile(configFile);
|
|
8052
|
+
logger.info(`Received ${signal} — stopping worker...`);
|
|
8259
8053
|
if (reaperTimer) clearInterval(reaperTimer);
|
|
8260
8054
|
if (runDirTimer) clearInterval(runDirTimer);
|
|
8261
8055
|
// Stop the #144 liveness watchdog so it can't kick off a re-discovery
|
|
8262
8056
|
// mid-teardown (which would resurrect the channel we're about to close).
|
|
8263
8057
|
if (agenticWatchdog) { try { agenticWatchdog.stop(); } catch { /* best effort */ } agenticWatchdog = null; }
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
logger.info('All workers stopped.');
|
|
8058
|
+
try {
|
|
8059
|
+
await SupervisorEffect.runPromise(SupervisorFiber.interrupt(supervisorFiber));
|
|
8060
|
+
logger.info('Worker stopped.');
|
|
8061
|
+
} catch (err) {
|
|
8062
|
+
logger.warn(`supervisor shutdown error — runtime loop may not have shut down cleanly: ${err?.message || err}`);
|
|
8270
8063
|
}
|
|
8271
|
-
// Deregister from the visibility channel LAST, so the worker disappears
|
|
8272
|
-
//
|
|
8273
|
-
//
|
|
8064
|
+
// Deregister from the visibility channel LAST, so the worker disappears from
|
|
8065
|
+
// the page only once its jobs have drained. Best-effort — a channel teardown
|
|
8066
|
+
// must never hang shutdown.
|
|
8274
8067
|
if (workChannel) {
|
|
8275
|
-
// Stop the buffer monitor first so its sampler can't fire mid-teardown.
|
|
8276
8068
|
try {
|
|
8277
8069
|
bufferMonitor?.stop();
|
|
8278
8070
|
} catch { /* best effort */ }
|
|
@@ -12692,8 +12484,6 @@ export {
|
|
|
12692
12484
|
resolveLivenessOverrides,
|
|
12693
12485
|
parsePsTime,
|
|
12694
12486
|
ensureAcpFlag,
|
|
12695
|
-
startLockExtender,
|
|
12696
|
-
createSingleFlight,
|
|
12697
12487
|
provisionRepo,
|
|
12698
12488
|
finalizeGit,
|
|
12699
12489
|
describeGitFailure,
|