c8ctl-plugin-nano 1.51.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic-endpoint.mjs +27 -4
- package/c8ctl-plugin.js +317 -649
- package/package.json +10 -9
- package/supervisor.dist.js +2 -2
- package/work-relay.mjs +147 -0
package/c8ctl-plugin.js
CHANGED
|
@@ -63,9 +63,9 @@ import { createInterface } from 'node:readline/promises';
|
|
|
63
63
|
import { StringDecoder } from 'node:string_decoder';
|
|
64
64
|
import { createInterface as createReadline, cursorTo as rlCursorTo, moveCursor as rlMoveCursor, clearScreenDown as rlClearScreenDown } from 'node:readline';
|
|
65
65
|
import { platformForHost } from './platforms.mjs';
|
|
66
|
-
import {
|
|
67
|
-
import {
|
|
68
|
-
import {
|
|
66
|
+
import { redactAgenticUrl, buildAgenticUrl } from './work-channel.mjs';
|
|
67
|
+
import { createHostRelaySession, roleTerminalMode } from './work-relay.mjs';
|
|
68
|
+
import { resolveBufferCapacity } from './work-buffer.mjs';
|
|
69
69
|
// Canonical ACP → transcript wire bridge (nanobpm/nano-ide#534), consumed through
|
|
70
70
|
// the single agentic import surface. `acpUpdateToTranscriptChunk(update)` maps one
|
|
71
71
|
// raw ACP `session/update` to the exact transcript-chunk bytes the cockpit decodes,
|
|
@@ -136,11 +136,6 @@ const READINESS_TIMEOUT_MS = 60_000;
|
|
|
136
136
|
const READINESS_POLL_MS = 500;
|
|
137
137
|
const HEALTH_TIMEOUT_MS = 1_500;
|
|
138
138
|
const STOP_GRACE_MS = 8_000;
|
|
139
|
-
// Backoff applied when a poller fails a lease fast because the worker is already
|
|
140
|
-
// running another job (issue #142 single-flight). Long enough that the deferred
|
|
141
|
-
// job doesn't tight-loop re-activating while the first runs, short enough that it
|
|
142
|
-
// is picked up promptly once the worker frees up.
|
|
143
|
-
const WORKER_BUSY_RETRY_BACKOFF_MS = 5_000;
|
|
144
139
|
// Upper bound on one `--auto` engine-read reconcile (enumerate deployed
|
|
145
140
|
// definitions + fetch each BPMN). A read that stalls past this is treated as a
|
|
146
141
|
// transient failure so the running poller set is KEPT and, crucially, shutdown
|
|
@@ -3288,14 +3283,12 @@ async function createAgenticEndpoint(opts) {
|
|
|
3288
3283
|
//
|
|
3289
3284
|
// Returns everything a JS caller needs to run `makeSupervisor` — the assembled
|
|
3290
3285
|
// `deps`, the seeded `registry` (so workers can be add/remove'd live), and the
|
|
3291
|
-
// `makeSupervisor` + `Effect` handles from the bundle. NOTE (
|
|
3292
|
-
//
|
|
3293
|
-
//
|
|
3294
|
-
//
|
|
3295
|
-
//
|
|
3296
|
-
//
|
|
3297
|
-
// this seam unblocks. Constructing deps is side-effect-free (no socket opens, no
|
|
3298
|
-
// activation) until the caller forks `supervisor.run`.
|
|
3286
|
+
// `makeSupervisor` + `Effect` handles from the bundle. NOTE (issue #172): the
|
|
3287
|
+
// hot-path flip this seam unblocks is now DONE — `workAgent` runs
|
|
3288
|
+
// `makeSupervisor(deps).run` as the single per-host owner, having retired the
|
|
3289
|
+
// per-type SDK pollers, the process-wide `singleFlight`, the per-process reconcile
|
|
3290
|
+
// crawl, and the per-job lock extender. Constructing deps stays side-effect-free
|
|
3291
|
+
// (no socket opens, no activation) until the caller forks `supervisor.run`.
|
|
3299
3292
|
//
|
|
3300
3293
|
// @param {object} opts
|
|
3301
3294
|
// @param {{ run(job): Promise<void> }} opts.runner raw job runner (required to run)
|
|
@@ -5529,80 +5522,12 @@ function baseAgentEnv(profile, job) {
|
|
|
5529
5522
|
};
|
|
5530
5523
|
}
|
|
5531
5524
|
|
|
5532
|
-
|
|
5533
|
-
|
|
5534
|
-
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
* serving N job types could lease and run up to N jobs at once — each holding its
|
|
5539
|
-
* own PTY + git workspace + broker lock-extender — the exact failure the
|
|
5540
|
-
* "one job per worker" invariant exists to prevent.
|
|
5541
|
-
*
|
|
5542
|
-
* This is a capacity-1, non-blocking mutex shared by EVERY per-type poller: the
|
|
5543
|
-
* first poller to `tryAcquire()` runs its job to completion (releasing in a
|
|
5544
|
-
* `finally`); any other poller that finds the permit already held must NOT begin
|
|
5545
|
-
* a second job (the caller fails the lease fast so the broker re-queues it rather
|
|
5546
|
-
* than leaving it "claimed but idle"). `tryAcquire`/`release` are synchronous
|
|
5547
|
-
* check-and-set, so the single-threaded event loop makes them race-free across
|
|
5548
|
-
* the concurrently-invoked async job handlers.
|
|
5549
|
-
*/
|
|
5550
|
-
function createSingleFlight() {
|
|
5551
|
-
let held = false;
|
|
5552
|
-
return {
|
|
5553
|
-
/** Take the permit if free; returns false when a job is already in flight. */
|
|
5554
|
-
tryAcquire() {
|
|
5555
|
-
if (held) return false;
|
|
5556
|
-
held = true;
|
|
5557
|
-
return true;
|
|
5558
|
-
},
|
|
5559
|
-
/** Release the permit. Idempotent: redundant calls are safe no-ops, though the normal path releases once per acquire (in a `finally`). */
|
|
5560
|
-
release() {
|
|
5561
|
-
held = false;
|
|
5562
|
-
},
|
|
5563
|
-
/** True while a job holds the permit. */
|
|
5564
|
-
get busy() {
|
|
5565
|
-
return held;
|
|
5566
|
-
},
|
|
5567
|
-
};
|
|
5568
|
-
}
|
|
5569
|
-
|
|
5570
|
-
/**
|
|
5571
|
-
* Keep a leased job's broker activation lock ahead of *now* while the harness is
|
|
5572
|
-
* running, so a long agent run never has its lock lapse and get re-activated (a
|
|
5573
|
-
* second worker starting → the classic stale complete/fail 409). The lock is NOT
|
|
5574
|
-
* hardcoded up front: we refresh it to `windowMs` — a duration-from-now, per the
|
|
5575
|
-
* UpdateJobTimeout contract ("the duration of the new timeout in ms, starting
|
|
5576
|
-
* from the current moment"), so calls SET rather than accumulate — every
|
|
5577
|
-
* `intervalMs`. The deadline therefore stays a bounded `windowMs` ahead of now.
|
|
5578
|
-
* The instant we stop refreshing (harness exit / idle-kill / hard cap) the lock
|
|
5579
|
-
* lapses within `windowMs` and the broker reclaims the job — fast node-loss
|
|
5580
|
-
* recovery. Because the harness is always killed locally before we stop, the lock
|
|
5581
|
-
* strictly outlives our local run, so a reclaim never races a still-running agent.
|
|
5582
|
-
*
|
|
5583
|
-
* Returns a stop() to call once the run settles. Extension failures are logged
|
|
5584
|
-
* and swallowed — a transient network blip must not crash the job handler. Older
|
|
5585
|
-
* SDKs without `modifyJobTimeout` degrade to the fixed initial lock (a no-op stop).
|
|
5586
|
-
*/
|
|
5587
|
-
function startLockExtender(job, windowMs, intervalMs, tag, logger) {
|
|
5588
|
-
if (!(windowMs > 0) || !(intervalMs > 0)) {
|
|
5589
|
-
return () => {};
|
|
5590
|
-
}
|
|
5591
|
-
if (typeof job?.modifyJobTimeout !== 'function') {
|
|
5592
|
-
logger?.warn?.(`${tag}: job.modifyJobTimeout unavailable — activation lock will NOT be auto-extended; a run longer than ${windowMs}ms risks being reclaimed and executed twice`);
|
|
5593
|
-
return () => {};
|
|
5594
|
-
}
|
|
5595
|
-
const extend = () => Promise.resolve()
|
|
5596
|
-
.then(() => job.modifyJobTimeout({ newTimeoutMs: windowMs }))
|
|
5597
|
-
.catch((err) => logger?.warn?.(`${tag}: lock extend failed — ${err?.message ?? err}`));
|
|
5598
|
-
// Renew immediately so the harness starts with a full, fresh window no matter
|
|
5599
|
-
// how much of the initial activation lease provisioning (clone/checkout) ate.
|
|
5600
|
-
extend();
|
|
5601
|
-
const timer = setInterval(extend, intervalMs);
|
|
5602
|
-
// Never let the heartbeat keep the process alive on shutdown.
|
|
5603
|
-
if (typeof timer.unref === 'function') timer.unref();
|
|
5604
|
-
return () => clearInterval(timer);
|
|
5605
|
-
}
|
|
5525
|
+
// Issue #172 retired `createSingleFlight` (the process-wide capacity-1 mutex the
|
|
5526
|
+
// per-type SDK pollers shared) and `startLockExtender` (the per-job broker-lock
|
|
5527
|
+
// heartbeat). Both are now owned by the single-owner supervisor runtime: race-free
|
|
5528
|
+
// per-type slot accounting lives in `supervisor/src/registry.ts` (this worker
|
|
5529
|
+
// registers with capacity 1), and the lock lifecycle — extend-winner-before-start
|
|
5530
|
+
// plus a `Schedule`-driven heartbeat — lives in `supervisor/src/dispatch.ts`.
|
|
5606
5531
|
|
|
5607
5532
|
/**
|
|
5608
5533
|
* Run a single activated job through the profile's CLI command (one-shot),
|
|
@@ -7032,26 +6957,21 @@ async function workAgent(req, flags) {
|
|
|
7032
6957
|
};
|
|
7033
6958
|
// One job per worker, hard-wired (there is deliberately no --max-parallel
|
|
7034
6959
|
// flag): an agent harness holds a PTY + a git workspace for the whole life of
|
|
7035
|
-
// a job, so a worker must never lease a second job concurrently.
|
|
7036
|
-
//
|
|
7037
|
-
//
|
|
7038
|
-
|
|
7039
|
-
//
|
|
7040
|
-
// only serializes ONE job-type poller, but this process runs one poller per
|
|
7041
|
-
// job type, so nothing stops N pollers from each leasing + running a job
|
|
7042
|
-
// concurrently. This capacity-1 mutex, shared by every poller's jobHandler,
|
|
7043
|
-
// enforces the real "one job per worker" invariant: while any job is in flight
|
|
7044
|
-
// on any job type, no other poller starts a second one.
|
|
7045
|
-
const singleFlight = createSingleFlight();
|
|
6960
|
+
// a job, so a worker must never lease a second job concurrently. Issue #172:
|
|
6961
|
+
// this is now enforced by registering this worker with capacity 1 in the
|
|
6962
|
+
// single-owner runtime's shared registry (one worker, capacity 1, across all
|
|
6963
|
+
// job types) — retiring the process-wide capacity-1 single-flight the per-type
|
|
6964
|
+
// SDK pollers needed. Race-free slot accounting lives in the runtime registry.
|
|
7046
6965
|
// The broker job-activation lock is NOT hardcoded up front. A fixed timeout is
|
|
7047
6966
|
// impossible to size for an agent: too short reclaims a still-working job (a
|
|
7048
6967
|
// second agent starts + the stale complete/fail is rejected 409), too long
|
|
7049
|
-
// strands a dead worker's job. Instead the
|
|
7050
|
-
// `recovery-window` ahead of *now* while the harness runs
|
|
7051
|
-
//
|
|
7052
|
-
//
|
|
7053
|
-
// `idle-timeout` (max silence before
|
|
7054
|
-
// lock is held only while the agent is
|
|
6968
|
+
// strands a dead worker's job. Instead the runtime's dispatch lifecycle keeps
|
|
6969
|
+
// the lock a bounded `recovery-window` ahead of *now* while the harness runs
|
|
6970
|
+
// (extend-winner-before-start + a Schedule heartbeat in dispatch.ts), so long
|
|
6971
|
+
// runs never lose their lock, and a dead/killed worker's job is reclaimed
|
|
6972
|
+
// within one window. Liveness is enforced by `idle-timeout` (max silence before
|
|
6973
|
+
// the harness is killed as wedged), so the lock is held only while the agent is
|
|
6974
|
+
// alive AND producing output.
|
|
7055
6975
|
const recoveryWindowMs = intFlag(flags?.['recovery-window'], 5 * 60_000);
|
|
7056
6976
|
const idleTimeoutMs = intFlag(flags?.['idle-timeout'], 5 * 60_000);
|
|
7057
6977
|
// `--job-timeout` is now an OPTIONAL absolute hard cap on total harness runtime
|
|
@@ -7250,7 +7170,7 @@ async function workAgent(req, flags) {
|
|
|
7250
7170
|
}
|
|
7251
7171
|
const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
|
|
7252
7172
|
logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
|
|
7253
|
-
logger.info(` one job per worker (
|
|
7173
|
+
logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
|
|
7254
7174
|
// Warm the gh-token cache now, off the job-handling path: githubCloneToken()
|
|
7255
7175
|
// may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
|
|
7256
7176
|
// credential fallback, and doing that inside a job handler would block the
|
|
@@ -7316,58 +7236,54 @@ async function workAgent(req, flags) {
|
|
|
7316
7236
|
/* best effort — activity is advisory, never fail a job over it */
|
|
7317
7237
|
}
|
|
7318
7238
|
};
|
|
7319
|
-
// The
|
|
7320
|
-
//
|
|
7321
|
-
|
|
7322
|
-
|
|
7323
|
-
|
|
7324
|
-
|
|
7325
|
-
//
|
|
7326
|
-
//
|
|
7327
|
-
//
|
|
7328
|
-
|
|
7329
|
-
|
|
7330
|
-
|
|
7331
|
-
|
|
7332
|
-
|
|
7333
|
-
//
|
|
7334
|
-
//
|
|
7335
|
-
//
|
|
7336
|
-
|
|
7337
|
-
|
|
7338
|
-
|
|
7339
|
-
|
|
7340
|
-
|
|
7341
|
-
|
|
7342
|
-
|
|
7343
|
-
|
|
7344
|
-
|
|
7345
|
-
|
|
7346
|
-
// Maintain `activeJobs` unconditionally: it feeds both the supervisor activity
|
|
7347
|
-
// file (gated inside writeActivity) AND the agentic presence frame's live
|
|
7348
|
-
// jobKey set, so a standalone worker (no NANO_SUPERVISOR_ACTIVITY_FILE) still
|
|
7349
|
-
// reports its current jobs on the visibility page.
|
|
7239
|
+
// The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
|
|
7240
|
+
// host connection carries presence/steer/ownership/transcript for EVERY
|
|
7241
|
+
// supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
|
|
7242
|
+
// (where each process opened its own socket for a single identity). Built below
|
|
7243
|
+
// from the resolved connect config and injected into the runtime as
|
|
7244
|
+
// `deps.agenticEndpoint`; `agenticPlane` is late-bound to the running
|
|
7245
|
+
// supervisor's presence/steer/transcript seams once `makeSupervisor` returns, so
|
|
7246
|
+
// the per-job runner streams a job's terminal + accepts steer over that one
|
|
7247
|
+
// connection instead of a per-process channel.
|
|
7248
|
+
/** @type {import('./supervisor.dist.js').AgenticEndpoint | null} */
|
|
7249
|
+
let agenticEndpoint = null;
|
|
7250
|
+
/** @type {{ register: () => void, deregister: (reason?: string) => void, relaySessionFor: (jobKey: string|number) => (object|null) } | null} */
|
|
7251
|
+
let agenticPlane = null;
|
|
7252
|
+
// The presence attributes this worker announces on `register` (ENROLMENT
|
|
7253
|
+
// attributes, not routing tokens — jobKeys are carried by the explicit
|
|
7254
|
+
// claim/release ownership frames the dispatch lifecycle emits, never smuggled in
|
|
7255
|
+
// here). Reused for the initial seed and any resync.
|
|
7256
|
+
const agenticCapability = {
|
|
7257
|
+
cognition: profile.rank,
|
|
7258
|
+
family: profile.model || undefined,
|
|
7259
|
+
host: hostname(),
|
|
7260
|
+
};
|
|
7261
|
+
// Maintain `activeJobs` unconditionally: it feeds the supervisor activity file
|
|
7262
|
+
// (gated inside writeActivity) so a standalone worker still reports its current
|
|
7263
|
+
// jobs. Live presence/ownership jobKeys are now owned by the runtime — the
|
|
7264
|
+
// dispatch claim/release lifecycle keyed by this worker's instance drives the
|
|
7265
|
+
// cockpit's jobKeys — so the recorders no longer poke a per-process channel.
|
|
7350
7266
|
const recordJobStart = (job, jobType) => {
|
|
7351
7267
|
activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now() });
|
|
7352
7268
|
writeActivity();
|
|
7353
|
-
workChannel?.refreshPresence();
|
|
7354
7269
|
};
|
|
7355
7270
|
const recordJobEnd = (job) => {
|
|
7356
7271
|
activeJobs.delete(String(job.jobKey));
|
|
7357
7272
|
writeActivity();
|
|
7358
|
-
workChannel?.refreshPresence();
|
|
7359
7273
|
};
|
|
7360
7274
|
// Seed an initial idle marker so status reports 'idle' immediately after spawn.
|
|
7361
7275
|
writeActivity();
|
|
7362
7276
|
|
|
7363
|
-
// ---- Agentic visibility
|
|
7364
|
-
//
|
|
7365
|
-
//
|
|
7366
|
-
//
|
|
7367
|
-
//
|
|
7368
|
-
//
|
|
7369
|
-
//
|
|
7370
|
-
//
|
|
7277
|
+
// ---- Agentic visibility connection (ADR 0056; issue #173) -----------------
|
|
7278
|
+
// Resolve WHERE this worker's presence is announced, then hand the connect
|
|
7279
|
+
// config to the single-owner supervisor runtime as its ONE multiplexed host
|
|
7280
|
+
// connection (built below). The supervisor announces presence (identity, host),
|
|
7281
|
+
// heartbeats it, claims/releases jobs, and streams transcript + accepts steer
|
|
7282
|
+
// for EVERY supervised agent over that single connection — replacing the retired
|
|
7283
|
+
// per-`work`-process `createWorkChannel` fan-out where each process opened its
|
|
7284
|
+
// own socket for a single identity. The sibling data planes (C3 PTY relay #42,
|
|
7285
|
+
// steer #163) now ride this connection via the runtime's transcript/steer seams
|
|
7286
|
+
// rather than a per-process channel.
|
|
7371
7287
|
//
|
|
7372
7288
|
// Local-first (security opt-in): visibility is ON BY DEFAULT. In LOCAL mode the
|
|
7373
7289
|
// worker joins with the well-known LOCAL token and no credential; SECURE mode
|
|
@@ -7413,226 +7329,67 @@ async function workAgent(req, flags) {
|
|
|
7413
7329
|
// the socket opens (or without a channel at all).
|
|
7414
7330
|
writeActivity();
|
|
7415
7331
|
// Track the live connection state on the activity marker so the supervisor
|
|
7416
|
-
// shows connected↔disconnected transitions (#99).
|
|
7417
|
-
//
|
|
7418
|
-
//
|
|
7332
|
+
// shows connected↔disconnected transitions (#99). Diagnostics ride the contract
|
|
7333
|
+
// `agentic.message` field (not `reason`): the endpoint's connection observer
|
|
7334
|
+
// only reports a state (`connected`/`disconnected`) — the underlying close does
|
|
7335
|
+
// not surface a reason — so a hub drop records a generic 'connection dropped'
|
|
7336
|
+
// message, while an endpoint-construction failure records the normalized error
|
|
7337
|
+
// (see the catch below). A fresh (re)connect clears any stale message.
|
|
7419
7338
|
const markAgentic = (status, message = null) => { agenticState = { ...agenticState, status, message }; writeActivity(); };
|
|
7420
|
-
//
|
|
7421
|
-
//
|
|
7422
|
-
//
|
|
7423
|
-
//
|
|
7424
|
-
//
|
|
7425
|
-
|
|
7339
|
+
// Build the ONE multiplexed host-connection endpoint (issue #173) for the
|
|
7340
|
+
// resolved connect config and hand it to the single-owner runtime as
|
|
7341
|
+
// `deps.agenticEndpoint`. This RETIRES the per-`work`-process createWorkChannel
|
|
7342
|
+
// fan-out: the supervisor now owns exactly one connection carrying presence
|
|
7343
|
+
// (register/heartbeat/deregister projected from the registry), ownership
|
|
7344
|
+
// (claim/release per job), inbound steer, and transcript for EVERY supervised
|
|
7345
|
+
// agent — each frame carrying its `instance` explicitly. The endpoint composes
|
|
7346
|
+
// the raw-JS wire (`agentic-endpoint.mjs` → the single agentic import surface)
|
|
7347
|
+
// with the bundle's Effect adapter; additive negotiation still degrades
|
|
7348
|
+
// claim/release/steer to a no-op against an older hub. Reconnect + resync (and
|
|
7349
|
+
// teardown-on-interruption) are owned by the runtime's `superviseAgentic`, so the
|
|
7350
|
+
// retired #133 self-heal loop and #144/#147 liveness watchdog are gone — the
|
|
7351
|
+
// marker's agentic status is driven by the endpoint's connection observer below.
|
|
7352
|
+
// Constructing the endpoint opens NO socket (the runtime's supervision calls the
|
|
7353
|
+
// connect factory), so this stays cheap and side-effect-free until `run` forks.
|
|
7354
|
+
if (agenticCfg) {
|
|
7426
7355
|
try {
|
|
7427
|
-
|
|
7428
|
-
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
|
|
7356
|
+
agenticEndpoint = await createAgenticEndpoint({
|
|
7357
|
+
url: agenticCfg.url,
|
|
7358
|
+
token: agenticCfg.token,
|
|
7359
|
+
credential: agenticCfg.credential,
|
|
7360
|
+
// Keep the activity marker's agentic status honest across the single
|
|
7361
|
+
// connection's open/drop transitions (#99) — the direct analogue of the
|
|
7362
|
+
// retired createWorkChannel onConnect/onDisconnect marker wiring.
|
|
7363
|
+
onConnectionState: (state) => {
|
|
7364
|
+
if (state === 'connected') markAgentic('connected');
|
|
7365
|
+
else markAgentic('disconnected', 'connection dropped');
|
|
7434
7366
|
},
|
|
7435
|
-
listJobKeys: () => [...activeJobs.keys()],
|
|
7436
|
-
url: cfg.url,
|
|
7437
|
-
token: cfg.token,
|
|
7438
|
-
credential: cfg.credential,
|
|
7439
|
-
bufferCapacity: cfg.bufferCapacity,
|
|
7440
|
-
// #147: de-synchronise reconnect attempts with equal-jitter backoff so a
|
|
7441
|
-
// fleet dropped on the same lossy link does not reconnect in lockstep and
|
|
7442
|
-
// re-congest it. Wraps the client lib's own exponential policy (which has
|
|
7443
|
-
// no jitter of its own); the base delays/factor stay the lib's defaults.
|
|
7444
|
-
schedule: makeJitteredReconnectSchedule(),
|
|
7445
7367
|
logger,
|
|
7446
7368
|
});
|
|
7447
|
-
const shown = redactAgenticUrl(buildAgenticUrl(cfg.url, {}));
|
|
7448
|
-
const mode = cfg.secure ? 'secure' : 'local';
|
|
7449
|
-
if (cfg.discovered) {
|
|
7450
|
-
const d = cfg.discovered;
|
|
7451
|
-
logger.info(` agentic channel: auto-discovered ${d.project} on the app's /agentic port ${wsHostPart(d.host)}:${d.port} (bypassing the WS-incapable console proxy).`);
|
|
7452
|
-
}
|
|
7453
|
-
logger.info(` agentic channel (${mode}): announcing presence as ${workerName} on ${shown}`);
|
|
7454
|
-
// onConnect fires only for listeners present at first open, so also
|
|
7455
|
-
// reconcile the already-open case synchronously via connected(). If the
|
|
7456
|
-
// socket opened and then dropped inside the createWorkChannel() await window
|
|
7457
|
-
// (before these listeners existed), connected() is false but everConnected()
|
|
7458
|
-
// is true — record that as `disconnected` rather than leaving it stuck at
|
|
7459
|
-
// `connecting`.
|
|
7460
|
-
// #144: track the drop clock alongside presence — a (re)connect clears it,
|
|
7461
|
-
// a disconnect starts it (first drop wins, so the watchdog measures from the
|
|
7462
|
-
// ORIGINAL drop, not the latest of a reconnect storm). The watchdog reads
|
|
7463
|
-
// this to decide when the client lib has failed to self-heal.
|
|
7464
|
-
workChannel.onConnect(() => {
|
|
7465
|
-
markAgentic('connected');
|
|
7466
|
-
agenticDisconnectedSince = null;
|
|
7467
|
-
agenticConnectedSince = Date.now();
|
|
7468
|
-
// First open drains the buffered REGISTER → presence lands; seed the
|
|
7469
|
-
// presence-health clock so the #147 trigger measures from here (#147).
|
|
7470
|
-
agenticPresenceHealthyAt = Date.now();
|
|
7471
|
-
});
|
|
7472
|
-
workChannel.onReconnect(() => {
|
|
7473
|
-
markAgentic('connected');
|
|
7474
|
-
agenticDisconnectedSince = null;
|
|
7475
|
-
agenticConnectedSince = Date.now();
|
|
7476
|
-
// Deliberately do NOT advance agenticPresenceHealthyAt here: a reconnect
|
|
7477
|
-
// only CLAIMS presence (re-announces). On a lossy link the socket may
|
|
7478
|
-
// re-drop `1006` before presence actually lands, so the watchdog confirms
|
|
7479
|
-
// it only once a connection HOLDS for the grace window — a reconnect that
|
|
7480
|
-
// immediately re-drops must not mask an unrecovered presence (#147).
|
|
7481
|
-
});
|
|
7482
|
-
workChannel.onDisconnect((info) => {
|
|
7483
|
-
markAgentic('disconnected', normalizeAgenticMessage(info));
|
|
7484
|
-
if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
|
|
7485
|
-
agenticConnectedSince = null;
|
|
7486
|
-
});
|
|
7487
|
-
if (workChannel.connected()) {
|
|
7488
|
-
markAgentic('connected');
|
|
7489
|
-
agenticDisconnectedSince = null;
|
|
7490
|
-
agenticConnectedSince = Date.now();
|
|
7491
|
-
if (agenticPresenceHealthyAt == null) agenticPresenceHealthyAt = Date.now();
|
|
7492
|
-
} else if (workChannel.everConnected()) {
|
|
7493
|
-
markAgentic('disconnected');
|
|
7494
|
-
if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
|
|
7495
|
-
agenticConnectedSince = null;
|
|
7496
|
-
}
|
|
7497
7369
|
} catch (err) {
|
|
7498
|
-
// Never let
|
|
7499
|
-
|
|
7500
|
-
//
|
|
7501
|
-
|
|
7502
|
-
// The contract diagnostic field is `agentic.message` (#99), matching the
|
|
7503
|
-
// live-disconnect path above — keep the key consistent, not `reason`.
|
|
7370
|
+
// Never let endpoint construction stop the worker from doing its job — the
|
|
7371
|
+
// agentic plane is best-effort visibility. Record the failure on the marker
|
|
7372
|
+
// and run the supervisor with no agenticEndpoint (presence/steer become no-ops).
|
|
7373
|
+
agenticEndpoint = null;
|
|
7504
7374
|
agenticState = { ...agenticState, status: 'disconnected', message: normalizeAgenticMessage(err) };
|
|
7505
7375
|
writeActivity();
|
|
7506
|
-
logger.warn(` agentic
|
|
7507
|
-
return;
|
|
7376
|
+
logger.warn(` agentic endpoint unavailable (${err?.message || err}); continuing without visibility.`);
|
|
7508
7377
|
}
|
|
7509
|
-
// C4 (#43): observe the client's built-in outbound buffer across the
|
|
7510
|
-
// channel lifecycle — surface a high-water mark and warn when the bound
|
|
7511
|
-
// is hit so a hub outage that starts shedding frames is never silent. The
|
|
7512
|
-
// monitor is observability-only, so keep it OUTSIDE the channel try/catch:
|
|
7513
|
-
// a monitor failure must never null out a healthy channel and take down
|
|
7514
|
-
// presence/visibility.
|
|
7515
|
-
if (workChannel) {
|
|
7516
|
-
try {
|
|
7517
|
-
bufferMonitor = createBufferMonitor(workChannel, {
|
|
7518
|
-
capacity: cfg.bufferCapacity,
|
|
7519
|
-
logger,
|
|
7520
|
-
});
|
|
7521
|
-
} catch (err) {
|
|
7522
|
-
bufferMonitor = null;
|
|
7523
|
-
logger.warn(` agentic buffer monitor unavailable (${err?.message || err}); channel presence still active.`);
|
|
7524
|
-
}
|
|
7525
|
-
}
|
|
7526
|
-
};
|
|
7527
|
-
|
|
7528
|
-
// (A) Background self-heal loop, shared by the cold-start advisory path (#133)
|
|
7529
|
-
// AND the #144 liveness watchdog. Re-run discovery on a jittered backoff and,
|
|
7530
|
-
// on the first `connect` target, (re)open the channel WITHOUT a restart. The
|
|
7531
|
-
// `agenticSelfHealing` guard makes it idempotent: the watchdog can call it
|
|
7532
|
-
// after tearing a stale channel down without racing a still-running cold-start
|
|
7533
|
-
// loop. A shared cache lets a brief blip reuse the last known-good hub (#133-C).
|
|
7534
|
-
const armAgenticSelfHeal = () => {
|
|
7535
|
-
if (agenticSelfHealing) return; // a re-discovery loop is already running
|
|
7536
|
-
if (workChannel !== null) return; // a channel already exists — nothing to heal
|
|
7537
|
-
agenticSelfHealing = true;
|
|
7538
|
-
const hubCache = new Map();
|
|
7539
|
-
rediscoverAgenticUntilConnected({
|
|
7540
|
-
resolveTarget: () => resolveAgenticTarget({ camunda, logger, cache: hubCache }),
|
|
7541
|
-
onConnect: async (target) => {
|
|
7542
|
-
agenticCfg = target.config;
|
|
7543
|
-
agenticState = agenticStateForTarget(target, safeAgenticDisplayUrl);
|
|
7544
|
-
writeActivity();
|
|
7545
|
-
logger.info(' agentic channel: background re-discovery succeeded — (re)opening channel.');
|
|
7546
|
-
await openAgenticChannel(agenticCfg);
|
|
7547
|
-
// openAgenticChannel swallows its own open failures (it nulls
|
|
7548
|
-
// workChannel and returns rather than throwing), so a failed open must
|
|
7549
|
-
// be re-thrown here — otherwise the self-heal loop treats this attempt
|
|
7550
|
-
// as success and stops retrying with workChannel still null (#133).
|
|
7551
|
-
if (workChannel === null) {
|
|
7552
|
-
throw new Error('agentic channel failed to open after background re-discovery');
|
|
7553
|
-
}
|
|
7554
|
-
},
|
|
7555
|
-
// Stop as soon as a channel exists (loop won this or a prior attempt did).
|
|
7556
|
-
shouldContinue: () => workChannel === null,
|
|
7557
|
-
logger,
|
|
7558
|
-
})
|
|
7559
|
-
.catch(() => { /* best-effort self-heal — never surfaces an error */ })
|
|
7560
|
-
.finally(() => { agenticSelfHealing = false; });
|
|
7561
|
-
};
|
|
7562
|
-
|
|
7563
|
-
// (B) #144 liveness watchdog: force-heal a wedged channel. When a channel that
|
|
7564
|
-
// HAS connected drops and the client lib's own reconnect never brings it back
|
|
7565
|
-
// within the stale threshold (a half-open drop after a server restart/crash/
|
|
7566
|
-
// partition, or a reconnect that keeps failing), the client sits `disconnected`
|
|
7567
|
-
// forever and the worker vanishes from the Workers view until a supervisor
|
|
7568
|
-
// restart. This tears the wedged channel down (so `shouldContinue` re-arms) and
|
|
7569
|
-
// re-runs full discovery + reopen instead of trusting the client lib alone.
|
|
7570
|
-
const healStaleAgenticChannel = async () => {
|
|
7571
|
-
const stale = workChannel;
|
|
7572
|
-
if (!stale) return;
|
|
7573
|
-
workChannel = null; // re-arms armAgenticSelfHeal()'s shouldContinue gate
|
|
7574
|
-
agenticDisconnectedSince = null; // reset the clock; the fresh open restarts it
|
|
7575
|
-
agenticConnectedSince = null; // #147: the fresh open re-seeds it
|
|
7576
|
-
agenticPresenceHealthyAt = null; // #147: the fresh open re-confirms presence
|
|
7577
|
-
try { bufferMonitor?.stop(); } catch { /* best effort */ }
|
|
7578
|
-
bufferMonitor = null;
|
|
7579
|
-
markAgentic('disconnected', 'stale channel — re-discovering hub');
|
|
7580
|
-
// Deregister + close the wedged client so it stops its own doomed reconnect
|
|
7581
|
-
// attempts and we don't leak two clients once the fresh one connects.
|
|
7582
|
-
try { await stale.stop('stale channel — re-discovering'); } catch { /* best effort */ }
|
|
7583
|
-
armAgenticSelfHeal();
|
|
7584
|
-
};
|
|
7585
|
-
|
|
7586
|
-
const startAgenticWatchdog = () => {
|
|
7587
|
-
if (agenticWatchdog) return;
|
|
7588
|
-
const staleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_STALE_MS, DEFAULT_AGENTIC_STALE_MS));
|
|
7589
|
-
const intervalMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_WATCHDOG_MS, DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS));
|
|
7590
|
-
const presenceStaleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_STALE_MS, DEFAULT_AGENTIC_PRESENCE_STALE_MS));
|
|
7591
|
-
const presenceGraceMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_GRACE_MS, DEFAULT_AGENTIC_PRESENCE_GRACE_MS));
|
|
7592
|
-
agenticWatchdog = startAgenticChannelWatchdog({
|
|
7593
|
-
getChannel: () => workChannel,
|
|
7594
|
-
disconnectedSince: () => agenticDisconnectedSince,
|
|
7595
|
-
// #147: presence-keyed trigger — heal reconnect-churn that never re-lands
|
|
7596
|
-
// presence, not just a sustained socket drop.
|
|
7597
|
-
connectedSince: () => agenticConnectedSince,
|
|
7598
|
-
presenceHealthySince: () => agenticPresenceHealthyAt,
|
|
7599
|
-
onPresenceHealthy: () => { agenticPresenceHealthyAt = Date.now(); },
|
|
7600
|
-
presenceStaleAfterMs,
|
|
7601
|
-
presenceGraceMs,
|
|
7602
|
-
onStale: healStaleAgenticChannel,
|
|
7603
|
-
staleAfterMs,
|
|
7604
|
-
intervalMs,
|
|
7605
|
-
logger,
|
|
7606
|
-
});
|
|
7607
|
-
};
|
|
7608
|
-
|
|
7609
|
-
if (agenticCfg) {
|
|
7610
|
-
await openAgenticChannel(agenticCfg);
|
|
7611
|
-
// Guard the connected channel: if it later drops and the client lib can't
|
|
7612
|
-
// recover it, the watchdog forces a full re-discovery + reopen (#144).
|
|
7613
|
-
startAgenticWatchdog();
|
|
7614
|
-
} else if (agenticTarget.status === 'advisory') {
|
|
7615
|
-
// A cold-start discovery miss leaves the worker `advisory`; the self-heal
|
|
7616
|
-
// loop upgrades it to `connected` without a restart (#133), and once a
|
|
7617
|
-
// channel exists the watchdog keeps it alive across later drops (#144).
|
|
7618
|
-
armAgenticSelfHeal();
|
|
7619
|
-
startAgenticWatchdog();
|
|
7620
7378
|
}
|
|
7621
|
-
|
|
7622
7379
|
// C3 (#42): the role's live-terminal mode — a full PTY (streamed on the relay
|
|
7623
7380
|
// lane when a relay session exists, steerable) or a plain pipe. Honors the
|
|
7624
7381
|
// vocab's per-role opt-in read off the hire profile (`terminal: pty|pipe`),
|
|
7625
7382
|
// with an env override for a one-off worker (`NANO_AGENTIC_TERMINAL`). The PTY
|
|
7626
7383
|
// itself is allocated locally regardless of enrollment; relay streaming (and
|
|
7627
|
-
// steer-in) only engages when the worker is enrolled
|
|
7628
|
-
// without
|
|
7629
|
-
//
|
|
7384
|
+
// steer-in) only engages when the worker is enrolled (a live agentic endpoint),
|
|
7385
|
+
// so without it there's simply no relay tap — the harness still runs on the
|
|
7386
|
+
// chosen local transport.
|
|
7630
7387
|
const envTerminal = (process.env.NANO_AGENTIC_TERMINAL || '').trim().toLowerCase();
|
|
7631
7388
|
const roleTerminal = (envTerminal === 'pty' || envTerminal === 'pipe')
|
|
7632
7389
|
? envTerminal
|
|
7633
7390
|
: roleTerminalMode(profile);
|
|
7634
|
-
if (
|
|
7635
|
-
logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the
|
|
7391
|
+
if (agenticEndpoint) {
|
|
7392
|
+
logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
|
|
7636
7393
|
}
|
|
7637
7394
|
|
|
7638
7395
|
// #110: the role's harness protocol (pipe|acp) and ACP permission policy
|
|
@@ -7646,46 +7403,20 @@ async function workAgent(req, flags) {
|
|
|
7646
7403
|
const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
|
|
7647
7404
|
const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
|
|
7648
7405
|
|
|
7649
|
-
//
|
|
7650
|
-
//
|
|
7651
|
-
//
|
|
7652
|
-
|
|
7653
|
-
|
|
7654
|
-
|
|
7655
|
-
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7662
|
-
|
|
7663
|
-
// anything — so the broker re-queues it (retries preserved) instead of it
|
|
7664
|
-
// sitting "claimed but idle" while the first job runs. Gating here, at the
|
|
7665
|
-
// point activation surfaces as a handler call, is the cross-poller gate
|
|
7666
|
-
// the per-type maxParallelJobs cannot provide.
|
|
7667
|
-
if (!singleFlight.tryAcquire()) {
|
|
7668
|
-
// Not a failure — preserve the broker-provided retries verbatim so
|
|
7669
|
-
// re-dispatch doesn't decrement (or resurrect) the job. Keep a real 0
|
|
7670
|
-
// as 0 (an already-incidentable job must stay that way); only default
|
|
7671
|
-
// to 1 when the count is missing/invalid.
|
|
7672
|
-
const rawRetries = Number(job.retries);
|
|
7673
|
-
const retries = Number.isInteger(rawRetries) && rawRetries >= 0 ? rawRetries : 1;
|
|
7674
|
-
logger.info(`[${jobType}] job ${job.jobKey} deferred — worker already running another job; releasing lease for re-dispatch.`);
|
|
7675
|
-
return job.fail({
|
|
7676
|
-
errorMessage: 'worker busy: one job per worker (single-flight across all job types)',
|
|
7677
|
-
retries,
|
|
7678
|
-
retryBackOff: WORKER_BUSY_RETRY_BACKOFF_MS,
|
|
7679
|
-
});
|
|
7680
|
-
}
|
|
7681
|
-
recordJobStart(job, jobType);
|
|
7682
|
-
// Auto-extend the broker lock for the whole life of this job (harness run
|
|
7683
|
-
// + git finalize + complete/fail), stopped in the outer finally. The lock
|
|
7684
|
-
// is held only while the harness stays alive and productive — a silent
|
|
7685
|
-
// hang is killed by the idle-timeout, which resolves runAgentJob and stops
|
|
7686
|
-
// the extension, so the broker can reclaim the job.
|
|
7687
|
-
let stopLockExtender = () => {};
|
|
7688
|
-
try {
|
|
7406
|
+
// The agent job runner (issue #172 hot-path flip). The single-owner supervisor
|
|
7407
|
+
// runtime dispatches each activated job to this `run(job)`; it executes the
|
|
7408
|
+
// harness exactly as the retired per-type SDK jobHandler did, but SETTLES via the
|
|
7409
|
+
// injected `settle` seam (engine complete/fail) because the plain ActivatedJob
|
|
7410
|
+
// carries no SDK `job.complete()/job.fail()`. Capacity (one job per worker), the
|
|
7411
|
+
// activation long-polls, the lock lifecycle, and reconcile are all owned by the
|
|
7412
|
+
// runtime now — retiring the per-type pollers, the process-wide single-flight,
|
|
7413
|
+
// the per-process 1+N reconcile crawl, and the per-job lock extender.
|
|
7414
|
+
let settle;
|
|
7415
|
+
const runner = {
|
|
7416
|
+
run: async (job) => {
|
|
7417
|
+
const jobType = job.type;
|
|
7418
|
+
recordJobStart(job, jobType);
|
|
7419
|
+
try {
|
|
7689
7420
|
logger.info(`[${jobType}] job ${job.jobKey} (instance ${job.processInstanceKey ?? '-'}) → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
|
|
7690
7421
|
|
|
7691
7422
|
// Disk-budget admission shed: if the engine data root is below the free
|
|
@@ -7697,7 +7428,7 @@ async function workAgent(req, flags) {
|
|
|
7697
7428
|
const freeMb = budget.free != null ? Math.round(budget.free / 1_048_576) : '?';
|
|
7698
7429
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7699
7430
|
logger.warn(`[${jobType}] job ${job.jobKey} shed — low disk (${freeMb}MB free); retries left ${retries}`);
|
|
7700
|
-
return
|
|
7431
|
+
return settle.fail(job.jobKey, { errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
|
|
7701
7432
|
}
|
|
7702
7433
|
}
|
|
7703
7434
|
|
|
@@ -7729,7 +7460,7 @@ async function workAgent(req, flags) {
|
|
|
7729
7460
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7730
7461
|
const msg = err instanceof ProvisionError ? err.message : `prompt resource fetch failed: ${err.message}`;
|
|
7731
7462
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7732
|
-
return
|
|
7463
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7733
7464
|
}
|
|
7734
7465
|
|
|
7735
7466
|
// Assemble + normalize the task envelope from headers (defaults) and
|
|
@@ -7740,27 +7471,24 @@ async function workAgent(req, flags) {
|
|
|
7740
7471
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7741
7472
|
const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
|
|
7742
7473
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7743
|
-
return
|
|
7474
|
+
return settle.fail(job.jobKey, { errorMessage: msg, retries });
|
|
7744
7475
|
}
|
|
7745
7476
|
|
|
7746
7477
|
// #130: per-task liveness overrides. Precedence: envelope override →
|
|
7747
7478
|
// worker-flag default → built-in default, each clamped to a sane max so
|
|
7748
7479
|
// a task can't request an unbounded window. Absent envelope fields leave
|
|
7749
|
-
// the worker-flag behaviour unchanged. These drive
|
|
7750
|
-
// liveness (idle/recovery)
|
|
7751
|
-
//
|
|
7480
|
+
// the worker-flag behaviour unchanged. These drive the harness idle
|
|
7481
|
+
// liveness (idle/recovery/hard-cap) so a JVM-heavy task can widen its own
|
|
7482
|
+
// window without a global flag change. NOTE: after the hot-path flip
|
|
7483
|
+
// (#172) the broker lock cadence/window is owned by the supervisor
|
|
7484
|
+
// runtime dispatch config at the worker level (the `dispatch` config's
|
|
7485
|
+
// `recoveryWindowMs` / `extendIntervalMs`), so a per-task override no
|
|
7486
|
+
// longer widens the broker lock window — only the harness liveness.
|
|
7752
7487
|
const {
|
|
7753
7488
|
idleTimeoutMs: effectiveIdleTimeoutMs,
|
|
7754
7489
|
recoveryWindowMs: effectiveRecoveryWindowMs,
|
|
7755
7490
|
hardCapMs: effectiveHardCapMs,
|
|
7756
7491
|
} = resolveLivenessOverrides(envelope.task, { idleTimeoutMs, recoveryWindowMs, hardCapMs });
|
|
7757
|
-
// Recompute the lock-extend cadence from the effective recovery window
|
|
7758
|
-
// (same ~1/3-of-window rule as at startup) so a widened window still
|
|
7759
|
-
// renews comfortably before it lapses.
|
|
7760
|
-
const effectiveLockExtendIntervalMs = Math.min(
|
|
7761
|
-
Math.max(5_000, Math.floor(effectiveRecoveryWindowMs / 3)),
|
|
7762
|
-
Math.max(1, Math.floor(effectiveRecoveryWindowMs * 0.75)),
|
|
7763
|
-
);
|
|
7764
7492
|
if (
|
|
7765
7493
|
effectiveIdleTimeoutMs !== idleTimeoutMs ||
|
|
7766
7494
|
effectiveRecoveryWindowMs !== recoveryWindowMs ||
|
|
@@ -7790,7 +7518,7 @@ async function workAgent(req, flags) {
|
|
|
7790
7518
|
: 'repository.url is missing';
|
|
7791
7519
|
const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
|
|
7792
7520
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7793
|
-
return
|
|
7521
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7794
7522
|
}
|
|
7795
7523
|
}
|
|
7796
7524
|
|
|
@@ -7800,12 +7528,10 @@ async function workAgent(req, flags) {
|
|
|
7800
7528
|
const hasRepo = !isContainer && !!envelope.repository?.url;
|
|
7801
7529
|
let runDir = null;
|
|
7802
7530
|
let provisioned = null;
|
|
7803
|
-
//
|
|
7804
|
-
//
|
|
7805
|
-
//
|
|
7806
|
-
//
|
|
7807
|
-
// duplicate-activation / stale-409 race. The `finally` below stops it.
|
|
7808
|
-
stopLockExtender = startLockExtender(job, effectiveRecoveryWindowMs, effectiveLockExtendIntervalMs, `[${jobType}] job ${job.jobKey}`, logger);
|
|
7531
|
+
// The broker activation lock is owned by the single-owner runtime's dispatch
|
|
7532
|
+
// lifecycle (supervisor/src/dispatch.ts): it extends the winner to the
|
|
7533
|
+
// recovery window BEFORE this runner starts and heartbeats it on a Schedule
|
|
7534
|
+
// for the whole run, so the retired per-job startLockExtender is gone here.
|
|
7809
7535
|
let cwd;
|
|
7810
7536
|
let extraEnv;
|
|
7811
7537
|
let repoToken = null;
|
|
@@ -7847,7 +7573,7 @@ async function workAgent(req, flags) {
|
|
|
7847
7573
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7848
7574
|
const msg = err instanceof ProvisionError ? err.message : `provisioning error: ${err.message}`;
|
|
7849
7575
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7850
|
-
return
|
|
7576
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7851
7577
|
}
|
|
7852
7578
|
} else if (!isContainer) {
|
|
7853
7579
|
// Repo-less host job (issue #129, hardening 1): nothing is provisioned,
|
|
@@ -7873,24 +7599,21 @@ async function workAgent(req, flags) {
|
|
|
7873
7599
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
7874
7600
|
const msg = `could not create a temp workspace under the runs root: ${err.message}`;
|
|
7875
7601
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
7876
|
-
return
|
|
7602
|
+
return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
7877
7603
|
}
|
|
7878
7604
|
}
|
|
7879
7605
|
|
|
7880
7606
|
let result;
|
|
7881
7607
|
let gitResult = null;
|
|
7882
|
-
//
|
|
7883
|
-
// harness terminal
|
|
7884
|
-
//
|
|
7885
|
-
//
|
|
7608
|
+
// The per-job live-terminal relay session (issue #173): streams this job's
|
|
7609
|
+
// harness terminal over the single-owner supervisor's ONE multiplexed host
|
|
7610
|
+
// connection, keyed by this worker's instance + the jobKey, and accepts
|
|
7611
|
+
// cockpit steer-in fanned back to this job's PTY by the runtime's steer
|
|
7612
|
+
// router. Only when the worker is enrolled (a live agentic plane); closed
|
|
7613
|
+
// in the finally so its steer subscription never leaks across jobs.
|
|
7886
7614
|
let relaySession = null;
|
|
7887
|
-
if (
|
|
7888
|
-
|
|
7889
|
-
relaySession = createRelaySession({ channel: workChannel, jobKey: job.jobKey, logger });
|
|
7890
|
-
} catch (err) {
|
|
7891
|
-
relaySession = null;
|
|
7892
|
-
logger.warn(`[${jobType}] job ${job.jobKey}: relay session unavailable (${err?.message || err}); continuing without live terminal.`);
|
|
7893
|
-
}
|
|
7615
|
+
if (agenticPlane) {
|
|
7616
|
+
relaySession = agenticPlane.relaySessionFor(job.jobKey);
|
|
7894
7617
|
}
|
|
7895
7618
|
// Private structured-result channel: hand the agent a file (outside any
|
|
7896
7619
|
// repo clone so it can't be `git add`ed) to write its job-result vars to.
|
|
@@ -8029,7 +7752,7 @@ async function workAgent(req, flags) {
|
|
|
8029
7752
|
const resultKeys = Object.keys(resultVars);
|
|
8030
7753
|
if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
|
|
8031
7754
|
else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
|
|
8032
|
-
return await
|
|
7755
|
+
return await settle.complete(job.jobKey, {
|
|
8033
7756
|
...resultVars,
|
|
8034
7757
|
[AGENT_RESULT_KEY]: resultEnvelope,
|
|
8035
7758
|
output: result.stdout,
|
|
@@ -8046,258 +7769,205 @@ async function workAgent(req, flags) {
|
|
|
8046
7769
|
|| (result.stderr || '').trim() + (result.stderrTruncated && (result.stderr || '').trim() ? ' [stderr truncated]' : '')
|
|
8047
7770
|
|| (result.signal ? `terminated by signal ${result.signal}` : `exit code ${result.exitCode}`);
|
|
8048
7771
|
logger.warn(`[${jobType}] job ${job.jobKey} failed (${detail}); retries left ${retries}`);
|
|
8049
|
-
return await
|
|
7772
|
+
return await settle.fail(job.jobKey, {
|
|
8050
7773
|
errorMessage: `agent "${profile.name}" failed: ${detail}`.slice(0, 2000),
|
|
8051
7774
|
retries,
|
|
8052
7775
|
variables: { [AGENT_RESULT_KEY]: resultEnvelope },
|
|
8053
7776
|
});
|
|
8054
|
-
|
|
8055
|
-
|
|
8056
|
-
recordJobEnd(job);
|
|
8057
|
-
// Release the process-wide single-flight permit LAST, once this job's
|
|
8058
|
-
// lock-extender is stopped and its bookkeeping cleared, so another
|
|
8059
|
-
// poller can only begin after this job is fully settled.
|
|
8060
|
-
singleFlight.release();
|
|
8061
|
-
}
|
|
8062
|
-
},
|
|
8063
|
-
});
|
|
8064
|
-
|
|
8065
|
-
// Live worker registry keyed by job type, so the profile watcher can add or
|
|
8066
|
-
// drain individual pollers without disturbing the others. `draining` is the
|
|
8067
|
-
// shutdown latch (shared with the watcher so a reconcile can't race a stop).
|
|
8068
|
-
const workers = new Map();
|
|
8069
|
-
let draining = false;
|
|
8070
|
-
|
|
8071
|
-
const drainWorker = async (w) => {
|
|
8072
|
-
try {
|
|
8073
|
-
if (typeof w.stopGracefully === 'function') {
|
|
8074
|
-
await w.stopGracefully({ waitUpToMs: STOP_GRACE_MS });
|
|
8075
|
-
} else if (typeof w.stop === 'function') {
|
|
8076
|
-
await w.stop();
|
|
7777
|
+
} finally {
|
|
7778
|
+
recordJobEnd(job);
|
|
8077
7779
|
}
|
|
8078
|
-
|
|
8079
|
-
} catch {
|
|
8080
|
-
return false; // best-effort: never let one worker's stop failure hang us
|
|
8081
|
-
}
|
|
8082
|
-
};
|
|
8083
|
-
|
|
8084
|
-
const spawnJobType = (jobType) => {
|
|
8085
|
-
if (workers.has(jobType)) return false;
|
|
8086
|
-
workers.set(jobType, makeWorker(jobType));
|
|
8087
|
-
return true;
|
|
7780
|
+
},
|
|
8088
7781
|
};
|
|
8089
7782
|
|
|
8090
|
-
|
|
8091
|
-
|
|
8092
|
-
//
|
|
8093
|
-
//
|
|
8094
|
-
|
|
8095
|
-
|
|
8096
|
-
|
|
8097
|
-
|
|
8098
|
-
|
|
8099
|
-
|
|
8100
|
-
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
|
|
8104
|
-
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8120
|
-
|
|
8121
|
-
|
|
8122
|
-
|
|
7783
|
+
// Compose the runtime deps over the plugin's real edges (issue #156 seam) and
|
|
7784
|
+
// register THIS worker (capacity 1) into the shared registry. In --auto the
|
|
7785
|
+
// runtime's reconcile loop rewrites this worker's serviceable types from the
|
|
7786
|
+
// engine read (`autoWorkerId`); otherwise the profile watch below does.
|
|
7787
|
+
const composed = await createSupervisorDeps({
|
|
7788
|
+
runner,
|
|
7789
|
+
camunda,
|
|
7790
|
+
restConfig,
|
|
7791
|
+
worker: workerName,
|
|
7792
|
+
workers: [{ id: workerName, types: jobTypes, capacity: 1 }],
|
|
7793
|
+
autoWorkerId: autoMode ? workerName : undefined,
|
|
7794
|
+
scope: autoScope,
|
|
7795
|
+
// The ONE multiplexed host connection (issue #173): the runtime owns its
|
|
7796
|
+
// connect/reconnect/resync + teardown lifecycle. Omitted (undefined) when the
|
|
7797
|
+
// agentic target didn't resolve to a connect — the runtime then runs with no
|
|
7798
|
+
// agentic scope and presence/steer degrade to no-ops.
|
|
7799
|
+
agenticEndpoint: agenticEndpoint || undefined,
|
|
7800
|
+
config: {
|
|
7801
|
+
activation: { requestTimeoutMs: pollTimeoutMs },
|
|
7802
|
+
dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
|
|
7803
|
+
},
|
|
7804
|
+
});
|
|
7805
|
+
settle = composed.settle;
|
|
7806
|
+
const {
|
|
7807
|
+
deps: supervisorDeps,
|
|
7808
|
+
registry: workerRegistry,
|
|
7809
|
+
makeSupervisor: makeSupervisorRuntime,
|
|
7810
|
+
Effect: SupervisorEffect,
|
|
7811
|
+
Fiber: SupervisorFiber,
|
|
7812
|
+
} = composed;
|
|
7813
|
+
|
|
7814
|
+
// Run the single per-host owner. `Effect.runFork` keeps the process alive on the
|
|
7815
|
+
// runtime's own Schedule cadences (reconcile + activation/idle loop), so even an
|
|
7816
|
+
// --auto worker that starts with zero types stays up and fills them in on the
|
|
7817
|
+
// next reconcile — the same liveness the retired ref'd auto-poll timer provided.
|
|
7818
|
+
const supervisor = await SupervisorEffect.runPromise(makeSupervisorRuntime(supervisorDeps));
|
|
7819
|
+
const supervisorFiber = SupervisorEffect.runFork(supervisor.run);
|
|
7820
|
+
|
|
7821
|
+
// Seed this worker's presence into the runtime's ownership registry (issue
|
|
7822
|
+
// #173) and late-bind the per-job relay seam to the running supervisor. The
|
|
7823
|
+
// presence-projection fiber announces (register) then heartbeats this instance
|
|
7824
|
+
// over the one multiplexed connection, and drops it (deregister) when it leaves
|
|
7825
|
+
// the registry; ownership jobKeys are driven by the dispatch claim/release
|
|
7826
|
+
// lifecycle keyed by this same instance. Every hop is best-effort — a
|
|
7827
|
+
// visibility seam must never fail the worker.
|
|
7828
|
+
if (agenticEndpoint) {
|
|
7829
|
+
const agenticEncoder = new TextEncoder();
|
|
7830
|
+
const seedPresence = () => {
|
|
8123
7831
|
try {
|
|
8124
|
-
|
|
8125
|
-
return { jobTypes: [...new Set([...autoTypes, ...extraJobTypes])] };
|
|
7832
|
+
SupervisorEffect.runSync(supervisor.ownership.register(workerName, agenticCapability));
|
|
8126
7833
|
} catch (err) {
|
|
8127
|
-
|
|
7834
|
+
logger.warn(` agentic presence seed failed (${err?.message || err}); continuing.`);
|
|
8128
7835
|
}
|
|
8129
|
-
}
|
|
8130
|
-
|
|
8131
|
-
|
|
8132
|
-
|
|
8133
|
-
|
|
8134
|
-
//
|
|
8135
|
-
//
|
|
8136
|
-
|
|
8137
|
-
|
|
8138
|
-
|
|
8139
|
-
|
|
8140
|
-
|
|
8141
|
-
|
|
8142
|
-
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8146
|
-
|
|
8147
|
-
|
|
8148
|
-
|
|
8149
|
-
|
|
8150
|
-
|
|
8151
|
-
|
|
8152
|
-
|
|
8153
|
-
|
|
8154
|
-
|
|
8155
|
-
|
|
8156
|
-
|
|
8157
|
-
|
|
8158
|
-
|
|
8159
|
-
|
|
8160
|
-
|
|
8161
|
-
|
|
8162
|
-
|
|
8163
|
-
|
|
8164
|
-
|
|
8165
|
-
|
|
8166
|
-
|
|
8167
|
-
|
|
8168
|
-
|
|
8169
|
-
|
|
8170
|
-
|
|
8171
|
-
|
|
8172
|
-
const desired = await desiredJobTypes();
|
|
8173
|
-
if (desired.skip) {
|
|
8174
|
-
if (autoMode) {
|
|
8175
|
-
logger.warn(`--auto reconcile skipped — ${desired.skip}; keeping the current ${workers.size} worker(s) running.`);
|
|
8176
|
-
} else if (desired.skip === 'deleted') {
|
|
8177
|
-
logger.warn(`Profile "${name}" is gone from config — keeping the current ${workers.size} worker(s) running.`);
|
|
8178
|
-
} else {
|
|
8179
|
-
logger.warn(`Profile "${name}" reload skipped — ${desired.skip}; keeping current workers.`);
|
|
7836
|
+
};
|
|
7837
|
+
seedPresence();
|
|
7838
|
+
agenticPlane = {
|
|
7839
|
+
register: seedPresence,
|
|
7840
|
+
// Graceful teardown: emit an explicit deregister for this identity over the
|
|
7841
|
+
// live connection, then drop it from the registry so the projection stops
|
|
7842
|
+
// heartbeating it. Best-effort — teardown must never hang or throw.
|
|
7843
|
+
deregister: (reason) => {
|
|
7844
|
+
try { SupervisorEffect.runSync(supervisor.presence.deregister(workerName, reason)); } catch { /* best effort */ }
|
|
7845
|
+
try { SupervisorEffect.runSync(supervisor.ownership.deregister(workerName)); } catch { /* best effort */ }
|
|
7846
|
+
},
|
|
7847
|
+
// Build a per-job relay session over the supervisor's transcript + steer
|
|
7848
|
+
// seams (issue #173), shape-compatible with the retired createWorkChannel
|
|
7849
|
+
// relay session so `runAgentJob` consumes it unchanged. Transcript rides the
|
|
7850
|
+
// one connection keyed by this instance + jobKey; inbound steer is fanned
|
|
7851
|
+
// back to this job's PTY by the runtime's per-instance steer router.
|
|
7852
|
+
relaySessionFor: (jobKey) => {
|
|
7853
|
+
try {
|
|
7854
|
+
return createHostRelaySession({
|
|
7855
|
+
instance: workerName,
|
|
7856
|
+
jobKey,
|
|
7857
|
+
publish: (text) => {
|
|
7858
|
+
// Fire-and-forget over the live handle; a frame between a drop and
|
|
7859
|
+
// the next reconnect is a harmless no-op (best-effort semantics).
|
|
7860
|
+
try { SupervisorEffect.runSync(supervisor.transcript(workerName, String(jobKey), agenticEncoder.encode(text))); } catch { /* best effort */ }
|
|
7861
|
+
},
|
|
7862
|
+
subscribeSteer: (onChunk) => {
|
|
7863
|
+
// The router keys sinks by instance, not jobKey, so it hands every
|
|
7864
|
+
// steer frame for this instance to the active sink. Filter by the
|
|
7865
|
+
// session's own jobKey before delivering: even at capacity=1 a
|
|
7866
|
+
// late/queued steer frame for a PRIOR job could otherwise land in
|
|
7867
|
+
// the NEXT job's PTY. Drop any frame whose jobKey isn't this job's.
|
|
7868
|
+
const sink = (jk, chunk) => SupervisorEffect.sync(() => {
|
|
7869
|
+
if (String(jk) === String(jobKey)) onChunk(chunk);
|
|
7870
|
+
});
|
|
7871
|
+
try { SupervisorEffect.runSync(supervisor.steerRouter.register(workerName, sink)); } catch { /* best effort */ }
|
|
7872
|
+
return () => { try { SupervisorEffect.runSync(supervisor.steerRouter.unregister(workerName)); } catch { /* best effort */ } };
|
|
7873
|
+
},
|
|
7874
|
+
logger,
|
|
7875
|
+
});
|
|
7876
|
+
} catch (err) {
|
|
7877
|
+
logger.warn(` host relay session unavailable (${err?.message || err}); continuing without live terminal.`);
|
|
7878
|
+
return null;
|
|
8180
7879
|
}
|
|
8181
|
-
|
|
8182
|
-
|
|
8183
|
-
|
|
8184
|
-
if (added.length === 0 && removed.length === 0) return;
|
|
8185
|
-
const source = autoMode ? 'engine deployed set' : `Profile "${name}"`;
|
|
8186
|
-
logger.info(`${source} changed — reconciling job types (+${added.length} / -${removed.length}).`);
|
|
8187
|
-
for (const jt of added) {
|
|
8188
|
-
spawnJobType(jt);
|
|
8189
|
-
logger.info(` + now listening on ${jt}`);
|
|
8190
|
-
}
|
|
8191
|
-
await Promise.all(
|
|
8192
|
-
removed.map(async (jt) => {
|
|
8193
|
-
const w = workers.get(jt);
|
|
8194
|
-
logger.info(` - draining ${jt} …`);
|
|
8195
|
-
const ok = await drainWorker(w);
|
|
8196
|
-
if (ok) {
|
|
8197
|
-
// Only drop it from the registry once it has actually stopped, so a
|
|
8198
|
-
// failed drain stays tracked and gets retried on the next reconcile
|
|
8199
|
-
// pass (or on shutdown) instead of leaking an untracked poller.
|
|
8200
|
-
workers.delete(jt);
|
|
8201
|
-
logger.info(` - stopped ${jt}`);
|
|
8202
|
-
} else {
|
|
8203
|
-
logger.warn(` - ${jt} did not stop cleanly; keeping it tracked so it is retried on the next reconcile or shutdown.`);
|
|
8204
|
-
}
|
|
8205
|
-
}),
|
|
8206
|
-
);
|
|
8207
|
-
logger.info(` now listening on ${workers.size} job type(s): ${[...workers.keys()].join(' ')}`);
|
|
8208
|
-
};
|
|
7880
|
+
},
|
|
7881
|
+
};
|
|
7882
|
+
}
|
|
8209
7883
|
|
|
8210
|
-
//
|
|
8211
|
-
//
|
|
8212
|
-
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
|
|
8220
|
-
//
|
|
8221
|
-
//
|
|
8222
|
-
|
|
8223
|
-
|
|
8224
|
-
|
|
8225
|
-
}, AUTO_POLL_INTERVAL_MS);
|
|
8226
|
-
// Deliberately REF'd (unlike the reaper/run-dir hygiene timers, which are
|
|
8227
|
-
// unref'd): in `--auto` this poll IS the retry loop, and it must keep the
|
|
8228
|
-
// process alive even with zero pollers. When the INITIAL engine read fails
|
|
8229
|
-
// (transient miss, or the engine isn't up yet) the worker registers 0
|
|
8230
|
-
// pollers; nothing else holds the event loop open (the SDK client with no
|
|
8231
|
-
// job workers doesn't, and the hygiene timers are unref'd), so an unref'd
|
|
8232
|
-
// poll timer would let the process exit 0 — the observed crash-loop under a
|
|
8233
|
-
// supervisor (jwulf/c8ctl-plugin-nano#93). Keeping it ref'd makes the worker
|
|
8234
|
-
// stay up and re-read on the next poll, exactly as the initial-read warning
|
|
8235
|
-
// promises. Shutdown clears it (clearInterval), so Ctrl-C/SIGTERM still exit.
|
|
8236
|
-
} else {
|
|
8237
|
-
// `watchFile` (polling stat) is deliberate over `fs.watch`: it survives the
|
|
8238
|
-
// atomic temp+rename that `writeConfig` does (fs.watch would rebind to the old
|
|
8239
|
-
// inode and go silent), and it's uniform across platforms. Profile edits are
|
|
8240
|
-
// rare + manual, so a ~1.5s poll latency is fine.
|
|
7884
|
+
// Non-auto live retype: the runtime's reconcile loop only rewrites the --auto
|
|
7885
|
+
// worker's types (from the engine read), so keep watching the profile file to
|
|
7886
|
+
// honour `nano assign` — a profile edit rewrites THIS worker's serviceable types
|
|
7887
|
+
// in the shared registry without a restart (a single `registry.setTypes` call,
|
|
7888
|
+
// the direct analogue of the retired per-process reconcile). In --auto the
|
|
7889
|
+
// runtime owns the desired set, so no profile watch is installed.
|
|
7890
|
+
const configFile = getConfigFile();
|
|
7891
|
+
const WATCH_INTERVAL_MS = 1500;
|
|
7892
|
+
let draining = false;
|
|
7893
|
+
if (!autoMode) {
|
|
7894
|
+
// Serialize reloads on a chain (never overlap a `setTypes` write) and coalesce
|
|
7895
|
+
// with a monotonic generation guard, so a slow older reload can never apply a
|
|
7896
|
+
// stale job-type set after a newer edit has already superseded it.
|
|
7897
|
+
let reloadSeq = 0;
|
|
7898
|
+
let reloadChain = Promise.resolve();
|
|
8241
7899
|
watchFile(configFile, { interval: WATCH_INTERVAL_MS }, (curr, prev) => {
|
|
7900
|
+
// A callback can already be queued when teardown flips `draining`; bail so we
|
|
7901
|
+
// never write to the shared registry (or race its teardown) during shutdown.
|
|
7902
|
+
if (draining) return;
|
|
8242
7903
|
// Fires each interval; act only on real changes. Compare mtime, ctime and
|
|
8243
|
-
// size, not mtime alone:
|
|
8244
|
-
//
|
|
8245
|
-
// differ, and an mtime-only guard would skip a genuine profile update.
|
|
7904
|
+
// size, not mtime alone: coarse-mtime filesystems (or two edits in one tick)
|
|
7905
|
+
// can leave mtimeMs unchanged while size/ctimeMs differ.
|
|
8246
7906
|
if (
|
|
8247
7907
|
curr.mtimeMs === prev.mtimeMs &&
|
|
8248
7908
|
curr.ctimeMs === prev.ctimeMs &&
|
|
8249
7909
|
curr.size === prev.size
|
|
8250
7910
|
) return;
|
|
8251
|
-
|
|
8252
|
-
|
|
8253
|
-
|
|
8254
|
-
|
|
7911
|
+
const mySeq = ++reloadSeq;
|
|
7912
|
+
reloadChain = reloadChain.then(async () => {
|
|
7913
|
+
if (draining) return;
|
|
7914
|
+
// A newer edit already landed while we were queued — skip this stale
|
|
7915
|
+
// reload so its (older) job-type set never lands after the newer one.
|
|
7916
|
+
if (mySeq !== reloadSeq) return;
|
|
7917
|
+
let stored;
|
|
7918
|
+
try {
|
|
7919
|
+
stored = readHiresStrict()[name];
|
|
7920
|
+
} catch (err) {
|
|
7921
|
+
// config exists but doesn't parse (torn write) — keep current types
|
|
7922
|
+
logger.warn(`Profile "${name}" reload skipped — ${configFile} unreadable/unparseable (keeping current job types): ${err?.message || err}`);
|
|
7923
|
+
return;
|
|
7924
|
+
}
|
|
7925
|
+
if (!stored) {
|
|
7926
|
+
// profile vanished — keep serving the current types
|
|
7927
|
+
logger.warn(`Profile "${name}" reload skipped — profile no longer present in ${configFile} (keeping current job types)`);
|
|
7928
|
+
return;
|
|
7929
|
+
}
|
|
7930
|
+
const norm = normalizeStoredProfile(name, stored);
|
|
7931
|
+
if (norm.error) {
|
|
7932
|
+
// invalid edit — keep current types
|
|
7933
|
+
logger.warn(`Profile "${name}" reload skipped — invalid profile edit (keeping current job types): ${norm.error}`);
|
|
7934
|
+
return;
|
|
7935
|
+
}
|
|
7936
|
+
const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
|
|
7937
|
+
const desired = [...new Set([...m, ...extraJobTypes])];
|
|
7938
|
+
if (draining) return; // teardown began while we were reading — don't write
|
|
7939
|
+
if (mySeq !== reloadSeq) return; // superseded during the async read — skip
|
|
7940
|
+
await SupervisorEffect.runPromise(workerRegistry.setTypes(workerName, desired));
|
|
7941
|
+
logger.info(`Profile "${name}" changed — now servicing ${desired.length} job type(s): ${desired.join(' ')}`);
|
|
7942
|
+
}).catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
|
|
8255
7943
|
});
|
|
8256
7944
|
}
|
|
8257
7945
|
|
|
8258
|
-
// Keep the process alive until a stop signal, then
|
|
7946
|
+
// Keep the process alive until a stop signal, then interrupt the runtime loop
|
|
7947
|
+
// and tear down visibility. Interrupting the supervisor fiber runs the runtime's
|
|
7948
|
+
// bracketed teardown (release slots, stop the heartbeat + agentic scope).
|
|
8259
7949
|
await new Promise((resolve) => {
|
|
8260
7950
|
const stop = async (signal) => {
|
|
8261
7951
|
if (draining) return;
|
|
8262
7952
|
draining = true;
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
// `workers` — this prevents double-stops, missed drains, or a wrong worker
|
|
8266
|
-
// count on exit.
|
|
8267
|
-
if (autoPollTimer) clearInterval(autoPollTimer);
|
|
8268
|
-
else unwatchFile(configFile);
|
|
8269
|
-
if (inFlightReconcile) {
|
|
8270
|
-
logger.info('Waiting for in-flight reconcile to finish before shutdown…');
|
|
8271
|
-
await inFlightReconcile;
|
|
8272
|
-
}
|
|
8273
|
-
const list = [...workers.values()];
|
|
8274
|
-
logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
|
|
7953
|
+
if (!autoMode) unwatchFile(configFile);
|
|
7954
|
+
logger.info(`Received ${signal} — stopping worker...`);
|
|
8275
7955
|
if (reaperTimer) clearInterval(reaperTimer);
|
|
8276
7956
|
if (runDirTimer) clearInterval(runDirTimer);
|
|
8277
|
-
//
|
|
8278
|
-
//
|
|
8279
|
-
|
|
8280
|
-
|
|
8281
|
-
|
|
8282
|
-
|
|
8283
|
-
|
|
8284
|
-
|
|
8285
|
-
logger.info('All workers stopped.');
|
|
7957
|
+
// Deregister this worker's presence BEFORE interrupting the runtime: emit
|
|
7958
|
+
// the explicit deregister while the multiplexed host connection is still
|
|
7959
|
+
// live, so the worker disappears from the cockpit cleanly rather than
|
|
7960
|
+
// lingering until its heartbeat lapses. Best-effort — teardown must never
|
|
7961
|
+
// hang. Interrupting the fiber then runs the runtime's bracketed teardown
|
|
7962
|
+
// (release slots, stop the heartbeat + tear down the agentic scope).
|
|
7963
|
+
if (agenticPlane) {
|
|
7964
|
+
try { agenticPlane.deregister(`worker stopped (${signal})`); } catch { /* best effort */ }
|
|
8286
7965
|
}
|
|
8287
|
-
|
|
8288
|
-
|
|
8289
|
-
|
|
8290
|
-
|
|
8291
|
-
|
|
8292
|
-
try {
|
|
8293
|
-
bufferMonitor?.stop();
|
|
8294
|
-
} catch { /* best effort */ }
|
|
8295
|
-
try {
|
|
8296
|
-
await workChannel.stop(`worker stopped (${signal})`);
|
|
8297
|
-
logger.info('Deregistered from the agentic visibility channel.');
|
|
8298
|
-
} catch (err) {
|
|
8299
|
-
logger.warn(`agentic channel deregister failed: ${err?.message || err}`);
|
|
8300
|
-
}
|
|
7966
|
+
try {
|
|
7967
|
+
await SupervisorEffect.runPromise(SupervisorFiber.interrupt(supervisorFiber));
|
|
7968
|
+
logger.info('Worker stopped.');
|
|
7969
|
+
} catch (err) {
|
|
7970
|
+
logger.warn(`supervisor shutdown error — runtime loop may not have shut down cleanly: ${err?.message || err}`);
|
|
8301
7971
|
}
|
|
8302
7972
|
resolve();
|
|
8303
7973
|
};
|
|
@@ -12708,8 +12378,6 @@ export {
|
|
|
12708
12378
|
resolveLivenessOverrides,
|
|
12709
12379
|
parsePsTime,
|
|
12710
12380
|
ensureAcpFlag,
|
|
12711
|
-
startLockExtender,
|
|
12712
|
-
createSingleFlight,
|
|
12713
12381
|
provisionRepo,
|
|
12714
12382
|
finalizeGit,
|
|
12715
12383
|
describeGitFailure,
|