c8ctl-plugin-nano 1.51.0 → 1.53.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -63,9 +63,9 @@ import { createInterface } from 'node:readline/promises';
63
63
  import { StringDecoder } from 'node:string_decoder';
64
64
  import { createInterface as createReadline, cursorTo as rlCursorTo, moveCursor as rlMoveCursor, clearScreenDown as rlClearScreenDown } from 'node:readline';
65
65
  import { platformForHost } from './platforms.mjs';
66
- import { createWorkChannel, redactAgenticUrl, buildAgenticUrl } from './work-channel.mjs';
67
- import { createRelaySession, roleTerminalMode } from './work-relay.mjs';
68
- import { createBufferMonitor, resolveBufferCapacity } from './work-buffer.mjs';
66
+ import { redactAgenticUrl, buildAgenticUrl } from './work-channel.mjs';
67
+ import { createHostRelaySession, roleTerminalMode } from './work-relay.mjs';
68
+ import { resolveBufferCapacity } from './work-buffer.mjs';
69
69
  // Canonical ACP → transcript wire bridge (nanobpm/nano-ide#534), consumed through
70
70
  // the single agentic import surface. `acpUpdateToTranscriptChunk(update)` maps one
71
71
  // raw ACP `session/update` to the exact transcript-chunk bytes the cockpit decodes,
@@ -136,11 +136,6 @@ const READINESS_TIMEOUT_MS = 60_000;
136
136
  const READINESS_POLL_MS = 500;
137
137
  const HEALTH_TIMEOUT_MS = 1_500;
138
138
  const STOP_GRACE_MS = 8_000;
139
- // Backoff applied when a poller fails a lease fast because the worker is already
140
- // running another job (issue #142 single-flight). Long enough that the deferred
141
- // job doesn't tight-loop re-activating while the first runs, short enough that it
142
- // is picked up promptly once the worker frees up.
143
- const WORKER_BUSY_RETRY_BACKOFF_MS = 5_000;
144
139
  // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
145
140
  // definitions + fetch each BPMN). A read that stalls past this is treated as a
146
141
  // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -3288,14 +3283,12 @@ async function createAgenticEndpoint(opts) {
3288
3283
  //
3289
3284
  // Returns everything a JS caller needs to run `makeSupervisor` — the assembled
3290
3285
  // `deps`, the seeded `registry` (so workers can be add/remove'd live), and the
3291
- // `makeSupervisor` + `Effect` handles from the bundle. NOTE (deferred, issue
3292
- // #156): the actual hot-path flip deleting the per-type SDK pollers, the
3293
- // process-wide `singleFlight`, and the per-process reconcile crawl in `workAgent`
3294
- // and running `makeSupervisor(deps).run` as the single per-host owner is NOT
3295
- // wired here; it deletes battle-tested crash-safety code and can only be
3296
- // validated against a live engine, so it is intentionally left as the follow-up
3297
- // this seam unblocks. Constructing deps is side-effect-free (no socket opens, no
3298
- // activation) until the caller forks `supervisor.run`.
3286
+ // `makeSupervisor` + `Effect` handles from the bundle. NOTE (issue #172): the
3287
+ // hot-path flip this seam unblocks is now DONE — `workAgent` runs
3288
+ // `makeSupervisor(deps).run` as the single per-host owner, having retired the
3289
+ // per-type SDK pollers, the process-wide `singleFlight`, the per-process reconcile
3290
+ // crawl, and the per-job lock extender. Constructing deps stays side-effect-free
3291
+ // (no socket opens, no activation) until the caller forks `supervisor.run`.
3299
3292
  //
3300
3293
  // @param {object} opts
3301
3294
  // @param {{ run(job): Promise<void> }} opts.runner raw job runner (required to run)
@@ -5529,80 +5522,12 @@ function baseAgentEnv(profile, job) {
5529
5522
  };
5530
5523
  }
5531
5524
 
5532
- /**
5533
- * Process-wide single-flight guard (issue #142).
5534
- *
5535
- * `maxParallelJobs = 1` only caps concurrency WITHIN one job-type poller, but a
5536
- * single `work` process runs one poller per job type (rank×capability matrix, or
5537
- * every deployed agent type under `--auto`). Without a shared gate a worker
5538
- * serving N job types could lease and run up to N jobs at once — each holding its
5539
- * own PTY + git workspace + broker lock-extender — the exact failure the
5540
- * "one job per worker" invariant exists to prevent.
5541
- *
5542
- * This is a capacity-1, non-blocking mutex shared by EVERY per-type poller: the
5543
- * first poller to `tryAcquire()` runs its job to completion (releasing in a
5544
- * `finally`); any other poller that finds the permit already held must NOT begin
5545
- * a second job (the caller fails the lease fast so the broker re-queues it rather
5546
- * than leaving it "claimed but idle"). `tryAcquire`/`release` are synchronous
5547
- * check-and-set, so the single-threaded event loop makes them race-free across
5548
- * the concurrently-invoked async job handlers.
5549
- */
5550
- function createSingleFlight() {
5551
- let held = false;
5552
- return {
5553
- /** Take the permit if free; returns false when a job is already in flight. */
5554
- tryAcquire() {
5555
- if (held) return false;
5556
- held = true;
5557
- return true;
5558
- },
5559
- /** Release the permit. Idempotent: redundant calls are safe no-ops, though the normal path releases once per acquire (in a `finally`). */
5560
- release() {
5561
- held = false;
5562
- },
5563
- /** True while a job holds the permit. */
5564
- get busy() {
5565
- return held;
5566
- },
5567
- };
5568
- }
5569
-
5570
- /**
5571
- * Keep a leased job's broker activation lock ahead of *now* while the harness is
5572
- * running, so a long agent run never has its lock lapse and get re-activated (a
5573
- * second worker starting → the classic stale complete/fail 409). The lock is NOT
5574
- * hardcoded up front: we refresh it to `windowMs` — a duration-from-now, per the
5575
- * UpdateJobTimeout contract ("the duration of the new timeout in ms, starting
5576
- * from the current moment"), so calls SET rather than accumulate — every
5577
- * `intervalMs`. The deadline therefore stays a bounded `windowMs` ahead of now.
5578
- * The instant we stop refreshing (harness exit / idle-kill / hard cap) the lock
5579
- * lapses within `windowMs` and the broker reclaims the job — fast node-loss
5580
- * recovery. Because the harness is always killed locally before we stop, the lock
5581
- * strictly outlives our local run, so a reclaim never races a still-running agent.
5582
- *
5583
- * Returns a stop() to call once the run settles. Extension failures are logged
5584
- * and swallowed — a transient network blip must not crash the job handler. Older
5585
- * SDKs without `modifyJobTimeout` degrade to the fixed initial lock (a no-op stop).
5586
- */
5587
- function startLockExtender(job, windowMs, intervalMs, tag, logger) {
5588
- if (!(windowMs > 0) || !(intervalMs > 0)) {
5589
- return () => {};
5590
- }
5591
- if (typeof job?.modifyJobTimeout !== 'function') {
5592
- logger?.warn?.(`${tag}: job.modifyJobTimeout unavailable — activation lock will NOT be auto-extended; a run longer than ${windowMs}ms risks being reclaimed and executed twice`);
5593
- return () => {};
5594
- }
5595
- const extend = () => Promise.resolve()
5596
- .then(() => job.modifyJobTimeout({ newTimeoutMs: windowMs }))
5597
- .catch((err) => logger?.warn?.(`${tag}: lock extend failed — ${err?.message ?? err}`));
5598
- // Renew immediately so the harness starts with a full, fresh window no matter
5599
- // how much of the initial activation lease provisioning (clone/checkout) ate.
5600
- extend();
5601
- const timer = setInterval(extend, intervalMs);
5602
- // Never let the heartbeat keep the process alive on shutdown.
5603
- if (typeof timer.unref === 'function') timer.unref();
5604
- return () => clearInterval(timer);
5605
- }
5525
+ // Issue #172 retired `createSingleFlight` (the process-wide capacity-1 mutex the
5526
+ // per-type SDK pollers shared) and `startLockExtender` (the per-job broker-lock
5527
+ // heartbeat). Both are now owned by the single-owner supervisor runtime: race-free
5528
+ // per-type slot accounting lives in `supervisor/src/registry.ts` (this worker
5529
+ // registers with capacity 1), and the lock lifecycle extend-winner-before-start
5530
+ // plus a `Schedule`-driven heartbeat lives in `supervisor/src/dispatch.ts`.
5606
5531
 
5607
5532
  /**
5608
5533
  * Run a single activated job through the profile's CLI command (one-shot),
@@ -7032,26 +6957,21 @@ async function workAgent(req, flags) {
7032
6957
  };
7033
6958
  // One job per worker, hard-wired (there is deliberately no --max-parallel
7034
6959
  // flag): an agent harness holds a PTY + a git workspace for the whole life of
7035
- // a job, so a worker must never lease a second job concurrently. The @camunda8
7036
- // SDK derives maxJobsToActivate = maxParallelJobs - activeJobs, so 1 means
7037
- // "activate one job, then stop polling until it completes".
7038
- const maxParallelJobs = 1;
7039
- // Process-wide single-flight guard (issue #142). The SDK's maxParallelJobs=1
7040
- // only serializes ONE job-type poller, but this process runs one poller per
7041
- // job type, so nothing stops N pollers from each leasing + running a job
7042
- // concurrently. This capacity-1 mutex, shared by every poller's jobHandler,
7043
- // enforces the real "one job per worker" invariant: while any job is in flight
7044
- // on any job type, no other poller starts a second one.
7045
- const singleFlight = createSingleFlight();
6960
+ // a job, so a worker must never lease a second job concurrently. Issue #172:
6961
+ // this is now enforced by registering this worker with capacity 1 in the
6962
+ // single-owner runtime's shared registry (one worker, capacity 1, across all
6963
+ // job types) — retiring the process-wide capacity-1 single-flight the per-type
6964
+ // SDK pollers needed. Race-free slot accounting lives in the runtime registry.
7046
6965
  // The broker job-activation lock is NOT hardcoded up front. A fixed timeout is
7047
6966
  // impossible to size for an agent: too short reclaims a still-working job (a
7048
6967
  // second agent starts + the stale complete/fail is rejected 409), too long
7049
- // strands a dead worker's job. Instead the worker keeps the lock a bounded
7050
- // `recovery-window` ahead of *now* while the harness runs (see
7051
- // startLockExtender), so long runs never lose their lock, and a dead/killed
7052
- // worker's job is reclaimed within one window. Liveness is enforced by
7053
- // `idle-timeout` (max silence before the harness is killed as wedged), so the
7054
- // lock is held only while the agent is alive AND producing output.
6968
+ // strands a dead worker's job. Instead the runtime's dispatch lifecycle keeps
6969
+ // the lock a bounded `recovery-window` ahead of *now* while the harness runs
6970
+ // (extend-winner-before-start + a Schedule heartbeat in dispatch.ts), so long
6971
+ // runs never lose their lock, and a dead/killed worker's job is reclaimed
6972
+ // within one window. Liveness is enforced by `idle-timeout` (max silence before
6973
+ // the harness is killed as wedged), so the lock is held only while the agent is
6974
+ // alive AND producing output.
7055
6975
  const recoveryWindowMs = intFlag(flags?.['recovery-window'], 5 * 60_000);
7056
6976
  const idleTimeoutMs = intFlag(flags?.['idle-timeout'], 5 * 60_000);
7057
6977
  // `--job-timeout` is now an OPTIONAL absolute hard cap on total harness runtime
@@ -7250,7 +7170,7 @@ async function workAgent(req, flags) {
7250
7170
  }
7251
7171
  const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
7252
7172
  logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
7253
- logger.info(` one job per worker (single-flight across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
7173
+ logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
7254
7174
  // Warm the gh-token cache now, off the job-handling path: githubCloneToken()
7255
7175
  // may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
7256
7176
  // credential fallback, and doing that inside a job handler would block the
@@ -7316,58 +7236,54 @@ async function workAgent(req, flags) {
7316
7236
  /* best effort — activity is advisory, never fail a job over it */
7317
7237
  }
7318
7238
  };
7319
- // The agentic-visibility channel, wired below. Declared here so the job
7320
- // recorders can refresh presence with the live job set as jobs start/end.
7321
- /** @type {import('./work-channel.mjs').WorkChannel | null} */
7322
- let workChannel = null;
7323
- /** @type {import('./work-buffer.mjs').BufferMonitor | null} */
7324
- let bufferMonitor = null;
7325
- // #144 liveness watchdog state. `agenticDisconnectedSince` is the epoch-ms the
7326
- // channel last dropped (null whenever it is up or has never opened); the
7327
- // watchdog uses it to force a full re-discovery + reopen when the client lib's
7328
- // own reconnect fails to bring a previously-connected channel back within the
7329
- // stale threshold. `agenticWatchdog` is the running timer handle (stopped on
7330
- // shutdown); `agenticSelfHealing` guards against two concurrent re-discovery
7331
- // loops (the cold-start one and a watchdog-triggered one).
7332
- let agenticDisconnectedSince = null;
7333
- // #147 presence-keyed watchdog state. `agenticConnectedSince` is the epoch-ms
7334
- // the channel last (re)connected (null while down); the watchdog uses it to
7335
- // require a stable connection to have held for a grace window before counting
7336
- // presence as confirmed. `agenticPresenceHealthyAt` is the epoch-ms presence
7337
- // was last confirmed healthy — advanced on the first connect (buffered REGISTER
7338
- // drains) and by the watchdog whenever a stable connection is observed, and
7339
- // reset by a heal. When it ages past the presence-stale threshold — even while
7340
- // the socket flaps `1006` reconnects — the watchdog forces a re-discovery.
7341
- let agenticConnectedSince = null;
7342
- let agenticPresenceHealthyAt = null;
7343
- /** @type {{ stop: () => void } | null} */
7344
- let agenticWatchdog = null;
7345
- let agenticSelfHealing = false;
7346
- // Maintain `activeJobs` unconditionally: it feeds both the supervisor activity
7347
- // file (gated inside writeActivity) AND the agentic presence frame's live
7348
- // jobKey set, so a standalone worker (no NANO_SUPERVISOR_ACTIVITY_FILE) still
7349
- // reports its current jobs on the visibility page.
7239
+ // The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
7240
+ // host connection carries presence/steer/ownership/transcript for EVERY
7241
+ // supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
7242
+ // (where each process opened its own socket for a single identity). Built below
7243
+ // from the resolved connect config and injected into the runtime as
7244
+ // `deps.agenticEndpoint`; `agenticPlane` is late-bound to the running
7245
+ // supervisor's presence/steer/transcript seams once `makeSupervisor` returns, so
7246
+ // the per-job runner streams a job's terminal + accepts steer over that one
7247
+ // connection instead of a per-process channel.
7248
+ /** @type {import('./supervisor.dist.js').AgenticEndpoint | null} */
7249
+ let agenticEndpoint = null;
7250
+ /** @type {{ register: () => void, deregister: (reason?: string) => void, relaySessionFor: (jobKey: string|number) => (object|null) } | null} */
7251
+ let agenticPlane = null;
7252
+ // The presence attributes this worker announces on `register` (ENROLMENT
7253
+ // attributes, not routing tokens jobKeys are carried by the explicit
7254
+ // claim/release ownership frames the dispatch lifecycle emits, never smuggled in
7255
+ // here). Reused for the initial seed and any resync.
7256
+ const agenticCapability = {
7257
+ cognition: profile.rank,
7258
+ family: profile.model || undefined,
7259
+ host: hostname(),
7260
+ };
7261
+ // Maintain `activeJobs` unconditionally: it feeds the supervisor activity file
7262
+ // (gated inside writeActivity) so a standalone worker still reports its current
7263
+ // jobs. Live presence/ownership jobKeys are now owned by the runtime — the
7264
+ // dispatch claim/release lifecycle keyed by this worker's instance drives the
7265
+ // cockpit's jobKeys — so the recorders no longer poke a per-process channel.
7350
7266
  const recordJobStart = (job, jobType) => {
7351
7267
  activeJobs.set(String(job.jobKey), { type: jobType, since: Date.now() });
7352
7268
  writeActivity();
7353
- workChannel?.refreshPresence();
7354
7269
  };
7355
7270
  const recordJobEnd = (job) => {
7356
7271
  activeJobs.delete(String(job.jobKey));
7357
7272
  writeActivity();
7358
- workChannel?.refreshPresence();
7359
7273
  };
7360
7274
  // Seed an initial idle marker so status reports 'idle' immediately after spawn.
7361
7275
  writeActivity();
7362
7276
 
7363
- // ---- Agentic visibility channel (ADR 0056 slice C2, #41) ----------------
7364
- // Connect this worker to the app's same-port `/agentic` channel and announce
7365
- // presence (identity, host, live jobs), heartbeat, and deregister on exit, so
7366
- // it appears live on the Workforce visibility page. This is the SINGLE place
7367
- // the connected+authenticated channel client is instantiated in `work`: the
7368
- // sibling slices C3 (PTY relay, #42) and C4 (buffer, #43) attach to the
7369
- // accessors on `workChannel` (relay-lane sink + connect/disconnect/reconnect
7370
- // lifecycle events) rather than opening their own connection.
7277
+ // ---- Agentic visibility connection (ADR 0056; issue #173) -----------------
7278
+ // Resolve WHERE this worker's presence is announced, then hand the connect
7279
+ // config to the single-owner supervisor runtime as its ONE multiplexed host
7280
+ // connection (built below). The supervisor announces presence (identity, host),
7281
+ // heartbeats it, claims/releases jobs, and streams transcript + accepts steer
7282
+ // for EVERY supervised agent over that single connection replacing the retired
7283
+ // per-`work`-process `createWorkChannel` fan-out where each process opened its
7284
+ // own socket for a single identity. The sibling data planes (C3 PTY relay #42,
7285
+ // steer #163) now ride this connection via the runtime's transcript/steer seams
7286
+ // rather than a per-process channel.
7371
7287
  //
7372
7288
  // Local-first (security opt-in): visibility is ON BY DEFAULT. In LOCAL mode the
7373
7289
  // worker joins with the well-known LOCAL token and no credential; SECURE mode
@@ -7413,226 +7329,67 @@ async function workAgent(req, flags) {
7413
7329
  // the socket opens (or without a channel at all).
7414
7330
  writeActivity();
7415
7331
  // Track the live connection state on the activity marker so the supervisor
7416
- // shows connected↔disconnected transitions (#99). A close carries a normalized
7417
- // diagnostic under the contract `agentic.message` field (not `reason`) so a hub
7418
- // drop explains WHY; a fresh (re)connect clears any stale message.
7332
+ // shows connected↔disconnected transitions (#99). Diagnostics ride the contract
7333
+ // `agentic.message` field (not `reason`): the endpoint's connection observer
7334
+ // only reports a state (`connected`/`disconnected`) the underlying close does
7335
+ // not surface a reason — so a hub drop records a generic 'connection dropped'
7336
+ // message, while an endpoint-construction failure records the normalized error
7337
+ // (see the catch below). A fresh (re)connect clears any stale message.
7419
7338
  const markAgentic = (status, message = null) => { agenticState = { ...agenticState, status, message }; writeActivity(); };
7420
- // Open (or re-open) the visibility channel for a resolved connect config. This
7421
- // is the SINGLE place the connected+authenticated client is instantiated the
7422
- // initial connect path and the background self-heal loop (#133) both call it,
7423
- // (re)assigning the shared `workChannel`/`bufferMonitor` closures the job
7424
- // recorders and shutdown path already track.
7425
- const openAgenticChannel = async (cfg) => {
7339
+ // Build the ONE multiplexed host-connection endpoint (issue #173) for the
7340
+ // resolved connect config and hand it to the single-owner runtime as
7341
+ // `deps.agenticEndpoint`. This RETIRES the per-`work`-process createWorkChannel
7342
+ // fan-out: the supervisor now owns exactly one connection carrying presence
7343
+ // (register/heartbeat/deregister projected from the registry), ownership
7344
+ // (claim/release per job), inbound steer, and transcript for EVERY supervised
7345
+ // agent — each frame carrying its `instance` explicitly. The endpoint composes
7346
+ // the raw-JS wire (`agentic-endpoint.mjs` → the single agentic import surface)
7347
+ // with the bundle's Effect adapter; additive negotiation still degrades
7348
+ // claim/release/steer to a no-op against an older hub. Reconnect + resync (and
7349
+ // teardown-on-interruption) are owned by the runtime's `superviseAgentic`, so the
7350
+ // retired #133 self-heal loop and #144/#147 liveness watchdog are gone — the
7351
+ // marker's agentic status is driven by the endpoint's connection observer below.
7352
+ // Constructing the endpoint opens NO socket (the runtime's supervision calls the
7353
+ // connect factory), so this stays cheap and side-effect-free until `run` forks.
7354
+ if (agenticCfg) {
7426
7355
  try {
7427
- workChannel = await createWorkChannel({
7428
- instance: workerName,
7429
- host: hostname(),
7430
- capability: {
7431
- cognition: profile.rank,
7432
- family: profile.model || undefined,
7433
- host: hostname(),
7356
+ agenticEndpoint = await createAgenticEndpoint({
7357
+ url: agenticCfg.url,
7358
+ token: agenticCfg.token,
7359
+ credential: agenticCfg.credential,
7360
+ // Keep the activity marker's agentic status honest across the single
7361
+ // connection's open/drop transitions (#99) — the direct analogue of the
7362
+ // retired createWorkChannel onConnect/onDisconnect marker wiring.
7363
+ onConnectionState: (state) => {
7364
+ if (state === 'connected') markAgentic('connected');
7365
+ else markAgentic('disconnected', 'connection dropped');
7434
7366
  },
7435
- listJobKeys: () => [...activeJobs.keys()],
7436
- url: cfg.url,
7437
- token: cfg.token,
7438
- credential: cfg.credential,
7439
- bufferCapacity: cfg.bufferCapacity,
7440
- // #147: de-synchronise reconnect attempts with equal-jitter backoff so a
7441
- // fleet dropped on the same lossy link does not reconnect in lockstep and
7442
- // re-congest it. Wraps the client lib's own exponential policy (which has
7443
- // no jitter of its own); the base delays/factor stay the lib's defaults.
7444
- schedule: makeJitteredReconnectSchedule(),
7445
7367
  logger,
7446
7368
  });
7447
- const shown = redactAgenticUrl(buildAgenticUrl(cfg.url, {}));
7448
- const mode = cfg.secure ? 'secure' : 'local';
7449
- if (cfg.discovered) {
7450
- const d = cfg.discovered;
7451
- logger.info(` agentic channel: auto-discovered ${d.project} on the app's /agentic port ${wsHostPart(d.host)}:${d.port} (bypassing the WS-incapable console proxy).`);
7452
- }
7453
- logger.info(` agentic channel (${mode}): announcing presence as ${workerName} on ${shown}`);
7454
- // onConnect fires only for listeners present at first open, so also
7455
- // reconcile the already-open case synchronously via connected(). If the
7456
- // socket opened and then dropped inside the createWorkChannel() await window
7457
- // (before these listeners existed), connected() is false but everConnected()
7458
- // is true — record that as `disconnected` rather than leaving it stuck at
7459
- // `connecting`.
7460
- // #144: track the drop clock alongside presence — a (re)connect clears it,
7461
- // a disconnect starts it (first drop wins, so the watchdog measures from the
7462
- // ORIGINAL drop, not the latest of a reconnect storm). The watchdog reads
7463
- // this to decide when the client lib has failed to self-heal.
7464
- workChannel.onConnect(() => {
7465
- markAgentic('connected');
7466
- agenticDisconnectedSince = null;
7467
- agenticConnectedSince = Date.now();
7468
- // First open drains the buffered REGISTER → presence lands; seed the
7469
- // presence-health clock so the #147 trigger measures from here (#147).
7470
- agenticPresenceHealthyAt = Date.now();
7471
- });
7472
- workChannel.onReconnect(() => {
7473
- markAgentic('connected');
7474
- agenticDisconnectedSince = null;
7475
- agenticConnectedSince = Date.now();
7476
- // Deliberately do NOT advance agenticPresenceHealthyAt here: a reconnect
7477
- // only CLAIMS presence (re-announces). On a lossy link the socket may
7478
- // re-drop `1006` before presence actually lands, so the watchdog confirms
7479
- // it only once a connection HOLDS for the grace window — a reconnect that
7480
- // immediately re-drops must not mask an unrecovered presence (#147).
7481
- });
7482
- workChannel.onDisconnect((info) => {
7483
- markAgentic('disconnected', normalizeAgenticMessage(info));
7484
- if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
7485
- agenticConnectedSince = null;
7486
- });
7487
- if (workChannel.connected()) {
7488
- markAgentic('connected');
7489
- agenticDisconnectedSince = null;
7490
- agenticConnectedSince = Date.now();
7491
- if (agenticPresenceHealthyAt == null) agenticPresenceHealthyAt = Date.now();
7492
- } else if (workChannel.everConnected()) {
7493
- markAgentic('disconnected');
7494
- if (agenticDisconnectedSince == null) agenticDisconnectedSince = Date.now();
7495
- agenticConnectedSince = null;
7496
- }
7497
7369
  } catch (err) {
7498
- // Never let a channel failure stop the worker from doing its actual job.
7499
- workChannel = null;
7500
- // Retain the failure reason on the marker so the supervisor can show WHY
7501
- // presence dropped (bad URL, refused socket, …), not just `disconnected`.
7502
- // The contract diagnostic field is `agentic.message` (#99), matching the
7503
- // live-disconnect path above — keep the key consistent, not `reason`.
7370
+ // Never let endpoint construction stop the worker from doing its job — the
7371
+ // agentic plane is best-effort visibility. Record the failure on the marker
7372
+ // and run the supervisor with no agenticEndpoint (presence/steer become no-ops).
7373
+ agenticEndpoint = null;
7504
7374
  agenticState = { ...agenticState, status: 'disconnected', message: normalizeAgenticMessage(err) };
7505
7375
  writeActivity();
7506
- logger.warn(` agentic channel unavailable (${err?.message || err}); continuing without visibility.`);
7507
- return;
7376
+ logger.warn(` agentic endpoint unavailable (${err?.message || err}); continuing without visibility.`);
7508
7377
  }
7509
- // C4 (#43): observe the client's built-in outbound buffer across the
7510
- // channel lifecycle — surface a high-water mark and warn when the bound
7511
- // is hit so a hub outage that starts shedding frames is never silent. The
7512
- // monitor is observability-only, so keep it OUTSIDE the channel try/catch:
7513
- // a monitor failure must never null out a healthy channel and take down
7514
- // presence/visibility.
7515
- if (workChannel) {
7516
- try {
7517
- bufferMonitor = createBufferMonitor(workChannel, {
7518
- capacity: cfg.bufferCapacity,
7519
- logger,
7520
- });
7521
- } catch (err) {
7522
- bufferMonitor = null;
7523
- logger.warn(` agentic buffer monitor unavailable (${err?.message || err}); channel presence still active.`);
7524
- }
7525
- }
7526
- };
7527
-
7528
- // (A) Background self-heal loop, shared by the cold-start advisory path (#133)
7529
- // AND the #144 liveness watchdog. Re-run discovery on a jittered backoff and,
7530
- // on the first `connect` target, (re)open the channel WITHOUT a restart. The
7531
- // `agenticSelfHealing` guard makes it idempotent: the watchdog can call it
7532
- // after tearing a stale channel down without racing a still-running cold-start
7533
- // loop. A shared cache lets a brief blip reuse the last known-good hub (#133-C).
7534
- const armAgenticSelfHeal = () => {
7535
- if (agenticSelfHealing) return; // a re-discovery loop is already running
7536
- if (workChannel !== null) return; // a channel already exists — nothing to heal
7537
- agenticSelfHealing = true;
7538
- const hubCache = new Map();
7539
- rediscoverAgenticUntilConnected({
7540
- resolveTarget: () => resolveAgenticTarget({ camunda, logger, cache: hubCache }),
7541
- onConnect: async (target) => {
7542
- agenticCfg = target.config;
7543
- agenticState = agenticStateForTarget(target, safeAgenticDisplayUrl);
7544
- writeActivity();
7545
- logger.info(' agentic channel: background re-discovery succeeded — (re)opening channel.');
7546
- await openAgenticChannel(agenticCfg);
7547
- // openAgenticChannel swallows its own open failures (it nulls
7548
- // workChannel and returns rather than throwing), so a failed open must
7549
- // be re-thrown here — otherwise the self-heal loop treats this attempt
7550
- // as success and stops retrying with workChannel still null (#133).
7551
- if (workChannel === null) {
7552
- throw new Error('agentic channel failed to open after background re-discovery');
7553
- }
7554
- },
7555
- // Stop as soon as a channel exists (loop won this or a prior attempt did).
7556
- shouldContinue: () => workChannel === null,
7557
- logger,
7558
- })
7559
- .catch(() => { /* best-effort self-heal — never surfaces an error */ })
7560
- .finally(() => { agenticSelfHealing = false; });
7561
- };
7562
-
7563
- // (B) #144 liveness watchdog: force-heal a wedged channel. When a channel that
7564
- // HAS connected drops and the client lib's own reconnect never brings it back
7565
- // within the stale threshold (a half-open drop after a server restart/crash/
7566
- // partition, or a reconnect that keeps failing), the client sits `disconnected`
7567
- // forever and the worker vanishes from the Workers view until a supervisor
7568
- // restart. This tears the wedged channel down (so `shouldContinue` re-arms) and
7569
- // re-runs full discovery + reopen instead of trusting the client lib alone.
7570
- const healStaleAgenticChannel = async () => {
7571
- const stale = workChannel;
7572
- if (!stale) return;
7573
- workChannel = null; // re-arms armAgenticSelfHeal()'s shouldContinue gate
7574
- agenticDisconnectedSince = null; // reset the clock; the fresh open restarts it
7575
- agenticConnectedSince = null; // #147: the fresh open re-seeds it
7576
- agenticPresenceHealthyAt = null; // #147: the fresh open re-confirms presence
7577
- try { bufferMonitor?.stop(); } catch { /* best effort */ }
7578
- bufferMonitor = null;
7579
- markAgentic('disconnected', 'stale channel — re-discovering hub');
7580
- // Deregister + close the wedged client so it stops its own doomed reconnect
7581
- // attempts and we don't leak two clients once the fresh one connects.
7582
- try { await stale.stop('stale channel — re-discovering'); } catch { /* best effort */ }
7583
- armAgenticSelfHeal();
7584
- };
7585
-
7586
- const startAgenticWatchdog = () => {
7587
- if (agenticWatchdog) return;
7588
- const staleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_STALE_MS, DEFAULT_AGENTIC_STALE_MS));
7589
- const intervalMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_WATCHDOG_MS, DEFAULT_AGENTIC_WATCHDOG_INTERVAL_MS));
7590
- const presenceStaleAfterMs = Math.max(5_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_STALE_MS, DEFAULT_AGENTIC_PRESENCE_STALE_MS));
7591
- const presenceGraceMs = Math.max(1_000, intFlag(process.env.NANO_AGENTIC_PRESENCE_GRACE_MS, DEFAULT_AGENTIC_PRESENCE_GRACE_MS));
7592
- agenticWatchdog = startAgenticChannelWatchdog({
7593
- getChannel: () => workChannel,
7594
- disconnectedSince: () => agenticDisconnectedSince,
7595
- // #147: presence-keyed trigger — heal reconnect-churn that never re-lands
7596
- // presence, not just a sustained socket drop.
7597
- connectedSince: () => agenticConnectedSince,
7598
- presenceHealthySince: () => agenticPresenceHealthyAt,
7599
- onPresenceHealthy: () => { agenticPresenceHealthyAt = Date.now(); },
7600
- presenceStaleAfterMs,
7601
- presenceGraceMs,
7602
- onStale: healStaleAgenticChannel,
7603
- staleAfterMs,
7604
- intervalMs,
7605
- logger,
7606
- });
7607
- };
7608
-
7609
- if (agenticCfg) {
7610
- await openAgenticChannel(agenticCfg);
7611
- // Guard the connected channel: if it later drops and the client lib can't
7612
- // recover it, the watchdog forces a full re-discovery + reopen (#144).
7613
- startAgenticWatchdog();
7614
- } else if (agenticTarget.status === 'advisory') {
7615
- // A cold-start discovery miss leaves the worker `advisory`; the self-heal
7616
- // loop upgrades it to `connected` without a restart (#133), and once a
7617
- // channel exists the watchdog keeps it alive across later drops (#144).
7618
- armAgenticSelfHeal();
7619
- startAgenticWatchdog();
7620
7378
  }
7621
-
7622
7379
  // C3 (#42): the role's live-terminal mode — a full PTY (streamed on the relay
7623
7380
  // lane when a relay session exists, steerable) or a plain pipe. Honors the
7624
7381
  // vocab's per-role opt-in read off the hire profile (`terminal: pty|pipe`),
7625
7382
  // with an env override for a one-off worker (`NANO_AGENTIC_TERMINAL`). The PTY
7626
7383
  // itself is allocated locally regardless of enrollment; relay streaming (and
7627
- // steer-in) only engages when the worker is enrolled on the channel, so
7628
- // without the channel there's simply no relay tap — the harness still runs on
7629
- // the chosen local transport.
7384
+ // steer-in) only engages when the worker is enrolled (a live agentic endpoint),
7385
+ // so without it there's simply no relay tap — the harness still runs on the
7386
+ // chosen local transport.
7630
7387
  const envTerminal = (process.env.NANO_AGENTIC_TERMINAL || '').trim().toLowerCase();
7631
7388
  const roleTerminal = (envTerminal === 'pty' || envTerminal === 'pipe')
7632
7389
  ? envTerminal
7633
7390
  : roleTerminalMode(profile);
7634
- if (workChannel) {
7635
- logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the relay lane.`);
7391
+ if (agenticEndpoint) {
7392
+ logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
7636
7393
  }
7637
7394
 
7638
7395
  // #110: the role's harness protocol (pipe|acp) and ACP permission policy
@@ -7646,46 +7403,20 @@ async function workAgent(req, flags) {
7646
7403
  const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
7647
7404
  const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
7648
7405
 
7649
- // A per-job-type worker factory. Captures all the CLI-local + profile context
7650
- // in closure scope so the profile watcher below can (re)spawn a poller for any
7651
- // job type on demand without re-reading the flags.
7652
- const makeWorker = (jobType) =>
7653
- camunda.createJobWorker({
7654
- jobType,
7655
- workerName: `${workerName}:${jobType}`,
7656
- maxParallelJobs,
7657
- jobTimeoutMs: recoveryWindowMs,
7658
- pollTimeoutMs,
7659
- jobHandler: async (job) => {
7660
- // Process-wide single-flight (issue #142): if another job is already
7661
- // running on ANY poller, do not start a second harness. Fail this lease
7662
- // FAST — before recording it active, extending its lock, or provisioning
7663
- // anything — so the broker re-queues it (retries preserved) instead of it
7664
- // sitting "claimed but idle" while the first job runs. Gating here, at the
7665
- // point activation surfaces as a handler call, is the cross-poller gate
7666
- // the per-type maxParallelJobs cannot provide.
7667
- if (!singleFlight.tryAcquire()) {
7668
- // Not a failure — preserve the broker-provided retries verbatim so
7669
- // re-dispatch doesn't decrement (or resurrect) the job. Keep a real 0
7670
- // as 0 (an already-incidentable job must stay that way); only default
7671
- // to 1 when the count is missing/invalid.
7672
- const rawRetries = Number(job.retries);
7673
- const retries = Number.isInteger(rawRetries) && rawRetries >= 0 ? rawRetries : 1;
7674
- logger.info(`[${jobType}] job ${job.jobKey} deferred — worker already running another job; releasing lease for re-dispatch.`);
7675
- return job.fail({
7676
- errorMessage: 'worker busy: one job per worker (single-flight across all job types)',
7677
- retries,
7678
- retryBackOff: WORKER_BUSY_RETRY_BACKOFF_MS,
7679
- });
7680
- }
7681
- recordJobStart(job, jobType);
7682
- // Auto-extend the broker lock for the whole life of this job (harness run
7683
- // + git finalize + complete/fail), stopped in the outer finally. The lock
7684
- // is held only while the harness stays alive and productive — a silent
7685
- // hang is killed by the idle-timeout, which resolves runAgentJob and stops
7686
- // the extension, so the broker can reclaim the job.
7687
- let stopLockExtender = () => {};
7688
- try {
7406
+ // The agent job runner (issue #172 hot-path flip). The single-owner supervisor
7407
+ // runtime dispatches each activated job to this `run(job)`; it executes the
7408
+ // harness exactly as the retired per-type SDK jobHandler did, but SETTLES via the
7409
+ // injected `settle` seam (engine complete/fail) because the plain ActivatedJob
7410
+ // carries no SDK `job.complete()/job.fail()`. Capacity (one job per worker), the
7411
+ // activation long-polls, the lock lifecycle, and reconcile are all owned by the
7412
+ // runtime now — retiring the per-type pollers, the process-wide single-flight,
7413
+ // the per-process 1+N reconcile crawl, and the per-job lock extender.
7414
+ let settle;
7415
+ const runner = {
7416
+ run: async (job) => {
7417
+ const jobType = job.type;
7418
+ recordJobStart(job, jobType);
7419
+ try {
7689
7420
  logger.info(`[${jobType}] job ${job.jobKey} (instance ${job.processInstanceKey ?? '-'}) → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
7690
7421
 
7691
7422
  // Disk-budget admission shed: if the engine data root is below the free
@@ -7697,7 +7428,7 @@ async function workAgent(req, flags) {
7697
7428
  const freeMb = budget.free != null ? Math.round(budget.free / 1_048_576) : '?';
7698
7429
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7699
7430
  logger.warn(`[${jobType}] job ${job.jobKey} shed — low disk (${freeMb}MB free); retries left ${retries}`);
7700
- return job.fail({ errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
7431
+ return settle.fail(job.jobKey, { errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
7701
7432
  }
7702
7433
  }
7703
7434
 
@@ -7729,7 +7460,7 @@ async function workAgent(req, flags) {
7729
7460
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7730
7461
  const msg = err instanceof ProvisionError ? err.message : `prompt resource fetch failed: ${err.message}`;
7731
7462
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7732
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7463
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7733
7464
  }
7734
7465
 
7735
7466
  // Assemble + normalize the task envelope from headers (defaults) and
@@ -7740,27 +7471,24 @@ async function workAgent(req, flags) {
7740
7471
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7741
7472
  const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
7742
7473
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7743
- return job.fail({ errorMessage: msg, retries });
7474
+ return settle.fail(job.jobKey, { errorMessage: msg, retries });
7744
7475
  }
7745
7476
 
7746
7477
  // #130: per-task liveness overrides. Precedence: envelope override →
7747
7478
  // worker-flag default → built-in default, each clamped to a sane max so
7748
7479
  // a task can't request an unbounded window. Absent envelope fields leave
7749
- // the worker-flag behaviour unchanged. These drive BOTH the harness idle
7750
- // liveness (idle/recovery) AND the broker lock recovery window, so a
7751
- // JVM-heavy task can widen its own window without a global flag change.
7480
+ // the worker-flag behaviour unchanged. These drive the harness idle
7481
+ // liveness (idle/recovery/hard-cap) so a JVM-heavy task can widen its own
7482
+ // window without a global flag change. NOTE: after the hot-path flip
7483
+ // (#172) the broker lock cadence/window is owned by the supervisor
7484
+ // runtime dispatch config at the worker level (the `dispatch` config's
7485
+ // `recoveryWindowMs` / `extendIntervalMs`), so a per-task override no
7486
+ // longer widens the broker lock window — only the harness liveness.
7752
7487
  const {
7753
7488
  idleTimeoutMs: effectiveIdleTimeoutMs,
7754
7489
  recoveryWindowMs: effectiveRecoveryWindowMs,
7755
7490
  hardCapMs: effectiveHardCapMs,
7756
7491
  } = resolveLivenessOverrides(envelope.task, { idleTimeoutMs, recoveryWindowMs, hardCapMs });
7757
- // Recompute the lock-extend cadence from the effective recovery window
7758
- // (same ~1/3-of-window rule as at startup) so a widened window still
7759
- // renews comfortably before it lapses.
7760
- const effectiveLockExtendIntervalMs = Math.min(
7761
- Math.max(5_000, Math.floor(effectiveRecoveryWindowMs / 3)),
7762
- Math.max(1, Math.floor(effectiveRecoveryWindowMs * 0.75)),
7763
- );
7764
7492
  if (
7765
7493
  effectiveIdleTimeoutMs !== idleTimeoutMs ||
7766
7494
  effectiveRecoveryWindowMs !== recoveryWindowMs ||
@@ -7790,7 +7518,7 @@ async function workAgent(req, flags) {
7790
7518
  : 'repository.url is missing';
7791
7519
  const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
7792
7520
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7793
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7521
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7794
7522
  }
7795
7523
  }
7796
7524
 
@@ -7800,12 +7528,10 @@ async function workAgent(req, flags) {
7800
7528
  const hasRepo = !isContainer && !!envelope.repository?.url;
7801
7529
  let runDir = null;
7802
7530
  let provisioned = null;
7803
- // Start refreshing the broker activation lock BEFORE any potentially-long
7804
- // work (host git clone/checkout can outlast the initial window). Starting
7805
- // here ahead of provisionRepo guarantees the first renewal is queued
7806
- // before the clone, so the lock can't lapse mid-provision and trigger the
7807
- // duplicate-activation / stale-409 race. The `finally` below stops it.
7808
- stopLockExtender = startLockExtender(job, effectiveRecoveryWindowMs, effectiveLockExtendIntervalMs, `[${jobType}] job ${job.jobKey}`, logger);
7531
+ // The broker activation lock is owned by the single-owner runtime's dispatch
7532
+ // lifecycle (supervisor/src/dispatch.ts): it extends the winner to the
7533
+ // recovery window BEFORE this runner starts and heartbeats it on a Schedule
7534
+ // for the whole run, so the retired per-job startLockExtender is gone here.
7809
7535
  let cwd;
7810
7536
  let extraEnv;
7811
7537
  let repoToken = null;
@@ -7847,7 +7573,7 @@ async function workAgent(req, flags) {
7847
7573
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7848
7574
  const msg = err instanceof ProvisionError ? err.message : `provisioning error: ${err.message}`;
7849
7575
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7850
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7576
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7851
7577
  }
7852
7578
  } else if (!isContainer) {
7853
7579
  // Repo-less host job (issue #129, hardening 1): nothing is provisioned,
@@ -7873,24 +7599,21 @@ async function workAgent(req, flags) {
7873
7599
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7874
7600
  const msg = `could not create a temp workspace under the runs root: ${err.message}`;
7875
7601
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7876
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7602
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7877
7603
  }
7878
7604
  }
7879
7605
 
7880
7606
  let result;
7881
7607
  let gitResult = null;
7882
- // C3 (#42): the per-job live-terminal relay session. Streams this job's
7883
- // harness terminal on the relay lane tagged with its jobKey, and accepts
7884
- // steer-in. Only when the worker is enrolled on the channel; closed in
7885
- // the finally so its inbound-frame subscription never leaks across jobs.
7608
+ // The per-job live-terminal relay session (issue #173): streams this job's
7609
+ // harness terminal over the single-owner supervisor's ONE multiplexed host
7610
+ // connection, keyed by this worker's instance + the jobKey, and accepts
7611
+ // cockpit steer-in fanned back to this job's PTY by the runtime's steer
7612
+ // router. Only when the worker is enrolled (a live agentic plane); closed
7613
+ // in the finally so its steer subscription never leaks across jobs.
7886
7614
  let relaySession = null;
7887
- if (workChannel) {
7888
- try {
7889
- relaySession = createRelaySession({ channel: workChannel, jobKey: job.jobKey, logger });
7890
- } catch (err) {
7891
- relaySession = null;
7892
- logger.warn(`[${jobType}] job ${job.jobKey}: relay session unavailable (${err?.message || err}); continuing without live terminal.`);
7893
- }
7615
+ if (agenticPlane) {
7616
+ relaySession = agenticPlane.relaySessionFor(job.jobKey);
7894
7617
  }
7895
7618
  // Private structured-result channel: hand the agent a file (outside any
7896
7619
  // repo clone so it can't be `git add`ed) to write its job-result vars to.
@@ -8029,7 +7752,7 @@ async function workAgent(req, flags) {
8029
7752
  const resultKeys = Object.keys(resultVars);
8030
7753
  if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
8031
7754
  else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
8032
- return await job.complete({
7755
+ return await settle.complete(job.jobKey, {
8033
7756
  ...resultVars,
8034
7757
  [AGENT_RESULT_KEY]: resultEnvelope,
8035
7758
  output: result.stdout,
@@ -8046,258 +7769,205 @@ async function workAgent(req, flags) {
8046
7769
  || (result.stderr || '').trim() + (result.stderrTruncated && (result.stderr || '').trim() ? ' [stderr truncated]' : '')
8047
7770
  || (result.signal ? `terminated by signal ${result.signal}` : `exit code ${result.exitCode}`);
8048
7771
  logger.warn(`[${jobType}] job ${job.jobKey} failed (${detail}); retries left ${retries}`);
8049
- return await job.fail({
7772
+ return await settle.fail(job.jobKey, {
8050
7773
  errorMessage: `agent "${profile.name}" failed: ${detail}`.slice(0, 2000),
8051
7774
  retries,
8052
7775
  variables: { [AGENT_RESULT_KEY]: resultEnvelope },
8053
7776
  });
8054
- } finally {
8055
- stopLockExtender();
8056
- recordJobEnd(job);
8057
- // Release the process-wide single-flight permit LAST, once this job's
8058
- // lock-extender is stopped and its bookkeeping cleared, so another
8059
- // poller can only begin after this job is fully settled.
8060
- singleFlight.release();
8061
- }
8062
- },
8063
- });
8064
-
8065
- // Live worker registry keyed by job type, so the profile watcher can add or
8066
- // drain individual pollers without disturbing the others. `draining` is the
8067
- // shutdown latch (shared with the watcher so a reconcile can't race a stop).
8068
- const workers = new Map();
8069
- let draining = false;
8070
-
8071
- const drainWorker = async (w) => {
8072
- try {
8073
- if (typeof w.stopGracefully === 'function') {
8074
- await w.stopGracefully({ waitUpToMs: STOP_GRACE_MS });
8075
- } else if (typeof w.stop === 'function') {
8076
- await w.stop();
7777
+ } finally {
7778
+ recordJobEnd(job);
8077
7779
  }
8078
- return true;
8079
- } catch {
8080
- return false; // best-effort: never let one worker's stop failure hang us
8081
- }
8082
- };
8083
-
8084
- const spawnJobType = (jobType) => {
8085
- if (workers.has(jobType)) return false;
8086
- workers.set(jobType, makeWorker(jobType));
8087
- return true;
7780
+ },
8088
7781
  };
8089
7782
 
8090
- for (const jobType of jobTypes) spawnJobType(jobType);
8091
-
8092
- // ---- Live reconcile: keep the poller set in step with the desired job-type
8093
- // set start pollers for added types, gracefully drain pollers for removed
8094
- // types without a restart and without disturbing unchanged types' in-flight
8095
- // work. The DESIRED set comes from one of two sources depending on mode:
8096
- // - default: the watched profile's rank×capability matrix (∪ --job-type),
8097
- // reconciled when the on-disk profile changes (e.g. `nano assign`);
8098
- // - --auto: the engine's deployed *agent* job types, reconciled by polling
8099
- // the engine (the deployed set changes as apps deploy/undeploy). ----
8100
- const configFile = getConfigFile();
8101
- const WATCH_INTERVAL_MS = 1500;
8102
- // How often `--auto` re-reads the engine's deployed agent job types to pick up
8103
- // newly deployed / undeployed agent processes. Deploys are occasional, so a
8104
- // few seconds of latency is fine; the read is a couple of cheap C8 REST calls.
8105
- const AUTO_POLL_INTERVAL_MS = 5000;
8106
- let reconciling = false;
8107
- // Set when a profile change arrives while a reconcile is already in flight, so
8108
- // we run one more pass after the current drain completes instead of dropping
8109
- // the update until the next change fires.
8110
- let reconcileRequested = false;
8111
- // Handle to the in-flight reconcile so shutdown can wait for it to finish
8112
- // before snapshotting `workers` (avoids double-stops / missed drains).
8113
- let inFlightReconcile = null;
8114
-
8115
- // Desired job types. In `--auto` this is the engine's deployed agent job types
8116
- // (∪ --job-type extras), read fresh each pass; a transient engine-read failure
8117
- // returns { skip } so the running set is KEPT, never torn down. Otherwise it is
8118
- // the CURRENT on-disk profile's matrix (∪ extras), with { skip } for a
8119
- // transient/torn read, a vanished profile, or an invalid edit — callers must
8120
- // then KEEP the running set, never tear down.
8121
- const desiredJobTypes = async () => {
8122
- if (autoMode) {
7783
+ // Compose the runtime deps over the plugin's real edges (issue #156 seam) and
7784
+ // register THIS worker (capacity 1) into the shared registry. In --auto the
7785
+ // runtime's reconcile loop rewrites this worker's serviceable types from the
7786
+ // engine read (`autoWorkerId`); otherwise the profile watch below does.
7787
+ const composed = await createSupervisorDeps({
7788
+ runner,
7789
+ camunda,
7790
+ restConfig,
7791
+ worker: workerName,
7792
+ workers: [{ id: workerName, types: jobTypes, capacity: 1 }],
7793
+ autoWorkerId: autoMode ? workerName : undefined,
7794
+ scope: autoScope,
7795
+ // The ONE multiplexed host connection (issue #173): the runtime owns its
7796
+ // connect/reconnect/resync + teardown lifecycle. Omitted (undefined) when the
7797
+ // agentic target didn't resolve to a connect the runtime then runs with no
7798
+ // agentic scope and presence/steer degrade to no-ops.
7799
+ agenticEndpoint: agenticEndpoint || undefined,
7800
+ config: {
7801
+ activation: { requestTimeoutMs: pollTimeoutMs },
7802
+ dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
7803
+ },
7804
+ });
7805
+ settle = composed.settle;
7806
+ const {
7807
+ deps: supervisorDeps,
7808
+ registry: workerRegistry,
7809
+ makeSupervisor: makeSupervisorRuntime,
7810
+ Effect: SupervisorEffect,
7811
+ Fiber: SupervisorFiber,
7812
+ } = composed;
7813
+
7814
+ // Run the single per-host owner. `Effect.runFork` keeps the process alive on the
7815
+ // runtime's own Schedule cadences (reconcile + activation/idle loop), so even an
7816
+ // --auto worker that starts with zero types stays up and fills them in on the
7817
+ // next reconcile — the same liveness the retired ref'd auto-poll timer provided.
7818
+ const supervisor = await SupervisorEffect.runPromise(makeSupervisorRuntime(supervisorDeps));
7819
+ const supervisorFiber = SupervisorEffect.runFork(supervisor.run);
7820
+
7821
+ // Seed this worker's presence into the runtime's ownership registry (issue
7822
+ // #173) and late-bind the per-job relay seam to the running supervisor. The
7823
+ // presence-projection fiber announces (register) then heartbeats this instance
7824
+ // over the one multiplexed connection, and drops it (deregister) when it leaves
7825
+ // the registry; ownership jobKeys are driven by the dispatch claim/release
7826
+ // lifecycle keyed by this same instance. Every hop is best-effort — a
7827
+ // visibility seam must never fail the worker.
7828
+ if (agenticEndpoint) {
7829
+ const agenticEncoder = new TextEncoder();
7830
+ const seedPresence = () => {
8123
7831
  try {
8124
- const autoTypes = await resolveAutoJobTypes({ restConfig, scope: autoScope });
8125
- return { jobTypes: [...new Set([...autoTypes, ...extraJobTypes])] };
7832
+ SupervisorEffect.runSync(supervisor.ownership.register(workerName, agenticCapability));
8126
7833
  } catch (err) {
8127
- return { skip: `engine read failed: ${err?.message || err}` };
7834
+ logger.warn(` agentic presence seed failed (${err?.message || err}); continuing.`);
8128
7835
  }
8129
- }
8130
- let stored;
8131
- try {
8132
- stored = readHiresStrict()[name];
8133
- } catch {
8134
- // config.json exists but doesn't parse (e.g. a torn write): the profile is
8135
- // NOT necessarily gone, so don't claim it was deleted — skip this pass.
8136
- return { skip: 'config unreadable' };
8137
- }
8138
- if (!stored) return { skip: 'deleted' };
8139
- const norm = normalizeStoredProfile(name, stored);
8140
- if (norm.error) return { skip: norm.error };
8141
- const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
8142
- return { jobTypes: [...new Set([...m, ...extraJobTypes])] };
8143
- };
8144
-
8145
- const reconcile = () => {
8146
- if (draining) return inFlightReconcile || Promise.resolve();
8147
- if (reconciling) {
8148
- // A change landed mid-reconcile — remember it so the current pass loops
8149
- // once more rather than leaving the worker set stale until the next edit.
8150
- // Return the ACTUAL in-flight promise (not a fresh short-lived one) so a
8151
- // caller including shutdown waits for the real reconcile to finish.
8152
- reconcileRequested = true;
8153
- return inFlightReconcile || Promise.resolve();
8154
- }
8155
- reconciling = true;
8156
- reconcileRequested = false;
8157
- inFlightReconcile = (async () => {
8158
- try {
8159
- do {
8160
- reconcileRequested = false;
8161
- await runReconcilePass();
8162
- } while (reconcileRequested && !draining);
8163
- } finally {
8164
- reconciling = false;
8165
- inFlightReconcile = null;
8166
- }
8167
- })();
8168
- return inFlightReconcile;
8169
- };
8170
-
8171
- const runReconcilePass = async () => {
8172
- const desired = await desiredJobTypes();
8173
- if (desired.skip) {
8174
- if (autoMode) {
8175
- logger.warn(`--auto reconcile skipped — ${desired.skip}; keeping the current ${workers.size} worker(s) running.`);
8176
- } else if (desired.skip === 'deleted') {
8177
- logger.warn(`Profile "${name}" is gone from config — keeping the current ${workers.size} worker(s) running.`);
8178
- } else {
8179
- logger.warn(`Profile "${name}" reload skipped — ${desired.skip}; keeping current workers.`);
7836
+ };
7837
+ seedPresence();
7838
+ agenticPlane = {
7839
+ register: seedPresence,
7840
+ // Graceful teardown: emit an explicit deregister for this identity over the
7841
+ // live connection, then drop it from the registry so the projection stops
7842
+ // heartbeating it. Best-effort teardown must never hang or throw.
7843
+ deregister: (reason) => {
7844
+ try { SupervisorEffect.runSync(supervisor.presence.deregister(workerName, reason)); } catch { /* best effort */ }
7845
+ try { SupervisorEffect.runSync(supervisor.ownership.deregister(workerName)); } catch { /* best effort */ }
7846
+ },
7847
+ // Build a per-job relay session over the supervisor's transcript + steer
7848
+ // seams (issue #173), shape-compatible with the retired createWorkChannel
7849
+ // relay session so `runAgentJob` consumes it unchanged. Transcript rides the
7850
+ // one connection keyed by this instance + jobKey; inbound steer is fanned
7851
+ // back to this job's PTY by the runtime's per-instance steer router.
7852
+ relaySessionFor: (jobKey) => {
7853
+ try {
7854
+ return createHostRelaySession({
7855
+ instance: workerName,
7856
+ jobKey,
7857
+ publish: (text) => {
7858
+ // Fire-and-forget over the live handle; a frame between a drop and
7859
+ // the next reconnect is a harmless no-op (best-effort semantics).
7860
+ try { SupervisorEffect.runSync(supervisor.transcript(workerName, String(jobKey), agenticEncoder.encode(text))); } catch { /* best effort */ }
7861
+ },
7862
+ subscribeSteer: (onChunk) => {
7863
+ // The router keys sinks by instance, not jobKey, so it hands every
7864
+ // steer frame for this instance to the active sink. Filter by the
7865
+ // session's own jobKey before delivering: even at capacity=1 a
7866
+ // late/queued steer frame for a PRIOR job could otherwise land in
7867
+ // the NEXT job's PTY. Drop any frame whose jobKey isn't this job's.
7868
+ const sink = (jk, chunk) => SupervisorEffect.sync(() => {
7869
+ if (String(jk) === String(jobKey)) onChunk(chunk);
7870
+ });
7871
+ try { SupervisorEffect.runSync(supervisor.steerRouter.register(workerName, sink)); } catch { /* best effort */ }
7872
+ return () => { try { SupervisorEffect.runSync(supervisor.steerRouter.unregister(workerName)); } catch { /* best effort */ } };
7873
+ },
7874
+ logger,
7875
+ });
7876
+ } catch (err) {
7877
+ logger.warn(` host relay session unavailable (${err?.message || err}); continuing without live terminal.`);
7878
+ return null;
8180
7879
  }
8181
- return;
8182
- }
8183
- const { added, removed } = diffJobTypes([...workers.keys()], desired.jobTypes);
8184
- if (added.length === 0 && removed.length === 0) return;
8185
- const source = autoMode ? 'engine deployed set' : `Profile "${name}"`;
8186
- logger.info(`${source} changed — reconciling job types (+${added.length} / -${removed.length}).`);
8187
- for (const jt of added) {
8188
- spawnJobType(jt);
8189
- logger.info(` + now listening on ${jt}`);
8190
- }
8191
- await Promise.all(
8192
- removed.map(async (jt) => {
8193
- const w = workers.get(jt);
8194
- logger.info(` - draining ${jt} …`);
8195
- const ok = await drainWorker(w);
8196
- if (ok) {
8197
- // Only drop it from the registry once it has actually stopped, so a
8198
- // failed drain stays tracked and gets retried on the next reconcile
8199
- // pass (or on shutdown) instead of leaking an untracked poller.
8200
- workers.delete(jt);
8201
- logger.info(` - stopped ${jt}`);
8202
- } else {
8203
- logger.warn(` - ${jt} did not stop cleanly; keeping it tracked so it is retried on the next reconcile or shutdown.`);
8204
- }
8205
- }),
8206
- );
8207
- logger.info(` now listening on ${workers.size} job type(s): ${[...workers.keys()].join(' ')}`);
8208
- };
7880
+ },
7881
+ };
7882
+ }
8209
7883
 
8210
- // Reconcile trigger. In `--auto` a periodic engine poll re-reads the deployed
8211
- // agent job types; otherwise a profile-file watch fires on profile edits.
8212
- let autoPollTimer = null;
8213
- if (autoMode) {
8214
- // Self-standing interval poll (not watchFile) since the desired set is
8215
- // derived from the engine, not the on-disk profile. Skip a tick while a
8216
- // reconcile is already in flight: calling reconcile() then would set
8217
- // reconcileRequested and make the in-flight pass loop back-to-back, so an
8218
- // engine read that consistently outlasts AUTO_POLL_INTERVAL_MS would run
8219
- // reconciles as fast as the read completes and hammer the broker. Skipping
8220
- // keeps polling rate-limited to the configured interval regardless of
8221
- // engine-read latency; the next tick re-reads the latest engine state.
8222
- autoPollTimer = setInterval(() => {
8223
- if (inFlightReconcile) return;
8224
- reconcile().catch((err) => logger.warn(`--auto reconcile failed: ${err?.message || err}`));
8225
- }, AUTO_POLL_INTERVAL_MS);
8226
- // Deliberately REF'd (unlike the reaper/run-dir hygiene timers, which are
8227
- // unref'd): in `--auto` this poll IS the retry loop, and it must keep the
8228
- // process alive even with zero pollers. When the INITIAL engine read fails
8229
- // (transient miss, or the engine isn't up yet) the worker registers 0
8230
- // pollers; nothing else holds the event loop open (the SDK client with no
8231
- // job workers doesn't, and the hygiene timers are unref'd), so an unref'd
8232
- // poll timer would let the process exit 0 — the observed crash-loop under a
8233
- // supervisor (jwulf/c8ctl-plugin-nano#93). Keeping it ref'd makes the worker
8234
- // stay up and re-read on the next poll, exactly as the initial-read warning
8235
- // promises. Shutdown clears it (clearInterval), so Ctrl-C/SIGTERM still exit.
8236
- } else {
8237
- // `watchFile` (polling stat) is deliberate over `fs.watch`: it survives the
8238
- // atomic temp+rename that `writeConfig` does (fs.watch would rebind to the old
8239
- // inode and go silent), and it's uniform across platforms. Profile edits are
8240
- // rare + manual, so a ~1.5s poll latency is fine.
7884
+ // Non-auto live retype: the runtime's reconcile loop only rewrites the --auto
7885
+ // worker's types (from the engine read), so keep watching the profile file to
7886
+ // honour `nano assign` — a profile edit rewrites THIS worker's serviceable types
7887
+ // in the shared registry without a restart (a single `registry.setTypes` call,
7888
+ // the direct analogue of the retired per-process reconcile). In --auto the
7889
+ // runtime owns the desired set, so no profile watch is installed.
7890
+ const configFile = getConfigFile();
7891
+ const WATCH_INTERVAL_MS = 1500;
7892
+ let draining = false;
7893
+ if (!autoMode) {
7894
+ // Serialize reloads on a chain (never overlap a `setTypes` write) and coalesce
7895
+ // with a monotonic generation guard, so a slow older reload can never apply a
7896
+ // stale job-type set after a newer edit has already superseded it.
7897
+ let reloadSeq = 0;
7898
+ let reloadChain = Promise.resolve();
8241
7899
  watchFile(configFile, { interval: WATCH_INTERVAL_MS }, (curr, prev) => {
7900
+ // A callback can already be queued when teardown flips `draining`; bail so we
7901
+ // never write to the shared registry (or race its teardown) during shutdown.
7902
+ if (draining) return;
8242
7903
  // Fires each interval; act only on real changes. Compare mtime, ctime and
8243
- // size, not mtime alone: on filesystems with coarse mtime resolution (or two
8244
- // edits within one mtime tick) mtimeMs can be unchanged while size/ctimeMs
8245
- // differ, and an mtime-only guard would skip a genuine profile update.
7904
+ // size, not mtime alone: coarse-mtime filesystems (or two edits in one tick)
7905
+ // can leave mtimeMs unchanged while size/ctimeMs differ.
8246
7906
  if (
8247
7907
  curr.mtimeMs === prev.mtimeMs &&
8248
7908
  curr.ctimeMs === prev.ctimeMs &&
8249
7909
  curr.size === prev.size
8250
7910
  ) return;
8251
- // `reconcile()` owns the `inFlightReconcile` handle: a change arriving while
8252
- // a reconcile is already running coalesces into the current pass and returns
8253
- // that same in-flight promise, so shutdown always waits for the real one.
8254
- reconcile().catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
7911
+ const mySeq = ++reloadSeq;
7912
+ reloadChain = reloadChain.then(async () => {
7913
+ if (draining) return;
7914
+ // A newer edit already landed while we were queued — skip this stale
7915
+ // reload so its (older) job-type set never lands after the newer one.
7916
+ if (mySeq !== reloadSeq) return;
7917
+ let stored;
7918
+ try {
7919
+ stored = readHiresStrict()[name];
7920
+ } catch (err) {
7921
+ // config exists but doesn't parse (torn write) — keep current types
7922
+ logger.warn(`Profile "${name}" reload skipped — ${configFile} unreadable/unparseable (keeping current job types): ${err?.message || err}`);
7923
+ return;
7924
+ }
7925
+ if (!stored) {
7926
+ // profile vanished — keep serving the current types
7927
+ logger.warn(`Profile "${name}" reload skipped — profile no longer present in ${configFile} (keeping current job types)`);
7928
+ return;
7929
+ }
7930
+ const norm = normalizeStoredProfile(name, stored);
7931
+ if (norm.error) {
7932
+ // invalid edit — keep current types
7933
+ logger.warn(`Profile "${name}" reload skipped — invalid profile edit (keeping current job types): ${norm.error}`);
7934
+ return;
7935
+ }
7936
+ const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
7937
+ const desired = [...new Set([...m, ...extraJobTypes])];
7938
+ if (draining) return; // teardown began while we were reading — don't write
7939
+ if (mySeq !== reloadSeq) return; // superseded during the async read — skip
7940
+ await SupervisorEffect.runPromise(workerRegistry.setTypes(workerName, desired));
7941
+ logger.info(`Profile "${name}" changed — now servicing ${desired.length} job type(s): ${desired.join(' ')}`);
7942
+ }).catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
8255
7943
  });
8256
7944
  }
8257
7945
 
8258
- // Keep the process alive until a stop signal, then drain gracefully.
7946
+ // Keep the process alive until a stop signal, then interrupt the runtime loop
7947
+ // and tear down visibility. Interrupting the supervisor fiber runs the runtime's
7948
+ // bracketed teardown (release slots, stop the heartbeat + agentic scope).
8259
7949
  await new Promise((resolve) => {
8260
7950
  const stop = async (signal) => {
8261
7951
  if (draining) return;
8262
7952
  draining = true;
8263
- // Stop the reconcile trigger first so no new reconcile can be triggered,
8264
- // then wait for any in-flight reconcile to finish before snapshotting
8265
- // `workers` — this prevents double-stops, missed drains, or a wrong worker
8266
- // count on exit.
8267
- if (autoPollTimer) clearInterval(autoPollTimer);
8268
- else unwatchFile(configFile);
8269
- if (inFlightReconcile) {
8270
- logger.info('Waiting for in-flight reconcile to finish before shutdown…');
8271
- await inFlightReconcile;
8272
- }
8273
- const list = [...workers.values()];
8274
- logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
7953
+ if (!autoMode) unwatchFile(configFile);
7954
+ logger.info(`Received ${signal} stopping worker...`);
8275
7955
  if (reaperTimer) clearInterval(reaperTimer);
8276
7956
  if (runDirTimer) clearInterval(runDirTimer);
8277
- // Stop the #144 liveness watchdog so it can't kick off a re-discovery
8278
- // mid-teardown (which would resurrect the channel we're about to close).
8279
- if (agenticWatchdog) { try { agenticWatchdog.stop(); } catch { /* best effort */ } agenticWatchdog = null; }
8280
- const results = await Promise.all(list.map(drainWorker));
8281
- const stopFailures = results.filter((ok) => !ok).length;
8282
- if (stopFailures > 0) {
8283
- logger.warn(`${stopFailures} of ${list.length} worker(s) did not stop cleanly; some connections may still be open.`);
8284
- } else {
8285
- logger.info('All workers stopped.');
7957
+ // Deregister this worker's presence BEFORE interrupting the runtime: emit
7958
+ // the explicit deregister while the multiplexed host connection is still
7959
+ // live, so the worker disappears from the cockpit cleanly rather than
7960
+ // lingering until its heartbeat lapses. Best-effort — teardown must never
7961
+ // hang. Interrupting the fiber then runs the runtime's bracketed teardown
7962
+ // (release slots, stop the heartbeat + tear down the agentic scope).
7963
+ if (agenticPlane) {
7964
+ try { agenticPlane.deregister(`worker stopped (${signal})`); } catch { /* best effort */ }
8286
7965
  }
8287
- // Deregister from the visibility channel LAST, so the worker disappears
8288
- // from the page only once its jobs have drained. Best-effort — a channel
8289
- // teardown must never hang shutdown.
8290
- if (workChannel) {
8291
- // Stop the buffer monitor first so its sampler can't fire mid-teardown.
8292
- try {
8293
- bufferMonitor?.stop();
8294
- } catch { /* best effort */ }
8295
- try {
8296
- await workChannel.stop(`worker stopped (${signal})`);
8297
- logger.info('Deregistered from the agentic visibility channel.');
8298
- } catch (err) {
8299
- logger.warn(`agentic channel deregister failed: ${err?.message || err}`);
8300
- }
7966
+ try {
7967
+ await SupervisorEffect.runPromise(SupervisorFiber.interrupt(supervisorFiber));
7968
+ logger.info('Worker stopped.');
7969
+ } catch (err) {
7970
+ logger.warn(`supervisor shutdown error runtime loop may not have shut down cleanly: ${err?.message || err}`);
8301
7971
  }
8302
7972
  resolve();
8303
7973
  };
@@ -12708,8 +12378,6 @@ export {
12708
12378
  resolveLivenessOverrides,
12709
12379
  parsePsTime,
12710
12380
  ensureAcpFlag,
12711
- startLockExtender,
12712
- createSingleFlight,
12713
12381
  provisionRepo,
12714
12382
  finalizeGit,
12715
12383
  describeGitFailure,