c8ctl-plugin-nano 1.50.0 → 1.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -136,11 +136,6 @@ const READINESS_TIMEOUT_MS = 60_000;
136
136
  const READINESS_POLL_MS = 500;
137
137
  const HEALTH_TIMEOUT_MS = 1_500;
138
138
  const STOP_GRACE_MS = 8_000;
139
- // Backoff applied when a poller fails a lease fast because the worker is already
140
- // running another job (issue #142 single-flight). Long enough that the deferred
141
- // job doesn't tight-loop re-activating while the first runs, short enough that it
142
- // is picked up promptly once the worker frees up.
143
- const WORKER_BUSY_RETRY_BACKOFF_MS = 5_000;
144
139
  // Upper bound on one `--auto` engine-read reconcile (enumerate deployed
145
140
  // definitions + fetch each BPMN). A read that stalls past this is treated as a
146
141
  // transient failure so the running poller set is KEPT and, crucially, shutdown
@@ -3288,14 +3283,12 @@ async function createAgenticEndpoint(opts) {
3288
3283
  //
3289
3284
  // Returns everything a JS caller needs to run `makeSupervisor` — the assembled
3290
3285
  // `deps`, the seeded `registry` (so workers can be add/remove'd live), and the
3291
- // `makeSupervisor` + `Effect` handles from the bundle. NOTE (deferred, issue
3292
- // #156): the actual hot-path flip — deleting the per-type SDK pollers, the
3293
- // process-wide `singleFlight`, and the per-process reconcile crawl in `workAgent`
3294
- // and running `makeSupervisor(deps).run` as the single per-host owner — is NOT
3295
- // wired here; it deletes battle-tested crash-safety code and can only be
3296
- // validated against a live engine, so it is intentionally left as the follow-up
3297
- // this seam unblocks. Constructing deps is side-effect-free (no socket opens, no
3298
- // activation) until the caller forks `supervisor.run`.
3286
+ // `makeSupervisor` + `Effect` handles from the bundle. NOTE (issue #172): the
3287
+ // hot-path flip this seam unblocks is now DONE — `workAgent` runs
3288
+ // `makeSupervisor(deps).run` as the single per-host owner, having retired the
3289
+ // per-type SDK pollers, the process-wide `singleFlight`, the per-process reconcile
3290
+ // crawl, and the per-job lock extender. Constructing deps stays side-effect-free
3291
+ // (no socket opens, no activation) until the caller forks `supervisor.run`.
3299
3292
  //
3300
3293
  // @param {object} opts
3301
3294
  // @param {{ run(job): Promise<void> }} opts.runner raw job runner (required to run)
@@ -3312,7 +3305,7 @@ async function createAgenticEndpoint(opts) {
3312
3305
  // @param {{ searchProcessDefinitionKeys: Function, getProcessDefinitionXml: Function }} [opts.reconcileReader] raw reconcile reader overriding the default keep-alive `defaultC8RestReader`
3313
3306
  // @param {typeof fetch} [opts.fetchImpl] injected fetch (tests)
3314
3307
  // @param {NodeJS.ProcessEnv} [opts.env]
3315
- // @returns {Promise<{ deps: object, registry: object, makeSupervisor: Function, Effect: object, Fiber: object }>}
3308
+ // @returns {Promise<{ deps: object, registry: object, settle: { complete: Function, fail: Function }, makeSupervisor: Function, Effect: object, Fiber: object }>}
3316
3309
  async function createSupervisorDeps(opts = {}) {
3317
3310
  const {
3318
3311
  runner,
@@ -3373,9 +3366,8 @@ async function createSupervisorDeps(opts = {}) {
3373
3366
  };
3374
3367
  }
3375
3368
 
3376
- const engine = rt.makeEngineClient(
3377
- createRawEngineClient({ baseUrl: rc.baseUrl, token: rc.token, authHeaders: resolvedAuthHeaders, worker, fetchImpl }),
3378
- );
3369
+ const rawEngine = createRawEngineClient({ baseUrl: rc.baseUrl, token: rc.token, authHeaders: resolvedAuthHeaders, worker, fetchImpl });
3370
+ const engine = rt.makeEngineClient(rawEngine);
3379
3371
  // `reconcileReader` (a raw `{ searchProcessDefinitionKeys, getProcessDefinitionXml }`)
3380
3372
  // may be injected to override the default keep-alive `httpC8RestReader` — the
3381
3373
  // caller may already hold one, and a test drives the crawl without a socket.
@@ -3402,7 +3394,24 @@ async function createSupervisorDeps(opts = {}) {
3402
3394
  config: scope ? { ...config, scope } : config,
3403
3395
  });
3404
3396
 
3405
- return { deps, registry, makeSupervisor: rt.makeSupervisor, Effect: rt.Effect, Fiber: rt.Fiber };
3397
+ // The `settle` seam (issue #156, escalation answer (a)): the runner settles a
3398
+ // finished job through the SAME engine client the supervisor activates/extends
3399
+ // with — one base URL, one (rotating) auth resolver — so a `complete`/`fail`
3400
+ // can never drift from the activation's credentials. A plain `ActivatedJob`
3401
+ // carries no `job.complete()`/`job.fail()` (unlike the SDK job object), so the
3402
+ // supervisor path settles via these `{ complete, fail }` calls. This is the
3403
+ // direct analogue of returning `registry` for live worker add/remove: the
3404
+ // caller wires it into `runAgentJob`'s completion, replacing the SDK job object.
3405
+ // Route through the LIFTED `engine` port (not `rawEngine`) + `Effect.runPromise`
3406
+ // so a settle rejection surfaces as the normalized `SupervisorError` the rest of
3407
+ // the runtime uses (activate/extendLock), and any future port instrumentation
3408
+ // covers the settle path too.
3409
+ const settle = {
3410
+ complete: (jobKey, variables) => rt.Effect.runPromise(engine.complete(jobKey, variables)),
3411
+ fail: (jobKey, opts2) => rt.Effect.runPromise(engine.fail(jobKey, opts2)),
3412
+ };
3413
+
3414
+ return { deps, registry, settle, makeSupervisor: rt.makeSupervisor, Effect: rt.Effect, Fiber: rt.Fiber };
3406
3415
  }
3407
3416
 
3408
3417
  // Build the content endpoint. Per issue #63 / nano-bpm #759 the non-binary
@@ -5513,80 +5522,12 @@ function baseAgentEnv(profile, job) {
5513
5522
  };
5514
5523
  }
5515
5524
 
5516
- /**
5517
- * Process-wide single-flight guard (issue #142).
5518
- *
5519
- * `maxParallelJobs = 1` only caps concurrency WITHIN one job-type poller, but a
5520
- * single `work` process runs one poller per job type (rank×capability matrix, or
5521
- * every deployed agent type under `--auto`). Without a shared gate a worker
5522
- * serving N job types could lease and run up to N jobs at once — each holding its
5523
- * own PTY + git workspace + broker lock-extender — the exact failure the
5524
- * "one job per worker" invariant exists to prevent.
5525
- *
5526
- * This is a capacity-1, non-blocking mutex shared by EVERY per-type poller: the
5527
- * first poller to `tryAcquire()` runs its job to completion (releasing in a
5528
- * `finally`); any other poller that finds the permit already held must NOT begin
5529
- * a second job (the caller fails the lease fast so the broker re-queues it rather
5530
- * than leaving it "claimed but idle"). `tryAcquire`/`release` are synchronous
5531
- * check-and-set, so the single-threaded event loop makes them race-free across
5532
- * the concurrently-invoked async job handlers.
5533
- */
5534
- function createSingleFlight() {
5535
- let held = false;
5536
- return {
5537
- /** Take the permit if free; returns false when a job is already in flight. */
5538
- tryAcquire() {
5539
- if (held) return false;
5540
- held = true;
5541
- return true;
5542
- },
5543
- /** Release the permit. Idempotent: redundant calls are safe no-ops, though the normal path releases once per acquire (in a `finally`). */
5544
- release() {
5545
- held = false;
5546
- },
5547
- /** True while a job holds the permit. */
5548
- get busy() {
5549
- return held;
5550
- },
5551
- };
5552
- }
5553
-
5554
- /**
5555
- * Keep a leased job's broker activation lock ahead of *now* while the harness is
5556
- * running, so a long agent run never has its lock lapse and get re-activated (a
5557
- * second worker starting → the classic stale complete/fail 409). The lock is NOT
5558
- * hardcoded up front: we refresh it to `windowMs` — a duration-from-now, per the
5559
- * UpdateJobTimeout contract ("the duration of the new timeout in ms, starting
5560
- * from the current moment"), so calls SET rather than accumulate — every
5561
- * `intervalMs`. The deadline therefore stays a bounded `windowMs` ahead of now.
5562
- * The instant we stop refreshing (harness exit / idle-kill / hard cap) the lock
5563
- * lapses within `windowMs` and the broker reclaims the job — fast node-loss
5564
- * recovery. Because the harness is always killed locally before we stop, the lock
5565
- * strictly outlives our local run, so a reclaim never races a still-running agent.
5566
- *
5567
- * Returns a stop() to call once the run settles. Extension failures are logged
5568
- * and swallowed — a transient network blip must not crash the job handler. Older
5569
- * SDKs without `modifyJobTimeout` degrade to the fixed initial lock (a no-op stop).
5570
- */
5571
- function startLockExtender(job, windowMs, intervalMs, tag, logger) {
5572
- if (!(windowMs > 0) || !(intervalMs > 0)) {
5573
- return () => {};
5574
- }
5575
- if (typeof job?.modifyJobTimeout !== 'function') {
5576
- logger?.warn?.(`${tag}: job.modifyJobTimeout unavailable — activation lock will NOT be auto-extended; a run longer than ${windowMs}ms risks being reclaimed and executed twice`);
5577
- return () => {};
5578
- }
5579
- const extend = () => Promise.resolve()
5580
- .then(() => job.modifyJobTimeout({ newTimeoutMs: windowMs }))
5581
- .catch((err) => logger?.warn?.(`${tag}: lock extend failed — ${err?.message ?? err}`));
5582
- // Renew immediately so the harness starts with a full, fresh window no matter
5583
- // how much of the initial activation lease provisioning (clone/checkout) ate.
5584
- extend();
5585
- const timer = setInterval(extend, intervalMs);
5586
- // Never let the heartbeat keep the process alive on shutdown.
5587
- if (typeof timer.unref === 'function') timer.unref();
5588
- return () => clearInterval(timer);
5589
- }
5525
+ // Issue #172 retired `createSingleFlight` (the process-wide capacity-1 mutex the
5526
+ // per-type SDK pollers shared) and `startLockExtender` (the per-job broker-lock
5527
+ // heartbeat). Both are now owned by the single-owner supervisor runtime: race-free
5528
+ // per-type slot accounting lives in `supervisor/src/registry.ts` (this worker
5529
+ // registers with capacity 1), and the lock lifecycle — extend-winner-before-start
5530
+ // plus a `Schedule`-driven heartbeat — lives in `supervisor/src/dispatch.ts`.
5590
5531
 
5591
5532
  /**
5592
5533
  * Run a single activated job through the profile's CLI command (one-shot),
@@ -7016,26 +6957,21 @@ async function workAgent(req, flags) {
7016
6957
  };
7017
6958
  // One job per worker, hard-wired (there is deliberately no --max-parallel
7018
6959
  // flag): an agent harness holds a PTY + a git workspace for the whole life of
7019
- // a job, so a worker must never lease a second job concurrently. The @camunda8
7020
- // SDK derives maxJobsToActivate = maxParallelJobs - activeJobs, so 1 means
7021
- // "activate one job, then stop polling until it completes".
7022
- const maxParallelJobs = 1;
7023
- // Process-wide single-flight guard (issue #142). The SDK's maxParallelJobs=1
7024
- // only serializes ONE job-type poller, but this process runs one poller per
7025
- // job type, so nothing stops N pollers from each leasing + running a job
7026
- // concurrently. This capacity-1 mutex, shared by every poller's jobHandler,
7027
- // enforces the real "one job per worker" invariant: while any job is in flight
7028
- // on any job type, no other poller starts a second one.
7029
- const singleFlight = createSingleFlight();
6960
+ // a job, so a worker must never lease a second job concurrently. Issue #172:
6961
+ // this is now enforced by registering this worker with capacity 1 in the
6962
+ // single-owner runtime's shared registry (one worker, capacity 1, across all
6963
+ // job types) — retiring the process-wide capacity-1 single-flight the per-type
6964
+ // SDK pollers needed. Race-free slot accounting lives in the runtime registry.
7030
6965
  // The broker job-activation lock is NOT hardcoded up front. A fixed timeout is
7031
6966
  // impossible to size for an agent: too short reclaims a still-working job (a
7032
6967
  // second agent starts + the stale complete/fail is rejected 409), too long
7033
- // strands a dead worker's job. Instead the worker keeps the lock a bounded
7034
- // `recovery-window` ahead of *now* while the harness runs (see
7035
- // startLockExtender), so long runs never lose their lock, and a dead/killed
7036
- // worker's job is reclaimed within one window. Liveness is enforced by
7037
- // `idle-timeout` (max silence before the harness is killed as wedged), so the
7038
- // lock is held only while the agent is alive AND producing output.
6968
+ // strands a dead worker's job. Instead the runtime's dispatch lifecycle keeps
6969
+ // the lock a bounded `recovery-window` ahead of *now* while the harness runs
6970
+ // (extend-winner-before-start + a Schedule heartbeat in dispatch.ts), so long
6971
+ // runs never lose their lock, and a dead/killed worker's job is reclaimed
6972
+ // within one window. Liveness is enforced by `idle-timeout` (max silence before
6973
+ // the harness is killed as wedged), so the lock is held only while the agent is
6974
+ // alive AND producing output.
7039
6975
  const recoveryWindowMs = intFlag(flags?.['recovery-window'], 5 * 60_000);
7040
6976
  const idleTimeoutMs = intFlag(flags?.['idle-timeout'], 5 * 60_000);
7041
6977
  // `--job-timeout` is now an OPTIONAL absolute hard cap on total harness runtime
@@ -7234,7 +7170,7 @@ async function workAgent(req, flags) {
7234
7170
  }
7235
7171
  const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
7236
7172
  logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
7237
- logger.info(` one job per worker (single-flight across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
7173
+ logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
7238
7174
  // Warm the gh-token cache now, off the job-handling path: githubCloneToken()
7239
7175
  // may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
7240
7176
  // credential fallback, and doing that inside a job handler would block the
@@ -7630,46 +7566,20 @@ async function workAgent(req, flags) {
7630
7566
  const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
7631
7567
  const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
7632
7568
 
7633
- // A per-job-type worker factory. Captures all the CLI-local + profile context
7634
- // in closure scope so the profile watcher below can (re)spawn a poller for any
7635
- // job type on demand without re-reading the flags.
7636
- const makeWorker = (jobType) =>
7637
- camunda.createJobWorker({
7638
- jobType,
7639
- workerName: `${workerName}:${jobType}`,
7640
- maxParallelJobs,
7641
- jobTimeoutMs: recoveryWindowMs,
7642
- pollTimeoutMs,
7643
- jobHandler: async (job) => {
7644
- // Process-wide single-flight (issue #142): if another job is already
7645
- // running on ANY poller, do not start a second harness. Fail this lease
7646
- // FAST — before recording it active, extending its lock, or provisioning
7647
- // anything — so the broker re-queues it (retries preserved) instead of it
7648
- // sitting "claimed but idle" while the first job runs. Gating here, at the
7649
- // point activation surfaces as a handler call, is the cross-poller gate
7650
- // the per-type maxParallelJobs cannot provide.
7651
- if (!singleFlight.tryAcquire()) {
7652
- // Not a failure — preserve the broker-provided retries verbatim so
7653
- // re-dispatch doesn't decrement (or resurrect) the job. Keep a real 0
7654
- // as 0 (an already-incidentable job must stay that way); only default
7655
- // to 1 when the count is missing/invalid.
7656
- const rawRetries = Number(job.retries);
7657
- const retries = Number.isInteger(rawRetries) && rawRetries >= 0 ? rawRetries : 1;
7658
- logger.info(`[${jobType}] job ${job.jobKey} deferred — worker already running another job; releasing lease for re-dispatch.`);
7659
- return job.fail({
7660
- errorMessage: 'worker busy: one job per worker (single-flight across all job types)',
7661
- retries,
7662
- retryBackOff: WORKER_BUSY_RETRY_BACKOFF_MS,
7663
- });
7664
- }
7665
- recordJobStart(job, jobType);
7666
- // Auto-extend the broker lock for the whole life of this job (harness run
7667
- // + git finalize + complete/fail), stopped in the outer finally. The lock
7668
- // is held only while the harness stays alive and productive — a silent
7669
- // hang is killed by the idle-timeout, which resolves runAgentJob and stops
7670
- // the extension, so the broker can reclaim the job.
7671
- let stopLockExtender = () => {};
7672
- try {
7569
+ // The agent job runner (issue #172 hot-path flip). The single-owner supervisor
7570
+ // runtime dispatches each activated job to this `run(job)`; it executes the
7571
+ // harness exactly as the retired per-type SDK jobHandler did, but SETTLES via the
7572
+ // injected `settle` seam (engine complete/fail) because the plain ActivatedJob
7573
+ // carries no SDK `job.complete()/job.fail()`. Capacity (one job per worker), the
7574
+ // activation long-polls, the lock lifecycle, and reconcile are all owned by the
7575
+ // runtime now — retiring the per-type pollers, the process-wide single-flight,
7576
+ // the per-process 1+N reconcile crawl, and the per-job lock extender.
7577
+ let settle;
7578
+ const runner = {
7579
+ run: async (job) => {
7580
+ const jobType = job.type;
7581
+ recordJobStart(job, jobType);
7582
+ try {
7673
7583
  logger.info(`[${jobType}] job ${job.jobKey} (instance ${job.processInstanceKey ?? '-'}) → ${buildAgentCommandLine(profile.command, effectiveArgs)}`);
7674
7584
 
7675
7585
  // Disk-budget admission shed: if the engine data root is below the free
@@ -7681,7 +7591,7 @@ async function workAgent(req, flags) {
7681
7591
  const freeMb = budget.free != null ? Math.round(budget.free / 1_048_576) : '?';
7682
7592
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7683
7593
  logger.warn(`[${jobType}] job ${job.jobKey} shed — low disk (${freeMb}MB free); retries left ${retries}`);
7684
- return job.fail({ errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
7594
+ return settle.fail(job.jobKey, { errorMessage: `disk budget exceeded (only ${freeMb}MB free)`, retries, retryBackOff: 30_000 });
7685
7595
  }
7686
7596
  }
7687
7597
 
@@ -7713,7 +7623,7 @@ async function workAgent(req, flags) {
7713
7623
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7714
7624
  const msg = err instanceof ProvisionError ? err.message : `prompt resource fetch failed: ${err.message}`;
7715
7625
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7716
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7626
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7717
7627
  }
7718
7628
 
7719
7629
  // Assemble + normalize the task envelope from headers (defaults) and
@@ -7724,27 +7634,24 @@ async function workAgent(req, flags) {
7724
7634
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7725
7635
  const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
7726
7636
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7727
- return job.fail({ errorMessage: msg, retries });
7637
+ return settle.fail(job.jobKey, { errorMessage: msg, retries });
7728
7638
  }
7729
7639
 
7730
7640
  // #130: per-task liveness overrides. Precedence: envelope override →
7731
7641
  // worker-flag default → built-in default, each clamped to a sane max so
7732
7642
  // a task can't request an unbounded window. Absent envelope fields leave
7733
- // the worker-flag behaviour unchanged. These drive BOTH the harness idle
7734
- // liveness (idle/recovery) AND the broker lock recovery window, so a
7735
- // JVM-heavy task can widen its own window without a global flag change.
7643
+ // the worker-flag behaviour unchanged. These drive the harness idle
7644
+ // liveness (idle/recovery/hard-cap) so a JVM-heavy task can widen its own
7645
+ // window without a global flag change. NOTE: after the hot-path flip
7646
+ // (#172) the broker lock cadence/window is owned by the supervisor
7647
+ // runtime dispatch config at the worker level (the `dispatch` config's
7648
+ // `recoveryWindowMs` / `extendIntervalMs`), so a per-task override no
7649
+ // longer widens the broker lock window — only the harness liveness.
7736
7650
  const {
7737
7651
  idleTimeoutMs: effectiveIdleTimeoutMs,
7738
7652
  recoveryWindowMs: effectiveRecoveryWindowMs,
7739
7653
  hardCapMs: effectiveHardCapMs,
7740
7654
  } = resolveLivenessOverrides(envelope.task, { idleTimeoutMs, recoveryWindowMs, hardCapMs });
7741
- // Recompute the lock-extend cadence from the effective recovery window
7742
- // (same ~1/3-of-window rule as at startup) so a widened window still
7743
- // renews comfortably before it lapses.
7744
- const effectiveLockExtendIntervalMs = Math.min(
7745
- Math.max(5_000, Math.floor(effectiveRecoveryWindowMs / 3)),
7746
- Math.max(1, Math.floor(effectiveRecoveryWindowMs * 0.75)),
7747
- );
7748
7655
  if (
7749
7656
  effectiveIdleTimeoutMs !== idleTimeoutMs ||
7750
7657
  effectiveRecoveryWindowMs !== recoveryWindowMs ||
@@ -7774,7 +7681,7 @@ async function workAgent(req, flags) {
7774
7681
  : 'repository.url is missing';
7775
7682
  const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
7776
7683
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7777
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7684
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7778
7685
  }
7779
7686
  }
7780
7687
 
@@ -7784,12 +7691,10 @@ async function workAgent(req, flags) {
7784
7691
  const hasRepo = !isContainer && !!envelope.repository?.url;
7785
7692
  let runDir = null;
7786
7693
  let provisioned = null;
7787
- // Start refreshing the broker activation lock BEFORE any potentially-long
7788
- // work (host git clone/checkout can outlast the initial window). Starting
7789
- // here — ahead of provisionRepo — guarantees the first renewal is queued
7790
- // before the clone, so the lock can't lapse mid-provision and trigger the
7791
- // duplicate-activation / stale-409 race. The `finally` below stops it.
7792
- stopLockExtender = startLockExtender(job, effectiveRecoveryWindowMs, effectiveLockExtendIntervalMs, `[${jobType}] job ${job.jobKey}`, logger);
7694
+ // The broker activation lock is owned by the single-owner runtime's dispatch
7695
+ // lifecycle (supervisor/src/dispatch.ts): it extends the winner to the
7696
+ // recovery window BEFORE this runner starts and heartbeats it on a Schedule
7697
+ // for the whole run, so the retired per-job startLockExtender is gone here.
7793
7698
  let cwd;
7794
7699
  let extraEnv;
7795
7700
  let repoToken = null;
@@ -7831,7 +7736,7 @@ async function workAgent(req, flags) {
7831
7736
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7832
7737
  const msg = err instanceof ProvisionError ? err.message : `provisioning error: ${err.message}`;
7833
7738
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7834
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7739
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7835
7740
  }
7836
7741
  } else if (!isContainer) {
7837
7742
  // Repo-less host job (issue #129, hardening 1): nothing is provisioned,
@@ -7857,7 +7762,7 @@ async function workAgent(req, flags) {
7857
7762
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
7858
7763
  const msg = `could not create a temp workspace under the runs root: ${err.message}`;
7859
7764
  logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
7860
- return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7765
+ return settle.fail(job.jobKey, { errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
7861
7766
  }
7862
7767
  }
7863
7768
 
@@ -8013,7 +7918,7 @@ async function workAgent(req, flags) {
8013
7918
  const resultKeys = Object.keys(resultVars);
8014
7919
  if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
8015
7920
  else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
8016
- return await job.complete({
7921
+ return await settle.complete(job.jobKey, {
8017
7922
  ...resultVars,
8018
7923
  [AGENT_RESULT_KEY]: resultEnvelope,
8019
7924
  output: result.stdout,
@@ -8030,249 +7935,136 @@ async function workAgent(req, flags) {
8030
7935
  || (result.stderr || '').trim() + (result.stderrTruncated && (result.stderr || '').trim() ? ' [stderr truncated]' : '')
8031
7936
  || (result.signal ? `terminated by signal ${result.signal}` : `exit code ${result.exitCode}`);
8032
7937
  logger.warn(`[${jobType}] job ${job.jobKey} failed (${detail}); retries left ${retries}`);
8033
- return await job.fail({
7938
+ return await settle.fail(job.jobKey, {
8034
7939
  errorMessage: `agent "${profile.name}" failed: ${detail}`.slice(0, 2000),
8035
7940
  retries,
8036
7941
  variables: { [AGENT_RESULT_KEY]: resultEnvelope },
8037
7942
  });
8038
- } finally {
8039
- stopLockExtender();
8040
- recordJobEnd(job);
8041
- // Release the process-wide single-flight permit LAST, once this job's
8042
- // lock-extender is stopped and its bookkeeping cleared, so another
8043
- // poller can only begin after this job is fully settled.
8044
- singleFlight.release();
8045
- }
8046
- },
8047
- });
8048
-
8049
- // Live worker registry keyed by job type, so the profile watcher can add or
8050
- // drain individual pollers without disturbing the others. `draining` is the
8051
- // shutdown latch (shared with the watcher so a reconcile can't race a stop).
8052
- const workers = new Map();
8053
- let draining = false;
8054
-
8055
- const drainWorker = async (w) => {
8056
- try {
8057
- if (typeof w.stopGracefully === 'function') {
8058
- await w.stopGracefully({ waitUpToMs: STOP_GRACE_MS });
8059
- } else if (typeof w.stop === 'function') {
8060
- await w.stop();
7943
+ } finally {
7944
+ recordJobEnd(job);
8061
7945
  }
8062
- return true;
8063
- } catch {
8064
- return false; // best-effort: never let one worker's stop failure hang us
8065
- }
8066
- };
8067
-
8068
- const spawnJobType = (jobType) => {
8069
- if (workers.has(jobType)) return false;
8070
- workers.set(jobType, makeWorker(jobType));
8071
- return true;
7946
+ },
8072
7947
  };
8073
7948
 
8074
- for (const jobType of jobTypes) spawnJobType(jobType);
8075
-
8076
- // ---- Live reconcile: keep the poller set in step with the desired job-type
8077
- // set — start pollers for added types, gracefully drain pollers for removed
8078
- // types — without a restart and without disturbing unchanged types' in-flight
8079
- // work. The DESIRED set comes from one of two sources depending on mode:
8080
- // - default: the watched profile's rank×capability matrix (∪ --job-type),
8081
- // reconciled when the on-disk profile changes (e.g. `nano assign`);
8082
- // - --auto: the engine's deployed *agent* job types, reconciled by polling
8083
- // the engine (the deployed set changes as apps deploy/undeploy). ----
7949
+ // Compose the runtime deps over the plugin's real edges (issue #156 seam) and
7950
+ // register THIS worker (capacity 1) into the shared registry. In --auto the
7951
+ // runtime's reconcile loop rewrites this worker's serviceable types from the
7952
+ // engine read (`autoWorkerId`); otherwise the profile watch below does.
7953
+ const composed = await createSupervisorDeps({
7954
+ runner,
7955
+ camunda,
7956
+ restConfig,
7957
+ worker: workerName,
7958
+ workers: [{ id: workerName, types: jobTypes, capacity: 1 }],
7959
+ autoWorkerId: autoMode ? workerName : undefined,
7960
+ scope: autoScope,
7961
+ config: {
7962
+ activation: { requestTimeoutMs: pollTimeoutMs },
7963
+ dispatch: { recoveryWindowMs, extendIntervalMs: lockExtendIntervalMs },
7964
+ },
7965
+ });
7966
+ settle = composed.settle;
7967
+ const {
7968
+ deps: supervisorDeps,
7969
+ registry: workerRegistry,
7970
+ makeSupervisor: makeSupervisorRuntime,
7971
+ Effect: SupervisorEffect,
7972
+ Fiber: SupervisorFiber,
7973
+ } = composed;
7974
+
7975
+ // Run the single per-host owner. `Effect.runFork` keeps the process alive on the
7976
+ // runtime's own Schedule cadences (reconcile + activation/idle loop), so even an
7977
+ // --auto worker that starts with zero types stays up and fills them in on the
7978
+ // next reconcile — the same liveness the retired ref'd auto-poll timer provided.
7979
+ const supervisor = await SupervisorEffect.runPromise(makeSupervisorRuntime(supervisorDeps));
7980
+ const supervisorFiber = SupervisorEffect.runFork(supervisor.run);
7981
+
7982
+ // Non-auto live retype: the runtime's reconcile loop only rewrites the --auto
7983
+ // worker's types (from the engine read), so keep watching the profile file to
7984
+ // honour `nano assign` — a profile edit rewrites THIS worker's serviceable types
7985
+ // in the shared registry without a restart (a single `registry.setTypes` call,
7986
+ // the direct analogue of the retired per-process reconcile). In --auto the
7987
+ // runtime owns the desired set, so no profile watch is installed.
8084
7988
  const configFile = getConfigFile();
8085
7989
  const WATCH_INTERVAL_MS = 1500;
8086
- // How often `--auto` re-reads the engine's deployed agent job types to pick up
8087
- // newly deployed / undeployed agent processes. Deploys are occasional, so a
8088
- // few seconds of latency is fine; the read is a couple of cheap C8 REST calls.
8089
- const AUTO_POLL_INTERVAL_MS = 5000;
8090
- let reconciling = false;
8091
- // Set when a profile change arrives while a reconcile is already in flight, so
8092
- // we run one more pass after the current drain completes instead of dropping
8093
- // the update until the next change fires.
8094
- let reconcileRequested = false;
8095
- // Handle to the in-flight reconcile so shutdown can wait for it to finish
8096
- // before snapshotting `workers` (avoids double-stops / missed drains).
8097
- let inFlightReconcile = null;
8098
-
8099
- // Desired job types. In `--auto` this is the engine's deployed agent job types
8100
- // (∪ --job-type extras), read fresh each pass; a transient engine-read failure
8101
- // returns { skip } so the running set is KEPT, never torn down. Otherwise it is
8102
- // the CURRENT on-disk profile's matrix (∪ extras), with { skip } for a
8103
- // transient/torn read, a vanished profile, or an invalid edit — callers must
8104
- // then KEEP the running set, never tear down.
8105
- const desiredJobTypes = async () => {
8106
- if (autoMode) {
8107
- try {
8108
- const autoTypes = await resolveAutoJobTypes({ restConfig, scope: autoScope });
8109
- return { jobTypes: [...new Set([...autoTypes, ...extraJobTypes])] };
8110
- } catch (err) {
8111
- return { skip: `engine read failed: ${err?.message || err}` };
8112
- }
8113
- }
8114
- let stored;
8115
- try {
8116
- stored = readHiresStrict()[name];
8117
- } catch {
8118
- // config.json exists but doesn't parse (e.g. a torn write): the profile is
8119
- // NOT necessarily gone, so don't claim it was deleted — skip this pass.
8120
- return { skip: 'config unreadable' };
8121
- }
8122
- if (!stored) return { skip: 'deleted' };
8123
- const norm = normalizeStoredProfile(name, stored);
8124
- if (norm.error) return { skip: norm.error };
8125
- const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
8126
- return { jobTypes: [...new Set([...m, ...extraJobTypes])] };
8127
- };
8128
-
8129
- const reconcile = () => {
8130
- if (draining) return inFlightReconcile || Promise.resolve();
8131
- if (reconciling) {
8132
- // A change landed mid-reconcile — remember it so the current pass loops
8133
- // once more rather than leaving the worker set stale until the next edit.
8134
- // Return the ACTUAL in-flight promise (not a fresh short-lived one) so a
8135
- // caller — including shutdown — waits for the real reconcile to finish.
8136
- reconcileRequested = true;
8137
- return inFlightReconcile || Promise.resolve();
8138
- }
8139
- reconciling = true;
8140
- reconcileRequested = false;
8141
- inFlightReconcile = (async () => {
8142
- try {
8143
- do {
8144
- reconcileRequested = false;
8145
- await runReconcilePass();
8146
- } while (reconcileRequested && !draining);
8147
- } finally {
8148
- reconciling = false;
8149
- inFlightReconcile = null;
8150
- }
8151
- })();
8152
- return inFlightReconcile;
8153
- };
8154
-
8155
- const runReconcilePass = async () => {
8156
- const desired = await desiredJobTypes();
8157
- if (desired.skip) {
8158
- if (autoMode) {
8159
- logger.warn(`--auto reconcile skipped — ${desired.skip}; keeping the current ${workers.size} worker(s) running.`);
8160
- } else if (desired.skip === 'deleted') {
8161
- logger.warn(`Profile "${name}" is gone from config — keeping the current ${workers.size} worker(s) running.`);
8162
- } else {
8163
- logger.warn(`Profile "${name}" reload skipped — ${desired.skip}; keeping current workers.`);
8164
- }
8165
- return;
8166
- }
8167
- const { added, removed } = diffJobTypes([...workers.keys()], desired.jobTypes);
8168
- if (added.length === 0 && removed.length === 0) return;
8169
- const source = autoMode ? 'engine deployed set' : `Profile "${name}"`;
8170
- logger.info(`${source} changed — reconciling job types (+${added.length} / -${removed.length}).`);
8171
- for (const jt of added) {
8172
- spawnJobType(jt);
8173
- logger.info(` + now listening on ${jt}`);
8174
- }
8175
- await Promise.all(
8176
- removed.map(async (jt) => {
8177
- const w = workers.get(jt);
8178
- logger.info(` - draining ${jt} …`);
8179
- const ok = await drainWorker(w);
8180
- if (ok) {
8181
- // Only drop it from the registry once it has actually stopped, so a
8182
- // failed drain stays tracked and gets retried on the next reconcile
8183
- // pass (or on shutdown) instead of leaking an untracked poller.
8184
- workers.delete(jt);
8185
- logger.info(` - stopped ${jt}`);
8186
- } else {
8187
- logger.warn(` - ${jt} did not stop cleanly; keeping it tracked so it is retried on the next reconcile or shutdown.`);
8188
- }
8189
- }),
8190
- );
8191
- logger.info(` now listening on ${workers.size} job type(s): ${[...workers.keys()].join(' ')}`);
8192
- };
8193
-
8194
- // Reconcile trigger. In `--auto` a periodic engine poll re-reads the deployed
8195
- // agent job types; otherwise a profile-file watch fires on profile edits.
8196
- let autoPollTimer = null;
8197
- if (autoMode) {
8198
- // Self-standing interval poll (not watchFile) since the desired set is
8199
- // derived from the engine, not the on-disk profile. Skip a tick while a
8200
- // reconcile is already in flight: calling reconcile() then would set
8201
- // reconcileRequested and make the in-flight pass loop back-to-back, so an
8202
- // engine read that consistently outlasts AUTO_POLL_INTERVAL_MS would run
8203
- // reconciles as fast as the read completes and hammer the broker. Skipping
8204
- // keeps polling rate-limited to the configured interval regardless of
8205
- // engine-read latency; the next tick re-reads the latest engine state.
8206
- autoPollTimer = setInterval(() => {
8207
- if (inFlightReconcile) return;
8208
- reconcile().catch((err) => logger.warn(`--auto reconcile failed: ${err?.message || err}`));
8209
- }, AUTO_POLL_INTERVAL_MS);
8210
- // Deliberately REF'd (unlike the reaper/run-dir hygiene timers, which are
8211
- // unref'd): in `--auto` this poll IS the retry loop, and it must keep the
8212
- // process alive even with zero pollers. When the INITIAL engine read fails
8213
- // (transient miss, or the engine isn't up yet) the worker registers 0
8214
- // pollers; nothing else holds the event loop open (the SDK client with no
8215
- // job workers doesn't, and the hygiene timers are unref'd), so an unref'd
8216
- // poll timer would let the process exit 0 — the observed crash-loop under a
8217
- // supervisor (jwulf/c8ctl-plugin-nano#93). Keeping it ref'd makes the worker
8218
- // stay up and re-read on the next poll, exactly as the initial-read warning
8219
- // promises. Shutdown clears it (clearInterval), so Ctrl-C/SIGTERM still exit.
8220
- } else {
8221
- // `watchFile` (polling stat) is deliberate over `fs.watch`: it survives the
8222
- // atomic temp+rename that `writeConfig` does (fs.watch would rebind to the old
8223
- // inode and go silent), and it's uniform across platforms. Profile edits are
8224
- // rare + manual, so a ~1.5s poll latency is fine.
7990
+ let draining = false;
7991
+ if (!autoMode) {
7992
+ // Serialize reloads on a chain (never overlap a `setTypes` write) and coalesce
7993
+ // with a monotonic generation guard, so a slow older reload can never apply a
7994
+ // stale job-type set after a newer edit has already superseded it.
7995
+ let reloadSeq = 0;
7996
+ let reloadChain = Promise.resolve();
8225
7997
  watchFile(configFile, { interval: WATCH_INTERVAL_MS }, (curr, prev) => {
7998
+ // A callback can already be queued when teardown flips `draining`; bail so we
7999
+ // never write to the shared registry (or race its teardown) during shutdown.
8000
+ if (draining) return;
8226
8001
  // Fires each interval; act only on real changes. Compare mtime, ctime and
8227
- // size, not mtime alone: on filesystems with coarse mtime resolution (or two
8228
- // edits within one mtime tick) mtimeMs can be unchanged while size/ctimeMs
8229
- // differ, and an mtime-only guard would skip a genuine profile update.
8002
+ // size, not mtime alone: coarse-mtime filesystems (or two edits in one tick)
8003
+ // can leave mtimeMs unchanged while size/ctimeMs differ.
8230
8004
  if (
8231
8005
  curr.mtimeMs === prev.mtimeMs &&
8232
8006
  curr.ctimeMs === prev.ctimeMs &&
8233
8007
  curr.size === prev.size
8234
8008
  ) return;
8235
- // `reconcile()` owns the `inFlightReconcile` handle: a change arriving while
8236
- // a reconcile is already running coalesces into the current pass and returns
8237
- // that same in-flight promise, so shutdown always waits for the real one.
8238
- reconcile().catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
8009
+ const mySeq = ++reloadSeq;
8010
+ reloadChain = reloadChain.then(async () => {
8011
+ if (draining) return;
8012
+ // A newer edit already landed while we were queued — skip this stale
8013
+ // reload so its (older) job-type set never lands after the newer one.
8014
+ if (mySeq !== reloadSeq) return;
8015
+ let stored;
8016
+ try {
8017
+ stored = readHiresStrict()[name];
8018
+ } catch (err) {
8019
+ // config exists but doesn't parse (torn write) — keep current types
8020
+ logger.warn(`Profile "${name}" reload skipped — ${configFile} unreadable/unparseable (keeping current job types): ${err?.message || err}`);
8021
+ return;
8022
+ }
8023
+ if (!stored) {
8024
+ // profile vanished — keep serving the current types
8025
+ logger.warn(`Profile "${name}" reload skipped — profile no longer present in ${configFile} (keeping current job types)`);
8026
+ return;
8027
+ }
8028
+ const norm = normalizeStoredProfile(name, stored);
8029
+ if (norm.error) {
8030
+ // invalid edit — keep current types
8031
+ logger.warn(`Profile "${name}" reload skipped — invalid profile edit (keeping current job types): ${norm.error}`);
8032
+ return;
8033
+ }
8034
+ const m = jobTypeMatrix(norm.profile.rank, norm.profile.capabilities);
8035
+ const desired = [...new Set([...m, ...extraJobTypes])];
8036
+ if (draining) return; // teardown began while we were reading — don't write
8037
+ if (mySeq !== reloadSeq) return; // superseded during the async read — skip
8038
+ await SupervisorEffect.runPromise(workerRegistry.setTypes(workerName, desired));
8039
+ logger.info(`Profile "${name}" changed — now servicing ${desired.length} job type(s): ${desired.join(' ')}`);
8040
+ }).catch((err) => logger.warn(`profile reload failed: ${err?.message || err}`));
8239
8041
  });
8240
8042
  }
8241
8043
 
8242
- // Keep the process alive until a stop signal, then drain gracefully.
8044
+ // Keep the process alive until a stop signal, then interrupt the runtime loop
8045
+ // and tear down visibility. Interrupting the supervisor fiber runs the runtime's
8046
+ // bracketed teardown (release slots, stop the heartbeat + agentic scope).
8243
8047
  await new Promise((resolve) => {
8244
8048
  const stop = async (signal) => {
8245
8049
  if (draining) return;
8246
8050
  draining = true;
8247
- // Stop the reconcile trigger first so no new reconcile can be triggered,
8248
- // then wait for any in-flight reconcile to finish before snapshotting
8249
- // `workers` — this prevents double-stops, missed drains, or a wrong worker
8250
- // count on exit.
8251
- if (autoPollTimer) clearInterval(autoPollTimer);
8252
- else unwatchFile(configFile);
8253
- if (inFlightReconcile) {
8254
- logger.info('Waiting for in-flight reconcile to finish before shutdown…');
8255
- await inFlightReconcile;
8256
- }
8257
- const list = [...workers.values()];
8258
- logger.info(`Received ${signal} — stopping ${list.length} worker(s)...`);
8051
+ if (!autoMode) unwatchFile(configFile);
8052
+ logger.info(`Received ${signal} — stopping worker...`);
8259
8053
  if (reaperTimer) clearInterval(reaperTimer);
8260
8054
  if (runDirTimer) clearInterval(runDirTimer);
8261
8055
  // Stop the #144 liveness watchdog so it can't kick off a re-discovery
8262
8056
  // mid-teardown (which would resurrect the channel we're about to close).
8263
8057
  if (agenticWatchdog) { try { agenticWatchdog.stop(); } catch { /* best effort */ } agenticWatchdog = null; }
8264
- const results = await Promise.all(list.map(drainWorker));
8265
- const stopFailures = results.filter((ok) => !ok).length;
8266
- if (stopFailures > 0) {
8267
- logger.warn(`${stopFailures} of ${list.length} worker(s) did not stop cleanly; some connections may still be open.`);
8268
- } else {
8269
- logger.info('All workers stopped.');
8058
+ try {
8059
+ await SupervisorEffect.runPromise(SupervisorFiber.interrupt(supervisorFiber));
8060
+ logger.info('Worker stopped.');
8061
+ } catch (err) {
8062
+ logger.warn(`supervisor shutdown error — runtime loop may not have shut down cleanly: ${err?.message || err}`);
8270
8063
  }
8271
- // Deregister from the visibility channel LAST, so the worker disappears
8272
- // from the page only once its jobs have drained. Best-effort — a channel
8273
- // teardown must never hang shutdown.
8064
+ // Deregister from the visibility channel LAST, so the worker disappears from
8065
+ // the page only once its jobs have drained. Best-effort — a channel teardown
8066
+ // must never hang shutdown.
8274
8067
  if (workChannel) {
8275
- // Stop the buffer monitor first so its sampler can't fire mid-teardown.
8276
8068
  try {
8277
8069
  bufferMonitor?.stop();
8278
8070
  } catch { /* best effort */ }
@@ -12692,8 +12484,6 @@ export {
12692
12484
  resolveLivenessOverrides,
12693
12485
  parsePsTime,
12694
12486
  ensureAcpFlag,
12695
- startLockExtender,
12696
- createSingleFlight,
12697
12487
  provisionRepo,
12698
12488
  finalizeGit,
12699
12489
  describeGitFailure,