c8ctl-plugin-nano 1.70.2 → 1.70.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -90,11 +90,115 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
90
90
  // so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
91
91
  // a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
92
92
  import { resolveEffectiveEnvelope } from './agent-resume.mjs';
93
- import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
93
+ import { checkpointConfig, checkpointEligibility, checkpointRef, recoveredCheckpointRef, predecessorCheckpointRef, resumeRefIsExplicit, pushNamedRef, observeRemoteRefSha, makeOrphanProbe, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
94
94
 
95
95
  const requireFromHere = createRequire(import.meta.url);
96
96
  const pluginDir = dirname(fileURLToPath(import.meta.url));
97
97
 
98
+ /**
99
+ * Build the ordered list of `require` resolvers used to read the HOST c8ctl / Camunda
100
+ * SDK package manifests (issue #289 review). The host SDK lives in the host **c8ctl
101
+ * CLI's** dependency graph, NOT this plugin's. In the managed layout the plugin loads
102
+ * from `<data>/plugins/node_modules`, a SIBLING of `@camunda8/cli`; in the global
103
+ * layout the SDK is nested BENEATH the CLI package. Node resolution never searches a
104
+ * sibling package's `node_modules`, so a probe rooted only at this module commonly
105
+ * reads `null` for both versions — exactly when it needs to name the old host SDK.
106
+ * Resolve from the host graph instead:
107
+ * 1. a `require` rooted at the real c8ctl entry (`C8CTL_NANO_ENTRY` / `process.argv[1]`),
108
+ * canonicalized through symlinks — the CLI and its nested SDK live in THIS graph;
109
+ * 2. a `require` rooted at the resolved `@camunda8/cli` package itself, so an SDK
110
+ * nested beneath the CLI (not hoisted to the entry root) still resolves;
111
+ * 3. this module's own graph as a last-resort fallback (works in a hoisted/global
112
+ * layout where the SDK is visible from here).
113
+ * Every step is guarded: an unresolvable root is skipped, never thrown.
114
+ *
115
+ * The entry is `realpathSync`'d first because a globally installed CLI is launched
116
+ * through a launcher symlink (e.g. `/usr/local/bin/c8ctl` → `.../lib/node_modules/@camunda8/cli/bin/...`),
117
+ * which Node preserves verbatim in `process.argv[1]`. Rooting `createRequire` at the
118
+ * raw symlink would search `/usr/local/bin/node_modules` and miss the nested host SDK
119
+ * entirely — exactly the global layout this probe exists to diagnose.
120
+ */
121
+ export function buildHostSdkResolvers(entryArg) {
122
+ const resolvers = [];
123
+ const seen = new Set();
124
+ const push = (rq) => {
125
+ if (typeof rq === 'function' && !seen.has(rq)) {
126
+ seen.add(rq);
127
+ resolvers.push(rq);
128
+ }
129
+ };
130
+ const rawEntry = entryArg !== undefined ? entryArg : (process.env.C8CTL_NANO_ENTRY || process.argv[1]);
131
+ let entry = rawEntry;
132
+ if (entry) {
133
+ try {
134
+ entry = realpathSync(entry);
135
+ } catch {
136
+ /* not a real path yet (or already canonical) — fall back to the raw entry */
137
+ }
138
+ }
139
+ let entryRequire = null;
140
+ if (entry) {
141
+ try {
142
+ entryRequire = createRequire(entry);
143
+ push(entryRequire);
144
+ } catch {
145
+ /* unresolvable entry — skip */
146
+ }
147
+ }
148
+ for (const rq of [entryRequire, requireFromHere]) {
149
+ if (!rq) continue;
150
+ try {
151
+ const cliPkg = rq.resolve('@camunda8/cli/package.json');
152
+ push(createRequire(cliPkg));
153
+ } catch {
154
+ /* CLI not resolvable from this root — skip */
155
+ }
156
+ }
157
+ push(requireFromHere);
158
+ return resolvers;
159
+ }
160
+
161
+ /**
162
+ * Best-effort host c8ctl / Camunda SDK versions (issue #289). The host SDK lives in the
163
+ * host c8ctl CLI's dependency graph, which this plugin's (sibling) `node_modules` cannot
164
+ * reach, so the probe walks an ordered chain of resolvers rooted at the host CLI entry /
165
+ * package first and this module only as a fallback (see `buildHostSdkResolvers`) — the
166
+ * first resolver to yield a version wins. Every read is guarded: an unresolvable package
167
+ * (hoisting layout, bundling) simply yields null rather than throwing. Tests may inject a
168
+ * single `requireFn` (used as the sole resolver) or an explicit `resolvers` array.
169
+ */
170
+ export function probeHostSdkInfo(requireFn, resolvers) {
171
+ const chain = Array.isArray(resolvers) && resolvers.length > 0
172
+ ? resolvers
173
+ : requireFn
174
+ ? [requireFn]
175
+ : buildHostSdkResolvers();
176
+ const read = (spec) => {
177
+ for (const rq of chain) {
178
+ try {
179
+ const v = rq(`${spec}/package.json`)?.version;
180
+ if (typeof v === 'string' && v.trim() !== '') return String(v);
181
+ } catch {
182
+ /* try the next resolver in the chain */
183
+ }
184
+ }
185
+ return null;
186
+ };
187
+ return {
188
+ cliVersion: read('@camunda8/cli'),
189
+ sdkVersion: read('@camunda8/orchestration-cluster-api'),
190
+ };
191
+ }
192
+
193
+ /** Render host SDK versions for a log line (issue #289). Never throws. */
194
+ export function formatHostSdkInfo(info) {
195
+ const cli = info?.cliVersion ? `@camunda8/cli ${info.cliVersion}` : '@camunda8/cli (version unknown)';
196
+ const sdk = info?.sdkVersion
197
+ ? `@camunda8/orchestration-cluster-api ${info.sdkVersion}`
198
+ : '@camunda8/orchestration-cluster-api (version unknown)';
199
+ return `${cli}, ${sdk}`;
200
+ }
201
+
98
202
  /**
99
203
  * Read the bundled-binary marker (nanobpmn-binary.json) written by the upstream
100
204
  * release pipeline. Records which nanobpmn build the shipped binaries came from.
@@ -156,6 +260,24 @@ const READINESS_TIMEOUT_MS = 60_000;
156
260
  const READINESS_POLL_MS = 500;
157
261
  const HEALTH_TIMEOUT_MS = 1_500;
158
262
  const STOP_GRACE_MS = 8_000;
263
+ // #296: a `stop --force` SIGTERMs a `nano work` child, which (on an interrupted
264
+ // agent job) flushes a last WIP checkpoint and promotes it to a durable recovery
265
+ // ref (#295) before exiting. That flush+promote must fit inside the window the
266
+ // supervisor allows before its SIGKILL backstop, or the recovery ref is never
267
+ // written. This is the bounded budget the worker gives that force-stop recovery,
268
+ // and the EXTRA grace `forceStopWorker` waits beyond STOP_GRACE_MS before SIGKILL
269
+ // (a child that exits sooner is reaped immediately — this only lengthens the
270
+ // SIGKILL backstop for a worker that is actively preserving recovery).
271
+ const FORCE_STOP_RECOVERY_MS = 20_000;
272
+ // #296: the backstop deadline an OUTER daemon-stop path (the socket-unreachable
273
+ // `stop --force` fallback and `stopSupervisorProcess`) waits for the DAEMON — and
274
+ // its whole process group, which includes the workers — to exit before SIGKILLing
275
+ // the group. It must exceed what the daemon itself allows a worker before ITS
276
+ // SIGKILL backstop (`forceStopWorker` waits STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS),
277
+ // plus a small margin for the daemon to observe the worker's exit and shut down.
278
+ // Otherwise the outer path SIGKILLs the group ~18s early and kills a worker that is
279
+ // still flushing + promoting its durable recovery ref (#295).
280
+ const FORCE_STOP_DAEMON_GRACE_MS = STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS + 2_000;
159
281
  // #202: the signal a graceful `supervisor stop`/`workforce stop` sends each
160
282
  // `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
161
283
  // then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
@@ -352,6 +474,22 @@ function activityMarkerReadyFor(act, pid) {
352
474
  return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
353
475
  }
354
476
 
477
+ /**
478
+ * Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
479
+ * All three must hold (#293): the worker exists, its PID is alive, AND its
480
+ * activity marker proves its SIGUSR1 reload listener is installed (a finite
481
+ * `readyAt` + matching pid, via activityMarkerReadyFor). workAgent installs that
482
+ * listener only once its activation loop is up, which is the same handshake that
483
+ * stamps `readyAt`. Signalling a freshly spawned worker BEFORE it installs the
484
+ * handler delivers SIGUSR1 to Node's DEFAULT action — starting the inspector
485
+ * instead of reloading — so a not-yet-ready worker is skipped (it re-reads the
486
+ * fresh token at its own startup prime anyway). Pure so it can be unit-tested.
487
+ */
488
+ function reloadCredentialsReady(worker, activity, pidAlive) {
489
+ if (!worker || !pidAlive) return false;
490
+ return activityMarkerReadyFor(activity, worker.pid);
491
+ }
492
+
355
493
  /**
356
494
  * Deterministic control-socket path shared by the daemon and every client.
357
495
  * Derived from a hash of the (possibly overridden) state home so distinct
@@ -2757,6 +2895,19 @@ const RESERVED_RESULT_KEYS = new Set([
2757
2895
  // push-checkpoint / points world-restore at an arbitrary SHA, so reserve it
2758
2896
  // alongside the other host-owned git-result keys (thread 4117276817).
2759
2897
  'worldMarker',
2898
+ // `resumeFromCheckpoint` is the host/process-owned resume pointer (issue #295):
2899
+ // the plugin writes it at the PROCESS scope so a re-entered (new element
2900
+ // instance) activation can find the predecessor's WIP checkpoint. When it is a
2901
+ // bare STRING it is trusted as an EXPLICIT, occurrence-authoritative ref that
2902
+ // BYPASSES both the elementId correlation AND the ambiguity guard
2903
+ // (resumeRefIsExplicit). Because a successful agent result is merged into the
2904
+ // process variables, an unreserved `resumeFromCheckpoint` lets an untrusted
2905
+ // harness emit a bare-string ref that a later activation then trusts as
2906
+ // occurrence-authoritative — restoring an ARBITRARY WIP ref from the repository
2907
+ // with no correlation or guard. Reserve it so the agent result can never inject
2908
+ // or shadow the host/process-owned pointer (Copilot review on #296, thread
2909
+ // 4243776734).
2910
+ 'resumeFromCheckpoint',
2760
2911
  ]);
2761
2912
 
2762
2913
  // Parse `text` as a JSON object, returning it only when it is a plain object.
@@ -4451,10 +4602,28 @@ function makeSecretResolver(kind) {
4451
4602
  return null; // unknown → caller reports a clear error
4452
4603
  }
4453
4604
 
4605
+ // Force a fresh `gh auth token` read, bypassing the TTL cache, and
4606
+ // return the token (or null). Injectable into resolveJobSecrets so a default
4607
+ // github gh-cache MISS re-reads at job-resolve time — catching a `gh auth login`
4608
+ // the worker-lifetime refresh timer has not re-primed yet — instead of trusting
4609
+ // provisionRepo's (non-universal) auth-failure recovery (Copilot review on #294).
4610
+ // ASYNC (spawn, never spawnSync): resolveJobSecrets runs on the activated-job
4611
+ // path after recordJobStart, so a SYNCHRONOUS `gh auth token` + scopes probe
4612
+ // here would block the shared event loop (and the lock-extension heartbeat) for
4613
+ // up to ~15s — the very hot-path stall the async refresh exists to prevent
4614
+ // (Copilot review on #294). refreshGhAuthTokenAsync also defers to any refresh
4615
+ // already in flight instead of starting a second concurrent read.
4616
+ async function defaultForceRefreshGhAuthToken() {
4617
+ const r = await refreshGhAuthTokenAsync({ force: true });
4618
+ return (r && r.token) || null;
4619
+ }
4620
+
4454
4621
  // Resolve the names a job needs (setup.secretRefs, plus the repo/PR credential
4455
4622
  // when allowPr). Returns resolved values + a list of names that were missing so
4456
- // the caller can fail the job with a clear provisioning error.
4457
- function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli } = {}) {
4623
+ // the caller can fail the job with a clear provisioning error. ASYNC: the
4624
+ // gh-cache-miss force-refresh awaits a spawned (non-blocking) gh read, so the
4625
+ // caller must `await` this (the workAgent job runner already is async).
4626
+ async function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli, forceRefreshGhAuthToken = defaultForceRefreshGhAuthToken } = {}) {
4458
4627
  const names = new Set();
4459
4628
  for (const n of envelope.setup?.secretRefs || []) if (n) names.add(n);
4460
4629
  const resolved = {};
@@ -4466,7 +4635,10 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
4466
4635
  }
4467
4636
  // The github clone/push credential is resolved with a gh-CLI fallback so a
4468
4637
  // default GITHUB_TOKEN isn't reported "missing" merely because it isn't in the
4469
- // env when `gh auth login` provides it. A custom authRef stays strict.
4638
+ // env when `gh auth login` provides it. A default github credential whose gh
4639
+ // cache is a MISS is force-re-read here (bypassing the TTL cache); it is
4640
+ // reported missing only when that forced read ALSO finds no token. A custom
4641
+ // authRef stays strict.
4470
4642
  if (envelope.task?.allowPr) {
4471
4643
  const provider = envelope.repository?.provider || 'github';
4472
4644
  const authRef = envelope.repository?.authRef;
@@ -4476,16 +4648,35 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
4476
4648
  // so provisioning sheds rather than silently borrowing the default/gh token.
4477
4649
  if (!missing.includes('repository.authRef')) missing.push('repository.authRef');
4478
4650
  } else {
4479
- const ghAuthRef = ref.kind === 'custom'
4480
- ? ref.name
4481
- : (provider === 'github' ? 'GITHUB_TOKEN' : undefined);
4651
+ const ghAuthRef = ghCredentialEnvName({ provider, authRef });
4482
4652
  if (ghAuthRef) {
4483
4653
  names.add(ghAuthRef);
4484
- const token = githubCloneToken({ provider, authRef, secretResolver: resolver, ghAuthToken });
4654
+ const { token, source } = githubCloneTokenWithSource({ provider, authRef, secretResolver: resolver, ghAuthToken });
4485
4655
  const missingIdx = missing.indexOf(ghAuthRef);
4486
4656
  if (token) {
4487
4657
  resolved[ghAuthRef] = token;
4488
4658
  if (missingIdx !== -1) missing.splice(missingIdx, 1);
4659
+ } else if (source === 'gh-cache') {
4660
+ // A DEFAULT github credential whose gh-CLI cache is a MISS (null).
4661
+ // provisionRepo's null → fresh auth-failure recovery is NOT universal:
4662
+ // container jobs never call provisionRepo, and an anonymous clone of a
4663
+ // PUBLIC repo SUCCEEDS (no auth failure → no forced re-read), so the
4664
+ // harness would otherwise reach an `allowPr` job with GITHUB_TOKEN
4665
+ // ABSENT instead of failing early. FORCE a fresh `gh auth token` read
4666
+ // HERE (catching a `gh auth login` the worker-lifetime refresh timer
4667
+ // has not re-primed yet): populate + drop from `missing` only if a
4668
+ // token now exists; otherwise KEEP it missing so the job fails with a
4669
+ // clear provisioning error rather than silently handing the harness no
4670
+ // credential (Copilot review on #294). Priming the cache here also
4671
+ // feeds provisioning's own re-resolve the same fresh token. A
4672
+ // custom/invalid authRef reports source 'none' and still fails strict.
4673
+ const fresh = await forceRefreshGhAuthToken();
4674
+ if (fresh) {
4675
+ resolved[ghAuthRef] = fresh;
4676
+ if (missingIdx !== -1) missing.splice(missingIdx, 1);
4677
+ } else if (missingIdx === -1) {
4678
+ missing.push(ghAuthRef);
4679
+ }
4489
4680
  } else if (missingIdx === -1) {
4490
4681
  missing.push(ghAuthRef);
4491
4682
  }
@@ -4779,42 +4970,690 @@ function credArgs() {
4779
4970
  // a trimmed token, or null when gh is missing / not logged in. The token is fed to
4780
4971
  // git via GIT_ASKPASS only (never argv/URL/helper), preserving the ephemeral-token
4781
4972
  // guarantee.
4782
- // Memoized for the process lifetime: this is a synchronous spawnSync (up to a
4783
- // 10s timeout) that can be reached per job, so consult the CLI at most once per
4784
- // worker run rather than blocking a handler on every job. A sentinel
4785
- // distinguishes "not yet computed" from
4786
- // a cached null (gh missing / not logged in).
4787
4973
  //
4788
- // Memoization alone still lets the *first* job pay the synchronous spawn on the
4789
- // event loop, stalling any sibling handlers (and lock-extension heartbeats) for
4790
- // up to the timeout. So `nano work` primes this cache once at startup via
4791
- // `primeGhAuthToken()` — before the poll loop — moving the one unavoidable
4792
- // blocking spawn off the job-handling path entirely. Any later call is a warm
4793
- // cache hit; the memoization here is the safety net for paths that never primed.
4974
+ // Caching (issue #293): the lookup is a synchronous spawnSync (up to a 10s
4975
+ // timeout), so it cannot run on the per-job hot path — but a PROCESS-LIFETIME
4976
+ // memo goes stale the moment the operator refreshes the gh login (re-login, a new
4977
+ // scope, a rotation), and nothing told the worker (or the operator) it was now
4978
+ // handing every job the OLD token. The cache is therefore a RECORD
4979
+ // (`{ token, fingerprint, scopes, since, lastCheck }`) with a TTL:
4980
+ // - `ghAuthTokenFromCli()` (the per-job hot path) serves the PRIMED cache
4981
+ // record VERBATIM — even once `lastCheck` is older than
4982
+ // GH_AUTH_TOKEN_TTL_MS — and NEVER triggers a synchronous TTL re-read, so a
4983
+ // job never pays `gh auth token`'s spawn cost (Copilot review on #293). The
4984
+ // sole synchronous read is for a still-UNSET cache (a worker that never
4985
+ // primed), and even that defers to any async refresh already in flight. The
4986
+ // off-path refreshers below keep a primed record current.
4987
+ // - `nano work` PRIMES at startup (`primeGhAuthToken()`) AND re-primes on a
4988
+ // timer (`startGhAuthTokenRefresh`) so the TTL re-read happens OFF the job
4989
+ // path. The timer's refresh is ASYNC (spawn, never spawnSync) and DEFERRED
4990
+ // while a job is active, so it can never block the event loop / lock
4991
+ // heartbeat (Copilot review on #293) — a refreshed login is therefore
4992
+ // adopted within one TTL only when the worker is idle at a tick. Under
4993
+ // sustained back-to-back load the defer is BOUNDED by
4994
+ // GH_AUTH_TOKEN_MAX_DEFER_MS, after which the tick refreshes DESPITE an
4995
+ // active job (still async) to cap staleness at a few intervals. When the
4996
+ // identity changes it republishes the activity marker via an onChange
4997
+ // callback.
4998
+ // - A git/gh auth FAILURE (clone/push/PR-reconcile — see isGitAuthFailure)
4999
+ // forces one re-read (`refreshGhAuthToken({ force, budgetMs })`, bounded by
5000
+ // the caller's remaining lease budget so the synchronous re-read can't
5001
+ // overrun the lock heartbeat) and a single retry, so a rotated token
5002
+ // self-heals instead of failing the job. The re-read is
5003
+ // NON-destructive (it never invalidates the cache first), so a transient
5004
+ // re-read failure keeps the still-valid cached credential.
5005
+ // - `primeGhAuthToken({ logger })` logs the token's FINGERPRINT (a short sha256
5006
+ // prefix — never the value) and its OAuth scopes at startup and whenever the
5007
+ // token changes, so "this worker has a stale token" is a one-line check; the
5008
+ // same record rides the supervisor activity marker (`ghAuthTokenInfo()`) into
5009
+ // `supervisor status`, and `supervisor reload-credentials` re-reads it
5010
+ // fleet-wide (async; the old token is served until the new one lands)
5011
+ // without restarting workers.
5012
+ // A sentinel distinguishes "not yet computed" from a cached null (gh missing /
5013
+ // not logged in).
4794
5014
  const GH_AUTH_TOKEN_UNSET = Symbol('gh-auth-token-unset');
5015
+ // How long a cached gh token is trusted before a re-read. 5 minutes bounds the
5016
+ // staleness window after a `gh auth login`/`refresh` while keeping `gh auth
5017
+ // token` (a cheap keychain read) off the per-job path. Override with
5018
+ // NANO_GH_TOKEN_TTL_MS (0 = re-read on every lookup, honouring the spawn cost).
5019
+ const GH_AUTH_TOKEN_TTL_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_TTL_MS, 5 * 60_000));
5020
+ // Upper bound on how long the maintenance timer may KEEP DEFERRING its refresh
5021
+ // while jobs run back-to-back. The tick normally defers while a job is active
5022
+ // (`isIdle`) so it never even spawns `gh` mid-job — but that defer is only an
5023
+ // optimization, because the refresh is async (spawn, never spawnSync) and so
5024
+ // cannot block the event loop / lock heartbeat even when it DOES run mid-job.
5025
+ // Under SUSTAINED saturation the worker is never idle at a tick, so an unbounded
5026
+ // defer would leave an expired, non-null cache reaching every subsequent job for
5027
+ // the worker's whole busy lifetime — defeating issue #293's "keep the token
5028
+ // fresh for the worker's lifetime" goal (Copilot review on #294). Bound it: once
5029
+ // a PRIMED cache is stale beyond this window, the tick refreshes DESPITE an
5030
+ // active job (still async/non-blocking), capping staleness at a few intervals
5031
+ // instead of the busy lifetime. Derived from the TTL (4 intervals) and overridable
5032
+ // via NANO_GH_TOKEN_MAX_DEFER_MS. Priming an UNSET cache is NEVER forced mid-job
5033
+ // (the defer there still holds), so a worker that starts logged-out stays anonymous
5034
+ // until it goes idle, exactly as before.
5035
+ const GH_AUTH_TOKEN_MAX_DEFER_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_MAX_DEFER_MS, 4 * GH_AUTH_TOKEN_TTL_MS));
5036
+ // The gh subprocesses below must never block on an interactive prompt inside a
5037
+ // worker (a missing/expired login would otherwise hang the event loop for the
5038
+ // full timeout). GH_PROMPT_DISABLED fails fast instead; GH_NO_UPDATE_NOTIFIER
5039
+ // keeps gh's upgrade check out of the captured output.
5040
+ const GH_NONINTERACTIVE_ENV = () => ({ ...process.env, GH_PROMPT_DISABLED: '1', GH_NO_UPDATE_NOTIFIER: '1' });
4795
5041
  let ghAuthTokenCache = GH_AUTH_TOKEN_UNSET;
4796
- function ghAuthTokenFromCli() {
4797
- if (ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET) return ghAuthTokenCache;
4798
- let token = null;
5042
+ // A Promise while an ASYNC gh-token refresh (the periodic timer tick or the
5043
+ // `reload-credentials` SIGUSR1 handler) is actually spawning `gh` subprocesses;
5044
+ // null otherwise. This is the serialization point that keeps a refresh off the
5045
+ // job path (Copilot review on #293): while it is set, the SYNCHRONOUS per-job
5046
+ // lookup (`ghAuthTokenFromCli`) must NOT start its own blocking `spawnSync`
5047
+ // re-read — it hands out the currently-cached token and lets the in-flight
5048
+ // async refresh update the cache, so an async refresh overlapping a freshly
5049
+ // activated job can never degrade into a 15s event-loop/lease-heartbeat stall.
5050
+ // It also prevents the timer and the SIGUSR1 handler from racing two concurrent
5051
+ // reads: a second async refresh awaits the first instead of spawning again.
5052
+ let ghAuthRefreshInFlight = null;
5053
+ // Monotonic generation counter bumped on EVERY write to `ghAuthTokenCache`
5054
+ // (`writeGhAuthCache` below is the single mutation point). It fences a stale
5055
+ // async refresh against a newer forced recovery (Copilot review on #293): the
5056
+ // synchronous clone/push auth-failure recovery (a forced `refreshGhAuthToken({
5057
+ // force })`) CANNOT await an already-running async timer
5058
+ // refresh (sync code can't await the `ghAuthRefreshInFlight` promise). So an
5059
+ // async refresh can read the OLD token, yield across its scopes probe, the sync
5060
+ // recovery can adopt the ROTATED token in the meantime, and then the async
5061
+ // refresh's late `adoptGhAuthTokenAsync` would overwrite the cache with its
5062
+ // now-stale read — resurrecting the revoked credential. `refreshGhAuthTokenAsync`
5063
+ // snapshots this generation BEFORE it starts reading and `adoptGhAuthTokenAsync`
5064
+ // refuses to commit when the generation has advanced (a newer definitive write
5065
+ // landed), so an older read can never clobber a newer forced recovery.
5066
+ let ghAuthCacheGeneration = 0;
5067
+ // The single mutation point for `ghAuthTokenCache`. Every write (adopt, preserve,
5068
+ // invalidate/reset) goes through here so the generation fence above sees them all.
5069
+ function writeGhAuthCache(record) {
5070
+ ghAuthTokenCache = record;
5071
+ ghAuthCacheGeneration++;
5072
+ return record;
5073
+ }
5074
+ // Test-only seam: the `gh auth token` reader `refreshGhAuthToken` consults when
5075
+ // no per-call `run` is injected. Production leaves this unset (the real
5076
+ // spawnSync of the gh CLI); a test installs a stub so the clone/push
5077
+ // invalidation-and-retry paths — which refresh through the MODULE cache, not a
5078
+ // per-call `run` — can rotate the token deterministically. Always a function
5079
+ // returning a trimmed token-or-null; reset with `resetGhAuthTokenReader()`.
5080
+ let ghAuthTokenReader = null;
5081
+ function setGhAuthTokenReader(fn) { ghAuthTokenReader = typeof fn === 'function' ? fn : null; }
5082
+ function resetGhAuthTokenReader() { ghAuthTokenReader = null; }
5083
+
5084
+ // Sentinel distinguishing a TRANSIENT `gh auth token` read failure from a
5085
+ // definitive logout (Copilot review on #293). The readers used to collapse a
5086
+ // timeout / spawn error / unrecognised nonzero exit into `null`, which the
5087
+ // refresh path then adopted as a real "logged out" change — discarding a
5088
+ // previously usable cached token, stamping the cache fresh with `null`, and
5089
+ // dropping the worker (and its clone-auth recovery) to anonymous until the next
5090
+ // interval. A read that returns this sentinel instead means "could not
5091
+ // determine"; `adoptGhAuthToken*` preserves the prior credential rather than
5092
+ // wiping it, and only a DEFINITIVE result (a real token, or a confirmed logout)
5093
+ // ever changes the cache.
5094
+ const GH_AUTH_READ_FAILED = Symbol('gh-auth-read-failed');
5095
+ // gh's not-logged-in signatures ("no oauth token found for github.com",
5096
+ // "You are not logged into any GitHub hosts"). Matched on stderr ONLY for a
5097
+ // nonzero exit, so an unrelated nonzero failure (keychain error, transient
5098
+ // network) stays a transient read failure rather than a spurious logout.
5099
+ const GH_LOGGED_OUT_RE = /no oauth token|no token found|not logged in/i;
5100
+ // Map one completed (or failed) `gh auth token` invocation to:
5101
+ // - a trimmed token string → logged in
5102
+ // - null → the read DEFINITIVELY established no login (clean exit with empty
5103
+ // output, or a nonzero exit whose stderr carries gh's logged-out message)
5104
+ // - GH_AUTH_READ_FAILED → the read could not be completed (spawn error,
5105
+ // timeout — spawnSync sets `.error` — or a nonzero exit WITHOUT a
5106
+ // logged-out signal), so the caller must preserve any prior cached token.
5107
+ function classifyGhAuthRead({ error = null, status = null, stdout = '', stderr = '' } = {}) {
5108
+ if (error) return GH_AUTH_READ_FAILED;
5109
+ if (status === 0) return (String(stdout ?? '').trim()) || null;
5110
+ if (GH_LOGGED_OUT_RE.test(String(stderr ?? ''))) return null;
5111
+ return GH_AUTH_READ_FAILED;
5112
+ }
5113
+
5114
+ function readGhAuthTokenDefault(timeoutMs = 10_000) {
5115
+ let r;
5116
+ try {
5117
+ r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: Math.max(1, timeoutMs), env: GH_NONINTERACTIVE_ENV() });
5118
+ } catch {
5119
+ return GH_AUTH_READ_FAILED;
5120
+ }
5121
+ return classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
5122
+ }
5123
+
5124
+ // A short, non-reversible identity for a token so logs/status can show WHEN the
5125
+ // credential changed (and that two workers hold the SAME one) without ever
5126
+ // printing a usable fragment of the secret. 12 hex chars of sha256.
5127
+ function ghTokenFingerprint(token) {
5128
+ if (!token) return null;
5129
+ try { return createHash('sha256').update(String(token)).digest('hex').slice(0, 12); }
5130
+ catch { return null; }
5131
+ }
5132
+
5133
+ // Read the token's OAuth scopes from the live `X-OAuth-Scopes` response header
5134
+ // (the issue's own diagnostic). Best-effort and bounded: a single lightweight
5135
+ // api.github.com ping, failure → null (scopes simply go unreported). The token
5136
+ // rides the Authorization HEADER — never argv — so it can't leak via `ps`.
5137
+ function ghTokenScopes(token, { timeoutMs = 5_000, run = spawnSync } = {}) {
5138
+ if (!token) return null;
4799
5139
  try {
4800
- const r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: 10_000 });
4801
- const tok = r.status === 0 ? (r.stdout || '').trim() : '';
4802
- token = tok || null;
5140
+ const r = run('gh', ['api', '-i', '--include', 'user'], { encoding: 'utf8', timeout: timeoutMs, env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
5141
+ const out = `${r.stdout || ''}\n${r.stderr || ''}`;
5142
+ const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
5143
+ return m ? m[1].trim() : null;
4803
5144
  } catch {
5145
+ return null;
5146
+ }
5147
+ }
5148
+
5149
+ // Fold one (re)read result into the cache record. Shared by the sync and async
5150
+ // refresh paths so both maintain identical cache semantics. Returns
5151
+ // { token, changed, snapshotChanged } — `changed` is true when the token VALUE
5152
+ // differs from what the cache previously held (including null → token and token
5153
+ // → null), so a caller doing credential-rotation recovery fires exactly when the
5154
+ // credential the worker hands out actually rotates. `snapshotChanged` is the
5155
+ // broader "did the OBSERVABLE snapshot move" flag: `changed` OR a same-token
5156
+ // scopes self-repair (a prior null scopes reading re-probed to a real value) —
5157
+ // both of which change what `supervisor status`/the activity marker should show,
5158
+ // so the log + republish paths gate on IT, not on `changed` (Copilot review on
5159
+ // #293). `scopes` is read only when the token changed, on the first populate, or
5160
+ // to re-probe a cached null reading.
5161
+ function adoptGhAuthToken(token, { scopesProbe = ghTokenScopes } = {}) {
5162
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5163
+ const now = Date.now();
5164
+ if (token === GH_AUTH_READ_FAILED) {
5165
+ // Transient read failure: never discard a usable cached credential. Preserve
5166
+ // the prior record but bump `lastCheck` so the per-job hot path keeps serving
5167
+ // it via the TTL short-circuit (no blocking re-read on every lookup); the
5168
+ // off-path refresh timer retries on its own cadence. With no usable prior
5169
+ // token, fall back to anonymous (null) as before.
5170
+ if (prev && prev.token != null) {
5171
+ writeGhAuthCache({ ...prev, lastCheck: now });
5172
+ return { token: prev.token, changed: false, snapshotChanged: false };
5173
+ }
4804
5174
  token = null;
4805
5175
  }
4806
- ghAuthTokenCache = token;
4807
- return token;
5176
+ const changed = !prev || prev.token !== token;
5177
+ // On a CHANGE re-read the scopes; otherwise carry the previous reading forward
5178
+ // (a TTL re-read that finds the same token must not pay the scopes ping, and
5179
+ // must not silently DROP the scopes it already reported). BUT a prior probe
5180
+ // that failed transiently stored `scopes: null` — carrying that null forward
5181
+ // would leave the GH AUTH visibility unrepaired until the token rotates or the
5182
+ // worker restarts (Copilot review on #293). Re-probe when the cached scope
5183
+ // value is MISSING (null) while a token is present; reuse only a SUCCESSFUL
5184
+ // (non-null) prior reading.
5185
+ const scopes = token ? ((changed || !(prev && prev.scopes)) ? scopesProbe(token) : prev.scopes) : null;
5186
+ // `snapshotChanged` is true when the OBSERVABLE identity (the token-free
5187
+ // fingerprint + scopes that ghAuthTokenInfo/describeGhAuthToken surface) moved
5188
+ // — not only when the token VALUE rotated. A scopes self-repair (a prior
5189
+ // transient probe cached `scopes:null`, now re-probed to a real value) leaves
5190
+ // the token unchanged yet DOES change what `supervisor status` should show, so
5191
+ // the republish/log path must fire on it too; gating republishing on `changed`
5192
+ // alone leaves an idle worker reporting `?` scopes indefinitely (Copilot review
5193
+ // on #293).
5194
+ const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
5195
+ writeGhAuthCache({
5196
+ token,
5197
+ fingerprint: ghTokenFingerprint(token),
5198
+ scopes: scopes ?? null,
5199
+ // `since` tracks the TOKEN's age (when THIS value was first seen) so status
5200
+ // can answer "how stale is the credential"; `lastCheck` tracks the CACHE's
5201
+ // age (when we last asked gh) so the TTL is honoured.
5202
+ since: prev && prev.token === token && token != null ? prev.since : now,
5203
+ lastCheck: now,
5204
+ });
5205
+ return { token, changed, snapshotChanged };
5206
+ }
5207
+
5208
+ // The async twin of adoptGhAuthToken: identical cache semantics, but the scopes
5209
+ // probe is awaited so it may return a Promise (the async ghTokenScopesAsync).
5210
+ // `generation` (when provided by refreshGhAuthTokenAsync) is the cache generation
5211
+ // snapshotted BEFORE this refresh started reading. If it no longer matches at
5212
+ // commit time, a NEWER definitive write (a forced sync clone/push recovery, an
5213
+ // invalidate, or another refresh) landed while we were reading/probing — so this
5214
+ // read is stale and MUST NOT be committed, or it would resurrect the superseded
5215
+ // token (Copilot review on #293). We leave the newer cache in place and report
5216
+ // it as superseded.
5217
+ async function adoptGhAuthTokenAsync(token, { scopesProbe, generation = null } = {}) {
5218
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5219
+ const now = Date.now();
5220
+ const superseded = () => {
5221
+ const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5222
+ return { token: c ? c.token : null, changed: false, snapshotChanged: false, superseded: true };
5223
+ };
5224
+ if (token === GH_AUTH_READ_FAILED) {
5225
+ // Transient read failure (see adoptGhAuthToken): preserve the prior token and
5226
+ // bump `lastCheck` so the hot path keeps serving it; anonymous only with no prior.
5227
+ if (prev && prev.token != null) {
5228
+ if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
5229
+ writeGhAuthCache({ ...prev, lastCheck: now });
5230
+ return { token: prev.token, changed: false, snapshotChanged: false };
5231
+ }
5232
+ token = null;
5233
+ }
5234
+ const changed = !prev || prev.token !== token;
5235
+ // Same re-probe-on-missing rule as the sync twin: a same-token refresh reuses a
5236
+ // successful prior scope reading but RE-PROBES when the cached value is null
5237
+ // (a prior transient probe failure), so the GH AUTH visibility self-repairs.
5238
+ const scopes = token ? ((changed || !(prev && prev.scopes)) ? await scopesProbe(token) : prev.scopes) : null;
5239
+ // See adoptGhAuthToken: publish when the observable snapshot (fingerprint +
5240
+ // scopes) moved, not only when the token value rotated, so a same-token scopes
5241
+ // self-repair still republishes the activity marker (Copilot review on #293).
5242
+ const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
5243
+ // Generation fence: re-check AFTER the (awaited) scopes probe, the last point a
5244
+ // forced recovery could have slipped a newer credential in underneath us.
5245
+ if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
5246
+ writeGhAuthCache({
5247
+ token,
5248
+ fingerprint: ghTokenFingerprint(token),
5249
+ scopes: scopes ?? null,
5250
+ since: prev && prev.token === token && token != null ? prev.since : now,
5251
+ lastCheck: now,
5252
+ });
5253
+ return { token, changed, snapshotChanged };
5254
+ }
5255
+
5256
+ // One (re)read of `gh auth token`, folding the result into the cache record.
5257
+ // Returns { token, changed } (see adoptGhAuthToken). SYNCHRONOUS (spawnSync) —
5258
+ // safe off the job path (the startup prime and the clone/push auth-failure
5259
+ // recovery, where the caller is already blocked waiting on git) but it blocks
5260
+ // the event loop while it runs, so the periodic refresh AND the SIGUSR1
5261
+ // reload-credentials handler use refreshGhAuthTokenAsync instead.
5262
+ function refreshGhAuthToken({ force = false, run = null, scopesProbe, budgetMs = null } = {}) {
5263
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5264
+ const now = Date.now();
5265
+ if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
5266
+ return { token: prev.token, changed: false, snapshotChanged: false };
5267
+ }
5268
+ // When a caller on the ACTIVE-JOB path (the clone/push auth-failure recovery)
5269
+ // threads its remaining lease budget, bound BOTH synchronous gh subprocesses
5270
+ // (the `gh auth token` read AND the scopes probe) to it so this forced
5271
+ // spawnSync re-read can never block the event loop / lock heartbeat past the
5272
+ // caller's lease-safe window — the finalizeGit/clone git ops are already
5273
+ // netDeadline-bounded, and this read must participate in the same budget
5274
+ // (Copilot review on #294). Absent a budget (startup prime, UNSET-cache hot
5275
+ // path, unit tests) the historical 10s read / 5s scopes caps stand.
5276
+ const haveBudget = Number.isFinite(budgetMs) && budgetMs > 0;
5277
+ const deadline = haveBudget ? now + budgetMs : null;
5278
+ const readTimeout = haveBudget ? Math.max(1, Math.min(10_000, budgetMs)) : 10_000;
5279
+ let token = null;
5280
+ if (run) {
5281
+ // Explicit per-call spawn injection (unit tests of this function in isolation).
5282
+ try {
5283
+ const r = run('gh', ['auth', 'token'], { encoding: 'utf8', timeout: readTimeout, env: GH_NONINTERACTIVE_ENV() });
5284
+ token = classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
5285
+ } catch {
5286
+ token = GH_AUTH_READ_FAILED;
5287
+ }
5288
+ } else {
5289
+ // The module reader (production: the real gh CLI; tests: setGhAuthTokenReader).
5290
+ token = (ghAuthTokenReader || readGhAuthTokenDefault)(readTimeout);
5291
+ }
5292
+ // The scopes probe gets whatever of the budget the read left (still capped at
5293
+ // 5s); a read that consumed the whole budget leaves it ~1ms so the probe fails
5294
+ // fast to null (scopes are best-effort) rather than overrunning the lease.
5295
+ const scopesTimeout = haveBudget ? Math.max(1, Math.min(5_000, deadline - Date.now())) : 5_000;
5296
+ // The scopes probe defaults to the live `gh api -i user` ping — but when a
5297
+ // test installed a token READER (hermetic, no network) and did not inject a
5298
+ // scopesProbe, skip the ping so a stubbed refresh never hits the network.
5299
+ const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((t) => ghTokenScopes(t, { timeoutMs: scopesTimeout })));
5300
+ return adoptGhAuthToken(token, { scopesProbe: probe });
5301
+ }
5302
+
5303
+ // Async `gh auth token` read (spawn, never spawnSync) for the periodic refresh
5304
+ // path — the whole point of the timer is that it must NOT block the worker's
5305
+ // event loop (and with it the lock heartbeat) while a job runs. Same 10s bound
5306
+ // and non-interactive env as the sync reader; resolves a trimmed token-or-null.
5307
+ // `_spawn` is a test-only seam (production uses node:child_process.spawn).
5308
+ function readGhAuthTokenAsync({ _spawn = spawn } = {}) {
5309
+ return new Promise((resolve) => {
5310
+ let child;
5311
+ try {
5312
+ child = _spawn('gh', ['auth', 'token'], { env: GH_NONINTERACTIVE_ENV() });
5313
+ } catch {
5314
+ resolve(GH_AUTH_READ_FAILED);
5315
+ return;
5316
+ }
5317
+ let out = '';
5318
+ let err = '';
5319
+ let settled = false;
5320
+ const finish = (token) => {
5321
+ if (settled) return;
5322
+ settled = true;
5323
+ clearTimeout(timer);
5324
+ resolve(token);
5325
+ };
5326
+ // Same 10s bound as the sync reader. On expiry SIGTERM the child (matching
5327
+ // spawnSync's timeout semantics) and resolve the transient sentinel — a
5328
+ // wedged gh must never pin the refresh chain, NOR masquerade as a logout.
5329
+ const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(GH_AUTH_READ_FAILED); }, 10_000);
5330
+ if (typeof timer.unref === 'function') timer.unref();
5331
+ child.stdout?.on('data', (d) => { out += d; });
5332
+ // stderr is now CAPTURED (not just drained) so classifyGhAuthRead can tell a
5333
+ // definitive logout (gh's "no oauth token found" message) from a transient
5334
+ // nonzero failure that must preserve the cached credential.
5335
+ child.stderr?.on('data', (d) => { err += d; });
5336
+ child.on('error', () => finish(GH_AUTH_READ_FAILED));
5337
+ child.on('close', (code) => finish(classifyGhAuthRead({ status: code, stdout: out, stderr: err })));
5338
+ });
5339
+ }
5340
+
5341
+ // Async scopes probe mirroring ghTokenScopes (same 5s default bound, same
5342
+ // header parse, same GH_TOKEN-in-env delivery so the token never touches argv).
5343
+ // `_spawn` is a test-only seam.
5344
+ function ghTokenScopesAsync(token, { timeoutMs = 5_000, _spawn = spawn } = {}) {
5345
+ if (!token) return Promise.resolve(null);
5346
+ return new Promise((resolve) => {
5347
+ let child;
5348
+ try {
5349
+ child = _spawn('gh', ['api', '-i', '--include', 'user'], { env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
5350
+ } catch {
5351
+ resolve(null);
5352
+ return;
5353
+ }
5354
+ let out = '';
5355
+ let settled = false;
5356
+ const finish = (scopes) => {
5357
+ if (settled) return;
5358
+ settled = true;
5359
+ clearTimeout(timer);
5360
+ resolve(scopes);
5361
+ };
5362
+ const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(null); }, timeoutMs);
5363
+ if (typeof timer.unref === 'function') timer.unref();
5364
+ child.stdout?.on('data', (d) => { out += d; });
5365
+ child.stderr?.on('data', (d) => { out += d; });
5366
+ child.on('error', () => finish(null));
5367
+ child.on('close', () => {
5368
+ const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
5369
+ finish(m ? m[1].trim() : null);
5370
+ });
5371
+ });
5372
+ }
5373
+
5374
+ // The async twin of refreshGhAuthToken for the periodic timer: same TTL
5375
+ // short-circuit and the same cache record, but the gh subprocesses are spawned
5376
+ // (never spawnSync) so a refresh that DOES run while a job is active yields the
5377
+ // event loop instead of blocking the lock heartbeat (Copilot review on #293).
5378
+ // When a test installed a token READER (hermetic, no network) and did not
5379
+ // inject a scopesProbe, the scopes ping is skipped — mirroring the sync path.
5380
+ async function refreshGhAuthTokenAsync({ force = false, scopesProbe, _spawn } = {}) {
5381
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5382
+ const now = Date.now();
5383
+ if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
5384
+ return { token: prev.token, changed: false, snapshotChanged: false };
5385
+ }
5386
+ // Serialize against any refresh already spawning (another timer tick, or the
5387
+ // reload-credentials SIGUSR1 handler): await IT and report its (already
5388
+ // applied) cache rather than starting a second concurrent `gh` read. This is
5389
+ // what lets the synchronous per-job path safely defer to an in-flight async
5390
+ // refresh instead of blocking (Copilot review on #293).
5391
+ // LOOP (not a one-shot `if`): re-check the marker after EVERY await so forced
5392
+ // refreshes stay serialized. A plain `if` let multiple forced callers all await
5393
+ // the SAME in-flight promise and then fall through together, each overwriting
5394
+ // ghAuthRefreshInFlight with its own read; the first to settle then clears the
5395
+ // marker (its `finally` nulls it) while another forced read is still active, so
5396
+ // the hot path (ghAuthTokenFromCli) sees no marker and resumes a blocking
5397
+ // synchronous re-read — the very stall the marker exists to prevent. Re-checking
5398
+ // in a loop means a forced caller that finds a NEW refresh in flight (started by
5399
+ // another forced caller that fell through first) awaits THAT one too, so at most
5400
+ // one forced read runs at a time and the marker is never cleared out from under
5401
+ // an active read (Copilot review on #293).
5402
+ while (ghAuthRefreshInFlight) {
5403
+ try { await ghAuthRefreshInFlight; } catch { /* swallowed — the owner logs/handles it */ }
5404
+ // A NON-forced caller (a periodic timer tick) is happy with whatever the
5405
+ // in-flight refresh just applied. A FORCED caller (the reload-credentials
5406
+ // SIGUSR1 handler) must NOT be: the in-flight read may have captured the OLD
5407
+ // token microseconds before the operator's `gh auth refresh`/`login` landed,
5408
+ // so returning its result would report "reloaded" while leaving the stale
5409
+ // credential cached until the next TTL. Loop back: once no refresh is in
5410
+ // flight, fall through to start a fresh forced read AFTER the signal.
5411
+ if (!force) {
5412
+ const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5413
+ return { token: c ? c.token : null, changed: false, snapshotChanged: false };
5414
+ }
5415
+ }
5416
+ // Honor the module reader seam when a test installed one (mirroring the sync
5417
+ // path) so a stubbed refresh never spawns; production (reader null) spawns.
5418
+ // Snapshot the cache generation BEFORE the read begins: if a forced sync
5419
+ // recovery commits a newer credential while we read/probe, adoptGhAuthTokenAsync
5420
+ // sees the generation advance and refuses to clobber it (Copilot review on #293).
5421
+ const startGen = ghAuthCacheGeneration;
5422
+ const run = (async () => {
5423
+ const token = ghAuthTokenReader ? ((ghAuthTokenReader() || null)) : await readGhAuthTokenAsync({ _spawn });
5424
+ const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((tok) => ghTokenScopesAsync(tok, { _spawn })));
5425
+ return adoptGhAuthTokenAsync(token, { scopesProbe: probe, generation: startGen });
5426
+ })();
5427
+ // Publish the in-flight marker BEFORE the first await inside `run` has a chance
5428
+ // to yield, so a job activated on the very next tick already sees it. The
5429
+ // marker never rejects (so a concurrent awaiter can't throw on it); the real
5430
+ // result/rejection is surfaced to this caller below.
5431
+ ghAuthRefreshInFlight = run.then(() => {}, () => {});
5432
+ try {
5433
+ return await run;
5434
+ } finally {
5435
+ ghAuthRefreshInFlight = null;
5436
+ }
4808
5437
  }
4809
5438
 
4810
- // Warm the gh-token cache once, off the job-handling path. Safe to call any
4811
- // number of times: the first call performs the (possibly blocking) lookup, the
4812
- // rest are cache hits. Returns true once the cache is populated.
4813
- function primeGhAuthToken() {
4814
- ghAuthTokenFromCli();
5439
+ function ghAuthTokenFromCli() {
5440
+ // If an async refresh is already spawning (a timer tick or the
5441
+ // reload-credentials SIGUSR1 handler), do NOT start a blocking synchronous
5442
+ // re-read on this per-job path — that is exactly the 15s event-loop/lease
5443
+ // stall the async refresh exists to avoid (Copilot review on #293). Hand out
5444
+ // the currently-cached token; the in-flight refresh updates the cache for the
5445
+ // next lookup, and the clone/push auth-failure recovery still force-re-reads
5446
+ // if the handed-out token turns out stale. A not-yet-primed cache (UNSET)
5447
+ // yields null — the same anonymous fallback the refresh would briefly produce.
5448
+ if (ghAuthRefreshInFlight) {
5449
+ return ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache.token;
5450
+ }
5451
+ if (ghAuthTokenCache === GH_AUTH_TOKEN_UNSET) return refreshGhAuthToken({ force: true }).token;
5452
+ // A PRIMED cache is served verbatim — NEVER a synchronous TTL-triggered
5453
+ // re-read on this per-job hot path (Copilot review on #293). The async timer
5454
+ // DEFERS a tick while a job is active, so a skipped tick can leave the cache
5455
+ // past its TTL when the next job arrives; a `refreshGhAuthToken()` here would
5456
+ // then run `spawnSync('gh auth token')` + the synchronous scopes probe inline,
5457
+ // blocking the event loop (and the lock heartbeat) for up to 15s. Startup
5458
+ // priming, the async maintenance timer, and the clone/push auth-failure
5459
+ // recovery (which force-re-reads if the handed-out token turns out stale) own
5460
+ // every refresh; this path only hands out what they populated.
5461
+ return ghAuthTokenCache.token;
5462
+ }
5463
+
5464
+ // Drop the cached gh token so the next lookup re-reads the CLI. A cache-reset
5465
+ // primitive — exported and used by the tests to reset state between cases. NO
5466
+ // production path calls it anymore: both the git clone/push auth-failure
5467
+ // recovery (recoverGhTokenAfterAuthFailure) and the supervisor's
5468
+ // `reload-credentials` broadcast (SIGUSR1) were made deliberately
5469
+ // NON-destructive (a forced re-read that preserves/atomically-swaps the cached
5470
+ // record), so neither drops the cache first (Copilot review on #294).
5471
+ function invalidateGhAuthToken() {
5472
+ writeGhAuthCache(GH_AUTH_TOKEN_UNSET);
5473
+ }
5474
+
5475
+ // Recover from a git/gh auth failure on a gh-cache-derived credential: force a
5476
+ // fresh `gh auth token` read and return it IFF it differs from the credential
5477
+ // that just FAILED (`failingToken`), otherwise null.
5478
+ //
5479
+ // Deliberately does NOT `invalidateGhAuthToken()` first (Copilot review on
5480
+ // #294). An invalidate-before-reread blows the cached record away, so if the
5481
+ // forced re-read then fails TRANSIENTLY (gh momentarily unavailable), there is
5482
+ // no prior record left for `adoptGhAuthToken` to preserve — it falls back to
5483
+ // `{token:null}` and the worker is left "logged out", ERASING a still-usable —
5484
+ // and possibly NEWER (a `reload-credentials` SIGUSR1 swap that landed mid-job)
5485
+ // — cached credential. `refreshGhAuthToken({ force })` already re-reads past the
5486
+ // TTL, and `adoptGhAuthToken` PRESERVES the prior cached token on a transient
5487
+ // read failure, so a transient hiccup keeps the good token (and we even retry
5488
+ // with a newer cached one) instead of discarding it. This mirrors the SIGUSR1
5489
+ // reload-credentials handler, which is likewise deliberately NON-destructive.
5490
+ //
5491
+ // The retry is gated on `fresh !== failingToken` — NOT on the re-read's
5492
+ // `changed` flag (which compares against the CACHE): when the cache already
5493
+ // holds the newer token (the `reload-credentials` swap case), a forced re-read
5494
+ // returns it with `changed:false`, yet that newer token is exactly the
5495
+ // credential we want to retry with. `fresh !== failingToken` also covers the
5496
+ // genuine-denial case (the re-read returns the SAME failing token → null → no
5497
+ // retry) and the null→fresh recovery (a logged-out worker re-reads a real token
5498
+ // after `gh auth login`).
5499
+ // `budgetMs` (threaded by the clone/push recovery call sites from their
5500
+ // remaining lease budget — provTimeoutMs()/netTimeoutMs()) bounds the forced
5501
+ // synchronous `gh auth token` + scopes probe so this on-the-active-job-path
5502
+ // re-read participates in the caller's lease-safe window instead of blocking the
5503
+ // event loop / lock heartbeat for the full 15s (Copilot review on #294).
5504
+ function recoverGhTokenAfterAuthFailure(failingToken, { budgetMs = null } = {}) {
5505
+ const { token: fresh } = refreshGhAuthToken({ force: true, budgetMs });
5506
+ return fresh && fresh !== failingToken ? fresh : null;
5507
+ }
5508
+
5509
+ // A snapshot of the cached token's observable identity for logs and the
5510
+ // supervisor activity marker. Never carries the token value. `ageMs` is the
5511
+ // TOKEN's age (since this value was first seen); `lastCheckAgoMs` is how long
5512
+ // ago gh was last consulted (TTL freshness). Returns null when the cache has
5513
+ // never been populated (a worker that has not primed yet).
5514
+ function ghAuthTokenInfo(now = Date.now()) {
5515
+ const c = ghAuthTokenCache;
5516
+ if (c === GH_AUTH_TOKEN_UNSET) return null;
5517
+ return {
5518
+ present: c.token != null,
5519
+ fingerprint: c.fingerprint ?? null,
5520
+ scopes: c.scopes ?? null,
5521
+ since: c.since ?? null,
5522
+ ageMs: Number.isFinite(c.since) ? Math.max(0, now - c.since) : null,
5523
+ lastCheckAgoMs: Number.isFinite(c.lastCheck) ? Math.max(0, now - c.lastCheck) : null,
5524
+ };
5525
+ }
5526
+
5527
+ // One-line, token-free description for logs: `gho_…` →
5528
+ // "gh auth token: fingerprint ab12cd34ef56 (scopes: gist, read:org, repo; age 2m)".
5529
+ function describeGhAuthToken(info) {
5530
+ if (!info) return 'gh auth token: (not yet read)';
5531
+ if (!info.present) return 'gh auth token: none (gh missing or not logged in)';
5532
+ const bits = [];
5533
+ bits.push(`fingerprint ${info.fingerprint || 'unknown'}`);
5534
+ bits.push(info.scopes ? `scopes: ${info.scopes}` : 'scopes: unknown');
5535
+ if (info.ageMs != null) bits.push(`age ${formatDuration(info.ageMs)}`);
5536
+ return `gh auth token: ${bits.join('; ')}`;
5537
+ }
5538
+
5539
+ // Warm the gh-token cache, off the job-handling path. Safe to call any number of
5540
+ // times: within the TTL it is a no-op cache hit; past it (or after an
5541
+ // invalidation) it re-reads the CLI. With a `logger` it reports the token's
5542
+ // fingerprint + scopes on the FIRST populate and whenever the observable
5543
+ // snapshot moves — a rotated token OR a scopes self-repair (a prior null reading
5544
+ // re-probed to a real value) — the "which credential is this worker holding?"
5545
+ // line from issue #293. Returns true once the cache is populated.
5546
+ function primeGhAuthToken({ logger = null } = {}) {
5547
+ const { snapshotChanged } = refreshGhAuthToken();
5548
+ if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
5549
+ logger.info(describeGhAuthToken(ghAuthTokenInfo()));
5550
+ }
4815
5551
  return ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET;
4816
5552
  }
4817
5553
 
5554
+ // Keep the cache fresh for the worker's lifetime: re-prime on a TTL cadence so
5555
+ // a `gh auth login`/`refresh` the operator runs while the fleet is up is adopted
5556
+ // within one interval — WITHOUT ever blocking the job path (Copilot review on
5557
+ // #293). Two guards make that true:
5558
+ // 1. The refresh itself is ASYNC (refreshGhAuthTokenAsync — spawn, never
5559
+ // spawnSync), so even when a tick fires mid-job the gh subprocesses yield
5560
+ // the event loop instead of freezing the lock heartbeat for up to 15s.
5561
+ // 2. `isIdle` (optional) lets the worker DEFER a tick while a job is active:
5562
+ // the refresh is then simply adopted by the next idle tick (staleness is
5563
+ // bounded by the job length + one interval, and the sync per-job paths —
5564
+ // TTL lookup, auth-failure recovery — remain as the safety net). That defer
5565
+ // is itself BOUNDED (`maxStaleMs`): once a PRIMED cache is stale beyond the
5566
+ // bound, the tick refreshes DESPITE an active job (still async/non-blocking),
5567
+ // so sustained back-to-back jobs can't keep deferring the refresh for the
5568
+ // worker's whole busy lifetime and serve an expired non-null cache to every
5569
+ // job (Copilot review on #294). An UNSET cache is never primed mid-job — the
5570
+ // defer there still holds — so a logged-out worker stays anonymous until idle.
5571
+ // Ticks never overlap (a slow gh can't stack refreshes), a tick failure is
5572
+ // swallowed (the next tick retries), and the change log fires from here so a
5573
+ // rotation is visible even when no job runs. `onChange` (optional) fires on a
5574
+ // genuine observable-snapshot change — a rotated token OR a scopes self-repair
5575
+ // (a prior null scopes reading re-probed to a real value), both of which move
5576
+ // what `supervisor status` should show — so the caller can republish the
5577
+ // supervisor activity marker; an idle worker otherwise keeps reporting the
5578
+ // pre-refresh fingerprint/scopes indefinitely (Copilot review on #293).
5579
+ //
5580
+ // The timer is UNREF'd (like the other maintenance/watchdog intervals): this is
5581
+ // a background credential refresh, NOT a serving handle, so it must never be the
5582
+ // thing that keeps the process alive. `unref()` does not starve the cadence —
5583
+ // contrary to a common misconception, it does not deprioritise or defer the
5584
+ // timer; it only drops the timer's claim on the event loop's "keep the process
5585
+ // alive" accounting. A busy worker's poll/heartbeat fibers keep the loop running,
5586
+ // so an unref'd interval still fires exactly on cadence. The one case where
5587
+ // unref matters is the dangerous one: if the worker's supervisor runtime has
5588
+ // unexpectedly terminated (its serving handles gone), a REF'd refresh timer
5589
+ // would keep the PID alive forever, masking the dead worker so the daemon never
5590
+ // observes the exit and never restarts the non-serving worker — whereas an
5591
+ // unref'd timer lets the now-idle process exit and be restarted. (The `--auto`
5592
+ // poll timer is deliberately the opposite — ref'd — because it IS a serving
5593
+ // handle that must keep a 0-poller worker alive to retry.) The returned handle
5594
+ // carries a `stop()` (clearInterval) so tests can halt it deterministically.
5595
+ function startGhAuthTokenRefresh({ logger = null, intervalMs = GH_AUTH_TOKEN_TTL_MS, isIdle = null, onChange = null, maxStaleMs = GH_AUTH_TOKEN_MAX_DEFER_MS, _spawn } = {}) {
5596
+ // Clamp to a 1s floor so a test can pass a tiny interval; an explicit 0 is
5597
+ // honoured (→ the floor), not treated as "use the default".
5598
+ const raw = Number.isFinite(Number(intervalMs)) ? Number(intervalMs) : GH_AUTH_TOKEN_TTL_MS;
5599
+ const every = Math.max(1000, raw || 1000);
5600
+ // The bounded-defer ceiling: a non-finite/negative value disables the bound
5601
+ // (pure defer-while-busy, the pre-#294 behaviour).
5602
+ const staleBound = Number.isFinite(Number(maxStaleMs)) && Number(maxStaleMs) >= 0 ? Number(maxStaleMs) : Infinity;
5603
+ let inFlight = false;
5604
+ const tick = async () => {
5605
+ if (inFlight) return; // a slow gh must not stack overlapping refreshes
5606
+ if (isIdle && !isIdle()) {
5607
+ // A job is active — normally defer so we don't even spawn gh mid-job. But
5608
+ // bound that defer: a PRIMED cache stale beyond `staleBound` must refresh
5609
+ // anyway (still async/non-blocking) rather than reach every subsequent job
5610
+ // for the worker's whole busy lifetime under sustained saturation (Copilot
5611
+ // review on #294). An UNSET cache is never primed mid-job (no bound applies),
5612
+ // so a logged-out worker stays anonymous until it goes idle, as before.
5613
+ const cached = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5614
+ const staleBeyondBound = cached && (Date.now() - cached.lastCheck) >= staleBound;
5615
+ if (!staleBeyondBound) return; // defer to the next idle tick
5616
+ }
5617
+ inFlight = true;
5618
+ try {
5619
+ const { snapshotChanged } = await refreshGhAuthTokenAsync({ _spawn });
5620
+ if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
5621
+ logger.info(describeGhAuthToken(ghAuthTokenInfo()));
5622
+ }
5623
+ if (snapshotChanged && onChange) {
5624
+ try { onChange(); } catch { /* best effort — republishing is advisory */ }
5625
+ }
5626
+ } catch {
5627
+ // A failed tick (gh wedged, spawn error) must never crash the worker or
5628
+ // surface as an unhandled rejection — the next tick retries.
5629
+ } finally {
5630
+ inFlight = false;
5631
+ }
5632
+ };
5633
+ const t = setInterval(() => { tick(); }, every);
5634
+ // Unref so a dead worker runtime (serving handles gone) can exit and be
5635
+ // restarted — this maintenance refresh must not keep a non-serving PID alive
5636
+ // (see the header comment). Guarded for fake-timer test harnesses.
5637
+ if (typeof t.unref === 'function') t.unref();
5638
+ t.stop = () => clearInterval(t);
5639
+ return t;
5640
+ }
5641
+
5642
+ // Classify a git/gh failure as an AUTH (or auth-scope) failure — the cases where
5643
+ // the cached credential, not the request, is the problem: a 401/403, git's
5644
+ // credential prompts on an anonymous/rejected clone, or GitHub's OAuth-scope
5645
+ // refusal ("refusing to allow an OAuth App to create or update workflow …
5646
+ // without `workflow` scope" — the exact escalation from issue #293). Matched on
5647
+ // the ALREADY-REDACTED detail string (gitErrorDetail/describeGitFailure output),
5648
+ // so the token value is never inspected here. Deliberately conservative: a
5649
+ // non-fast-forward reject, a missing repo (404) or a network timeout are NOT
5650
+ // auth failures and must not trigger a token drop + retry.
5651
+ function isGitAuthFailure(detail) {
5652
+ const s = String(detail ?? '');
5653
+ if (!s) return false;
5654
+ return /HTTP\s*40[13]\b|\b40[13]\s*\((Unauthorized|Forbidden)\)|returned error:\s*40[13]\b|could not read (Username|Password)|Authentication failed|authentication required|Permission to .+ denied|refusing to allow an OAuth App|without .?workflow.? scope|Bad credentials|Resource not accessible by (integration|personal access token)/i.test(s);
5655
+ }
5656
+
4818
5657
  // Read the operator's own git identity from their GLOBAL config, using the real
4819
5658
  // host environment (process.env) rather than a job's sanitized gitEnv — so it
4820
5659
  // resolves even on the anonymous clone path, where gitEnv points
@@ -4942,6 +5781,82 @@ function normalizeAuthRef(authRef) {
4942
5781
  return { kind: 'custom', name: trimmed };
4943
5782
  }
4944
5783
 
5784
+ // The env-var NAME the github clone/push credential is forwarded to the harness
5785
+ // under. Single source of truth so `resolveJobSecrets` (which builds the
5786
+ // `resolved` secret map) and the clone-recovery rotation glue (which keeps that
5787
+ // map in sync when the token rotates) can NEVER drift on the key: a custom
5788
+ // authRef uses its own name; the default github credential uses GITHUB_TOKEN.
5789
+ // Returns null when no credential is forwarded (a non-github provider with no
5790
+ // custom ref, or a blank/invalid authRef).
5791
+ function ghCredentialEnvName({ provider, authRef }) {
5792
+ const prov = provider || 'github';
5793
+ const ref = normalizeAuthRef(authRef);
5794
+ if (ref.kind === 'invalid') return null;
5795
+ if (ref.kind === 'custom') return ref.name;
5796
+ return prov === 'github' ? 'GITHUB_TOKEN' : null;
5797
+ }
5798
+
5799
+ // When clone recovery ROTATES the cache-derived credential during provisioning
5800
+ // (a stale gh token re-read after an auth failure), the `resolved` secret map —
5801
+ // built BEFORE provisioning and later forwarded to the harness as `secretEnv` —
5802
+ // still holds the STALE token, so an allowPr harness's own `gh`/`git` runs would
5803
+ // use a credential the clone already proved dead (Copilot review on #293). Sync
5804
+ // the map so it hands the harness the SAME credential provisioning just adopted.
5805
+ // Guard rails: only a 'gh-cache'-sourced credential may be swapped (an explicit
5806
+ // `env`/`custom` identity is deliberate and never silently replaced), only when
5807
+ // a token actually exists, and ONLY when that key was already forwarded (never
5808
+ // inject a brand-new secret the job did not request). Pure: mutates and returns
5809
+ // `resolved`.
5810
+ function applyRotatedGhCredential({ resolved, provider, authRef, source, token }) {
5811
+ if (!resolved || source !== 'gh-cache' || !token) return resolved;
5812
+ const key = ghCredentialEnvName({ provider, authRef });
5813
+ if (key && Object.prototype.hasOwnProperty.call(resolved, key)) resolved[key] = token;
5814
+ return resolved;
5815
+ }
5816
+
5817
+ // Resolve the github clone/push credential AND record where it came from. The
5818
+ // default credential (env GITHUB_TOKEN) falls back to the gh CLI's stored token
5819
+ // so `gh auth login` setups work without exporting GITHUB_TOKEN. A custom
5820
+ // `authRef` is honored strictly (env/secret resolver only, no gh fallback) so a
5821
+ // misconfigured named secret surfaces as missing rather than silently borrowing
5822
+ // the operator's gh login. An authRef that is present but blank is a
5823
+ // misconfiguration and yields no token (never the gh fallback). `ghAuthToken`
5824
+ // is injectable for testing.
5825
+ // The returned `source` matters downstream: the #293 auth-failure recovery
5826
+ // (force-re-read `gh auth token`, retry once — NON-destructive, never drops the
5827
+ // cache first) is only legitimate when
5828
+ // the token CAME from that cache ('gh-cache'). Retrying an explicit
5829
+ // GITHUB_TOKEN ('env') or a custom authRef ('custom') failure with the
5830
+ // operator's unrelated gh login would silently swap the configured identity
5831
+ // mid-job (Copilot review on #293), so those sources never take the fallback.
5832
+ // A DEFAULT github lookup that found NO cached gh token ALSO reports
5833
+ // 'gh-cache' (with `token: null`) rather than 'none': it is still the gh-cache
5834
+ // credential path, merely a cache MISS. Tagging it 'gh-cache' keeps the
5835
+ // clone/push auth-failure recovery ELIGIBLE so a worker that cached
5836
+ // "logged out" can still self-heal (a forced re-read recovers null → fresh
5837
+ // token after the operator runs `gh auth login`) instead of failing the job
5838
+ // anonymous and waiting for the periodic TTL refresh (Copilot review on #293).
5839
+ // 'none' is reserved for a non-github provider and an invalid/blank authRef —
5840
+ // neither may borrow the gh login.
5841
+ // Returns { token, source: 'env' | 'custom' | 'gh-cache' | 'none' }.
5842
+ function githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
5843
+ const prov = provider || 'github';
5844
+ const ref = normalizeAuthRef(authRef);
5845
+ if (ref.kind === 'invalid') return { token: null, source: 'none' };
5846
+ const usesDefault = prov === 'github' && ref.kind === 'default';
5847
+ const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
5848
+ const resolved = name ? (secretResolver.resolve(name) || null) : null;
5849
+ if (resolved) return { token: resolved, source: ref.kind === 'custom' ? 'custom' : 'env' };
5850
+ if (!resolved && usesDefault) {
5851
+ const token = ghAuthToken() || null;
5852
+ // A hit AND a miss both report 'gh-cache' (see the header): the miss keeps
5853
+ // the auth-failure recovery eligible so a post-`gh auth login` forced
5854
+ // re-read can still recover this job.
5855
+ return { token, source: 'gh-cache' };
5856
+ }
5857
+ return { token: null, source: 'none' };
5858
+ }
5859
+
4945
5860
  // Resolve the github clone/push credential. The default credential (env
4946
5861
  // GITHUB_TOKEN) falls back to the gh CLI's stored token so `gh auth login`
4947
5862
  // setups work without exporting GITHUB_TOKEN. A custom `authRef` is honored
@@ -4949,16 +5864,10 @@ function normalizeAuthRef(authRef) {
4949
5864
  // secret surfaces as missing rather than silently borrowing the operator's gh
4950
5865
  // login. An authRef that is present but blank is a misconfiguration and yields no
4951
5866
  // token (never the gh fallback). `ghAuthToken` is injectable for testing.
4952
- // Returns a token or null.
5867
+ // Returns a token or null. (Callers that need the credential's SOURCE — e.g. to
5868
+ // gate the #293 gh-token auth-failure retry — use githubCloneTokenWithSource.)
4953
5869
  function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
4954
- const prov = provider || 'github';
4955
- const ref = normalizeAuthRef(authRef);
4956
- if (ref.kind === 'invalid') return null;
4957
- const usesDefault = prov === 'github' && ref.kind === 'default';
4958
- const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
4959
- let token = name ? (secretResolver.resolve(name) || null) : null;
4960
- if (!token && usesDefault) token = ghAuthToken() || null;
4961
- return token;
5870
+ return githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken }).token;
4962
5871
  }
4963
5872
 
4964
5873
  // Clone repo into <runDir>/workspace and check out / create the working branch.
@@ -4973,11 +5882,11 @@ function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghA
4973
5882
  // resume sides agree on which no-create push targets are recoverable (#270).
4974
5883
  const CONVENTIONAL_BASE_BRANCHES = new Set(['main', 'master']);
4975
5884
 
4976
- function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, _runGit = null }) {
5885
+ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, credentialSource = null, _runGit = null }) {
4977
5886
  // Test-only seam mirroring finalizeGit: route every git call through `runGitFn` so a
4978
5887
  // deterministic test can observe the per-call timeouts drawn from the shared
4979
5888
  // provisioning budget below. Defaults to the module `runGit` (production unchanged).
4980
- const runGitFn = _runGit || runGit;
5889
+ let runGitFn = _runGit || runGit;
4981
5890
  // #222: the git ops below are blocking spawnSync calls that cannot observe an
4982
5891
  // AbortSignal mid-call, so provisioning is made signal-aware by rechecking
4983
5892
  // BETWEEN operations: if a lock-loss race won, bail before the NEXT side effect
@@ -5003,7 +5912,9 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5003
5912
  // already-lost lease without starting any repository side effect.
5004
5913
  throwIfAborted('entry');
5005
5914
  const askpass = writeAskpass(runDir, token);
5006
- const gitEnv = {
5915
+ // `let` (not const): the #293 clone auth-failure retry swaps in a FRESH token
5916
+ // (rewritten askpass + GIT_TOKEN) and re-runs the clone with an updated env.
5917
+ let gitEnv = {
5007
5918
  ...process.env,
5008
5919
  GIT_TERMINAL_PROMPT: '0',
5009
5920
  GIT_CONFIG_NOSYSTEM: '1',
@@ -5091,8 +6002,80 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5091
6002
  const detailNote = detail && detail !== 'unknown error' ? ` — last git output: ${detail}` : '';
5092
6003
  throw new ProvisionError(`git clone timed out after ${effectiveTimeoutMs}ms — the repo may be too large, or the network stalled; raise the timeout (repository.cloneTimeoutMs in the envelope, or the worker's --clone-timeout flag) or scope the clone with filter/singleBranch/depth${detailNote}`);
5093
6004
  }
5094
- throw new ProvisionError(`git clone failed: ${gitErrorDetail(clone, token)}`);
6005
+ const detail = gitErrorDetail(clone, token);
6006
+ // #293: an AUTH failure here usually means the cached gh token went stale
6007
+ // (the operator re-logged in / rotated it after this worker primed). Force a
6008
+ // fresh re-read and retry the clone ONCE with it before reporting failure —
6009
+ // a rotated credential self-heals instead of shedding the job. Only when the
6010
+ // re-read yields a token DIFFERENT from the one that failed (a same-token
6011
+ // retry can't fix a genuine permission denial). The recovery is
6012
+ // NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
6013
+ // the cache first, so a transiently-failing re-read can't erase a still-valid
6014
+ // — possibly newer — cached credential (Copilot review on #294). The retry
6015
+ // re-runs the whole clone with the new token threaded through the askpass
6016
+ // env, re-anchored to the remaining provisioning budget.
6017
+ // GATED on the credential's SOURCE (Copilot review on #293): this fallback
6018
+ // replaces the failed token with the operator's `gh auth token`, so it is
6019
+ // only legitimate when the failed token CAME from that same gh cache
6020
+ // ('gh-cache'). An explicit GITHUB_TOKEN ('env') or custom authRef
6021
+ // ('custom') is a deliberately configured identity — silently swapping it
6022
+ // for the operator's gh login would violate githubCloneToken's documented
6023
+ // precedence and could act as a different identity, so those sources fail
6024
+ // without the gh retry.
6025
+ // The `token` guard is deliberately ABSENT (Copilot review on #293): a
6026
+ // default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
6027
+ // a worker that cached "logged out" recovers null → fresh token after the
6028
+ // operator runs `gh auth login` instead of failing the job anonymous. That
6029
+ // recovery must REBUILD the credential plumbing the anonymous path omitted
6030
+ // (the askpass helper, GIT_ASKPASS/GIT_TOKEN, and the username-embedded
6031
+ // remote) — a bare GIT_TOKEN swap leaves the clone unauthenticated.
6032
+ if (credentialSource === 'gh-cache' && isGitAuthFailure(detail)) {
6033
+ const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: provTimeoutMs() });
6034
+ if (fresh) {
6035
+ logger?.info?.(`git clone auth failed${cs} — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the clone once`);
6036
+ throwIfAborted('clone-auth-retry');
6037
+ // Rebuild the credential env for the fresh token. On the null→fresh
6038
+ // recovery the original env was the ANONYMOUS one (no askpass helper,
6039
+ // GIT_CONFIG_GLOBAL neutralized), so re-mint the askpass helper and
6040
+ // restore global config; on a stale-token rotation this is a no-op
6041
+ // re-point of the same GIT_TOKEN.
6042
+ const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
6043
+ const retryAskpass = writeAskpass(runDir, fresh);
6044
+ if (retryAskpass) {
6045
+ retryEnv.GIT_ASKPASS = retryAskpass;
6046
+ // The anonymous path set GIT_CONFIG_GLOBAL=/dev/null so a no-token
6047
+ // clone can't inherit operator credentials; the fresh token is ours,
6048
+ // so global config (e.g. http.proxy) may apply again.
6049
+ delete retryEnv.GIT_CONFIG_GLOBAL;
6050
+ }
6051
+ // Rebuild the clone argv so the remote carries the fresh credential's
6052
+ // username (authUrl embeds x-access-token only when a token is present);
6053
+ // the anonymous remote had no username, so git would never consult the
6054
+ // askpass helper for it. `workspaceDir` stays last; the remote is the
6055
+ // argument immediately before it (cloneArgs.push(remote, workspaceDir)).
6056
+ const retryCloneArgs = cloneArgs.slice();
6057
+ retryCloneArgs[retryCloneArgs.length - 2] = authUrl(repo.url, repo.provider || 'github', true);
6058
+ const retry = runGitFn(retryCloneArgs, { env: retryEnv, timeoutMs: provTimeoutMs() });
6059
+ if (retry.status === 0) {
6060
+ // Adopt the fresh token for the REST of provisioning + the returned
6061
+ // gitEnv (finalizeGit pushes with it) so the whole job runs on the
6062
+ // rotated credential.
6063
+ token = fresh;
6064
+ gitEnv = retryEnv;
6065
+ return provisionRepoAfterClone();
6066
+ }
6067
+ throw new ProvisionError(`git clone failed after a fresh gh token retry: ${gitErrorDetail(retry, fresh)}`);
6068
+ }
6069
+ }
6070
+ throw new ProvisionError(`git clone failed: ${detail}`);
5095
6071
  }
6072
+ return provisionRepoAfterClone();
6073
+
6074
+ // Everything after a successful clone — hoisted into a closure so the
6075
+ // auth-retry path above re-enters it without duplicating the body. The clone
6076
+ // result itself is not read below (only its success mattered); the workspace
6077
+ // on disk is the real output.
6078
+ function provisionRepoAfterClone() {
5096
6079
 
5097
6080
  // #222: the clone (the dominant blocking op and the first repo side effect) is
5098
6081
  // done. Recheck the abort signal before the remaining side effects/network ops
@@ -5626,7 +6609,16 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5626
6609
  // the authoritative result so eligibility (auto checkpointing) doesn't
5627
6610
  // under-detect an authenticated clone.
5628
6611
  const authenticated = !!token || isAuthenticatedRemote(repo.url);
5629
- return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
6612
+ // `effectiveToken` is the credential the workspace ACTUALLY runs on after any
6613
+ // #293 auth-failure retry rotated it mid-provisioning (the local `token` is
6614
+ // reassigned on adoption). The caller (workAgent) must use THIS — not its own
6615
+ // pre-provisioning resolution — for the post-provisioning credential users
6616
+ // (finalizeGit's PR reconcile/push, the checkpoint GC/ref-delete askpass), or
6617
+ // a recovered clone would keep failing with the stale credential (Copilot
6618
+ // review on #293). `credentialSource` is echoed back so finalizeGit can gate
6619
+ // its own push-retry on the same source rule.
6620
+ return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, effectiveToken: token || null, credentialSource, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
6621
+ } // end provisionRepoAfterClone
5630
6622
  }
5631
6623
 
5632
6624
  // Look up a PR for this branch (2a does NOT open it — the harness does, driven
@@ -5789,7 +6781,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5789
6781
  ...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
5790
6782
  ...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
5791
6783
  };
5792
- const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
6784
+ // #296 (F9): the checkpoint-path git runner must stay USABLE after the harness
6785
+ // abort. The force-stop RECOVERY path (flush + preserveRecovery, promoting the
6786
+ // durable recovery ref #295) runs only once `result.aborted` is observed — i.e.
6787
+ // AFTER the job's `abortSignal` has already fired. Binding this runner to
6788
+ // `abortSignal` (the round-4 F8 attempt) made `createGitRunner` fail every
6789
+ // recovery command (`[aborted]`), so the abort flush could not push its snapshot
6790
+ // and preserveRecovery could not even observe/create the recovery ref — recovery
6791
+ // was self-defeating. Instead drive it from a DEDICATED recovery-deadline
6792
+ // controller that is NEVER aborted by the harness stop, only when the recovery
6793
+ // budget (FORCE_STOP_RECOVERY_MS) expires. That still delivers F8's goal — an
6794
+ // in-flight push is torn down at the deadline so the chain settles and `close()`
6795
+ // drains inside the supervisor's SIGKILL window — without killing the recovery
6796
+ // before it can write the ref.
6797
+ const recoveryController = new AbortController();
6798
+ const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: recoveryController.signal });
5793
6799
  // The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
5794
6800
  // if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
5795
6801
  // git child is killed and later ops fail fast, so setup unwinds instead of holding
@@ -5800,6 +6806,40 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5800
6806
  // which time the workspace (and its askpass helper) has been reaped.
5801
6807
  const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
5802
6808
 
6809
+ // Bound every checkpoint-path SDK call: a timed-out call rejects; a late settle is
6810
+ // swallowed so it can never surface as an unhandled rejection. Hoisted here (ahead
6811
+ // of the GC block, which reuses them) so the #295 ambiguity guard below is bounded.
6812
+ const CP_SDK_TIMEOUT_MS = cfg.sdkTimeoutMs;
6813
+ const withDeadline = (p, ms, label, signal = null) => {
6814
+ let timer;
6815
+ let onAbort = null;
6816
+ // #296 (review): a bare Promise.race REJECTS on timeout but leaves the SDK
6817
+ // mutation in flight — a timed-out `resumeFromCheckpoint` write could then
6818
+ // complete AFTER a retry already wrote a newer pointer and overwrite it with
6819
+ // the stale ref. The SDK returns a CancelablePromise (toCancelable): cancel it
6820
+ // on timeout so the underlying fetch is aborted and the late settle can never
6821
+ // land. `p.cancel?.()` is a no-op for a plain (non-cancelable) promise, and a
6822
+ // cancelled SDK call rejects with CancelSdkError, which the `.catch(() => {})`
6823
+ // below swallows.
6824
+ // #296 (review r4245146247): the SAME stale-write hazard applies to the job's
6825
+ // ABORT (lease loss), not just the per-call timeout. An optional `signal` lets a
6826
+ // setup-path side-effect write be cancelled the instant the job loses its lease,
6827
+ // so a write left pending after a replacement activation published a newer
6828
+ // pointer is torn down and its late settle can never overwrite that newer value.
6829
+ const race = new Promise((_r, reject) => {
6830
+ timer = setTimeout(() => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} timed out after ${ms}ms`)); }, ms);
6831
+ timer?.unref?.();
6832
+ if (signal) {
6833
+ if (signal.aborted === true) { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); return; }
6834
+ onAbort = () => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); };
6835
+ signal.addEventListener('abort', onAbort, { once: true });
6836
+ }
6837
+ });
6838
+ Promise.resolve(p).catch(() => {});
6839
+ const cleanup = () => { clearTimeout(timer); if (signal && onAbort) { try { signal.removeEventListener('abort', onAbort); } catch { /* best effort */ } } };
6840
+ return Promise.race([Promise.resolve(p).finally(cleanup), race]);
6841
+ };
6842
+
5803
6843
  let prior = null;
5804
6844
  let restored = null;
5805
6845
  let startupFetchFailed = false;
@@ -5809,6 +6849,93 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5809
6849
  startupFetchFailed = true;
5810
6850
  log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
5811
6851
  }
6852
+ // #295: nothing at this run's deterministic ref, but a human may have answered an
6853
+ // SLA escalation and re-entered the task as a NEW element instance. The predecessor
6854
+ // recorded its ref in a process-scoped `resumeFromCheckpoint` variable (survives
6855
+ // re-entry); adopt it under THIS run's key so the restore/ownership/discard paths
6856
+ // below operate unchanged and no GC can reclaim it between activations.
6857
+ if (!prior && !startupFetchFailed) {
6858
+ // Correlate the shared process-scoped slot to THIS task. Copilot review on #296
6859
+ // (r4244212483): a SELF-RECORDED (object) predecessor is correlated only by the
6860
+ // STATIC elementId, which is NOT unique under parallel multi-instance of one
6861
+ // agent-task element (N occurrences share the elementId AND the one process-scoped
6862
+ // resume slot). Counting ACTIVE occurrences cannot establish occurrence identity
6863
+ // (index lag can hide a live sibling; a terminal sibling drops out of the ACTIVE
6864
+ // result yet may have been the slot's last writer), so NO engine query makes a
6865
+ // self-record adoption provably safe. We therefore adopt ONLY an EXPLICIT,
6866
+ // occurrence-authoritative ref — a bare STRING the process delivers naming THIS
6867
+ // occurrence's exact predecessor (nano-workforce#891, option 2). A self-recorded
6868
+ // object is NEVER auto-adopted (fail closed to a clean start); until #891 delivers
6869
+ // the explicit ref, a resume is transcript-only rather than risking a sibling's
6870
+ // state. `predecessorCheckpointRef` still parses/locates the candidate ref; the
6871
+ // explicit check below is what authorizes adopting it.
6872
+ const predRef = predecessorCheckpointRef(job?.variables, ref, job?.elementId ?? null);
6873
+ if (predRef) {
6874
+ const adoptAllowed = resumeRefIsExplicit(job?.variables);
6875
+ if (!adoptAllowed) {
6876
+ log.warn(`not adopting predecessor WIP checkpoint ${predRef} — it is an elementId-correlated self-record, which cannot be proven to name THIS occurrence (parallel multi-instance of one agent-task element shares the elementId and the process-scoped resume slot); starting clean. Resume is transcript-only until the process delivers an explicit occurrence-authoritative ref (nano-workforce#891).`);
6877
+ }
6878
+ let predCp = null;
6879
+ if (adoptAllowed) {
6880
+ try { predCp = await fetchCp({ git: startupGit, ref: predRef }); } catch (err) { log.debug(`could not fetch predecessor WIP checkpoint ${predRef} — ${oneLineLog(err?.message || err)}.`); }
6881
+ }
6882
+ if (predCp?.sha) {
6883
+ // #296 (review): after ANY adoption the process-scoped slot still names the
6884
+ // stale PREDECESSOR ref. If this resumed activation then succeeds WITHOUT
6885
+ // taking another checkpoint (a short run before the safety timer), its adopted
6886
+ // WIP ref is deleted on ack while the slot keeps pointing at the predecessor —
6887
+ // a later activation would re-restore that old snapshot. Advance the slot to
6888
+ // THIS run's key (with the sha we actually adopted) so the pointer tracks the
6889
+ // current owner. Factored into one helper so EVERY adoption branch advances it
6890
+ // (the create-only win AND the lost-race re-read, whose race winner's own
6891
+ // best-effort advance may have failed — this makes the advance idempotent
6892
+ // across the race rather than reliant on the winner). Best-effort; a failure
6893
+ // never blocks setup.
6894
+ const advanceResumePointer = async (adoptedSha) => {
6895
+ if (!adoptedSha || typeof camunda?.createElementInstanceVariables !== 'function' || !job?.processInstanceKey) return;
6896
+ // #296 (review r4245146247): recheck the job signal before this process-scoped
6897
+ // side effect. The startup git work above (fetch/push) may have straddled a
6898
+ // lease loss; once aborted, a replacement activation owns this pointer slot and
6899
+ // may already have published a newer value, so writing our (now stale) ref here
6900
+ // would overwrite it. Skip when aborted, and bind the SDK call to abortSignal so
6901
+ // a lease loss DURING the write cancels it before its late settle can land.
6902
+ if (abortSignal?.aborted === true) return;
6903
+ try {
6904
+ await withDeadline(camunda.createElementInstanceVariables({
6905
+ elementInstanceKey: String(job.processInstanceKey),
6906
+ variables: { resumeFromCheckpoint: { ref, sha: adoptedSha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch: provisioned.workingBranch || null, at: Date.now(), adoptedFrom: predRef } },
6907
+ local: true,
6908
+ }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', abortSignal || undefined);
6909
+ } catch (err) { log.debug(`could not advance the resumeFromCheckpoint pointer to ${ref} — ${oneLineLog(err?.message || err)}.`); }
6910
+ };
6911
+ // Create-only adoption (default expectSha=''): only write our key while it
6912
+ // is ABSENT, so a concurrent same-key retry that already adopted + advanced
6913
+ // the ref is never clobbered (Copilot review on #296).
6914
+ const copied = await pushNamedRef({ git: startupGit, ref, sha: predCp.sha });
6915
+ if (copied?.ok) {
6916
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = { ...predCp, ref }; }
6917
+ if (!prior) prior = { ...predCp, ref };
6918
+ logger.info?.(`${prefix} (${corr}): adopted predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} under this run's key ${ref} (SLA-escalation resume); restoring it below.`);
6919
+ await advanceResumePointer(predCp.sha);
6920
+ } else if (copied?.staleLease) {
6921
+ // A concurrent activation populated this key first — adopt what IS there
6922
+ // rather than force over it; the ref now holds the race winner's state.
6923
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = null; }
6924
+ if (prior) {
6925
+ logger.info?.(`${prefix} (${corr}): a concurrent activation already populated ${ref}; adopting its checkpoint ${prior.sha ? '@' + String(prior.sha).slice(0, 12) : ''} (SLA-escalation resume); restoring it below.`);
6926
+ // Advance the pointer to the re-read sha too: the race winner's own
6927
+ // best-effort advance may have failed, which would leave the slot naming
6928
+ // the bare predecessor and risk a later re-adoption of stale work.
6929
+ await advanceResumePointer(prior.sha);
6930
+ } else {
6931
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but ${ref} was concurrently moved and could not be re-read — the predecessor ref is retained for explicit recovery.`);
6932
+ }
6933
+ } else {
6934
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but could not copy it to ${ref} — ${copied?.error || 'unknown'}; the predecessor ref is retained for explicit recovery.`);
6935
+ }
6936
+ }
6937
+ }
6938
+ }
5812
6939
  if (prior) {
5813
6940
  const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
5814
6941
  if (r?.restored) {
@@ -5836,12 +6963,12 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5836
6963
 
5837
6964
  // Orphan GC for this remote, throttled per worker process, in the background.
5838
6965
  let gc = Promise.resolve();
6966
+ // #296: the GC sweep's abort controller, hoisted so the force-stop recovery
6967
+ // DEADLINE (abortRecovery) can tear the (up to 120s) sweep down too — otherwise
6968
+ // the single shared force-stop budget would bound only the recovery git, and a
6969
+ // still-running GC could hold close()'s drain past the supervisor's SIGKILL window.
6970
+ let gcAbort = null;
5839
6971
  const gcKey = provisioned.remote || provisioned.workspaceDir;
5840
- // Bound every checkpoint-path SDK call: the GC sweep and the variable write are
5841
- // awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
5842
- // call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
5843
- // late settle is swallowed so it can never surface as an unhandled rejection.
5844
- const CP_SDK_TIMEOUT_MS = 10000;
5845
6972
  // A single overall bound on the whole GC sweep. Individual ops are already
5846
6973
  // bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
5847
6974
  // but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
@@ -5852,23 +6979,27 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5852
6979
  // reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
5853
6980
  // 4099558206 / 4099762442).
5854
6981
  const GC_OVERALL_TIMEOUT_MS = 120000;
5855
- const withDeadline = (p, ms, label) => {
5856
- let timer;
5857
- const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
5858
- Promise.resolve(p).catch(() => {});
5859
- return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
5860
- };
5861
6982
  if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
5862
- const isTerminal = typeof camunda?.getElementInstance === 'function'
5863
- ? async (key) => {
5864
- try {
5865
- const ei = await withDeadline(
6983
+ // #295: a nano-managed ref (WIP or durable recovery) is an orphan only when its
6984
+ // element is terminal AND its PROCESS instance is finished too. An element
6985
+ // cancelled by an SLA boundary timer, inside a process that is still active (human
6986
+ // escalation pending), must NOT be reclaimed — the TTL is the final backstop.
6987
+ // `makeOrphanProbe` encodes that; it degrades to the legacy element-only check when
6988
+ // the host SDK exposes no `getProcessInstance`. The sweep applies this to BOTH the
6989
+ // `refs/nano/wip/*` and `refs/nano/recovered/*` namespaces so recovery refs cannot
6990
+ // grow without bound once their escalation is resolved.
6991
+ const isTerminal = makeOrphanProbe({
6992
+ getElementInstance: typeof camunda?.getElementInstance === 'function'
6993
+ ? (key) => withDeadline(
5866
6994
  camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
5867
- CP_SDK_TIMEOUT_MS, 'getElementInstance');
5868
- return ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED';
5869
- } catch { return false; }
5870
- }
5871
- : null;
6995
+ CP_SDK_TIMEOUT_MS, 'getElementInstance')
6996
+ : undefined,
6997
+ getProcessInstance: typeof camunda?.getProcessInstance === 'function'
6998
+ ? (pik) => withDeadline(
6999
+ camunda.getProcessInstance({ processInstanceKey: String(pik) }, { consistency: { waitUpToMs: 0 } }),
7000
+ CP_SDK_TIMEOUT_MS, 'getProcessInstance')
7001
+ : null,
7002
+ });
5872
7003
  // Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
5873
7004
  // job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
5874
7005
  // push-deletes stale ones, while the workspace is concurrently driven by the harness
@@ -5878,6 +7009,7 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5878
7009
  // decouples them; its dir is reaped when the sweep settles. On the overall deadline
5879
7010
  // we abort the sweep's cancellable git so the promise unwinds instead of hanging.
5880
7011
  const sweepAbort = new AbortController();
7012
+ gcAbort = sweepAbort;
5881
7013
  let sweepDir = null;
5882
7014
  try {
5883
7015
  if (!originUrl) throw new Error('workspace had no origin URL');
@@ -5901,10 +7033,10 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5901
7033
  // the sweep's git is still live. The deadline only bounds it by aborting the git.
5902
7034
  gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
5903
7035
  .then((r) => {
5904
- if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned WIP checkpoint ref(s): ${r.deleted.map((d) => `${d.ref} [${d.why}]`).join(', ')}.`);
5905
- else if (sweepAbort.signal.aborted) log.debug(`WIP checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
5906
- else if (r?.error) log.debug(`WIP checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
5907
- }, (err) => log.debug(`WIP checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
7036
+ if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned checkpoint ref(s): ${r.deleted.map((d) => `${d.ref}@${String(d.sha || '') || '?'} [${d.why}]`).join(', ')}.`);
7037
+ else if (sweepAbort.signal.aborted) log.debug(`checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
7038
+ else if (r?.error) log.debug(`checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
7039
+ }, (err) => log.debug(`checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
5908
7040
  .finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
5909
7041
  } catch (err) {
5910
7042
  if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
@@ -5918,17 +7050,80 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5918
7050
  // delete against it so a stale activation cannot delete a newer owner's WIP.
5919
7051
  let ownedSha = prior?.sha || '';
5920
7052
  let varWarned = false;
7053
+ let resumeVarWarned = false;
7054
+ // #296 (review): the in-flight ABORT-path metadata write (`setVar` fired
7055
+ // fire-and-forget on a force-stop flush, below). The force-stop recovery path
7056
+ // later writes the durable recovery ref to the SAME process-scoped
7057
+ // `resumeFromCheckpoint` slot. If this earlier WIP write is still in flight and
7058
+ // lands AFTER the recovery-pointer write, it overwrites the recovery pointer with
7059
+ // a stale WIP ref — and once the WIP TTL expires, auto-resume can no longer
7060
+ // discover the still-existing recovery ref. Track the promise so the recovery
7061
+ // path can AWAIT it (bounded) before writing the recovery pointer, guaranteeing
7062
+ // the recovery write lands last. The WIP write still lands on its own when
7063
+ // promotion fails (the fallback is retained); we only serialize the ordering.
7064
+ let abortMetaWrite = null;
5921
7065
  const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
5922
- ? async (res) => {
5923
- try {
5924
- await withDeadline(camunda.createElementInstanceVariables({
7066
+ ? (res, signal = null) => {
7067
+ // #296 (review r4245278256): bind these shared process-scoped writes to the
7068
+ // job abort signal for NON-`abort` checkpoints (periodic/`failed`). A periodic
7069
+ // or failed checkpoint can be AWAITING this mutation (`await setVar(res).promise`
7070
+ // below) when heartbeat lease loss interrupts the raw runner and releases the
7071
+ // slot; a replacement activation may then publish a newer `resumeFromCheckpoint`
7072
+ // pointer before this stale call completes and overwrites it — and the later
7073
+ // post-flush signal recheck cannot undo a write that already landed. Passing
7074
+ // `abortSignal` makes `withDeadline` cancel the in-flight SDK call the instant
7075
+ // the lease is lost, so the stale write is torn down and can never land late.
7076
+ // The `abort`-reason path deliberately passes NO signal: that write is owned by
7077
+ // the dedicated `settleAbortMetaWrite` flow (recovery-deadline governed), which
7078
+ // must still fire AFTER the job abort during the force-stop recovery window.
7079
+ // #296: run the element-scope and process-scope writes CONCURRENTLY, not
7080
+ // sequentially. Each carries its own CP_SDK_TIMEOUT_MS deadline, so a
7081
+ // sequential pair could stall for 2× the timeout (20s) — enough to exhaust
7082
+ // the whole FORCE_STOP_RECOVERY_MS budget during a force-stop flush and
7083
+ // starve the recovery git promotion. Racing them caps the worst case at a
7084
+ // single timeout, and each has an independent try/catch so one failing (or
7085
+ // being absent, e.g. no processInstanceKey) never blocks the other.
7086
+ // #296 (review, r4244281967): RETAIN the underlying CancelablePromise(s) so
7087
+ // the abort path can CANCEL them when the recovery deadline fires — not just
7088
+ // stop awaiting. Otherwise a fire-and-forget abort write whose own SDK timeout
7089
+ // outlasts the recovery window stays in flight after the worker yields and can
7090
+ // land LATE over a newer activation's `resumeFromCheckpoint`. The SDK returns a
7091
+ // CancelablePromise (toCancelable); `.cancel?.()` is a no-op for a plain promise
7092
+ // (the test stubs), so capturing the raw return is enough.
7093
+ const rawWrites = [
7094
+ camunda.createElementInstanceVariables({
5925
7095
  elementInstanceKey: String(job.elementInstanceKey),
5926
7096
  variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
5927
7097
  local: true,
5928
- }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables');
5929
- } catch (err) {
5930
- if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7098
+ }),
7099
+ ];
7100
+ const writes = [
7101
+ withDeadline(rawWrites[0], CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7102
+ if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7103
+ }),
7104
+ ];
7105
+ // #295: also record the ref at the PROCESS instance scope (local on the root
7106
+ // scope key = the process instance key). Unlike the local `agentCheckpoint`
7107
+ // above, this survives the element re-entry an SLA escalation triggers, so a
7108
+ // resumed run (a NEW element instance) can find and adopt its predecessor's
7109
+ // checkpoint via `resumeFromCheckpoint`. Best-effort and independent of the
7110
+ // local write; a failure never blocks the push.
7111
+ if (job?.processInstanceKey) {
7112
+ const rawProcess = camunda.createElementInstanceVariables({
7113
+ elementInstanceKey: String(job.processInstanceKey),
7114
+ variables: { resumeFromCheckpoint: { ref, sha: res.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch, at: res.at } },
7115
+ local: true,
7116
+ });
7117
+ rawWrites.push(rawProcess);
7118
+ writes.push(withDeadline(rawProcess, CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7119
+ if (!resumeVarWarned) { resumeVarWarned = true; log.warn(`could not set the resumeFromCheckpoint variable — ${oneLineLog(err?.message || err)}; a resumed run may not auto-find ${ref}.`); }
7120
+ }));
5931
7121
  }
7122
+ const promise = Promise.all(writes);
7123
+ // Cancel every in-flight underlying write (best-effort; no-op for plain
7124
+ // promises). Used by the abort path when the recovery deadline fires.
7125
+ const cancel = () => { for (const w of rawWrites) { try { w?.cancel?.(); } catch { /* best effort */ } } };
7126
+ return { promise, cancel };
5932
7127
  }
5933
7128
  : null;
5934
7129
 
@@ -5968,7 +7163,35 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5968
7163
  onCheckpoint: async (res) => {
5969
7164
  ownedSha = res.sha;
5970
7165
  log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
5971
- if (setVar) await setVar(res);
7166
+ if (!setVar) return;
7167
+ // #296: `ownedSha` (above) is the only state preserveRecovery() needs from
7168
+ // this callback, and it is already set. Only a FORCE-STOP (`abort`) flush
7169
+ // fires the metadata writes off the serialized flush/promote chain: that path
7170
+ // is bounded by FORCE_STOP_RECOVERY_MS and the worker returns early (no
7171
+ // settle), so awaiting a hung engine there could burn the whole recovery
7172
+ // budget before the recovery git promotion runs — and the durable recovery
7173
+ // ref, not these vars, is the durable artifact of a force-stopped run.
7174
+ // A `failed` flush is DIFFERENT: the job settles (settleJob.fail) right after
7175
+ // the flush returns, and a retry/reactivation can read variables before a
7176
+ // fire-and-forget `resumeFromCheckpoint` write is durable — worse, a delayed
7177
+ // process-scoped write could overwrite a NEWER activation's pointer. So the
7178
+ // failed path (and every normal checkpoint) AWAITS the write before
7179
+ // settlement, keeping the predecessor pointer durable before the job settles.
7180
+ // #296 (review r4245278256): and it passes `abortSignal`, so if heartbeat lease
7181
+ // loss fires DURING this awaited write the in-flight SDK call is cancelled — a
7182
+ // replacement activation then owns the slot and this run's now-stale pointer can
7183
+ // never land late over its newer value.
7184
+ // #296 (review): on the abort path, CAPTURE the fire-and-forget write (promise +
7185
+ // cancel) so the force-stop recovery path can await it before writing the
7186
+ // recovery pointer to the same slot — otherwise a slow WIP write lands after and
7187
+ // clobbers the recovery ref. `.promise.catch(() => {})` keeps it fire-and-forget
7188
+ // for the flush; the settled/rejected result is irrelevant to the caller. The
7189
+ // retained `cancel` lets settleAbortMetaWrite tear down the in-flight SDK call
7190
+ // when the recovery deadline fires (r4244281967), so it can never land late. This
7191
+ // path deliberately passes NO abortSignal to setVar: the abort write is owned by
7192
+ // settleAbortMetaWrite's recovery-deadline flow and must outlive the job abort.
7193
+ if (res.reason === 'abort') { const w = setVar(res); abortMetaWrite = { promise: w.promise.catch(() => {}), cancel: w.cancel }; return; }
7194
+ await setVar(res, abortSignal || undefined).promise;
5972
7195
  },
5973
7196
  });
5974
7197
  return {
@@ -5978,6 +7201,176 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5978
7201
  restored,
5979
7202
  priorNotRestored,
5980
7203
  flushTimeoutMs: cfg.flushTimeoutMs,
7204
+ // #296 (review): the git SIGKILL grace window, surfaced so the force-stop path
7205
+ // can reserve a real drain window for close() instead of the 1ms an expired
7206
+ // shared deadline would otherwise hand it (see the force-stop block).
7207
+ gitKillGraceMs: cfg.gitKillGraceMs,
7208
+ // #296 (F9): abort the recovery-deadline controller — tears down an in-flight
7209
+ // checkpoint/recovery git push so `stop()`/`close()` settle promptly. Driven by
7210
+ // the force-stop path once the recovery budget expires (NOT by the job's
7211
+ // abortSignal, which fires BEFORE recovery runs and would fail every recovery
7212
+ // command). Also aborts the background GC sweep so the ONE shared force-stop
7213
+ // deadline bounds recovery AND cleanup together — otherwise a still-running
7214
+ // (up to 120s) sweep could hold close()'s drain past the supervisor's SIGKILL
7215
+ // window even after recovery settled. Idempotent / best-effort; a no-op once
7216
+ // already aborted.
7217
+ abortRecovery() {
7218
+ try { recoveryController.abort(); } catch { /* best effort */ }
7219
+ try { gcAbort?.abort(); } catch { /* best effort */ }
7220
+ },
7221
+ // Adopt a credential the job ROTATED after this checkpointer was created —
7222
+ // e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
7223
+ // (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
7224
+ // its own askpass purely from `token`, so re-binding `token` here is enough
7225
+ // to make the cleanup push authenticate with the live credential instead of
7226
+ // the one the remote already rejected — otherwise the WIP ref lingers until a
7227
+ // later GC (Copilot review on #293). Ignored when the token is unchanged or
7228
+ // falsy; the live checkpointer has already stopped by the time this is called.
7229
+ updateCredential(nextToken) {
7230
+ if (nextToken && nextToken !== token) token = nextToken;
7231
+ },
7232
+ // #296 (review): await the in-flight ABORT-path metadata write (the WIP
7233
+ // `resumeFromCheckpoint` write fired fire-and-forget on a force-stop flush),
7234
+ // bounded, so the force-stop recovery path can guarantee ITS recovery-pointer
7235
+ // write to the same slot lands LAST (a slow WIP write must not overwrite the
7236
+ // durable recovery ref). Best-effort, never throws; resolves immediately when
7237
+ // no abort write is in flight. The caller bounds the wait inside the shared
7238
+ // force-stop budget, so a hung engine cannot stall the force-stop exit.
7239
+ // #296 (review, r4244116341): accept the recovery-deadline `signal` so this SDK
7240
+ // await is cancelled by the SAME deadline that tears down the recovery git — the
7241
+ // recover chain must SETTLE (not dangle) when the budget expires, so a stale
7242
+ // continuation can never write the old recovery pointer over a reactivated
7243
+ // worker's newer one.
7244
+ // #296 (review, r4244281967): stopping the AWAIT is not enough — the underlying
7245
+ // `createElementInstanceVariables` mutation would otherwise stay in flight past
7246
+ // the deadline and could land LATE over a newer activation's `resumeFromCheckpoint`.
7247
+ // So when `signal` aborts we also CANCEL the retained CancelablePromise(s)
7248
+ // (`abortMetaWrite.cancel()`), aborting the fetch so the late settle can never land.
7249
+ async settleAbortMetaWrite(signal = null) {
7250
+ const w = abortMetaWrite;
7251
+ if (!w) return;
7252
+ if (signal?.aborted) { try { w.cancel?.(); } catch { /* best effort */ } return; }
7253
+ if (!signal) { try { await w.promise; } catch { /* best effort */ } return; }
7254
+ let onAbort = null;
7255
+ try {
7256
+ await Promise.race([
7257
+ w.promise.catch(() => {}),
7258
+ new Promise((res) => { onAbort = () => { try { w.cancel?.(); } catch { /* best effort */ } res(); }; signal.addEventListener('abort', onAbort, { once: true }); }),
7259
+ ]);
7260
+ } catch { /* best effort */ } finally { if (onAbort) signal.removeEventListener('abort', onAbort); }
7261
+ },
7262
+ // #295: promote the latest snapshot to a durable recovery ref on an SLA
7263
+ // cancellation or a force-stop. `refs/nano/recovered/<elementInstanceKey>`
7264
+ // lives OUTSIDE the `refs/nano/wip/*` namespace, so it survives the
7265
+ // per-checkpoint WIP cleanup (`discardAfterAck`, which only deletes WIP refs)
7266
+ // and workspace reaping — long enough for the human escalation to be answered
7267
+ // and the work recovered. It is NOT exempt from the orphan GC, though: the
7268
+ // sweep reclaims it under the same process-terminal / 7d-TTL policy as a WIP
7269
+ // ref (#296), sparing it only while its process is still ACTIVE. Returns
7270
+ // `{ ref, sha }` (surfaced in the job outcome) or
7271
+ // null when there is nothing durable to preserve / the push failed. Best-effort;
7272
+ // never throws. The live WIP ref is kept too (this is an additional durable copy).
7273
+ async preserveRecovery(reason = 'recovery') {
7274
+ // Copilot review on #295: the caller invokes this right after a BOUNDED
7275
+ // `flush()`, which can RETURN while its forced checkpoint is still in flight
7276
+ // — and `ownedSha` is only updated later, by `onCheckpoint`, once that
7277
+ // checkpoint's push resolves. Reading `ownedSha` immediately could therefore
7278
+ // see `null`/the pre-flush SHA and promote an older snapshot (or nothing).
7279
+ // DRAIN the checkpoint chain first (`stop()` resolves once the in-flight
7280
+ // chain settles, letting `onCheckpoint` record the latest SHA) so the SHA
7281
+ // selected below is the newest snapshot this run actually pushed. This
7282
+ // mirrors the success path, which already `stop()`s before finalizeGit.
7283
+ try { await checkpointer.stop(); } catch { /* best effort — fall through to the last recorded ownedSha */ }
7284
+ if (!ownedSha) return null;
7285
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7286
+ if (!recRef) return null;
7287
+ try {
7288
+ // Fence the recovery promotion against the sha we OBSERVE on the ref now,
7289
+ // so a slower/stale activation that observed an older/absent value loses
7290
+ // the lease once a newer run moved it, instead of blind-forcing over
7291
+ // whatever is current (Copilot review on #296).
7292
+ const prevRecSha = await observeRemoteRefSha({ git, ref: recRef });
7293
+ // #296 (F7/F10): a bare CAS against the just-observed value is NOT ownership
7294
+ // — a superseded activation can observe the newer run's `shaB` here and
7295
+ // still overwrite it with its older `ownedShaA`. Mirror the WIP path's
7296
+ // `Nano-Checkpoint-Run` ownership check: only overwrite an EXISTING recovery
7297
+ // ref when it is POSITIVELY identified as either ours (a re-preservation by
7298
+ // this same run) or the run we explicitly superseded (`prior?.runId`). An
7299
+ // absent ref is a create-only write. FAIL CLOSED: if the owner cannot be
7300
+ // positively identified — the ownership fetch THREW/failed, or the ref
7301
+ // carries no `runId` trailer — refuse the overwrite rather than fall through
7302
+ // to the CAS, so a transient read failure can never let a stale activation
7303
+ // clobber a newer (or unknown) run's recovery point (F10). The WIP ref +
7304
+ // TTL remain the fallback in every refused case.
7305
+ if (prevRecSha) {
7306
+ let ownerRunId = '';
7307
+ let ownerKnown = false;
7308
+ try { ownerRunId = (await fetchCheckpoint({ git, ref: recRef }))?.runId || ''; ownerKnown = true; } catch { ownerKnown = false; }
7309
+ const myRun = String(runId || '');
7310
+ const supersededRun = String(prior?.runId || '');
7311
+ const mine = ownerKnown && ownerRunId && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7312
+ if (!mine) {
7313
+ const who = !ownerKnown ? 'could not be read (transient)' : (ownerRunId ? `owned by another run (${ownerRunId})` : 'carries no run identity');
7314
+ log.warn(`recovery checkpoint ${recRef} ${who} — refusing to overwrite it; the WIP ref ${ref} is still retained for recovery.`);
7315
+ return null;
7316
+ }
7317
+ }
7318
+ const r = await pushNamedRef({ git, ref: recRef, sha: ownedSha, expectSha: prevRecSha });
7319
+ if (r?.ok) {
7320
+ logger.info?.(`${prefix} (${corr}): preserved a durable recovery checkpoint ${recRef}@${ownedSha.slice(0, 12)} (${reason}) — it survives the per-checkpoint WIP cleanup and workspace reaping (the orphan GC spares it while its process stays active, up to the ${Math.round(cfg.ttlMs / 86400000)}d TTL backstop, which reclaims it regardless of process state) so the SLA-cancelled/force-stopped run can be recovered and resumed.`);
7321
+ return { ref: recRef, sha: ownedSha };
7322
+ }
7323
+ if (r?.staleLease) {
7324
+ // A newer activation already preserved a recovery point on this ref;
7325
+ // keep theirs. Our WIP ref is still retained as a fallback.
7326
+ log.warn(`recovery checkpoint ${recRef} was moved by a newer activation — keeping its recovery point; the WIP ref ${ref} is still retained for recovery.`);
7327
+ return null;
7328
+ }
7329
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(r?.error || 'unknown', token)}; the WIP ref ${ref} is still retained for recovery.`);
7330
+ } catch (err) {
7331
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(oneLineLog(err?.message || err), token)}; the WIP ref ${ref} is still retained for recovery.`);
7332
+ }
7333
+ return null;
7334
+ },
7335
+ // #296 (review, r4244827082): re-verify, IMMEDIATELY before the process-scoped
7336
+ // `resumeFromCheckpoint` recovery-pointer write, that THIS run still owns the
7337
+ // durable recovery ref it just preserved. `preserveRecovery` fences the GIT ref
7338
+ // (CAS + `Nano-Checkpoint-Run` ownership), but the subsequent process-scoped
7339
+ // variable write is unconditional last-writer-wins: a superseded activation that
7340
+ // won its ref earlier can still land its stale pointer AFTER a newer activation
7341
+ // (which re-activated the job on lease loss) published its own. Close that
7342
+ // residual race by re-observing the recovery ref now and confirming it still
7343
+ // resolves to the sha this run preserved AND is still owned by this run (or the
7344
+ // run it explicitly superseded) — the same ownership rule the GIT fence applies.
7345
+ // FAIL CLOSED: any read failure, a moved/absent ref, or an owner that is not
7346
+ // positively this run returns false, so the caller SKIPS the variable write and
7347
+ // leaves the newer activation's pointer intact (the pushed ref stays the durable
7348
+ // fallback). `expectedSha` is the sha this run's preserveRecovery returned.
7349
+ // #296 (review, r4244916101): accept a per-call `signal` and run the git reads
7350
+ // on a DEDICATED runner bound to it (AND the recovery deadline). The caller
7351
+ // races this against a 5s timeout; without a wired signal the losing branch
7352
+ // left the verify's git process live on the 60s runner — untracked by
7353
+ // checkpointer.stop()/gc — so recover() returned and the run dir was reaped
7354
+ // under a live git. Binding the runner to the per-call signal lets the caller
7355
+ // abort+await the verify on timeout so it is SETTLED before cleanup proceeds.
7356
+ async verifyRecoveryOwnership(expectedSha, { signal } = {}) {
7357
+ try {
7358
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7359
+ if (!recRef || !expectedSha) return false;
7360
+ // Combine the per-call abort with the recovery-deadline abort so EITHER
7361
+ // tears the git down (whichever fires first). AbortSignal.any is Node 20+.
7362
+ const signals = [recoveryController.signal, signal].filter(Boolean);
7363
+ const combined = signals.length > 1 ? AbortSignal.any(signals) : (signals[0] || undefined);
7364
+ const vgit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: combined });
7365
+ const curSha = await observeRemoteRefSha({ git: vgit, ref: recRef });
7366
+ if (!curSha || curSha !== expectedSha) return false;
7367
+ let ownerRunId = '';
7368
+ try { ownerRunId = (await fetchCheckpoint({ git: vgit, ref: recRef }))?.runId || ''; } catch { return false; }
7369
+ const myRun = String(runId || '');
7370
+ const supersededRun = String(prior?.runId || '');
7371
+ return Boolean(ownerRunId) && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7372
+ } catch { return false; }
7373
+ },
5981
7374
  // True when the workspace still holds checkpoint content NOT captured by a
5982
7375
  // commit, so the pushed branch does not represent it (an uncommitted restored
5983
7376
  // or working delta that lives only on the WIP ref). The discard-on-ack gate
@@ -5992,8 +7385,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5992
7385
  } catch { return true; }
5993
7386
  },
5994
7387
  // Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
5995
- // run dir can be reaped safely.
5996
- async close() { await checkpointer.stop(); await gc; },
7388
+ // run dir can be reaped safely. `close()` is bounded by `closeTimeoutMs` so a
7389
+ // force-stopped worker is never held past the supervisor's SIGKILL backstop by
7390
+ // the (up to 120s) GC sweep: on the deadline we stop awaiting and let the
7391
+ // caller proceed (the in-flight checkpoint git AND the GC sweep are both
7392
+ // cancelled by the recovery DEADLINE — abortRecovery() — so the chain settles on
7393
+ // its own). The force-stop caller passes only the REMAINING of the one shared
7394
+ // force-stop budget, so recovery + this drain never exceed it together. Default
7395
+ // 0 = unbounded (normal/failed paths fully drain).
7396
+ async close({ timeoutMs = 0 } = {}) {
7397
+ const drain = (async () => { await checkpointer.stop(); await gc; })();
7398
+ if (!timeoutMs) return drain;
7399
+ let t;
7400
+ const cap = new Promise((resolve) => { t = setTimeout(resolve, timeoutMs); t?.unref?.(); });
7401
+ try { return await Promise.race([drain, cap]); } finally { clearTimeout(t); }
7402
+ },
5997
7403
  async discard() {
5998
7404
  await checkpointer.stop();
5999
7405
  try {
@@ -6069,7 +7475,7 @@ function computeRelayCloseReason(result, runCompleted, gitResult) {
6069
7475
  // branch.push), and reconcile the agent-opened PR (when task.allowPr). A push
6070
7476
  // failure is reported (pushError) rather than thrown — the process model decides
6071
7477
  // what to do next, and re-running the agent would be non-idempotent.
6072
- function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, _runGit = null }) {
7478
+ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, credentialSource = null, _runGit = null, _pushGit = null, _reconcilePr = null, _postAttribution = null }) {
6073
7479
  const log = logger || getLogger(); // route observability through the injected logger (tests) or the host logger
6074
7480
  // Test-only seam: the late remote-reachability FILTER scans (movedStray /
6075
7481
  // unpushedBranchCommits / final stranded) AND the remote-tip VERIFICATION calls
@@ -6080,8 +7486,25 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6080
7486
  // probes cannot be made to fail deterministically. Defaults to the module `runGit`, so
6081
7487
  // production behaviour is unchanged.
6082
7488
  const lateScanGit = _runGit || runGit;
7489
+ // Test-only seam for the branch PUSH itself (the initial push AND the single
7490
+ // post-auth-failure retry). Routed separately from `lateScanGit` so a
7491
+ // hermetic test can drive the central push-recovery branches — changed-token
7492
+ // retry success, the unchanged-token guard, the credential-source gate, and
7493
+ // retry-still-fails — without a live remote, while the local enumeration and
7494
+ // late remote-reachability scans keep their own runner (Copilot review on
7495
+ // #293). Defaults to the module `runGit`, so production behaviour is unchanged.
7496
+ const pushGit = _pushGit || runGit;
7497
+ const reconcilePr = _reconcilePr || reconcileAgentPr;
7498
+ const postAttribution = _postAttribution || postAgentAttribution;
6083
7499
  const cs = corr ? ` [${corr}]` : ''; // #229 cross-channel correlation suffix
6084
7500
  const out = { branch: workingBranch, baseSha: startSha || null, headSha: null, commits: [], pushed: false, remote: null, pr: null };
7501
+ // The credential this finalize was HANDED. The push auth-failure recovery below
7502
+ // may rotate `token` to a freshly re-read gh credential mid-finalize; capturing
7503
+ // the original lets the return expose `rotatedToken` so the CALLER can propagate
7504
+ // the rotation to subsystems that captured the pre-rotation credential — notably
7505
+ // the WIP-checkpoint cleanup, whose `discardAfterAck()` would otherwise push a
7506
+ // ref deletion with the already-rejected token (Copilot review on #293).
7507
+ const originalToken = token;
6085
7508
 
6086
7509
  // Bound EVERY finalize git op against ONE shared deadline derived from the recovery
6087
7510
  // window (budgetMs) — the LOCAL graph scans below AND the sequential NETWORK ops
@@ -6465,7 +7888,100 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6465
7888
  // qualified heads ref resolves unambiguously; --set-upstream still tracks the
6466
7889
  // local branch.
6467
7890
  const pushTimeout = netTimeoutMs();
6468
- const push = runGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
7891
+ let push = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
7892
+ // #293: an AUTH rejection (401/403, or the OAuth `workflow`-scope refusal)
7893
+ // usually means the cached gh token went stale mid-run. Force a fresh
7894
+ // re-read and retry the push ONCE with it before stranding the commits — a
7895
+ // rotated credential self-heals instead of a `pushFailed`. Only when the
7896
+ // re-read yields a token DIFFERENT from the one that failed (a same-token
7897
+ // retry can't fix a genuine permission denial), and the retry runs with the
7898
+ // fresh token in GIT_TOKEN so the askpass hands it to git. The recovery is
7899
+ // NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
7900
+ // the cache first, so a transiently-failing re-read can't erase a still-valid
7901
+ // — possibly newer — cached credential (Copilot review on #294).
7902
+ // GATED on the credential's SOURCE (Copilot review on #293): only a token
7903
+ // that CAME from the gh cache ('gh-cache') may be replaced by a fresh
7904
+ // `gh auth token` read — an explicit GITHUB_TOKEN ('env') or custom authRef
7905
+ // ('custom') is a deliberately configured identity that must fail as
7906
+ // configured, never be silently swapped for the operator's gh login.
7907
+ // The `token` guard is deliberately ABSENT (Copilot review on #293): a
7908
+ // default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
7909
+ // a worker that cached "logged out" recovers null → fresh token after the
7910
+ // operator runs `gh auth login` instead of stranding the commits. That
7911
+ // recovery must REBUILD the credential plumbing the anonymous gitEnv lacked
7912
+ // (the askpass helper + GIT_ASKPASS, and global git config) — a bare
7913
+ // GIT_TOKEN swap leaves the push unauthenticated.
7914
+ // The credential that PRODUCED the ORIGINAL `push` — preserved separately so
7915
+ // the failure report below redacts the value that actually appeared in that
7916
+ // push's output, even once `token` is swapped to the fresh credential for
7917
+ // subsequent authenticated operations after a retry (Copilot review on #293).
7918
+ let pushFailureToken = token;
7919
+ // The timeout that bounded the reported `push` — promoted alongside
7920
+ // pushFailureToken when a failed retry is promoted to the reported result, so
7921
+ // describeGitFailure attributes the RETRY's timeout, not the original push's
7922
+ // (Copilot review on #293).
7923
+ let pushFailureTimeout = pushTimeout;
7924
+ if (push.status !== 0 && credentialSource === 'gh-cache' && isGitAuthFailure(describeGitFailure('git push', push, { token, timeoutMs: pushTimeout }))) {
7925
+ const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: netTimeoutMs() });
7926
+ if (fresh) {
7927
+ log.info?.(`finalizeGit${cs}: git push auth failed — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the push once`);
7928
+ const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
7929
+ // Reuse the clone-time askpass helper when gitEnv already carries one
7930
+ // (the token-backed path): the helper emits GIT_TOKEN at runtime, so
7931
+ // pointing GIT_TOKEN at the fresh credential is enough — no rewrite.
7932
+ // On the null→fresh recovery the original gitEnv was the ANONYMOUS one
7933
+ // (no askpass helper, GIT_CONFIG_GLOBAL neutralized), so mint one and
7934
+ // restore global config. Mint it in the PARENT run directory
7935
+ // (dirname(workspaceDir) === runDir), NEVER inside workspaceDir: writing
7936
+ // askpass.js/.sh into the checked-out repo after the final commits leaves
7937
+ // untracked files that make hasUncommittedChanges() retain the WIP
7938
+ // checkpoint and pollute a preserved workspace (Copilot review on #294).
7939
+ if (!retryEnv.GIT_ASKPASS) {
7940
+ const retryAskpass = writeAskpass(dirname(workspaceDir), fresh);
7941
+ if (retryAskpass) {
7942
+ retryEnv.GIT_ASKPASS = retryAskpass;
7943
+ delete retryEnv.GIT_CONFIG_GLOBAL;
7944
+ }
7945
+ }
7946
+ const retryTimeout = netTimeoutMs();
7947
+ const retry = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: retryEnv, timeoutMs: retryTimeout });
7948
+ // Rebind the git env to the FRESH credential whenever the retry ran —
7949
+ // regardless of its exit code — so the post-push remote-tip verification
7950
+ // below (`ls-remote`/`fetch`, all `env: gitEnv`) queries origin with the
7951
+ // credential the retry used, not the REJECTED old token. A retry that
7952
+ // exits nonzero because the server accepted the ref before a client-side
7953
+ // timeout would otherwise be verified with the stale token, ls-remote
7954
+ // would fail to authenticate, and a landed push would be misreported as
7955
+ // pushFailed/stranded (Copilot review on #293).
7956
+ gitEnv = retryEnv;
7957
+ if (retry.status === 0) {
7958
+ // Adopt the fresh token for the rest of finalization (the PR reconcile
7959
+ // + attribution below read `token`) and record the success.
7960
+ token = fresh;
7961
+ push = retry;
7962
+ } else {
7963
+ // The retry ALSO exited nonzero. It may still have LANDED the ref before
7964
+ // a client-side timeout (the transport-hiccup case verified below); if it
7965
+ // did, `landed` flips on and pushError is never reported. But if it did
7966
+ // NOT land, the RETRY — not the original stale-token rejection — is the
7967
+ // failure that actually stranded the commits: the original failure is
7968
+ // already RECOVERED (we re-read the token and retried), so reporting it
7969
+ // would point at a resolved cause instead of the real one. Promote `retry`
7970
+ // to the reported push result and carry ITS redaction credential (`fresh`,
7971
+ // which appeared in the retry's output) and ITS timeout into the final
7972
+ // error. The ORIGINAL rejected token must NOT be carried into the
7973
+ // subsequent authenticated operations (PR reconcile + attribution read
7974
+ // `token`): a landed retry would otherwise reconcile/attribute with the
7975
+ // rejected old token and fail despite the push. Adopt `fresh` for all
7976
+ // later ops; `gitEnv` is already rebound above (Copilot review on #293).
7977
+ push = retry;
7978
+ pushFailureToken = fresh;
7979
+ pushFailureTimeout = retryTimeout;
7980
+ token = fresh;
7981
+ log.warn?.(`finalizeGit${cs}: push still failed with the fresh gh token — ${oneLineLog(describeGitFailure('git push', retry, { token: fresh, timeoutMs: retryTimeout }))}`);
7982
+ }
7983
+ }
7984
+ }
6469
7985
  if (push.status === 0) out.pushed = true;
6470
7986
  else {
6471
7987
  // A nonzero `git push` does NOT prove the ref was not updated: a timeout or
@@ -6547,7 +8063,7 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6547
8063
  // new commit in this throwaway workspace. Surface the at-risk SHAs explicitly
6548
8064
  // (issue #231, defense #2) so the failure is a hard, actionable signal with
6549
8065
  // the commits to recover — not a soft pushError that reads as "completed".
6550
- out.pushError = describeGitFailure('git push', push, { token, timeoutMs: pushTimeout });
8066
+ out.pushError = describeGitFailure('git push', push, { token: pushFailureToken, timeoutMs: pushFailureTimeout });
6551
8067
  out.pushFailed = true;
6552
8068
  // Union the work-branch commit list into the stranded set: out.commits is now
6553
8069
  // the promoted branchCommits, but if HEAD sat off the branch and promotion
@@ -6627,13 +8143,18 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6627
8143
  // envelope + completion note carry only the `pushFailed`/`strandedCommits`
6628
8144
  // recovery signal, not a misleading PR reference.
6629
8145
  if (workingBranch && envelope.task?.allowPr && !out.pushFailed) {
6630
- out.pr = reconcileAgentPr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
8146
+ out.pr = reconcilePr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
6631
8147
  // Record the agent's authorship on the PR (commits carry the operator's
6632
8148
  // identity now, so this preserves the machine-generated provenance).
6633
8149
  if (out.pr?.found && out.pr.number && (envelope.repository?.provider || 'github') === 'github') {
6634
- out.attribution = postAgentAttribution({ workspaceDir, token, number: out.pr.number });
8150
+ out.attribution = postAttribution({ workspaceDir, token, number: out.pr.number });
6635
8151
  }
6636
8152
  }
8153
+ // Surface a mid-finalize credential rotation (the push auth-failure recovery
8154
+ // re-read a fresh gh token) so the caller can propagate it to subsystems that
8155
+ // captured the pre-rotation credential — the WIP-checkpoint cleanup in
8156
+ // particular (Copilot review on #293). Absent when the token was never rotated.
8157
+ if (token && token !== originalToken) out.rotatedToken = token;
6637
8158
  return out;
6638
8159
  }
6639
8160
 
@@ -8657,6 +10178,11 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
8657
10178
  if (git.branchMismatch) env.branchMismatch = git.branchMismatch;
8658
10179
  if (git.pr) env.pr = git.pr;
8659
10180
  if (git.error) env.gitError = git.error;
10181
+ // #295: a durable recovery ref (refs/nano/recovered/<eik>) an SLA-cancelled or
10182
+ // force-stopped run left behind — carried in the outcome so the escalation /
10183
+ // resume can recover the snapshot from it even after the workspace is reaped.
10184
+ if (git.recoveryRef) env.recoveryRef = git.recoveryRef;
10185
+ if (git.recoverySha) env.recoverySha = git.recoverySha;
8660
10186
  }
8661
10187
  return env;
8662
10188
  }
@@ -9578,9 +11104,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
9578
11104
  * `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
9579
11105
  * longer running here, so it rides the marker as a flagged, observational entry
9580
11106
  * without making an idle worker look busy or inflating the drain's in-flight count.
9581
- * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object }} fields
11107
+ * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
9582
11108
  */
9583
- function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }) {
11109
+ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
9584
11110
  const jobList = Array.isArray(jobs) ? jobs : [];
9585
11111
  return {
9586
11112
  pid,
@@ -9593,6 +11119,11 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }
9593
11119
  // SDK client and begun leasing), the producer stamps this; null until then.
9594
11120
  // The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
9595
11121
  readyAt: readyAt ?? null,
11122
+ // #293: the cached gh auth token's observable identity — { present,
11123
+ // fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
11124
+ // `supervisor status` can show the credential's age/scopes per worker. Null
11125
+ // on a worker that has not primed the cache yet (or predates the field).
11126
+ ghAuth: ghAuth ?? null,
9596
11127
  };
9597
11128
  }
9598
11129
 
@@ -10193,10 +11724,27 @@ async function workAgent(req, flags, ctx) {
10193
11724
  // for why per-job setup.env is deliberately excluded), so PATH resolves a stable
10194
11725
  // representative harness binary.
10195
11726
  const agentInstanceProbeOff = String(process.env.NANO_AGENT_INSTANCE || '').trim().toLowerCase() === 'off';
11727
+ // Harness transport protocol, resolved with env-override-then-profile precedence and
11728
+ // reduced to the EFFECTIVE protocol the worker actually runs (a container sandbox is
11729
+ // pipe-only regardless of `protocol`). Computed here — ahead of the durable-transcript
11730
+ // diagnostic below and the #275 mismatch refusal later — so the startup "ACP note" is
11731
+ // keyed on the transport the worker WILL use, not a brittle reparse of the command
11732
+ // line (issue #289 review).
11733
+ const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
11734
+ const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
11735
+ const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
10196
11736
  const sdkSupportsAgentInstance =
10197
11737
  !!camunda &&
10198
11738
  typeof camunda.createAgentInstance === 'function' &&
10199
11739
  typeof camunda.updateAgentInstance === 'function';
11740
+ // #289: best-effort host c8ctl / SDK versions, so the startup warning and the
11741
+ // per-job "producer unavailable" diagnostic can name the package that must be
11742
+ // upgraded (the macbook workers ran @camunda8/cli 4.1.0-alpha.2 bundling
11743
+ // @camunda8/orchestration-cluster-api 9.1.5, which has no AgentInstance verbs).
11744
+ // Resolved from the HOST c8ctl CLI's dependency graph (rooted at the real entry /
11745
+ // CLI package), not this plugin's sibling node_modules, so the nested host SDK
11746
+ // actually resolves (issue #289 review). Null when unresolvable.
11747
+ const hostSdkInfo = probeHostSdkInfo();
10200
11748
  // The producer's `usable` gate also requires the ACP classifier (agent-instance.mjs);
10201
11749
  // without it the producer is inert regardless of SDK support, so exclude the probe too.
10202
11750
  const acpClassifierAvailable = typeof agenticSessionAcp?.classifyUpdate === 'function';
@@ -10267,13 +11815,113 @@ async function workAgent(req, flags, ctx) {
10267
11815
  const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
10268
11816
  logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
10269
11817
  logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
11818
+ // #289: fail LOUDLY at worker start — not silently per job — when this worker
11819
+ // cannot record a durable AgentInstance transcript on an agentic profile. Every
11820
+ // `nano work` profile runs an agent harness, so any worker that services external
11821
+ // agent jobs needs the host c8ctl SDK's createAgentInstance/updateAgentInstance
11822
+ // verbs. When the host SDK lacks them (e.g. @camunda8/cli 4.1.0-alpha.2 bundling
11823
+ // @camunda8/orchestration-cluster-api 9.1.5), EVERY job silently records no
11824
+ // transcript — the exact macbook-fleet defect. Name the host versions and the fix so
11825
+ // it is actionable from the startup log alone, rather than a per-job "producer
11826
+ // unavailable" line that merges two causes. NANO_AGENT_INSTANCE=off suppresses the
11827
+ // warning (transcripts were intentionally disabled).
11828
+ if (!agentInstanceProbeOff) {
11829
+ if (!sdkSupportsAgentInstance) {
11830
+ logger.warn(
11831
+ `[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the host c8ctl SDK lacks ` +
11832
+ `createAgentInstance/updateAgentInstance, so NO AgentInstance transcript will be ` +
11833
+ `recorded for any job this worker runs. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}. ` +
11834
+ `Upgrade the host c8ctl (the plugin version is irrelevant — the SDK ships with c8ctl): ` +
11835
+ `@camunda8/cli must bundle @camunda8/orchestration-cluster-api >= 10.0.0-alpha.45 ` +
11836
+ `(the first with the AgentInstance verbs). Set NANO_AGENT_INSTANCE=off to silence this ` +
11837
+ `if transcripts are intentionally disabled.`,
11838
+ );
11839
+ } else if (!acpClassifierAvailable) {
11840
+ logger.warn(
11841
+ `[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the ACP classifier ` +
11842
+ `(agenticSessionAcp.classifyUpdate) is unavailable, so no AgentInstance transcript ` +
11843
+ `will be recorded. This is unexpected (it is a static import) — the plugin install may ` +
11844
+ `be corrupt. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}.`,
11845
+ );
11846
+ } else {
11847
+ // Transcripts CAN be recorded, but only for an ACP harness: a non-ACP harness
11848
+ // emits no session/update stream for the producer to classify, so it records
11849
+ // only the opening CONFIGURATION turn (no body). Surface this once at startup so
11850
+ // an operator running e.g. `copilot --allow-all` (no --acp) knows why a run has a
11851
+ // header-only transcript. Key the note on the EFFECTIVE transport the worker will
11852
+ // actually use (`effectiveProtocol === 'acp'`), NOT a reparse of the command line:
11853
+ // a `protocol: acp` profile is driven over ACP even when the command carries no
11854
+ // literal `--acp` (`ensureAcpFlag` appends it at spawn), and a hand-reparse also
11855
+ // misses supported selectors (`acp` subcommand, `*-acp` adapter, `--experimental-acp`)
11856
+ // while a container sandbox runs pipe-only regardless of `protocol` (issue #289 review).
11857
+ const protoNote = effectiveProtocol === 'acp'
11858
+ ? ''
11859
+ : ` NOTE: the harness does not run in ACP mode (effective protocol "${effectiveProtocol}") — a non-ACP harness emits no ACP session/update stream, so only the opening CONFIGURATION turn is recorded (no body). Run the harness in ACP mode (--protocol acp on the host executor) for a full transcript.`;
11860
+ logger.info(`[${name}] Durable agent transcripts enabled (host SDK: ${formatHostSdkInfo(hostSdkInfo)}).${protoNote}`);
11861
+ }
11862
+ }
10270
11863
  // Warm the gh-token cache now, off the job-handling path: githubCloneToken()
10271
11864
  // may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
10272
11865
  // credential fallback, and doing that inside a job handler would block the
10273
11866
  // event loop — stalling the lock heartbeat and delaying the job itself.
10274
11867
  // Priming here pays that cost once at startup so every later lookup is a warm
10275
- // cache hit.
10276
- primeGhAuthToken();
11868
+ // cache hit. Pass the logger so the token's fingerprint + scopes (never the
11869
+ // value) land in the worker log at startup — issue #293's "which credential is
11870
+ // this worker actually holding?" line.
11871
+ primeGhAuthToken({ logger });
11872
+ // …and keep it fresh for the worker's lifetime (issue #293): a process-lifetime
11873
+ // memo went silently stale the moment the operator refreshed the gh login, and
11874
+ // every job kept getting the OLD token until a restart. The TTL timer re-reads
11875
+ // `gh auth token` — ASYNCHRONOUSLY (spawn, never spawnSync) and DEFERRED while
11876
+ // a job is active (`isIdle`) — so the refresh can never block the event loop /
11877
+ // lock heartbeat the way a synchronous prime on the interval could (Copilot
11878
+ // review on #293). When the cached identity actually changes, `onChange`
11879
+ // republishes the activity marker so `supervisor status` shows the adopted
11880
+ // credential even while the worker sits idle. The supervisor's
11881
+ // `reload-credentials` broadcast (SIGUSR1) forces the same re-read on demand
11882
+ // (non-destructive — an atomic swap, never a drop).
11883
+ // `activeJobs`/`writeActivity` are declared below this point but are only
11884
+ // dereferenced when a tick fires (≥1s later) or a change lands — long after
11885
+ // the whole module body has run — so the closures capture the initialized
11886
+ // bindings (TDZ-safe). The handle is tracked so the stop-signal teardown can
11887
+ // clear it (a ref'd interval would otherwise keep the process alive past
11888
+ // SIGTERM/SIGINT — the --auto liveness test relies on the poll timer being
11889
+ // the sole keepalive).
11890
+ const ghRefreshTimer = startGhAuthTokenRefresh({
11891
+ logger,
11892
+ isIdle: () => activeJobs.size === 0,
11893
+ onChange: () => writeActivity(),
11894
+ });
11895
+ // `supervisor reload-credentials` asks every worker to re-read its gh token
11896
+ // without draining in-flight jobs. The daemon signals SIGUSR1; the handler
11897
+ // re-reads the CLI ASYNCHRONOUSLY (spawn, never spawnSync) and serialized
11898
+ // against the TTL timer, so even a worker mid-job keeps its event loop / lock
11899
+ // heartbeat responsive while the new credential is adopted. Deliberately NOT
11900
+ // the SIGUSR2 drain path — reloading a credential must not park the fleet.
11901
+ // Windows cannot deliver SIGUSR1 (Node terminates the process), so the daemon
11902
+ // refuses the op there. `writeActivityRef` is late-bound to `writeActivity`
11903
+ // once it is defined below so the handler can republish the token's
11904
+ // age/scopes once the async re-read settles.
11905
+ let writeActivityRef = null;
11906
+ if (process.platform !== 'win32') {
11907
+ process.on('SIGUSR1', () => {
11908
+ // Use the ASYNC refresh (spawn, never spawnSync) so re-reading/replacing
11909
+ // the cached credential can never pause the event loop / lock heartbeat of a
11910
+ // worker servicing a job — the whole promise of `reload-credentials` is
11911
+ // that it does NOT disrupt in-flight jobs (Copilot review on #293). It is
11912
+ // serialized against the periodic timer via `ghAuthRefreshInFlight`.
11913
+ // Deliberately NOT `invalidateGhAuthToken()` first: the async `force` read
11914
+ // re-reads `gh auth token` regardless of TTL and swaps the cache ATOMICALLY
11915
+ // when the new value lands, so the worker keeps serving the still-valid old
11916
+ // token for the ~10s read window instead of handing out null meanwhile.
11917
+ refreshGhAuthTokenAsync({ force: true })
11918
+ .then(({ changed }) => {
11919
+ logger.info(`Credentials reload requested — re-read the gh auth token: ${describeGhAuthToken(ghAuthTokenInfo())}${changed ? ' (token changed)' : ''}`);
11920
+ if (writeActivityRef) writeActivityRef();
11921
+ })
11922
+ .catch(() => { /* best effort — a wedged gh must not crash the worker */ });
11923
+ });
11924
+ }
10277
11925
  logger.info('Polling for work — press Ctrl-C to stop.');
10278
11926
 
10279
11927
  // When launched under the supervisor, report per-job activity — which job(s)
@@ -10458,7 +12106,10 @@ async function workAgent(req, flags, ctx) {
10458
12106
  // Bound the last-activation guard map (TTL + settle-aware size cap). Shared
10459
12107
  // with recordJobStart so a standalone worker still prunes it.
10460
12108
  pruneLastActivation(nowMs);
10461
- const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt });
12109
+ // #293: publish the cached gh token's identity (fingerprint/scopes/age —
12110
+ // never the value) so `supervisor status` can answer "is this worker holding
12111
+ // a stale token?" in one line.
12112
+ const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
10462
12113
  const tmp = `${activityFile}.${process.pid}.tmp`;
10463
12114
  try {
10464
12115
  mkdirSync(dirname(activityFile), { recursive: true });
@@ -10469,6 +12120,9 @@ async function workAgent(req, flags, ctx) {
10469
12120
  /* best effort — activity is advisory, never fail a job over it */
10470
12121
  }
10471
12122
  };
12123
+ // Late-bind the SIGUSR1 (reload-credentials) handler's republish hook now that
12124
+ // writeActivity exists (#293).
12125
+ writeActivityRef = writeActivity;
10472
12126
  // The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
10473
12127
  // host connection carries presence/steer/ownership/transcript for EVERY
10474
12128
  // supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
@@ -10745,14 +12399,14 @@ async function workAgent(req, flags, ctx) {
10745
12399
  logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
10746
12400
  }
10747
12401
 
10748
- // #110: the role's harness protocol (pipe|acp) and ACP permission policy
10749
- // (yolo|escalate|filter), resolved with the same env-override-then-profile
10750
- // precedence as terminal. `NANO_AGENTIC_PROTOCOL`/`NANO_AGENTIC_PERMISSION`
10751
- // override a one-off worker; otherwise the hire profile decides; else the safe
10752
- // defaults (pipe/yolo). escalate/filter are carried through verbatim — the
10753
- // acp-executor task enforces yolo and interim-handles the reserved policies.
10754
- const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
10755
- const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
12402
+ // #110: the role's ACP permission policy (yolo|escalate|filter), resolved with
12403
+ // the same env-override-then-profile precedence as terminal. The harness protocol
12404
+ // (`envProtocol`/`roleProtocol`/`effectiveProtocol`) is resolved earlier, ahead of
12405
+ // the durable-transcript startup diagnostic (issue #289 review).
12406
+ // `NANO_AGENTIC_PERMISSION` overrides a one-off worker; otherwise the hire profile
12407
+ // decides; else the safe default (yolo). escalate/filter are carried through
12408
+ // verbatim — the acp-executor task enforces yolo and interim-handles the reserved
12409
+ // policies.
10756
12410
  const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
10757
12411
  const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
10758
12412
 
@@ -10767,8 +12421,7 @@ async function workAgent(req, flags, ctx) {
10767
12421
  // container profile's command STILL pipes plain JSON to an ACP harness — the
10768
12422
  // same husk. `effectiveHarnessProtocol` reduces a container's protocol to `pipe`
10769
12423
  // so the refusal covers that pipe-only executor too, not just the host (thread
10770
- // 4118327651).
10771
- const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
12424
+ // 4118327651). `effectiveProtocol` is computed earlier (issue #289 review).
10772
12425
  const startupMismatch = detectProtocolMismatch({ command: profile.command, args: effectiveArgs, protocol: effectiveProtocol });
10773
12426
  if (startupMismatch) {
10774
12427
  // Non-restartable config failure: under the supervisor this must exit with
@@ -10894,7 +12547,14 @@ async function workAgent(req, flags, ctx) {
10894
12547
  // Assemble + normalize the task envelope from headers (defaults) and
10895
12548
  // variables (overrides), then resolve any secrets it references.
10896
12549
  const envelope = normalizeTaskEnvelope(job.customHeaders ?? {}, job.variables ?? {}, { basePromptOverride });
10897
- const { resolved, missing, names } = resolveJobSecrets(secretResolver, envelope);
12550
+ const { resolved, missing, names } = await resolveJobSecrets(secretResolver, envelope);
12551
+ // #222: recheck after the awaited secret resolution (stage: secrets) —
12552
+ // resolveJobSecrets AWAITS the gh-cache-miss forced refresh, so a
12553
+ // force-stop / lease loss can win DURING that await. Gate BEFORE the
12554
+ // missing-secret settlement below so an abort that landed mid-resolve
12555
+ // returns WITHOUT settling (which would race the force-stop yield),
12556
+ // mirroring the post-prompt-fetch gate above.
12557
+ if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'secrets', logger })) return;
10898
12558
  if (missing.length > 0) {
10899
12559
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
10900
12560
  const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
@@ -10998,7 +12658,21 @@ async function workAgent(req, flags, ctx) {
10998
12658
  } else if (agentInstanceProducer.retryPending) {
10999
12659
  logger.info(`[${jobType}] AgentInstance producer not yet active for external agent job (${aiCorr}) — the initial createAgentInstance did not mint a key in time (transient rejection, no-key response, or bounded-out create), but the create stays armed to retry on the ACP hot path, so a run that recovers can still record a durable transcript (job completion unaffected).`);
11000
12660
  } else {
11001
- logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: the host SDK lacks createAgentInstance/updateAgentInstance or the ACP classifier is unavailable; continuing without a durable transcript (job completion unaffected).`);
12661
+ // #289: report WHICH precondition failed (SDK create, SDK update, ACP
12662
+ // classifier, or not-external) plus the host SDK versions, instead of
12663
+ // merging the causes into one opaque line — so the no-transcript cause is
12664
+ // root-causable from this line alone. A PERMANENT create rejection (a 4xx
12665
+ // the engine will never mint on retry) is NOT a construction-time
12666
+ // precondition failure — activation WAS attempted and permanently
12667
+ // rejected — so report it distinctly rather than as "activation not
12668
+ // attempted: unknown precondition" (issue #289 review).
12669
+ const rejectedStatus = agentInstanceProducer.permanentlyRejected;
12670
+ if (rejectedStatus != null) {
12671
+ logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation attempted but permanently rejected (HTTP ${rejectedStatus}); host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
12672
+ } else {
12673
+ const reason = agentInstanceProducer.unavailableReason || 'unknown precondition';
12674
+ logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: ${reason}; host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
12675
+ }
11002
12676
  }
11003
12677
  } catch (err) {
11004
12678
  logger.warn(`[${jobType}] AgentInstance producer activate() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; continuing without a durable transcript (job completion unaffected).`);
@@ -11137,15 +12811,38 @@ async function workAgent(req, flags, ctx) {
11137
12811
  let cwd;
11138
12812
  let extraEnv;
11139
12813
  let repoToken = null;
12814
+ let repoCredentialSource = null;
11140
12815
  if (hasRepo) {
11141
12816
  const provider = envelope.repository.provider || 'github';
11142
12817
  const authRef = envelope.repository.authRef;
11143
- repoToken = githubCloneToken({ provider, authRef, secretResolver }); // absent → anonymous clone
12818
+ // Resolve the credential AND its source once: the source gates whether
12819
+ // a later auth failure may fall back to the operator's `gh auth token`
12820
+ // (only a 'gh-cache' token may — never an explicit GITHUB_TOKEN or a
12821
+ // custom authRef; Copilot review on #293).
12822
+ const cred = githubCloneTokenWithSource({ provider, authRef, secretResolver });
12823
+ repoToken = cred.token; // absent → anonymous clone
12824
+ repoCredentialSource = cred.source;
11144
12825
  try {
11145
12826
  mkdirSync(workerNsDir, { recursive: true });
11146
12827
  runDir = mkdtempSync(join(workerNsDir, 'run-'));
11147
12828
  liveRunDirs.add(runDir);
11148
- provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal });
12829
+ provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal, credentialSource: repoCredentialSource });
12830
+ // #293: provisioning's clone auth-retry may have ROTATED the token
12831
+ // (adopting the fresh gh token into the workspace's askpass env).
12832
+ // Adopt that effective credential for the rest of the job —
12833
+ // finalizeGit's push/PR-reconcile and the checkpoint GC/ref-delete
12834
+ // askpass — so they don't keep failing with the stale pre-provision
12835
+ // token (Copilot review on #293).
12836
+ if (provisioned.effectiveToken) repoToken = provisioned.effectiveToken;
12837
+ if (provisioned.credentialSource) repoCredentialSource = provisioned.credentialSource;
12838
+ // #293: when the rotation above swapped a gh-cache credential, the
12839
+ // `resolved` secret map (built pre-provisioning, forwarded to the
12840
+ // harness as `secretEnv`) still holds the STALE token — sync it so an
12841
+ // allowPr harness's own gh/git runs use the credential provisioning
12842
+ // just adopted, not the one the clone proved dead (Copilot review on
12843
+ // #293). No-op for an env/custom identity, an unchanged token, or a
12844
+ // key the job never forwarded.
12845
+ applyRotatedGhCredential({ resolved, provider, authRef, source: repoCredentialSource, token: repoToken });
11149
12846
  if (provisioned.baseFetchError) {
11150
12847
  logger.warn(`[${jobType}] job ${job.jobKey} base fetch failed (${aiCorr}) — ${oneLineLog(provisioned.baseFetchError)}; base...head diffs may be unavailable`);
11151
12848
  }
@@ -11244,6 +12941,28 @@ async function workAgent(req, flags, ctx) {
11244
12941
  // performs a last-chance `failed` flush so edits since the last periodic
11245
12942
  // checkpoint are not lost.
11246
12943
  let checkpointFinalized = false;
12944
+ // #296 (F8/F9): set on the worker force-stop path so the `finally` below bounds
12945
+ // its `close()` drain to the same FORCE_STOP_RECOVERY_MS budget — a force-stop
12946
+ // must never be held past the supervisor's SIGKILL backstop by the (up to
12947
+ // 120s) GC sweep. The in-flight checkpoint git is cancelled by the recovery
12948
+ // DEADLINE (checkpointing.abortRecovery(), fired from the force-stop block),
12949
+ // not the job's abortSignal, so the chain settles on its own after we stop
12950
+ // awaiting while recovery can still run after the harness abort.
12951
+ let forceStopping = false;
12952
+ // #296: the SINGLE absolute force-stop deadline (set when the force-stop
12953
+ // recovery block begins). The recover race AND the `finally`'s close() drain
12954
+ // BOTH bound against it, so recovery + cleanup together fit inside ONE
12955
+ // FORCE_STOP_RECOVERY_MS window rather than consuming it twice (recovery race,
12956
+ // then an independent close() wait) and risking a SIGKILL during cleanup
12957
+ // (Copilot review on #296). 0 until a force-stop begins.
12958
+ let forceStopDeadline = 0;
12959
+ // #295: the durable recovery ref (and SHA) a preserveRecovery() pushed on an
12960
+ // SLA cancellation / failed run, surfaced in the job outcome (io.nanobpm.agentResult)
12961
+ // — but ONLY on the failed/SLA-cancel path, which settles normally and so reaches
12962
+ // buildResultEnvelope. A FORCE-STOPPED run is yielded (settle.fail, retries
12963
+ // preserved) and returns early below WITHOUT an outcome envelope, so its
12964
+ // recoveryRefInfo is intentionally NOT reported there (Copilot review on #295).
12965
+ let recoveryRefInfo = null;
11247
12966
  // The per-job live-terminal relay session (issue #173): streams this job's
11248
12967
  // harness terminal over the single-owner supervisor's ONE multiplexed host
11249
12968
  // connection, keyed by this worker's instance + the jobKey, and accepts
@@ -11394,14 +13113,259 @@ async function workAgent(req, flags, ctx) {
11394
13113
  // that yield. The finally below still runs (container/run-dir/relay
11395
13114
  // cleanup); recordJobEnd fires in the outer finally.
11396
13115
  if (result && result.aborted) {
11397
- logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
13116
+ logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop or SLA/lease-loss cancellation) — harness killed; releasing the job (the broker re-activates it for retry when the run is still live).`);
11398
13117
  // #264: last-chance WIP checkpoint so the retry can pick up the
11399
13118
  // uncommitted work (bounded; the lease guards a newer owner).
11400
- if (checkpointer) { await checkpointer.flush('abort', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13119
+ // #295: then promote it to a durable recovery ref so a force-stopped or
13120
+ // SLA-cancelled run keeps a recovery point. The orphan GC spares that ref
13121
+ // while its process stays active, so it survives long enough to be
13122
+ // resumed. The ref is
13123
+ // pushed here, but it is NOT surfaced in a job outcome: this path yields
13124
+ // the job and returns early, never reaching buildResultEnvelope — a
13125
+ // force-stopped run produces no outcome envelope by design (Copilot
13126
+ // review on #295), so the recovery ref is recoverable from the pushed
13127
+ // refs/nano/recovered/<eik> ref itself, not from any job result.
13128
+ // #296: bound the WHOLE flush+promote to FORCE_STOP_RECOVERY_MS so it
13129
+ // reliably finishes (and this worker exits) inside the window the
13130
+ // supervisor's forceStopWorker now waits before its SIGKILL backstop —
13131
+ // otherwise the default flush (20s) + recovery git push (60s) could run
13132
+ // long and be SIGKILLed before refs/nano/recovered/<eik> is written.
13133
+ // Best-effort: if the budget elapses we give up on the recovery ref (the
13134
+ // WIP ref + TTL remain the fallback), never block the yield/exit.
13135
+ if (checkpointer) {
13136
+ forceStopping = true;
13137
+ const recoveryDeadline = FORCE_STOP_RECOVERY_MS;
13138
+ // #296: anchor the ONE shared force-stop deadline here. The recover race
13139
+ // below and the `finally`'s close() both bound against it, so the TOTAL
13140
+ // force-stop window is a single FORCE_STOP_RECOVERY_MS (not 2x), fitting
13141
+ // inside the supervisor's STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS backstop.
13142
+ forceStopDeadline = Date.now() + recoveryDeadline;
13143
+ // #296 (review): reserve a DRAIN window inside the deadline. abortRecovery()
13144
+ // only SIGTERMs the checkpoint git; createGitRunner then grants up to
13145
+ // gitKillGraceMs before SIGKILL. If abortRecovery() fires AT the deadline,
13146
+ // a stubborn git is still dying when close() hits its (expired) cap and the
13147
+ // finally reaps the workspace under a live process. Fire the abort EARLY —
13148
+ // gitKillGraceMs + a small margin before the deadline — so the SIGKILL lands
13149
+ // by `forceStopDeadline` and close() gets the remaining grace window to
13150
+ // actually drain the now-dead git, not the 1ms `Math.max(1, …)` would give.
13151
+ const killGrace = Math.max(0, Number(checkpointing?.gitKillGraceMs) || 0);
13152
+ const drainReserveMs = killGrace + 250;
13153
+ const abortAtMs = Math.max(1, recoveryDeadline - drainReserveMs);
13154
+ // #296 (F9): arm the recovery-deadline. When the budget expires, abort
13155
+ // the DEDICATED recovery git runner (not the job's abortSignal, which
13156
+ // has already fired and would fail every recovery command) so an
13157
+ // in-flight flush/preserveRecovery push is torn down and the chain
13158
+ // settles — letting the `finally`'s bounded `close()` drain inside the
13159
+ // supervisor's SIGKILL window. Fires `abortAtMs` (the deadline MINUS the
13160
+ // reserved drain window) so the git SIGKILL completes before the deadline;
13161
+ // unref'd so it never keeps the process alive.
13162
+ // #296 (review, r4244116341): the recover-race `Promise.race([recover,
13163
+ // budget])` below only bounds the AWAIT — when `budget` wins, the SDK
13164
+ // mutations inside `recover` (settleAbortMetaWrite's awaited WIP write and
13165
+ // the recovery-pointer write) keep running. The job then yields and a
13166
+ // reactivated worker can publish a NEWER `resumeFromCheckpoint`; a stale
13167
+ // `recover` continuation that completes afterward would clobber it with the
13168
+ // old recovery pointer. abortRecovery() cancels only git/GC, not these SDK
13169
+ // calls. So drive the SDK mutations off a DEDICATED recovery-deadline
13170
+ // signal, fired by the same deadline (and on budget expiry), and AWAIT
13171
+ // `recover` after the race so it is SETTLED/cancelled before we yield.
13172
+ const recoverySdkAbort = new AbortController();
13173
+ let deadlineTimer = setTimeout(() => {
13174
+ try { checkpointing.abortRecovery(); } catch { /* best effort */ }
13175
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13176
+ }, abortAtMs);
13177
+ deadlineTimer?.unref?.();
13178
+ const recover = (async () => {
13179
+ await checkpointer.flush('abort', { timeoutMs: recoveryDeadline }); checkpointFinalized = true;
13180
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('force-stop') || recoveryRefInfo; } catch { /* best effort */ }
13181
+ // #296 (review, AC3): an SLA boundary cancel reaches THIS aborted branch
13182
+ // too — the supervisor heartbeat treats the resulting lease-loss 404 as
13183
+ // terminal and interrupts the runner (supervisor/src/dispatch.ts), so
13184
+ // runAgentJob returns aborted:true. The early return below then skips
13185
+ // buildResultEnvelope, so recoveryRef/recoverySha are never reported in a
13186
+ // job outcome. Persist the recovery ref through the PROCESS-scoped
13187
+ // `resumeFromCheckpoint` slot instead: it survives the element re-entry an
13188
+ // SLA escalation triggers, so the resumed run's predecessorCheckpointRef
13189
+ // can discover the durable refs/nano/recovered/<eik> point. Bounded and
13190
+ // best-effort — never holds the force-stop exit; the pushed ref itself
13191
+ // remains the authoritative recovery handle.
13192
+ // #296 (review): the abort flush fired its OWN WIP `resumeFromCheckpoint`
13193
+ // write fire-and-forget (see onCheckpoint). If that write is still in
13194
+ // flight it could land AFTER the recovery-pointer write below and
13195
+ // overwrite the recovery ref with a stale WIP pointer (which the orphan
13196
+ // GC then reclaims on TTL, losing the recovery discovery). Await the WIP
13197
+ // write FIRST — under the recovery-deadline signal so the await is
13198
+ // cancelled when the budget expires — so the recovery pointer is
13199
+ // guaranteed to land last. Bounded by the recover-race budget; best-effort.
13200
+ try { await checkpointing.settleAbortMetaWrite?.(recoverySdkAbort.signal); } catch { /* best effort */ }
13201
+ // #296 (review, r4244116341): if the recovery deadline has already fired,
13202
+ // do NOT write the recovery pointer — a write that lands after this branch
13203
+ // yields could overwrite a reactivated worker's newer resumeFromCheckpoint.
13204
+ if (recoverySdkAbort.signal.aborted) return;
13205
+ if (recoveryRefInfo && typeof camunda?.createElementInstanceVariables === 'function' && job?.processInstanceKey) {
13206
+ try {
13207
+ // #296 (review, r4244827082): gate the process-scoped pointer write on
13208
+ // PROVEN current ownership of the recovery ref. preserveRecovery only
13209
+ // fences the GIT ref; this variable write is last-writer-wins, so a
13210
+ // superseded activation could otherwise land its stale pointer after a
13211
+ // newer activation (which re-activated the job on lease loss) published
13212
+ // its own. Re-verify the ref still resolves to the sha THIS run
13213
+ // preserved and is still owned by this run; skip the write when it has
13214
+ // moved (a newer activation's pointer stays authoritative). Bounded by
13215
+ // the recovery deadline; a failed/aborted verify fails closed (skip).
13216
+ let stillOwns = false;
13217
+ try {
13218
+ let vt; let vOnAbort = null;
13219
+ // #296 (review, r4244916101): drive the verify off its OWN
13220
+ // AbortController so that when the 5s branch wins we can CANCEL
13221
+ // the in-flight git reads AND await the verify to full settlement
13222
+ // in the finally below — before recover() returns and the run dir
13223
+ // is reaped. Without this the losing verify kept running on the
13224
+ // 60s runner (untracked by stop()/gc) under a reaped workspace.
13225
+ const verifyAbort = new AbortController();
13226
+ const verifyOp = Promise.resolve(
13227
+ checkpointing.verifyRecoveryOwnership?.(recoveryRefInfo.sha, { signal: verifyAbort.signal }),
13228
+ ).then((v) => v === true);
13229
+ stillOwns = await Promise.race([
13230
+ verifyOp,
13231
+ new Promise((res) => { vt = setTimeout(() => res(false), 5000); vt?.unref?.(); }),
13232
+ new Promise((res) => { vOnAbort = () => res(false); recoverySdkAbort.signal.addEventListener('abort', vOnAbort, { once: true }); }),
13233
+ ]).finally(() => {
13234
+ clearTimeout(vt);
13235
+ if (vOnAbort) recoverySdkAbort.signal.removeEventListener('abort', vOnAbort);
13236
+ // The race settled (timeout/abort/complete). If the verify is
13237
+ // still mid-git (timeout/abort won), cancel its runner so the
13238
+ // git process is SIGTERM→SIGKILLed now, not at abortAtMs.
13239
+ try { verifyAbort.abort(); } catch { /* best effort */ }
13240
+ });
13241
+ // AWAIT the verify to full settlement (its git is now aborted, so
13242
+ // this resolves promptly) so no live git outlives recover()/close().
13243
+ try { await verifyOp; } catch { /* best effort */ }
13244
+ } catch { stillOwns = false; }
13245
+ if (!stillOwns) {
13246
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: recovery ref ${recoveryRefInfo.ref} is no longer owned by this run (a newer activation superseded it) — skipping the process-scoped resumeFromCheckpoint write so the newer pointer stays authoritative.`);
13247
+ return;
13248
+ }
13249
+ // #296 (review, r4245472572): the Git-ref ownership check above cannot
13250
+ // fence THIS write across an SLA resume — a re-activated run is a NEW
13251
+ // element instance, so it owns a DIFFERENT refs/nano/recovered/<eik>
13252
+ // and this run still "owns" its old one (passing the check) even after
13253
+ // the new activation published a newer resumeFromCheckpoint. The write
13254
+ // is unconditional last-writer-wins, so it could then overwrite that
13255
+ // newer pointer. Two fences close the class (no atomic process-scoped
13256
+ // CAS exists — so the write is a best-effort DISCOVERY HINT, not an
13257
+ // authoritative fence, and these two narrowings reduce (not eliminate)
13258
+ // its staleness:
13259
+ // (1) OCCURRENCE-SCOPED READ-BACK — re-read the shared slot and skip
13260
+ // when it already names a DIFFERENT run's ref (a newer activation
13261
+ // beat us). Best-effort: a read failure/empty/absent result fails
13262
+ // OPEN (we still write) so a transient search outage never strands
13263
+ // recovery; eventual consistency (index lag) can hide a JUST-landed
13264
+ // newer write, which is exactly why fence (2) is also required.
13265
+ // (2) NATIVE ABORT SIGNAL on the write itself — the SDK options extend
13266
+ // RequestInit, so `signal` reaches fetch and aborts the in-flight
13267
+ // HTTP request the instant the recovery deadline fires, tearing the
13268
+ // mutation down at the transport (not just a best-effort .cancel()).
13269
+ // #296 (review, r4245620621): a read-then-write is NOT atomic, so a
13270
+ // replacement activation can still publish its ref between the read-back
13271
+ // and this write — the residual TOCTOU the reviewer notes is real. It is
13272
+ // BENIGN for recovery CORRECTNESS, which does not rest on this write:
13273
+ // the self-recorded OBJECT form of resumeFromCheckpoint is NEVER
13274
+ // auto-adopted on resume (`makeOccurrenceGuard` fails closed BY
13275
+ // CONSTRUCTION — only a process-delivered bare STRING is adopted), and
13276
+ // the authoritative durable recovery handle is the CAS-fenced git ref
13277
+ // refs/nano/recovered/<eik> (`preserveRecovery` → `pushNamedRef`
13278
+ // --force-with-lease). A stale object pointer therefore yields a
13279
+ // transcript-only resume (identical to an ABSENT pointer), never an
13280
+ // incorrect restore. These narrowings keep the HINT as fresh as is
13281
+ // cheaply possible; they are not relied on for a safety property.
13282
+ // #296 (review, r4245620587): the consistency options MUST be nested
13283
+ // under `consistency` (the real @camunda8 client throws "Missing
13284
+ // consistency options" otherwise — mirrors the 6994/6999 reads and
13285
+ // agent-resume's READ_CONSISTENCY) or the catch below would swallow the
13286
+ // throw and this read-back would silently fail OPEN on every call.
13287
+ if (typeof camunda?.searchVariables === 'function') {
13288
+ try {
13289
+ const found = await Promise.resolve(camunda.searchVariables(
13290
+ { filter: { name: 'resumeFromCheckpoint', scopeKey: String(job.processInstanceKey) }, truncateValues: false },
13291
+ { consistency: { waitUpToMs: 0 }, signal: recoverySdkAbort.signal },
13292
+ ));
13293
+ const items = Array.isArray(found?.items) ? found.items : (Array.isArray(found?.data?.items) ? found.data.items : []);
13294
+ let slotRef = null;
13295
+ for (const it of items) {
13296
+ try { const v = JSON.parse(it?.value ?? 'null'); slotRef = (v && typeof v === 'object' ? v.ref : (typeof v === 'string' ? v : null)) ?? slotRef; } catch { /* unparseable entry — ignore */ }
13297
+ }
13298
+ if (slotRef && slotRef !== recoveryRefInfo.ref) {
13299
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: the process-scoped resumeFromCheckpoint slot already names a newer activation's ref ${slotRef} (not this run's ${recoveryRefInfo.ref}) — skipping the stale overwrite so the newer pointer stays authoritative.`);
13300
+ return;
13301
+ }
13302
+ } catch { /* best effort — a read failure fails OPEN (write proceeds) */ }
13303
+ }
13304
+ const w = camunda.createElementInstanceVariables({
13305
+ elementInstanceKey: String(job.processInstanceKey),
13306
+ variables: { resumeFromCheckpoint: { ref: recoveryRefInfo.ref, sha: recoveryRefInfo.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, recovered: true, at: Date.now() } },
13307
+ local: true,
13308
+ signal: recoverySdkAbort.signal,
13309
+ });
13310
+ let wt; let onAbort = null;
13311
+ // Cancel the in-flight write when the recovery deadline fires (the SDK
13312
+ // returns a CancelablePromise), AND reject the race so the chain
13313
+ // settles — the mutation never lands late over a newer pointer.
13314
+ await Promise.race([
13315
+ Promise.resolve(w).finally(() => { clearTimeout(wt); if (onAbort) recoverySdkAbort.signal.removeEventListener('abort', onAbort); }),
13316
+ new Promise((_r, rej) => { wt = setTimeout(() => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write timed out')); }, 5000); wt?.unref?.(); }),
13317
+ new Promise((_r, rej) => { onAbort = () => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write aborted')); }; recoverySdkAbort.signal.addEventListener('abort', onAbort, { once: true }); }),
13318
+ ]);
13319
+ } catch { /* best effort — the pushed ref is the fallback */ }
13320
+ }
13321
+ })();
13322
+ let budgetTimer = null;
13323
+ const budget = new Promise((res) => { budgetTimer = setTimeout(res, recoveryDeadline); if (budgetTimer.unref) budgetTimer.unref(); });
13324
+ try { await Promise.race([recover, budget]); } catch { /* best effort */ }
13325
+ // Clear the recover-race budget timer (it has fired or the race settled).
13326
+ // deadlineTimer is deliberately NOT cleared: even when the recover race
13327
+ // settles early, abortRecovery() must still fire at abortAtMs to abort the
13328
+ // background GC sweep (gcAbort) — otherwise close() below would await a
13329
+ // sweep that can run up to GC_OVERALL_TIMEOUT_MS (120s), holding the
13330
+ // force-stop open past the supervisor's SIGKILL window (the F8 failure).
13331
+ if (budgetTimer) clearTimeout(budgetTimer);
13332
+ // #296 (review, r4244116341): the race above only bounds the await — when
13333
+ // `budget` wins, `recover` may still be mid-SDK-write. Abort the recovery
13334
+ // signal (cancels the in-flight SDK mutation and short-circuits recover's
13335
+ // remaining steps) and AWAIT `recover` so it is fully settled BEFORE we
13336
+ // yield the job. This guarantees no stale continuation writes the old
13337
+ // recovery pointer after a reactivated worker publishes a newer one. The
13338
+ // signal + each step's internal deadline keep this await bounded.
13339
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13340
+ try { await recover; } catch { /* best effort */ }
13341
+ checkpointFinalized = true;
13342
+ }
11401
13343
  return;
11402
13344
  }
11403
13345
  // #264: a failed run is retried — checkpoint what it left behind.
11404
- if (checkpointer && !result.ok) { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13346
+ // #295: an SLA-cancelled run surfaces here as a non-ok (timed-out) result;
13347
+ // preserve a durable recovery ref and report it in the job outcome below.
13348
+ if (checkpointer && !result.ok) {
13349
+ await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true;
13350
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('failed') || recoveryRefInfo; } catch { /* best effort */ }
13351
+ }
13352
+ // #296 (review, r4244991586): the failed-result path can outlive its lease.
13353
+ // `result.aborted` is only true when the harness observed the abort WHILE the
13354
+ // process ran; an abortSignal that fires AFTER runAgentJob() returned (with a
13355
+ // genuine non-ok result) but DURING the flush/promote above leaves
13356
+ // `result.aborted === false`. Effect has already interrupted the raw runner and
13357
+ // released the slot, so if we now fall through to settleJob.fail (and the
13358
+ // checkpoint-variable/envelope writes) we overlap a replacement activation with
13359
+ // STALE writes + a STALE settle. Recheck the job signal here — after the
13360
+ // post-run flush/promote, before any settlement — and switch to the
13361
+ // interrupted/yield path (return WITHOUT settling, exactly like the
13362
+ // result.aborted branch and settleRunError's abort idiom) when it has fired.
13363
+ // The recovery ref preserved above still stands (the durable handle); we simply
13364
+ // do not settle a job whose lease is already gone. The broker re-activates it.
13365
+ if (!result.ok && abortSignal?.aborted === true) {
13366
+ logger.warn(`[${jobType}] job ${job.jobKey}: abortSignal fired after the failed run returned (lease loss/force-stop won the race) — yielding WITHOUT settleJob.fail so a replacement activation does not overlap stale checkpoint/settle writes (the preserved recovery ref remains the durable handle).`);
13367
+ return;
13368
+ }
11405
13369
 
11406
13370
  // Gap 2 (#678): a clean run that emitted no machine-readable result gets
11407
13371
  // ONE bounded re-emit nudge in the same workspace, feeding back its own
@@ -11498,6 +13462,7 @@ async function workAgent(req, flags, ctx) {
11498
13462
  hasPrBranch: provisioned.hasPrBranch,
11499
13463
  envelope,
11500
13464
  token: repoToken,
13465
+ credentialSource: repoCredentialSource,
11501
13466
  logger,
11502
13467
  corr: aiCorr,
11503
13468
  keepRuns,
@@ -11542,6 +13507,21 @@ async function workAgent(req, flags, ctx) {
11542
13507
  } else if (provisioned) {
11543
13508
  gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
11544
13509
  }
13510
+ // #295: surface the durable recovery ref (pushed on an SLA cancellation /
13511
+ // failed run above) in the job outcome so the escalation can point a resume
13512
+ // at it. Attach to whatever gitResult was built for this (provisioned) job.
13513
+ if (recoveryRefInfo && gitResult) { gitResult.recoveryRef = recoveryRefInfo.ref; gitResult.recoverySha = recoveryRefInfo.sha; }
13514
+ // Propagate a mid-finalize credential rotation (finalizeGit's push
13515
+ // auth-failure recovery re-read a fresh gh token) to every subsystem that
13516
+ // captured the pre-rotation token: the outer `repoToken` (later redaction /
13517
+ // consumers) and the WIP-checkpoint cleanup, whose post-ack ref deletion
13518
+ // would otherwise authenticate with the credential origin already rejected
13519
+ // and leave the remote WIP ref behind until a GC sweep (Copilot review on
13520
+ // #293). No-op when the token was not rotated.
13521
+ if (gitResult?.rotatedToken) {
13522
+ repoToken = gitResult.rotatedToken;
13523
+ checkpointing?.updateCredential?.(gitResult.rotatedToken);
13524
+ }
11545
13525
  // #264: the work is durable on the pushed branch, so the WIP ref goes —
11546
13526
  // but only once the engine acks job.complete (below): until then a lost
11547
13527
  // lease or failed settle redelivers the job, and the next activation must
@@ -11577,7 +13557,24 @@ async function workAgent(req, flags, ctx) {
11577
13557
  // last periodic checkpoint. `flush` forces past the post-stop guard and is
11578
13558
  // bounded; the lease still protects a newer owner. Then close (drain).
11579
13559
  if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
11580
- try { await checkpointing.close(); } catch { /* best effort */ }
13560
+ // #296 (F8): on a force-stop, bound the drain to the REMAINING of the ONE
13561
+ // shared force-stop deadline (set when the recovery block began) — NOT a
13562
+ // fresh full FORCE_STOP_RECOVERY_MS. The recovery race may already have
13563
+ // consumed most of the budget, so a fresh window here could run the total
13564
+ // (recovery + cleanup) to ~2x the budget and be SIGKILLed mid-cleanup
13565
+ // (Copilot review on #296). The recovery DEADLINE (abortRecovery) tears down
13566
+ // both the checkpoint git AND the GC sweep — and it now fires gitKillGraceMs
13567
+ // BEFORE the deadline, so by the time we get here the git SIGKILL has landed
13568
+ // (or is in its grace window) and the drain settles on its own. The cap is the
13569
+ // REMAINING budget; the floor is the git kill grace (not 1ms) so that even when
13570
+ // the recover race consumed the whole window the drain still awaits the actual
13571
+ // SIGKILL of the already-SIGTERMed git instead of abandoning a live process to
13572
+ // have its workspace reaped under it (Copilot review on #296, 1ms finding).
13573
+ const closeFloorMs = Math.max(1, Number(checkpointing?.gitKillGraceMs) || 0);
13574
+ const closeOpts = forceStopping
13575
+ ? { timeoutMs: Math.max(closeFloorMs, forceStopDeadline - Date.now()) }
13576
+ : undefined;
13577
+ try { await checkpointing.close(closeOpts); } catch { /* best effort */ }
11581
13578
  }
11582
13579
  if (isContainer) liveRunIds.delete(runId);
11583
13580
  // #205: clear the in-flight job marker now the harness has stopped — the
@@ -12038,6 +14035,7 @@ async function workAgent(req, flags, ctx) {
12038
14035
  if (finished) return;
12039
14036
  finished = true;
12040
14037
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
14038
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12041
14039
  if (reaperTimer) clearInterval(reaperTimer);
12042
14040
  if (runDirTimer) clearInterval(runDirTimer);
12043
14041
  if (agenticPlane) {
@@ -12068,6 +14066,7 @@ async function workAgent(req, flags, ctx) {
12068
14066
  aborting = true;
12069
14067
  draining = true;
12070
14068
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
14069
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12071
14070
  logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
12072
14071
  // Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
12073
14072
  // clears the ownership registry, so we can yield each one afterwards.
@@ -12113,6 +14112,11 @@ async function workAgent(req, flags, ctx) {
12113
14112
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
12114
14113
  const n0 = inFlightCount();
12115
14114
  logger.info(`Received ${signal} — draining: polling stopped; waiting on ${n0} in-flight job(s) to finish (send SIGTERM to abort).`);
14115
+ // The refresh timer is ref'd (so it fires on cadence even under a busy
14116
+ // loop) and there may be NO in-flight job during a drain — clear it now
14117
+ // so it can't keep the process alive once the drain completes and the
14118
+ // runtime is interrupted.
14119
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12116
14120
  // Poll until every in-flight job has settled and released its slot, then
12117
14121
  // interrupt the now-idle runtime and exit. Wait INDEFINITELY — a force
12118
14122
  // abort (SIGTERM) is the only escape hatch.
@@ -12508,8 +14512,9 @@ function formatDuration(ms) {
12508
14512
  return `${d}d${h % 24}h`;
12509
14513
  }
12510
14514
 
12511
- /** Project a live/stored worker record to a status row (pure w.r.t. `now`). */
12512
- function summarizeSupervisorWorker(w, now = Date.now()) {
14515
+ /** Project a live/stored worker record to a status row (pure w.r.t. `now`).
14516
+ * `readActivity` is an injectable seam over readWorkerActivity (testing). */
14517
+ function summarizeSupervisorWorker(w, now = Date.now(), { readActivity = readWorkerActivity } = {}) {
12513
14518
  const alive = isPidAlive(w.pid);
12514
14519
  // Absolute base epoch (ms) the worker started, so an attached console can
12515
14520
  // re-age `uptimeMs` locally on each tick (see reageSupervisorStatus) without
@@ -12523,8 +14528,9 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12523
14528
  let activity = null; // { state: 'busy'|'idle', jobs: [{ key, type, sinceMs, sinceEpochMs }] }
12524
14529
  let engine = null; // job-polling engine base URL this worker reported (#99)
12525
14530
  let agentic = null; // { status, mode, url, discovered } agentic-channel state (#99)
14531
+ let ghAuth = null; // #293: { present, fingerprint, scopes, since, ageMs, lastCheckAgoMs } — never the token
12526
14532
  if (alive) {
12527
- const act = readWorkerActivity(w.id);
14533
+ const act = readActivity(w.id);
12528
14534
  if (act && act.pid === w.pid) {
12529
14535
  const jobs = Array.isArray(act.jobs)
12530
14536
  ? act.jobs
@@ -12556,6 +14562,24 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12556
14562
  // stale incarnation can't show a dead worker as connected to a hub.
12557
14563
  engine = typeof act.engine === 'string' && act.engine ? act.engine : null;
12558
14564
  agentic = act.agentic && typeof act.agentic === 'object' ? act.agentic : null;
14565
+ // #293: the gh token identity rides the same marker. Re-age the ticking
14566
+ // durations from the stamped `since`/marker `updatedAt` so a console that
14567
+ // re-ages locally (and a status snapshot read seconds later) shows a
14568
+ // current age, not the write-time one. Never carries the token value.
14569
+ if (act.ghAuth && typeof act.ghAuth === 'object') {
14570
+ const g = act.ghAuth;
14571
+ const ageMs = Number.isFinite(g.since) ? Math.max(0, now - g.since) : (Number.isFinite(g.ageMs) ? g.ageMs : null);
14572
+ const lastCheckAgoMs = Number.isFinite(act.updatedAt) && Number.isFinite(g.lastCheckAgoMs)
14573
+ ? Math.max(0, now - act.updatedAt + g.lastCheckAgoMs)
14574
+ : (Number.isFinite(g.lastCheckAgoMs) ? g.lastCheckAgoMs : null);
14575
+ // Coerce the identity strings at the (untrusted) marker boundary: a
14576
+ // malformed/tampered/older marker could carry a non-string `scopes` or
14577
+ // `fingerprint` (a number, array, object), and a downstream string op
14578
+ // (e.g. `scopes.split(',')` in supervisorGhAuthCell) would THROW and
14579
+ // crash `supervisor status` (Copilot review on #294). Keep only real
14580
+ // strings; anything else degrades to null (rendered as `?`/unknown).
14581
+ ghAuth = { present: g.present === true, fingerprint: typeof g.fingerprint === 'string' ? g.fingerprint : null, scopes: typeof g.scopes === 'string' ? g.scopes : null, since: Number.isFinite(g.since) ? g.since : null, ageMs, lastCheckAgoMs };
14582
+ }
12559
14583
  }
12560
14584
  // No marker (or a stale-pid one): leave activity null → rendered as unknown.
12561
14585
  }
@@ -12580,6 +14604,7 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12580
14604
  activity,
12581
14605
  engine,
12582
14606
  agentic,
14607
+ ghAuth,
12583
14608
  };
12584
14609
  }
12585
14610
 
@@ -12614,6 +14639,16 @@ function supervisorStatusSignature(workers) {
12614
14639
  // change is a real transition that must repaint attached consoles (#99).
12615
14640
  w.engine ?? '',
12616
14641
  w.agentic ? (w.agentic.status ?? '') : null,
14642
+ // #293: the gh credential IDENTITY — presence, fingerprint, scopes, and the
14643
+ // token's `since` epoch. A token rotation / scope change / logout on an
14644
+ // otherwise-idle worker is a real transition the attached console must
14645
+ // repaint for (the GH AUTH cell changes), so fold it into the signature.
14646
+ // Deliberately EXCLUDES the ticking durations (ageMs, lastCheckAgoMs) —
14647
+ // those merely elapse and are re-aged locally (see reageSupervisorStatus),
14648
+ // so including them would make every clock tick a spurious "change".
14649
+ w.ghAuth
14650
+ ? `${w.ghAuth.present ? 1 : 0}\u0000${w.ghAuth.fingerprint ?? ''}\u0000${w.ghAuth.scopes ?? ''}\u0000${w.ghAuth.since ?? ''}`
14651
+ : null,
12617
14652
  ]),
12618
14653
  );
12619
14654
  }
@@ -12676,6 +14711,32 @@ function supervisorAgenticCell(w) {
12676
14711
  return String(w.agentic.status);
12677
14712
  }
12678
14713
 
14714
+ /**
14715
+ * GH AUTH cell (#293): the cached gh token's age + scopes, so "this worker is
14716
+ * holding a stale token" is a one-line check instead of a manual
14717
+ * /proc/<pid>/environ vs `gh auth token` comparison. `-` for a down/stopping
14718
+ * worker, `?` for a live worker not (yet) reporting or on an older build whose
14719
+ * marker predates the field, `none` when gh is missing/not logged in. The
14720
+ * fingerprint is available in the status JSON (and the worker log) when two
14721
+ * workers need to be told apart; the table keeps the cell narrow (age + scopes).
14722
+ */
14723
+ function supervisorGhAuthCell(w) {
14724
+ if (w.state !== 'running') return '-';
14725
+ if (!w.activity) return '?'; // alive but not reporting
14726
+ const g = w.ghAuth;
14727
+ if (!g) return '?'; // marker predates the ghAuth field
14728
+ if (!g.present) return 'none';
14729
+ const age = g.ageMs != null ? formatDuration(g.ageMs) : '?';
14730
+ // Defensive against a malformed activity marker (untrusted JSON): only call
14731
+ // string ops on a real string, so a non-string `scopes` (a corrupt/older/
14732
+ // tampered marker) renders `?` instead of throwing and crashing the whole
14733
+ // status table (Copilot review on #294). statusFromActivity already coerces at
14734
+ // the marker boundary; this is the belt-and-braces at the render edge, which
14735
+ // also runs on frames built elsewhere (e.g. reageSupervisorStatus passthrough).
14736
+ const scopes = typeof g.scopes === 'string' && g.scopes ? g.scopes.split(',').map((s) => s.trim()).filter(Boolean).join(',') : '?';
14737
+ return `${age} [${scopes}]`;
14738
+ }
14739
+
12679
14740
  /**
12680
14741
  * Re-age a supervisor status snapshot to `now`, recomputing the ticking
12681
14742
  * durations (`uptimeMs`, per-job `sinceMs`) from the absolute base epochs the
@@ -12718,7 +14779,17 @@ function reageSupervisorStatus(status, now = Date.now()) {
12718
14779
  const state = jobs.some((j) => j && typeof j === 'object' && !j.settlementPending) ? 'busy' : 'idle';
12719
14780
  activity = { ...activity, jobs, state };
12720
14781
  }
12721
- return { ...w, uptimeMs, activity };
14782
+ // #293: re-age the gh credential's AGE from its absolute `since` epoch, the
14783
+ // same way uptime/job-age tick locally — otherwise an attached console that
14784
+ // re-ages the cached snapshot on every tick (never rebuilding it via
14785
+ // summarizeSupervisorWorker) would freeze the GH AUTH cell's age at its
14786
+ // snapshot-time value. `lastCheckAgoMs` is RELATIVE (no absolute base rides
14787
+ // the frame), so it cannot be re-aged here and is left as-is.
14788
+ let ghAuth = w.ghAuth;
14789
+ if (ghAuth && typeof ghAuth === 'object' && Number.isFinite(ghAuth.since)) {
14790
+ ghAuth = { ...ghAuth, ageMs: Math.max(0, now - ghAuth.since) };
14791
+ }
14792
+ return { ...w, uptimeMs, activity, ghAuth };
12722
14793
  }),
12723
14794
  };
12724
14795
  }
@@ -12878,6 +14949,7 @@ function formatSupervisorStatus(status) {
12878
14949
  state: String(w.state) === 'down' && w.configFailed === true ? 'down (config)' : String(w.state),
12879
14950
  engine: supervisorEngineCell(w),
12880
14951
  agentic: supervisorAgenticCell(w),
14952
+ ghAuth: supervisorGhAuthCell(w),
12881
14953
  job: supervisorJobCell(w),
12882
14954
  pid: w.pid ? String(w.pid) : '-',
12883
14955
  restarts: String(w.restarts),
@@ -12887,8 +14959,8 @@ function formatSupervisorStatus(status) {
12887
14959
  // ENGINE + AGENTIC sit early (just after STATE) so the pinned live view's
12888
14960
  // width clamp (which trims from the right) drops the least-critical columns
12889
14961
  // (LAST EXIT, UPTIME) first and keeps the visibility diagnostics visible.
12890
- const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
12891
- const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'job', 'pid', 'restarts', 'uptime', 'last'];
14962
+ const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', ghAuth: 'GH AUTH', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
14963
+ const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'ghAuth', 'job', 'pid', 'restarts', 'uptime', 'last'];
12892
14964
  const width = {};
12893
14965
  for (const c of cols) width[c] = Math.max(head[c].length, ...rows.map((r) => r[c].length));
12894
14966
  const fmt = (r) => ' ' + cols.map((c) => r[c].padEnd(width[c])).join(' ');
@@ -13353,7 +15425,12 @@ async function runSupervisorDaemon() {
13353
15425
  const pid = w.pid;
13354
15426
  if (w.child && pid) {
13355
15427
  try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
13356
- await waitForChildExit(w.child, STOP_GRACE_MS);
15428
+ // #296: wait STOP_GRACE_MS for a routine exit PLUS the bounded recovery budget,
15429
+ // so a worker that is flushing + promoting a durable recovery ref (#295) for an
15430
+ // interrupted agent job can finish before the SIGKILL backstop. A worker that
15431
+ // exits sooner is reaped at once — this only lengthens the ceiling for one that
15432
+ // is actively preserving recovery.
15433
+ await waitForChildExit(w.child, STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS);
13357
15434
  if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
13358
15435
  }
13359
15436
  };
@@ -13800,6 +15877,40 @@ async function runSupervisorDaemon() {
13800
15877
  attachClients.add(sock);
13801
15878
  sock.write(encodeFrame(statusFrame(false)));
13802
15879
  break;
15880
+ case 'reload-credentials': {
15881
+ // #293: tell every worker to re-read its gh auth token (SIGUSR1,
15882
+ // async — the still-valid old token is served until the new one
15883
+ // atomically lands) WITHOUT draining in-flight jobs — the operator
15884
+ // re-logged in and wants
15885
+ // the fleet to pick up the fresh credential now, not after a restart.
15886
+ // Windows cannot deliver SIGUSR1 (Node terminates the process), so
15887
+ // refuse there; the TTL refresh still adopts a rotated token.
15888
+ if (osPlatform() === 'win32') {
15889
+ sock.write(encodeFrame({ ok: false, error: 'reload-credentials is not supported on Windows (it relies on SIGUSR1, which Windows cannot deliver) — the workers\' TTL refresh still picks up a new token, or use `supervisor restart`', final: true }));
15890
+ break;
15891
+ }
15892
+ const rawCreds = Array.isArray(req.targets)
15893
+ ? req.targets.flatMap((t) => resolveTargets(t))
15894
+ : resolveTargets(req.target);
15895
+ const credIds = [...new Set(rawCreds)];
15896
+ const signaled = [];
15897
+ const failed = [];
15898
+ for (const id of credIds) {
15899
+ const w = workers.get(id);
15900
+ const alive = !!(w && isPidAlive(w.pid));
15901
+ // #293: only signal a worker whose SIGUSR1 (reload) listener is
15902
+ // PROVEN installed (reloadCredentialsReady) — otherwise the signal
15903
+ // hits Node's default action (start the inspector) and the daemon
15904
+ // would wrongly record "reloaded". Skipped workers land in `failed`
15905
+ // so `ok` stays honest; they pick up the fresh token at their own
15906
+ // startup prime.
15907
+ if (!reloadCredentialsReady(w, alive ? readWorkerActivity(id) : null, alive)) { failed.push(id); continue; }
15908
+ try { process.kill(w.pid, 'SIGUSR1'); signaled.push(id); }
15909
+ catch { failed.push(id); }
15910
+ }
15911
+ sock.write(encodeFrame({ ok: failed.length === 0, type: 'credentials-reloaded', signaled, failed, final: true }));
15912
+ break;
15913
+ }
13803
15914
  case 'stop': {
13804
15915
  // #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
13805
15916
  // jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
@@ -14305,6 +16416,45 @@ async function supervisorReloadCmd(req) {
14305
16416
  if (outcome === 'error') process.exit(1);
14306
16417
  }
14307
16418
 
16419
+ /**
16420
+ * Re-read every supervised worker's cached gh auth token WITHOUT draining
16421
+ * in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
16422
+ * worker re-reads `gh auth token` NON-DESTRUCTIVELY (a forced async re-read that
16423
+ * swaps the cache atomically when the new value lands — the still-valid old
16424
+ * token is served meanwhile, never dropped), and logs the new fingerprint/
16425
+ * scopes. Use after a `gh auth login`/`refresh` so the fleet picks up the fresh
16426
+ * credential immediately instead of waiting for the TTL refresh or a restart.
16427
+ */
16428
+ async function supervisorReloadCredentialsCmd(req) {
16429
+ const logger = getLogger();
16430
+ const target = req.positional[1] || 'all';
16431
+ const running = await liveSupervisor();
16432
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
16433
+ const socketPath = running.socket || getSupervisorSocketPath();
16434
+ let res;
16435
+ try {
16436
+ res = await supervisorRequest({ op: 'reload-credentials', target }, { socketPath });
16437
+ } catch (err) {
16438
+ logger.error(`Could not reach the supervisor control socket to reload credentials: ${err?.message || err}`);
16439
+ process.exit(1);
16440
+ }
16441
+ // A structured credentials-reloaded result carries `signaled`/`failed` arrays
16442
+ // even on a PARTIAL failure (daemon returns `ok:false` with no `error` when
16443
+ // some workers were down). Report those lists BEFORE the generic failure guard
16444
+ // — exactly as the rolling-reload client does — so an operator can see which
16445
+ // workers picked up the new token and which are down, instead of a bare
16446
+ // "reload-credentials failed". The generic guard then only catches a truly
16447
+ // unstructured failure (unreachable socket, the Windows refusal with `error`).
16448
+ if (!res) { logger.error('reload-credentials failed'); process.exit(1); }
16449
+ const hasStructured = Array.isArray(res.signaled) || Array.isArray(res.failed);
16450
+ if (res.ok === false && !hasStructured) { logger.error(res.error || 'reload-credentials failed'); process.exit(1); }
16451
+ const signaled = Array.isArray(res.signaled) ? res.signaled : [];
16452
+ const failed = Array.isArray(res.failed) ? res.failed : [];
16453
+ if (signaled.length > 0) logger.info(`Told ${signaled.length} worker(s) to re-read their gh auth token: ${signaled.join(', ')}. Each re-reads \`gh auth token\` (non-destructively — the still-valid old token is served until the new one lands) and logs the new fingerprint/scopes.`);
16454
+ else logger.warn('No running workers matched — nothing to reload.');
16455
+ if (failed.length > 0) { logger.error(`Could not signal: ${failed.join(', ')} (down or gone).`); process.exit(1); }
16456
+ }
16457
+
14308
16458
  /**
14309
16459
  * Count the in-flight jobs across a fleet snapshot (array of
14310
16460
  * `summarizeSupervisorWorker` results) — the number an operator is waiting on
@@ -14411,9 +16561,11 @@ async function supervisorStopCmd(force = false) {
14411
16561
 
14412
16562
  // Wait for the daemon process itself to exit. A drain waits INDEFINITELY (the
14413
16563
  // operator opted to wait); a force stop applies the grace window + a SIGKILL
14414
- // backstop so a wedged daemon/worker group can't linger.
16564
+ // backstop so a wedged daemon/worker group can't linger. The window must cover
16565
+ // the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
16566
+ // group while a worker is still promoting its durable recovery ref (#295).
14415
16567
  if (force) {
14416
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
16568
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
14417
16569
  while (Date.now() < deadline) {
14418
16570
  if (!isPidAlive(daemonPid)) break;
14419
16571
  await new Promise((r) => setTimeout(r, 150));
@@ -14515,6 +16667,10 @@ async function attachSupervisorConsole(state) {
14515
16667
  out(`• removed: ${frame.removed.join(', ') || '(none matched)'}.`);
14516
16668
  } else if (frame.type === 'restarted') {
14517
16669
  out(`• restarted: ${frame.restarted.join(', ') || '(none matched)'}.`);
16670
+ } else if (frame.type === 'credentials-reloaded') {
16671
+ const sig = Array.isArray(frame.signaled) ? frame.signaled : [];
16672
+ const fail = Array.isArray(frame.failed) ? frame.failed : [];
16673
+ out(`• credentials reload signaled: ${sig.join(', ') || '(none matched)'}${fail.length ? ` — could not signal: ${fail.join(', ')}` : ''}.`);
14518
16674
  } else if (frame.ok === false) {
14519
16675
  out(`! ${frame.error}`);
14520
16676
  }
@@ -14560,10 +16716,15 @@ async function attachSupervisorConsole(state) {
14560
16716
  case '': break;
14561
16717
  case 'help':
14562
16718
  out('Commands: status | add <profile> [--name <worker>] [--instances <n>] [work flags] |');
14563
- out(' remove <id|profile|all> | restart <id|profile|all> |');
16719
+ out(' remove <id|profile|all> | restart <id|profile|all> | reload-credentials [id|profile|all] |');
14564
16720
  out(' logs [id] | detach | stop | help');
14565
16721
  break;
14566
16722
  case 'status': sock.write(encodeFrame({ op: 'status' })); break;
16723
+ case 'reload-credentials': {
16724
+ const target = parts.shift() || 'all';
16725
+ sock.write(encodeFrame({ op: 'reload-credentials', target }));
16726
+ break;
16727
+ }
14567
16728
  case 'add': {
14568
16729
  const profile = parts.shift();
14569
16730
  if (!profile) { out('usage: add <profile> [--name <worker>] [--instances <n>] [work flags]'); break; }
@@ -14882,7 +17043,9 @@ function systemdUserAvailable() {
14882
17043
  async function stopSupervisorProcess(running) {
14883
17044
  try { await supervisorRequest({ op: 'stop' }); }
14884
17045
  catch { try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ } }
14885
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
17046
+ // #296: cover the daemon's own worker force-stop grace so the group SIGKILL below
17047
+ // does not reap a worker mid-recovery (promoting its durable recovery ref, #295).
17048
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
14886
17049
  while (Date.now() < deadline) {
14887
17050
  if (!isPidAlive(running.pid)) break;
14888
17051
  await new Promise((r) => setTimeout(r, 150));
@@ -15109,6 +17272,9 @@ async function supervisorCommand(req, flags, ctx) {
15109
17272
  case 'reload':
15110
17273
  await supervisorReloadCmd(req);
15111
17274
  return;
17275
+ case 'reload-credentials':
17276
+ await supervisorReloadCredentialsCmd(req);
17277
+ return;
15112
17278
  case 'stop':
15113
17279
  await supervisorStopCmd(coerceBool(flags?.force, false));
15114
17280
  return;
@@ -15117,7 +17283,7 @@ async function supervisorCommand(req, flags, ctx) {
15117
17283
  supervisorLogsCmd(req);
15118
17284
  return;
15119
17285
  default:
15120
- getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach`);
17286
+ getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
15121
17287
  process.exit(1);
15122
17288
  }
15123
17289
  }
@@ -17922,8 +20088,30 @@ export {
17922
20088
  namespaceHasLiveHarness,
17923
20089
  authUrl,
17924
20090
  githubCloneToken,
20091
+ githubCloneTokenWithSource,
17925
20092
  ghAuthTokenFromCli,
17926
20093
  primeGhAuthToken,
20094
+ refreshGhAuthToken,
20095
+ refreshGhAuthTokenAsync,
20096
+ invalidateGhAuthToken,
20097
+ recoverGhTokenAfterAuthFailure,
20098
+ setGhAuthTokenReader,
20099
+ resetGhAuthTokenReader,
20100
+ classifyGhAuthRead,
20101
+ GH_AUTH_READ_FAILED,
20102
+ adoptGhAuthToken,
20103
+ adoptGhAuthTokenAsync,
20104
+ readGhAuthTokenAsync,
20105
+ ghCredentialEnvName,
20106
+ applyRotatedGhCredential,
20107
+ ghAuthTokenInfo,
20108
+ describeGhAuthToken,
20109
+ ghTokenFingerprint,
20110
+ ghTokenScopes,
20111
+ ghTokenScopesAsync,
20112
+ isGitAuthFailure,
20113
+ startGhAuthTokenRefresh,
20114
+ GH_AUTH_TOKEN_TTL_MS,
17927
20115
  ghAuthEnv,
17928
20116
  redactToken,
17929
20117
  agentRunsRoot,
@@ -18004,10 +20192,12 @@ export {
18004
20192
  countSupervisorInFlight,
18005
20193
  supervisorEngineCell,
18006
20194
  supervisorAgenticCell,
20195
+ supervisorGhAuthCell,
18007
20196
  agenticStateForTarget,
18008
20197
  normalizeAgenticMessage,
18009
20198
  buildActivityPayload,
18010
20199
  activityMarkerReadyFor,
20200
+ reloadCredentialsReady,
18011
20201
  waitForChildExit,
18012
20202
  supervisorWorkerActivityFile,
18013
20203
  SETTLEMENT_PENDING_GHOST_TTL_MS,
@@ -18019,6 +20209,7 @@ export {
18019
20209
  supervisorStartCmd,
18020
20210
  supervisorAddCmd,
18021
20211
  supervisorReloadCmd,
20212
+ supervisorReloadCredentialsCmd,
18022
20213
  workforceReloadCmd,
18023
20214
  runningSupervisor,
18024
20215
  readSupervisorState,
@@ -18127,6 +20318,7 @@ export const metadata = {
18127
20318
  { command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
18128
20319
  { command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
18129
20320
  { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
20321
+ { command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
18130
20322
  { command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
18131
20323
  { command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
18132
20324
  { command: 'c8ctl nano workforce add qwen --instances 2 --roles pr-review,feature', description: "Add an entry mapped to explicit job types (<rank>:pr-review, <rank>:feature, where <rank> is the qwen hire's rank at start) — does not mutate the hired profile" },
@@ -18367,7 +20559,7 @@ function printUsage() {
18367
20559
  console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--terminal pty|pipe] [--protocol pipe|acp] [--permission yolo|escalate|filter] [--env NAME=VALUE ...] [--list]');
18368
20560
  console.log(' c8ctl nano assign <profileName> <cap[,cap...]> [--name <n>] [--capabilities <a,b>]');
18369
20561
  console.log(' c8ctl nano work <profileName> [--auto [--auto-scope <p>]] [--arg <switch> ...] [--recovery-window <ms>] [--idle-timeout <ms>] [--job-timeout <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
18370
- console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach] ... (manage many workers from one terminal)');
20562
+ console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach] ... (manage many workers from one terminal)');
18371
20563
  console.log(' c8ctl nano workforce [add|remove|list|start|status|stop|reload] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
18372
20564
  console.log('');
18373
20565
  console.log('Subcommands:');