c8ctl-plugin-nano 1.70.2 → 1.70.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -95,6 +95,110 @@ import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecret
95
95
  const requireFromHere = createRequire(import.meta.url);
96
96
  const pluginDir = dirname(fileURLToPath(import.meta.url));
97
97
 
98
+ /**
99
+ * Build the ordered list of `require` resolvers used to read the HOST c8ctl / Camunda
100
+ * SDK package manifests (issue #289 review). The host SDK lives in the host **c8ctl
101
+ * CLI's** dependency graph, NOT this plugin's. In the managed layout the plugin loads
102
+ * from `<data>/plugins/node_modules`, a SIBLING of `@camunda8/cli`; in the global
103
+ * layout the SDK is nested BENEATH the CLI package. Node resolution never searches a
104
+ * sibling package's `node_modules`, so a probe rooted only at this module commonly
105
+ * reads `null` for both versions — exactly when it needs to name the old host SDK.
106
+ * Resolve from the host graph instead:
107
+ * 1. a `require` rooted at the real c8ctl entry (`C8CTL_NANO_ENTRY` / `process.argv[1]`),
108
+ * canonicalized through symlinks — the CLI and its nested SDK live in THIS graph;
109
+ * 2. a `require` rooted at the resolved `@camunda8/cli` package itself, so an SDK
110
+ * nested beneath the CLI (not hoisted to the entry root) still resolves;
111
+ * 3. this module's own graph as a last-resort fallback (works in a hoisted/global
112
+ * layout where the SDK is visible from here).
113
+ * Every step is guarded: an unresolvable root is skipped, never thrown.
114
+ *
115
+ * The entry is `realpathSync`'d first because a globally installed CLI is launched
116
+ * through a launcher symlink (e.g. `/usr/local/bin/c8ctl` → `.../lib/node_modules/@camunda8/cli/bin/...`),
117
+ * which Node preserves verbatim in `process.argv[1]`. Rooting `createRequire` at the
118
+ * raw symlink would search `/usr/local/bin/node_modules` and miss the nested host SDK
119
+ * entirely — exactly the global layout this probe exists to diagnose.
120
+ */
121
+ export function buildHostSdkResolvers(entryArg) {
122
+ const resolvers = [];
123
+ const seen = new Set();
124
+ const push = (rq) => {
125
+ if (typeof rq === 'function' && !seen.has(rq)) {
126
+ seen.add(rq);
127
+ resolvers.push(rq);
128
+ }
129
+ };
130
+ const rawEntry = entryArg !== undefined ? entryArg : (process.env.C8CTL_NANO_ENTRY || process.argv[1]);
131
+ let entry = rawEntry;
132
+ if (entry) {
133
+ try {
134
+ entry = realpathSync(entry);
135
+ } catch {
136
+ /* not a real path yet (or already canonical) — fall back to the raw entry */
137
+ }
138
+ }
139
+ let entryRequire = null;
140
+ if (entry) {
141
+ try {
142
+ entryRequire = createRequire(entry);
143
+ push(entryRequire);
144
+ } catch {
145
+ /* unresolvable entry — skip */
146
+ }
147
+ }
148
+ for (const rq of [entryRequire, requireFromHere]) {
149
+ if (!rq) continue;
150
+ try {
151
+ const cliPkg = rq.resolve('@camunda8/cli/package.json');
152
+ push(createRequire(cliPkg));
153
+ } catch {
154
+ /* CLI not resolvable from this root — skip */
155
+ }
156
+ }
157
+ push(requireFromHere);
158
+ return resolvers;
159
+ }
160
+
161
+ /**
162
+ * Best-effort host c8ctl / Camunda SDK versions (issue #289). The host SDK lives in the
163
+ * host c8ctl CLI's dependency graph, which this plugin's (sibling) `node_modules` cannot
164
+ * reach, so the probe walks an ordered chain of resolvers rooted at the host CLI entry /
165
+ * package first and this module only as a fallback (see `buildHostSdkResolvers`) — the
166
+ * first resolver to yield a version wins. Every read is guarded: an unresolvable package
167
+ * (hoisting layout, bundling) simply yields null rather than throwing. Tests may inject a
168
+ * single `requireFn` (used as the sole resolver) or an explicit `resolvers` array.
169
+ */
170
+ export function probeHostSdkInfo(requireFn, resolvers) {
171
+ const chain = Array.isArray(resolvers) && resolvers.length > 0
172
+ ? resolvers
173
+ : requireFn
174
+ ? [requireFn]
175
+ : buildHostSdkResolvers();
176
+ const read = (spec) => {
177
+ for (const rq of chain) {
178
+ try {
179
+ const v = rq(`${spec}/package.json`)?.version;
180
+ if (typeof v === 'string' && v.trim() !== '') return String(v);
181
+ } catch {
182
+ /* try the next resolver in the chain */
183
+ }
184
+ }
185
+ return null;
186
+ };
187
+ return {
188
+ cliVersion: read('@camunda8/cli'),
189
+ sdkVersion: read('@camunda8/orchestration-cluster-api'),
190
+ };
191
+ }
192
+
193
+ /** Render host SDK versions for a log line (issue #289). Never throws. */
194
+ export function formatHostSdkInfo(info) {
195
+ const cli = info?.cliVersion ? `@camunda8/cli ${info.cliVersion}` : '@camunda8/cli (version unknown)';
196
+ const sdk = info?.sdkVersion
197
+ ? `@camunda8/orchestration-cluster-api ${info.sdkVersion}`
198
+ : '@camunda8/orchestration-cluster-api (version unknown)';
199
+ return `${cli}, ${sdk}`;
200
+ }
201
+
98
202
  /**
99
203
  * Read the bundled-binary marker (nanobpmn-binary.json) written by the upstream
100
204
  * release pipeline. Records which nanobpmn build the shipped binaries came from.
@@ -352,6 +456,22 @@ function activityMarkerReadyFor(act, pid) {
352
456
  return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
353
457
  }
354
458
 
459
+ /**
460
+ * Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
461
+ * All three must hold (#293): the worker exists, its PID is alive, AND its
462
+ * activity marker proves its SIGUSR1 reload listener is installed (a finite
463
+ * `readyAt` + matching pid, via activityMarkerReadyFor). workAgent installs that
464
+ * listener only once its activation loop is up, which is the same handshake that
465
+ * stamps `readyAt`. Signalling a freshly spawned worker BEFORE it installs the
466
+ * handler delivers SIGUSR1 to Node's DEFAULT action — starting the inspector
467
+ * instead of reloading — so a not-yet-ready worker is skipped (it re-reads the
468
+ * fresh token at its own startup prime anyway). Pure so it can be unit-tested.
469
+ */
470
+ function reloadCredentialsReady(worker, activity, pidAlive) {
471
+ if (!worker || !pidAlive) return false;
472
+ return activityMarkerReadyFor(activity, worker.pid);
473
+ }
474
+
355
475
  /**
356
476
  * Deterministic control-socket path shared by the daemon and every client.
357
477
  * Derived from a hash of the (possibly overridden) state home so distinct
@@ -4451,10 +4571,28 @@ function makeSecretResolver(kind) {
4451
4571
  return null; // unknown → caller reports a clear error
4452
4572
  }
4453
4573
 
4574
+ // Force a fresh `gh auth token` read, bypassing the TTL cache, and
4575
+ // return the token (or null). Injectable into resolveJobSecrets so a default
4576
+ // github gh-cache MISS re-reads at job-resolve time — catching a `gh auth login`
4577
+ // the worker-lifetime refresh timer has not re-primed yet — instead of trusting
4578
+ // provisionRepo's (non-universal) auth-failure recovery (Copilot review on #294).
4579
+ // ASYNC (spawn, never spawnSync): resolveJobSecrets runs on the activated-job
4580
+ // path after recordJobStart, so a SYNCHRONOUS `gh auth token` + scopes probe
4581
+ // here would block the shared event loop (and the lock-extension heartbeat) for
4582
+ // up to ~15s — the very hot-path stall the async refresh exists to prevent
4583
+ // (Copilot review on #294). refreshGhAuthTokenAsync also defers to any refresh
4584
+ // already in flight instead of starting a second concurrent read.
4585
+ async function defaultForceRefreshGhAuthToken() {
4586
+ const r = await refreshGhAuthTokenAsync({ force: true });
4587
+ return (r && r.token) || null;
4588
+ }
4589
+
4454
4590
  // Resolve the names a job needs (setup.secretRefs, plus the repo/PR credential
4455
4591
  // when allowPr). Returns resolved values + a list of names that were missing so
4456
- // the caller can fail the job with a clear provisioning error.
4457
- function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli } = {}) {
4592
+ // the caller can fail the job with a clear provisioning error. ASYNC: the
4593
+ // gh-cache-miss force-refresh awaits a spawned (non-blocking) gh read, so the
4594
+ // caller must `await` this (the workAgent job runner already is async).
4595
+ async function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli, forceRefreshGhAuthToken = defaultForceRefreshGhAuthToken } = {}) {
4458
4596
  const names = new Set();
4459
4597
  for (const n of envelope.setup?.secretRefs || []) if (n) names.add(n);
4460
4598
  const resolved = {};
@@ -4466,7 +4604,10 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
4466
4604
  }
4467
4605
  // The github clone/push credential is resolved with a gh-CLI fallback so a
4468
4606
  // default GITHUB_TOKEN isn't reported "missing" merely because it isn't in the
4469
- // env when `gh auth login` provides it. A custom authRef stays strict.
4607
+ // env when `gh auth login` provides it. A default github credential whose gh
4608
+ // cache is a MISS is force-re-read here (bypassing the TTL cache); it is
4609
+ // reported missing only when that forced read ALSO finds no token. A custom
4610
+ // authRef stays strict.
4470
4611
  if (envelope.task?.allowPr) {
4471
4612
  const provider = envelope.repository?.provider || 'github';
4472
4613
  const authRef = envelope.repository?.authRef;
@@ -4476,16 +4617,35 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
4476
4617
  // so provisioning sheds rather than silently borrowing the default/gh token.
4477
4618
  if (!missing.includes('repository.authRef')) missing.push('repository.authRef');
4478
4619
  } else {
4479
- const ghAuthRef = ref.kind === 'custom'
4480
- ? ref.name
4481
- : (provider === 'github' ? 'GITHUB_TOKEN' : undefined);
4620
+ const ghAuthRef = ghCredentialEnvName({ provider, authRef });
4482
4621
  if (ghAuthRef) {
4483
4622
  names.add(ghAuthRef);
4484
- const token = githubCloneToken({ provider, authRef, secretResolver: resolver, ghAuthToken });
4623
+ const { token, source } = githubCloneTokenWithSource({ provider, authRef, secretResolver: resolver, ghAuthToken });
4485
4624
  const missingIdx = missing.indexOf(ghAuthRef);
4486
4625
  if (token) {
4487
4626
  resolved[ghAuthRef] = token;
4488
4627
  if (missingIdx !== -1) missing.splice(missingIdx, 1);
4628
+ } else if (source === 'gh-cache') {
4629
+ // A DEFAULT github credential whose gh-CLI cache is a MISS (null).
4630
+ // provisionRepo's null → fresh auth-failure recovery is NOT universal:
4631
+ // container jobs never call provisionRepo, and an anonymous clone of a
4632
+ // PUBLIC repo SUCCEEDS (no auth failure → no forced re-read), so the
4633
+ // harness would otherwise reach an `allowPr` job with GITHUB_TOKEN
4634
+ // ABSENT instead of failing early. FORCE a fresh `gh auth token` read
4635
+ // HERE (catching a `gh auth login` the worker-lifetime refresh timer
4636
+ // has not re-primed yet): populate + drop from `missing` only if a
4637
+ // token now exists; otherwise KEEP it missing so the job fails with a
4638
+ // clear provisioning error rather than silently handing the harness no
4639
+ // credential (Copilot review on #294). Priming the cache here also
4640
+ // feeds provisioning's own re-resolve the same fresh token. A
4641
+ // custom/invalid authRef reports source 'none' and still fails strict.
4642
+ const fresh = await forceRefreshGhAuthToken();
4643
+ if (fresh) {
4644
+ resolved[ghAuthRef] = fresh;
4645
+ if (missingIdx !== -1) missing.splice(missingIdx, 1);
4646
+ } else if (missingIdx === -1) {
4647
+ missing.push(ghAuthRef);
4648
+ }
4489
4649
  } else if (missingIdx === -1) {
4490
4650
  missing.push(ghAuthRef);
4491
4651
  }
@@ -4779,42 +4939,690 @@ function credArgs() {
4779
4939
  // a trimmed token, or null when gh is missing / not logged in. The token is fed to
4780
4940
  // git via GIT_ASKPASS only (never argv/URL/helper), preserving the ephemeral-token
4781
4941
  // guarantee.
4782
- // Memoized for the process lifetime: this is a synchronous spawnSync (up to a
4783
- // 10s timeout) that can be reached per job, so consult the CLI at most once per
4784
- // worker run rather than blocking a handler on every job. A sentinel
4785
- // distinguishes "not yet computed" from
4786
- // a cached null (gh missing / not logged in).
4787
4942
  //
4788
- // Memoization alone still lets the *first* job pay the synchronous spawn on the
4789
- // event loop, stalling any sibling handlers (and lock-extension heartbeats) for
4790
- // up to the timeout. So `nano work` primes this cache once at startup via
4791
- // `primeGhAuthToken()` — before the poll loop — moving the one unavoidable
4792
- // blocking spawn off the job-handling path entirely. Any later call is a warm
4793
- // cache hit; the memoization here is the safety net for paths that never primed.
4943
+ // Caching (issue #293): the lookup is a synchronous spawnSync (up to a 10s
4944
+ // timeout), so it cannot run on the per-job hot path — but a PROCESS-LIFETIME
4945
+ // memo goes stale the moment the operator refreshes the gh login (re-login, a new
4946
+ // scope, a rotation), and nothing told the worker (or the operator) it was now
4947
+ // handing every job the OLD token. The cache is therefore a RECORD
4948
+ // (`{ token, fingerprint, scopes, since, lastCheck }`) with a TTL:
4949
+ // - `ghAuthTokenFromCli()` (the per-job hot path) serves the PRIMED cache
4950
+ // record VERBATIM — even once `lastCheck` is older than
4951
+ // GH_AUTH_TOKEN_TTL_MS — and NEVER triggers a synchronous TTL re-read, so a
4952
+ // job never pays `gh auth token`'s spawn cost (Copilot review on #293). The
4953
+ // sole synchronous read is for a still-UNSET cache (a worker that never
4954
+ // primed), and even that defers to any async refresh already in flight. The
4955
+ // off-path refreshers below keep a primed record current.
4956
+ // - `nano work` PRIMES at startup (`primeGhAuthToken()`) AND re-primes on a
4957
+ // timer (`startGhAuthTokenRefresh`) so the TTL re-read happens OFF the job
4958
+ // path. The timer's refresh is ASYNC (spawn, never spawnSync) and DEFERRED
4959
+ // while a job is active, so it can never block the event loop / lock
4960
+ // heartbeat (Copilot review on #293) — a refreshed login is therefore
4961
+ // adopted within one TTL only when the worker is idle at a tick. Under
4962
+ // sustained back-to-back load the defer is BOUNDED by
4963
+ // GH_AUTH_TOKEN_MAX_DEFER_MS, after which the tick refreshes DESPITE an
4964
+ // active job (still async) to cap staleness at a few intervals. When the
4965
+ // identity changes it republishes the activity marker via an onChange
4966
+ // callback.
4967
+ // - A git/gh auth FAILURE (clone/push/PR-reconcile — see isGitAuthFailure)
4968
+ // forces one re-read (`refreshGhAuthToken({ force, budgetMs })`, bounded by
4969
+ // the caller's remaining lease budget so the synchronous re-read can't
4970
+ // overrun the lock heartbeat) and a single retry, so a rotated token
4971
+ // self-heals instead of failing the job. The re-read is
4972
+ // NON-destructive (it never invalidates the cache first), so a transient
4973
+ // re-read failure keeps the still-valid cached credential.
4974
+ // - `primeGhAuthToken({ logger })` logs the token's FINGERPRINT (a short sha256
4975
+ // prefix — never the value) and its OAuth scopes at startup and whenever the
4976
+ // token changes, so "this worker has a stale token" is a one-line check; the
4977
+ // same record rides the supervisor activity marker (`ghAuthTokenInfo()`) into
4978
+ // `supervisor status`, and `supervisor reload-credentials` re-reads it
4979
+ // fleet-wide (async; the old token is served until the new one lands)
4980
+ // without restarting workers.
4981
+ // A sentinel distinguishes "not yet computed" from a cached null (gh missing /
4982
+ // not logged in).
4794
4983
  const GH_AUTH_TOKEN_UNSET = Symbol('gh-auth-token-unset');
4984
+ // How long a cached gh token is trusted before a re-read. 5 minutes bounds the
4985
+ // staleness window after a `gh auth login`/`refresh` while keeping `gh auth
4986
+ // token` (a cheap keychain read) off the per-job path. Override with
4987
+ // NANO_GH_TOKEN_TTL_MS (0 = re-read on every lookup, honouring the spawn cost).
4988
+ const GH_AUTH_TOKEN_TTL_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_TTL_MS, 5 * 60_000));
4989
+ // Upper bound on how long the maintenance timer may KEEP DEFERRING its refresh
4990
+ // while jobs run back-to-back. The tick normally defers while a job is active
4991
+ // (`isIdle`) so it never even spawns `gh` mid-job — but that defer is only an
4992
+ // optimization, because the refresh is async (spawn, never spawnSync) and so
4993
+ // cannot block the event loop / lock heartbeat even when it DOES run mid-job.
4994
+ // Under SUSTAINED saturation the worker is never idle at a tick, so an unbounded
4995
+ // defer would leave an expired, non-null cache reaching every subsequent job for
4996
+ // the worker's whole busy lifetime — defeating issue #293's "keep the token
4997
+ // fresh for the worker's lifetime" goal (Copilot review on #294). Bound it: once
4998
+ // a PRIMED cache is stale beyond this window, the tick refreshes DESPITE an
4999
+ // active job (still async/non-blocking), capping staleness at a few intervals
5000
+ // instead of the busy lifetime. Derived from the TTL (4 intervals) and overridable
5001
+ // via NANO_GH_TOKEN_MAX_DEFER_MS. Priming an UNSET cache is NEVER forced mid-job
5002
+ // (the defer there still holds), so a worker that starts logged-out stays anonymous
5003
+ // until it goes idle, exactly as before.
5004
+ const GH_AUTH_TOKEN_MAX_DEFER_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_MAX_DEFER_MS, 4 * GH_AUTH_TOKEN_TTL_MS));
5005
+ // The gh subprocesses below must never block on an interactive prompt inside a
5006
+ // worker (a missing/expired login would otherwise hang the event loop for the
5007
+ // full timeout). GH_PROMPT_DISABLED fails fast instead; GH_NO_UPDATE_NOTIFIER
5008
+ // keeps gh's upgrade check out of the captured output.
5009
+ const GH_NONINTERACTIVE_ENV = () => ({ ...process.env, GH_PROMPT_DISABLED: '1', GH_NO_UPDATE_NOTIFIER: '1' });
4795
5010
  let ghAuthTokenCache = GH_AUTH_TOKEN_UNSET;
4796
- function ghAuthTokenFromCli() {
4797
- if (ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET) return ghAuthTokenCache;
4798
- let token = null;
5011
+ // A Promise while an ASYNC gh-token refresh (the periodic timer tick or the
5012
+ // `reload-credentials` SIGUSR1 handler) is actually spawning `gh` subprocesses;
5013
+ // null otherwise. This is the serialization point that keeps a refresh off the
5014
+ // job path (Copilot review on #293): while it is set, the SYNCHRONOUS per-job
5015
+ // lookup (`ghAuthTokenFromCli`) must NOT start its own blocking `spawnSync`
5016
+ // re-read — it hands out the currently-cached token and lets the in-flight
5017
+ // async refresh update the cache, so an async refresh overlapping a freshly
5018
+ // activated job can never degrade into a 15s event-loop/lease-heartbeat stall.
5019
+ // It also prevents the timer and the SIGUSR1 handler from racing two concurrent
5020
+ // reads: a second async refresh awaits the first instead of spawning again.
5021
+ let ghAuthRefreshInFlight = null;
5022
+ // Monotonic generation counter bumped on EVERY write to `ghAuthTokenCache`
5023
+ // (`writeGhAuthCache` below is the single mutation point). It fences a stale
5024
+ // async refresh against a newer forced recovery (Copilot review on #293): the
5025
+ // synchronous clone/push auth-failure recovery (a forced `refreshGhAuthToken({
5026
+ // force })`) CANNOT await an already-running async timer
5027
+ // refresh (sync code can't await the `ghAuthRefreshInFlight` promise). So an
5028
+ // async refresh can read the OLD token, yield across its scopes probe, the sync
5029
+ // recovery can adopt the ROTATED token in the meantime, and then the async
5030
+ // refresh's late `adoptGhAuthTokenAsync` would overwrite the cache with its
5031
+ // now-stale read — resurrecting the revoked credential. `refreshGhAuthTokenAsync`
5032
+ // snapshots this generation BEFORE it starts reading and `adoptGhAuthTokenAsync`
5033
+ // refuses to commit when the generation has advanced (a newer definitive write
5034
+ // landed), so an older read can never clobber a newer forced recovery.
5035
+ let ghAuthCacheGeneration = 0;
5036
+ // The single mutation point for `ghAuthTokenCache`. Every write (adopt, preserve,
5037
+ // invalidate/reset) goes through here so the generation fence above sees them all.
5038
+ function writeGhAuthCache(record) {
5039
+ ghAuthTokenCache = record;
5040
+ ghAuthCacheGeneration++;
5041
+ return record;
5042
+ }
5043
+ // Test-only seam: the `gh auth token` reader `refreshGhAuthToken` consults when
5044
+ // no per-call `run` is injected. Production leaves this unset (the real
5045
+ // spawnSync of the gh CLI); a test installs a stub so the clone/push
5046
+ // invalidation-and-retry paths — which refresh through the MODULE cache, not a
5047
+ // per-call `run` — can rotate the token deterministically. Always a function
5048
+ // returning a trimmed token-or-null; reset with `resetGhAuthTokenReader()`.
5049
+ let ghAuthTokenReader = null;
5050
+ function setGhAuthTokenReader(fn) { ghAuthTokenReader = typeof fn === 'function' ? fn : null; }
5051
+ function resetGhAuthTokenReader() { ghAuthTokenReader = null; }
5052
+
5053
+ // Sentinel distinguishing a TRANSIENT `gh auth token` read failure from a
5054
+ // definitive logout (Copilot review on #293). The readers used to collapse a
5055
+ // timeout / spawn error / unrecognised nonzero exit into `null`, which the
5056
+ // refresh path then adopted as a real "logged out" change — discarding a
5057
+ // previously usable cached token, stamping the cache fresh with `null`, and
5058
+ // dropping the worker (and its clone-auth recovery) to anonymous until the next
5059
+ // interval. A read that returns this sentinel instead means "could not
5060
+ // determine"; `adoptGhAuthToken*` preserves the prior credential rather than
5061
+ // wiping it, and only a DEFINITIVE result (a real token, or a confirmed logout)
5062
+ // ever changes the cache.
5063
+ const GH_AUTH_READ_FAILED = Symbol('gh-auth-read-failed');
5064
+ // gh's not-logged-in signatures ("no oauth token found for github.com",
5065
+ // "You are not logged into any GitHub hosts"). Matched on stderr ONLY for a
5066
+ // nonzero exit, so an unrelated nonzero failure (keychain error, transient
5067
+ // network) stays a transient read failure rather than a spurious logout.
5068
+ const GH_LOGGED_OUT_RE = /no oauth token|no token found|not logged in/i;
5069
+ // Map one completed (or failed) `gh auth token` invocation to:
5070
+ // - a trimmed token string → logged in
5071
+ // - null → the read DEFINITIVELY established no login (clean exit with empty
5072
+ // output, or a nonzero exit whose stderr carries gh's logged-out message)
5073
+ // - GH_AUTH_READ_FAILED → the read could not be completed (spawn error,
5074
+ // timeout — spawnSync sets `.error` — or a nonzero exit WITHOUT a
5075
+ // logged-out signal), so the caller must preserve any prior cached token.
5076
+ function classifyGhAuthRead({ error = null, status = null, stdout = '', stderr = '' } = {}) {
5077
+ if (error) return GH_AUTH_READ_FAILED;
5078
+ if (status === 0) return (String(stdout ?? '').trim()) || null;
5079
+ if (GH_LOGGED_OUT_RE.test(String(stderr ?? ''))) return null;
5080
+ return GH_AUTH_READ_FAILED;
5081
+ }
5082
+
5083
+ function readGhAuthTokenDefault(timeoutMs = 10_000) {
5084
+ let r;
4799
5085
  try {
4800
- const r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: 10_000 });
4801
- const tok = r.status === 0 ? (r.stdout || '').trim() : '';
4802
- token = tok || null;
5086
+ r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: Math.max(1, timeoutMs), env: GH_NONINTERACTIVE_ENV() });
4803
5087
  } catch {
5088
+ return GH_AUTH_READ_FAILED;
5089
+ }
5090
+ return classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
5091
+ }
5092
+
5093
+ // A short, non-reversible identity for a token so logs/status can show WHEN the
5094
+ // credential changed (and that two workers hold the SAME one) without ever
5095
+ // printing a usable fragment of the secret. 12 hex chars of sha256.
5096
+ function ghTokenFingerprint(token) {
5097
+ if (!token) return null;
5098
+ try { return createHash('sha256').update(String(token)).digest('hex').slice(0, 12); }
5099
+ catch { return null; }
5100
+ }
5101
+
5102
+ // Read the token's OAuth scopes from the live `X-OAuth-Scopes` response header
5103
+ // (the issue's own diagnostic). Best-effort and bounded: a single lightweight
5104
+ // api.github.com ping, failure → null (scopes simply go unreported). The token
5105
+ // rides the Authorization HEADER — never argv — so it can't leak via `ps`.
5106
+ function ghTokenScopes(token, { timeoutMs = 5_000, run = spawnSync } = {}) {
5107
+ if (!token) return null;
5108
+ try {
5109
+ const r = run('gh', ['api', '-i', '--include', 'user'], { encoding: 'utf8', timeout: timeoutMs, env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
5110
+ const out = `${r.stdout || ''}\n${r.stderr || ''}`;
5111
+ const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
5112
+ return m ? m[1].trim() : null;
5113
+ } catch {
5114
+ return null;
5115
+ }
5116
+ }
5117
+
5118
+ // Fold one (re)read result into the cache record. Shared by the sync and async
5119
+ // refresh paths so both maintain identical cache semantics. Returns
5120
+ // { token, changed, snapshotChanged } — `changed` is true when the token VALUE
5121
+ // differs from what the cache previously held (including null → token and token
5122
+ // → null), so a caller doing credential-rotation recovery fires exactly when the
5123
+ // credential the worker hands out actually rotates. `snapshotChanged` is the
5124
+ // broader "did the OBSERVABLE snapshot move" flag: `changed` OR a same-token
5125
+ // scopes self-repair (a prior null scopes reading re-probed to a real value) —
5126
+ // both of which change what `supervisor status`/the activity marker should show,
5127
+ // so the log + republish paths gate on IT, not on `changed` (Copilot review on
5128
+ // #293). `scopes` is read only when the token changed, on the first populate, or
5129
+ // to re-probe a cached null reading.
5130
+ function adoptGhAuthToken(token, { scopesProbe = ghTokenScopes } = {}) {
5131
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5132
+ const now = Date.now();
5133
+ if (token === GH_AUTH_READ_FAILED) {
5134
+ // Transient read failure: never discard a usable cached credential. Preserve
5135
+ // the prior record but bump `lastCheck` so the per-job hot path keeps serving
5136
+ // it via the TTL short-circuit (no blocking re-read on every lookup); the
5137
+ // off-path refresh timer retries on its own cadence. With no usable prior
5138
+ // token, fall back to anonymous (null) as before.
5139
+ if (prev && prev.token != null) {
5140
+ writeGhAuthCache({ ...prev, lastCheck: now });
5141
+ return { token: prev.token, changed: false, snapshotChanged: false };
5142
+ }
5143
+ token = null;
5144
+ }
5145
+ const changed = !prev || prev.token !== token;
5146
+ // On a CHANGE re-read the scopes; otherwise carry the previous reading forward
5147
+ // (a TTL re-read that finds the same token must not pay the scopes ping, and
5148
+ // must not silently DROP the scopes it already reported). BUT a prior probe
5149
+ // that failed transiently stored `scopes: null` — carrying that null forward
5150
+ // would leave the GH AUTH visibility unrepaired until the token rotates or the
5151
+ // worker restarts (Copilot review on #293). Re-probe when the cached scope
5152
+ // value is MISSING (null) while a token is present; reuse only a SUCCESSFUL
5153
+ // (non-null) prior reading.
5154
+ const scopes = token ? ((changed || !(prev && prev.scopes)) ? scopesProbe(token) : prev.scopes) : null;
5155
+ // `snapshotChanged` is true when the OBSERVABLE identity (the token-free
5156
+ // fingerprint + scopes that ghAuthTokenInfo/describeGhAuthToken surface) moved
5157
+ // — not only when the token VALUE rotated. A scopes self-repair (a prior
5158
+ // transient probe cached `scopes:null`, now re-probed to a real value) leaves
5159
+ // the token unchanged yet DOES change what `supervisor status` should show, so
5160
+ // the republish/log path must fire on it too; gating republishing on `changed`
5161
+ // alone leaves an idle worker reporting `?` scopes indefinitely (Copilot review
5162
+ // on #293).
5163
+ const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
5164
+ writeGhAuthCache({
5165
+ token,
5166
+ fingerprint: ghTokenFingerprint(token),
5167
+ scopes: scopes ?? null,
5168
+ // `since` tracks the TOKEN's age (when THIS value was first seen) so status
5169
+ // can answer "how stale is the credential"; `lastCheck` tracks the CACHE's
5170
+ // age (when we last asked gh) so the TTL is honoured.
5171
+ since: prev && prev.token === token && token != null ? prev.since : now,
5172
+ lastCheck: now,
5173
+ });
5174
+ return { token, changed, snapshotChanged };
5175
+ }
5176
+
5177
+ // The async twin of adoptGhAuthToken: identical cache semantics, but the scopes
5178
+ // probe is awaited so it may return a Promise (the async ghTokenScopesAsync).
5179
+ // `generation` (when provided by refreshGhAuthTokenAsync) is the cache generation
5180
+ // snapshotted BEFORE this refresh started reading. If it no longer matches at
5181
+ // commit time, a NEWER definitive write (a forced sync clone/push recovery, an
5182
+ // invalidate, or another refresh) landed while we were reading/probing — so this
5183
+ // read is stale and MUST NOT be committed, or it would resurrect the superseded
5184
+ // token (Copilot review on #293). We leave the newer cache in place and report
5185
+ // it as superseded.
5186
+ async function adoptGhAuthTokenAsync(token, { scopesProbe, generation = null } = {}) {
5187
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5188
+ const now = Date.now();
5189
+ const superseded = () => {
5190
+ const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5191
+ return { token: c ? c.token : null, changed: false, snapshotChanged: false, superseded: true };
5192
+ };
5193
+ if (token === GH_AUTH_READ_FAILED) {
5194
+ // Transient read failure (see adoptGhAuthToken): preserve the prior token and
5195
+ // bump `lastCheck` so the hot path keeps serving it; anonymous only with no prior.
5196
+ if (prev && prev.token != null) {
5197
+ if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
5198
+ writeGhAuthCache({ ...prev, lastCheck: now });
5199
+ return { token: prev.token, changed: false, snapshotChanged: false };
5200
+ }
4804
5201
  token = null;
4805
5202
  }
4806
- ghAuthTokenCache = token;
4807
- return token;
5203
+ const changed = !prev || prev.token !== token;
5204
+ // Same re-probe-on-missing rule as the sync twin: a same-token refresh reuses a
5205
+ // successful prior scope reading but RE-PROBES when the cached value is null
5206
+ // (a prior transient probe failure), so the GH AUTH visibility self-repairs.
5207
+ const scopes = token ? ((changed || !(prev && prev.scopes)) ? await scopesProbe(token) : prev.scopes) : null;
5208
+ // See adoptGhAuthToken: publish when the observable snapshot (fingerprint +
5209
+ // scopes) moved, not only when the token value rotated, so a same-token scopes
5210
+ // self-repair still republishes the activity marker (Copilot review on #293).
5211
+ const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
5212
+ // Generation fence: re-check AFTER the (awaited) scopes probe, the last point a
5213
+ // forced recovery could have slipped a newer credential in underneath us.
5214
+ if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
5215
+ writeGhAuthCache({
5216
+ token,
5217
+ fingerprint: ghTokenFingerprint(token),
5218
+ scopes: scopes ?? null,
5219
+ since: prev && prev.token === token && token != null ? prev.since : now,
5220
+ lastCheck: now,
5221
+ });
5222
+ return { token, changed, snapshotChanged };
5223
+ }
5224
+
5225
+ // One (re)read of `gh auth token`, folding the result into the cache record.
5226
+ // Returns { token, changed } (see adoptGhAuthToken). SYNCHRONOUS (spawnSync) —
5227
+ // safe off the job path (the startup prime and the clone/push auth-failure
5228
+ // recovery, where the caller is already blocked waiting on git) but it blocks
5229
+ // the event loop while it runs, so the periodic refresh AND the SIGUSR1
5230
+ // reload-credentials handler use refreshGhAuthTokenAsync instead.
5231
+ function refreshGhAuthToken({ force = false, run = null, scopesProbe, budgetMs = null } = {}) {
5232
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5233
+ const now = Date.now();
5234
+ if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
5235
+ return { token: prev.token, changed: false, snapshotChanged: false };
5236
+ }
5237
+ // When a caller on the ACTIVE-JOB path (the clone/push auth-failure recovery)
5238
+ // threads its remaining lease budget, bound BOTH synchronous gh subprocesses
5239
+ // (the `gh auth token` read AND the scopes probe) to it so this forced
5240
+ // spawnSync re-read can never block the event loop / lock heartbeat past the
5241
+ // caller's lease-safe window — the finalizeGit/clone git ops are already
5242
+ // netDeadline-bounded, and this read must participate in the same budget
5243
+ // (Copilot review on #294). Absent a budget (startup prime, UNSET-cache hot
5244
+ // path, unit tests) the historical 10s read / 5s scopes caps stand.
5245
+ const haveBudget = Number.isFinite(budgetMs) && budgetMs > 0;
5246
+ const deadline = haveBudget ? now + budgetMs : null;
5247
+ const readTimeout = haveBudget ? Math.max(1, Math.min(10_000, budgetMs)) : 10_000;
5248
+ let token = null;
5249
+ if (run) {
5250
+ // Explicit per-call spawn injection (unit tests of this function in isolation).
5251
+ try {
5252
+ const r = run('gh', ['auth', 'token'], { encoding: 'utf8', timeout: readTimeout, env: GH_NONINTERACTIVE_ENV() });
5253
+ token = classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
5254
+ } catch {
5255
+ token = GH_AUTH_READ_FAILED;
5256
+ }
5257
+ } else {
5258
+ // The module reader (production: the real gh CLI; tests: setGhAuthTokenReader).
5259
+ token = (ghAuthTokenReader || readGhAuthTokenDefault)(readTimeout);
5260
+ }
5261
+ // The scopes probe gets whatever of the budget the read left (still capped at
5262
+ // 5s); a read that consumed the whole budget leaves it ~1ms so the probe fails
5263
+ // fast to null (scopes are best-effort) rather than overrunning the lease.
5264
+ const scopesTimeout = haveBudget ? Math.max(1, Math.min(5_000, deadline - Date.now())) : 5_000;
5265
+ // The scopes probe defaults to the live `gh api -i user` ping — but when a
5266
+ // test installed a token READER (hermetic, no network) and did not inject a
5267
+ // scopesProbe, skip the ping so a stubbed refresh never hits the network.
5268
+ const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((t) => ghTokenScopes(t, { timeoutMs: scopesTimeout })));
5269
+ return adoptGhAuthToken(token, { scopesProbe: probe });
5270
+ }
5271
+
5272
+ // Async `gh auth token` read (spawn, never spawnSync) for the periodic refresh
5273
+ // path — the whole point of the timer is that it must NOT block the worker's
5274
+ // event loop (and with it the lock heartbeat) while a job runs. Same 10s bound
5275
+ // and non-interactive env as the sync reader; resolves a trimmed token-or-null.
5276
+ // `_spawn` is a test-only seam (production uses node:child_process.spawn).
5277
+ function readGhAuthTokenAsync({ _spawn = spawn } = {}) {
5278
+ return new Promise((resolve) => {
5279
+ let child;
5280
+ try {
5281
+ child = _spawn('gh', ['auth', 'token'], { env: GH_NONINTERACTIVE_ENV() });
5282
+ } catch {
5283
+ resolve(GH_AUTH_READ_FAILED);
5284
+ return;
5285
+ }
5286
+ let out = '';
5287
+ let err = '';
5288
+ let settled = false;
5289
+ const finish = (token) => {
5290
+ if (settled) return;
5291
+ settled = true;
5292
+ clearTimeout(timer);
5293
+ resolve(token);
5294
+ };
5295
+ // Same 10s bound as the sync reader. On expiry SIGTERM the child (matching
5296
+ // spawnSync's timeout semantics) and resolve the transient sentinel — a
5297
+ // wedged gh must never pin the refresh chain, NOR masquerade as a logout.
5298
+ const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(GH_AUTH_READ_FAILED); }, 10_000);
5299
+ if (typeof timer.unref === 'function') timer.unref();
5300
+ child.stdout?.on('data', (d) => { out += d; });
5301
+ // stderr is now CAPTURED (not just drained) so classifyGhAuthRead can tell a
5302
+ // definitive logout (gh's "no oauth token found" message) from a transient
5303
+ // nonzero failure that must preserve the cached credential.
5304
+ child.stderr?.on('data', (d) => { err += d; });
5305
+ child.on('error', () => finish(GH_AUTH_READ_FAILED));
5306
+ child.on('close', (code) => finish(classifyGhAuthRead({ status: code, stdout: out, stderr: err })));
5307
+ });
5308
+ }
5309
+
5310
+ // Async scopes probe mirroring ghTokenScopes (same 5s default bound, same
5311
+ // header parse, same GH_TOKEN-in-env delivery so the token never touches argv).
5312
+ // `_spawn` is a test-only seam.
5313
+ function ghTokenScopesAsync(token, { timeoutMs = 5_000, _spawn = spawn } = {}) {
5314
+ if (!token) return Promise.resolve(null);
5315
+ return new Promise((resolve) => {
5316
+ let child;
5317
+ try {
5318
+ child = _spawn('gh', ['api', '-i', '--include', 'user'], { env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
5319
+ } catch {
5320
+ resolve(null);
5321
+ return;
5322
+ }
5323
+ let out = '';
5324
+ let settled = false;
5325
+ const finish = (scopes) => {
5326
+ if (settled) return;
5327
+ settled = true;
5328
+ clearTimeout(timer);
5329
+ resolve(scopes);
5330
+ };
5331
+ const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(null); }, timeoutMs);
5332
+ if (typeof timer.unref === 'function') timer.unref();
5333
+ child.stdout?.on('data', (d) => { out += d; });
5334
+ child.stderr?.on('data', (d) => { out += d; });
5335
+ child.on('error', () => finish(null));
5336
+ child.on('close', () => {
5337
+ const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
5338
+ finish(m ? m[1].trim() : null);
5339
+ });
5340
+ });
5341
+ }
5342
+
5343
+ // The async twin of refreshGhAuthToken for the periodic timer: same TTL
5344
+ // short-circuit and the same cache record, but the gh subprocesses are spawned
5345
+ // (never spawnSync) so a refresh that DOES run while a job is active yields the
5346
+ // event loop instead of blocking the lock heartbeat (Copilot review on #293).
5347
+ // When a test installed a token READER (hermetic, no network) and did not
5348
+ // inject a scopesProbe, the scopes ping is skipped — mirroring the sync path.
5349
+ async function refreshGhAuthTokenAsync({ force = false, scopesProbe, _spawn } = {}) {
5350
+ const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5351
+ const now = Date.now();
5352
+ if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
5353
+ return { token: prev.token, changed: false, snapshotChanged: false };
5354
+ }
5355
+ // Serialize against any refresh already spawning (another timer tick, or the
5356
+ // reload-credentials SIGUSR1 handler): await IT and report its (already
5357
+ // applied) cache rather than starting a second concurrent `gh` read. This is
5358
+ // what lets the synchronous per-job path safely defer to an in-flight async
5359
+ // refresh instead of blocking (Copilot review on #293).
5360
+ // LOOP (not a one-shot `if`): re-check the marker after EVERY await so forced
5361
+ // refreshes stay serialized. A plain `if` let multiple forced callers all await
5362
+ // the SAME in-flight promise and then fall through together, each overwriting
5363
+ // ghAuthRefreshInFlight with its own read; the first to settle then clears the
5364
+ // marker (its `finally` nulls it) while another forced read is still active, so
5365
+ // the hot path (ghAuthTokenFromCli) sees no marker and resumes a blocking
5366
+ // synchronous re-read — the very stall the marker exists to prevent. Re-checking
5367
+ // in a loop means a forced caller that finds a NEW refresh in flight (started by
5368
+ // another forced caller that fell through first) awaits THAT one too, so at most
5369
+ // one forced read runs at a time and the marker is never cleared out from under
5370
+ // an active read (Copilot review on #293).
5371
+ while (ghAuthRefreshInFlight) {
5372
+ try { await ghAuthRefreshInFlight; } catch { /* swallowed — the owner logs/handles it */ }
5373
+ // A NON-forced caller (a periodic timer tick) is happy with whatever the
5374
+ // in-flight refresh just applied. A FORCED caller (the reload-credentials
5375
+ // SIGUSR1 handler) must NOT be: the in-flight read may have captured the OLD
5376
+ // token microseconds before the operator's `gh auth refresh`/`login` landed,
5377
+ // so returning its result would report "reloaded" while leaving the stale
5378
+ // credential cached until the next TTL. Loop back: once no refresh is in
5379
+ // flight, fall through to start a fresh forced read AFTER the signal.
5380
+ if (!force) {
5381
+ const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5382
+ return { token: c ? c.token : null, changed: false, snapshotChanged: false };
5383
+ }
5384
+ }
5385
+ // Honor the module reader seam when a test installed one (mirroring the sync
5386
+ // path) so a stubbed refresh never spawns; production (reader null) spawns.
5387
+ // Snapshot the cache generation BEFORE the read begins: if a forced sync
5388
+ // recovery commits a newer credential while we read/probe, adoptGhAuthTokenAsync
5389
+ // sees the generation advance and refuses to clobber it (Copilot review on #293).
5390
+ const startGen = ghAuthCacheGeneration;
5391
+ const run = (async () => {
5392
+ const token = ghAuthTokenReader ? ((ghAuthTokenReader() || null)) : await readGhAuthTokenAsync({ _spawn });
5393
+ const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((tok) => ghTokenScopesAsync(tok, { _spawn })));
5394
+ return adoptGhAuthTokenAsync(token, { scopesProbe: probe, generation: startGen });
5395
+ })();
5396
+ // Publish the in-flight marker BEFORE the first await inside `run` has a chance
5397
+ // to yield, so a job activated on the very next tick already sees it. The
5398
+ // marker never rejects (so a concurrent awaiter can't throw on it); the real
5399
+ // result/rejection is surfaced to this caller below.
5400
+ ghAuthRefreshInFlight = run.then(() => {}, () => {});
5401
+ try {
5402
+ return await run;
5403
+ } finally {
5404
+ ghAuthRefreshInFlight = null;
5405
+ }
4808
5406
  }
4809
5407
 
4810
- // Warm the gh-token cache once, off the job-handling path. Safe to call any
4811
- // number of times: the first call performs the (possibly blocking) lookup, the
4812
- // rest are cache hits. Returns true once the cache is populated.
4813
- function primeGhAuthToken() {
4814
- ghAuthTokenFromCli();
5408
+ function ghAuthTokenFromCli() {
5409
+ // If an async refresh is already spawning (a timer tick or the
5410
+ // reload-credentials SIGUSR1 handler), do NOT start a blocking synchronous
5411
+ // re-read on this per-job path — that is exactly the 15s event-loop/lease
5412
+ // stall the async refresh exists to avoid (Copilot review on #293). Hand out
5413
+ // the currently-cached token; the in-flight refresh updates the cache for the
5414
+ // next lookup, and the clone/push auth-failure recovery still force-re-reads
5415
+ // if the handed-out token turns out stale. A not-yet-primed cache (UNSET)
5416
+ // yields null — the same anonymous fallback the refresh would briefly produce.
5417
+ if (ghAuthRefreshInFlight) {
5418
+ return ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache.token;
5419
+ }
5420
+ if (ghAuthTokenCache === GH_AUTH_TOKEN_UNSET) return refreshGhAuthToken({ force: true }).token;
5421
+ // A PRIMED cache is served verbatim — NEVER a synchronous TTL-triggered
5422
+ // re-read on this per-job hot path (Copilot review on #293). The async timer
5423
+ // DEFERS a tick while a job is active, so a skipped tick can leave the cache
5424
+ // past its TTL when the next job arrives; a `refreshGhAuthToken()` here would
5425
+ // then run `spawnSync('gh auth token')` + the synchronous scopes probe inline,
5426
+ // blocking the event loop (and the lock heartbeat) for up to 15s. Startup
5427
+ // priming, the async maintenance timer, and the clone/push auth-failure
5428
+ // recovery (which force-re-reads if the handed-out token turns out stale) own
5429
+ // every refresh; this path only hands out what they populated.
5430
+ return ghAuthTokenCache.token;
5431
+ }
5432
+
5433
+ // Drop the cached gh token so the next lookup re-reads the CLI. A cache-reset
5434
+ // primitive — exported and used by the tests to reset state between cases. NO
5435
+ // production path calls it anymore: both the git clone/push auth-failure
5436
+ // recovery (recoverGhTokenAfterAuthFailure) and the supervisor's
5437
+ // `reload-credentials` broadcast (SIGUSR1) were made deliberately
5438
+ // NON-destructive (a forced re-read that preserves/atomically-swaps the cached
5439
+ // record), so neither drops the cache first (Copilot review on #294).
5440
+ function invalidateGhAuthToken() {
5441
+ writeGhAuthCache(GH_AUTH_TOKEN_UNSET);
5442
+ }
5443
+
5444
+ // Recover from a git/gh auth failure on a gh-cache-derived credential: force a
5445
+ // fresh `gh auth token` read and return it IFF it differs from the credential
5446
+ // that just FAILED (`failingToken`), otherwise null.
5447
+ //
5448
+ // Deliberately does NOT `invalidateGhAuthToken()` first (Copilot review on
5449
+ // #294). An invalidate-before-reread blows the cached record away, so if the
5450
+ // forced re-read then fails TRANSIENTLY (gh momentarily unavailable), there is
5451
+ // no prior record left for `adoptGhAuthToken` to preserve — it falls back to
5452
+ // `{token:null}` and the worker is left "logged out", ERASING a still-usable —
5453
+ // and possibly NEWER (a `reload-credentials` SIGUSR1 swap that landed mid-job)
5454
+ // — cached credential. `refreshGhAuthToken({ force })` already re-reads past the
5455
+ // TTL, and `adoptGhAuthToken` PRESERVES the prior cached token on a transient
5456
+ // read failure, so a transient hiccup keeps the good token (and we even retry
5457
+ // with a newer cached one) instead of discarding it. This mirrors the SIGUSR1
5458
+ // reload-credentials handler, which is likewise deliberately NON-destructive.
5459
+ //
5460
+ // The retry is gated on `fresh !== failingToken` — NOT on the re-read's
5461
+ // `changed` flag (which compares against the CACHE): when the cache already
5462
+ // holds the newer token (the `reload-credentials` swap case), a forced re-read
5463
+ // returns it with `changed:false`, yet that newer token is exactly the
5464
+ // credential we want to retry with. `fresh !== failingToken` also covers the
5465
+ // genuine-denial case (the re-read returns the SAME failing token → null → no
5466
+ // retry) and the null→fresh recovery (a logged-out worker re-reads a real token
5467
+ // after `gh auth login`).
5468
+ // `budgetMs` (threaded by the clone/push recovery call sites from their
5469
+ // remaining lease budget — provTimeoutMs()/netTimeoutMs()) bounds the forced
5470
+ // synchronous `gh auth token` + scopes probe so this on-the-active-job-path
5471
+ // re-read participates in the caller's lease-safe window instead of blocking the
5472
+ // event loop / lock heartbeat for the full 15s (Copilot review on #294).
5473
+ function recoverGhTokenAfterAuthFailure(failingToken, { budgetMs = null } = {}) {
5474
+ const { token: fresh } = refreshGhAuthToken({ force: true, budgetMs });
5475
+ return fresh && fresh !== failingToken ? fresh : null;
5476
+ }
5477
+
5478
+ // A snapshot of the cached token's observable identity for logs and the
5479
+ // supervisor activity marker. Never carries the token value. `ageMs` is the
5480
+ // TOKEN's age (since this value was first seen); `lastCheckAgoMs` is how long
5481
+ // ago gh was last consulted (TTL freshness). Returns null when the cache has
5482
+ // never been populated (a worker that has not primed yet).
5483
+ function ghAuthTokenInfo(now = Date.now()) {
5484
+ const c = ghAuthTokenCache;
5485
+ if (c === GH_AUTH_TOKEN_UNSET) return null;
5486
+ return {
5487
+ present: c.token != null,
5488
+ fingerprint: c.fingerprint ?? null,
5489
+ scopes: c.scopes ?? null,
5490
+ since: c.since ?? null,
5491
+ ageMs: Number.isFinite(c.since) ? Math.max(0, now - c.since) : null,
5492
+ lastCheckAgoMs: Number.isFinite(c.lastCheck) ? Math.max(0, now - c.lastCheck) : null,
5493
+ };
5494
+ }
5495
+
5496
+ // One-line, token-free description for logs: `gho_…` →
5497
+ // "gh auth token: fingerprint ab12cd34ef56 (scopes: gist, read:org, repo; age 2m)".
5498
+ function describeGhAuthToken(info) {
5499
+ if (!info) return 'gh auth token: (not yet read)';
5500
+ if (!info.present) return 'gh auth token: none (gh missing or not logged in)';
5501
+ const bits = [];
5502
+ bits.push(`fingerprint ${info.fingerprint || 'unknown'}`);
5503
+ bits.push(info.scopes ? `scopes: ${info.scopes}` : 'scopes: unknown');
5504
+ if (info.ageMs != null) bits.push(`age ${formatDuration(info.ageMs)}`);
5505
+ return `gh auth token: ${bits.join('; ')}`;
5506
+ }
5507
+
5508
+ // Warm the gh-token cache, off the job-handling path. Safe to call any number of
5509
+ // times: within the TTL it is a no-op cache hit; past it (or after an
5510
+ // invalidation) it re-reads the CLI. With a `logger` it reports the token's
5511
+ // fingerprint + scopes on the FIRST populate and whenever the observable
5512
+ // snapshot moves — a rotated token OR a scopes self-repair (a prior null reading
5513
+ // re-probed to a real value) — the "which credential is this worker holding?"
5514
+ // line from issue #293. Returns true once the cache is populated.
5515
+ function primeGhAuthToken({ logger = null } = {}) {
5516
+ const { snapshotChanged } = refreshGhAuthToken();
5517
+ if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
5518
+ logger.info(describeGhAuthToken(ghAuthTokenInfo()));
5519
+ }
4815
5520
  return ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET;
4816
5521
  }
4817
5522
 
5523
+ // Keep the cache fresh for the worker's lifetime: re-prime on a TTL cadence so
5524
+ // a `gh auth login`/`refresh` the operator runs while the fleet is up is adopted
5525
+ // within one interval — WITHOUT ever blocking the job path (Copilot review on
5526
+ // #293). Two guards make that true:
5527
+ // 1. The refresh itself is ASYNC (refreshGhAuthTokenAsync — spawn, never
5528
+ // spawnSync), so even when a tick fires mid-job the gh subprocesses yield
5529
+ // the event loop instead of freezing the lock heartbeat for up to 15s.
5530
+ // 2. `isIdle` (optional) lets the worker DEFER a tick while a job is active:
5531
+ // the refresh is then simply adopted by the next idle tick (staleness is
5532
+ // bounded by the job length + one interval, and the sync per-job paths —
5533
+ // TTL lookup, auth-failure recovery — remain as the safety net). That defer
5534
+ // is itself BOUNDED (`maxStaleMs`): once a PRIMED cache is stale beyond the
5535
+ // bound, the tick refreshes DESPITE an active job (still async/non-blocking),
5536
+ // so sustained back-to-back jobs can't keep deferring the refresh for the
5537
+ // worker's whole busy lifetime and serve an expired non-null cache to every
5538
+ // job (Copilot review on #294). An UNSET cache is never primed mid-job — the
5539
+ // defer there still holds — so a logged-out worker stays anonymous until idle.
5540
+ // Ticks never overlap (a slow gh can't stack refreshes), a tick failure is
5541
+ // swallowed (the next tick retries), and the change log fires from here so a
5542
+ // rotation is visible even when no job runs. `onChange` (optional) fires on a
5543
+ // genuine observable-snapshot change — a rotated token OR a scopes self-repair
5544
+ // (a prior null scopes reading re-probed to a real value), both of which move
5545
+ // what `supervisor status` should show — so the caller can republish the
5546
+ // supervisor activity marker; an idle worker otherwise keeps reporting the
5547
+ // pre-refresh fingerprint/scopes indefinitely (Copilot review on #293).
5548
+ //
5549
+ // The timer is UNREF'd (like the other maintenance/watchdog intervals): this is
5550
+ // a background credential refresh, NOT a serving handle, so it must never be the
5551
+ // thing that keeps the process alive. `unref()` does not starve the cadence —
5552
+ // contrary to a common misconception, it does not deprioritise or defer the
5553
+ // timer; it only drops the timer's claim on the event loop's "keep the process
5554
+ // alive" accounting. A busy worker's poll/heartbeat fibers keep the loop running,
5555
+ // so an unref'd interval still fires exactly on cadence. The one case where
5556
+ // unref matters is the dangerous one: if the worker's supervisor runtime has
5557
+ // unexpectedly terminated (its serving handles gone), a REF'd refresh timer
5558
+ // would keep the PID alive forever, masking the dead worker so the daemon never
5559
+ // observes the exit and never restarts the non-serving worker — whereas an
5560
+ // unref'd timer lets the now-idle process exit and be restarted. (The `--auto`
5561
+ // poll timer is deliberately the opposite — ref'd — because it IS a serving
5562
+ // handle that must keep a 0-poller worker alive to retry.) The returned handle
5563
+ // carries a `stop()` (clearInterval) so tests can halt it deterministically.
5564
+ function startGhAuthTokenRefresh({ logger = null, intervalMs = GH_AUTH_TOKEN_TTL_MS, isIdle = null, onChange = null, maxStaleMs = GH_AUTH_TOKEN_MAX_DEFER_MS, _spawn } = {}) {
5565
+ // Clamp to a 1s floor so a test can pass a tiny interval; an explicit 0 is
5566
+ // honoured (→ the floor), not treated as "use the default".
5567
+ const raw = Number.isFinite(Number(intervalMs)) ? Number(intervalMs) : GH_AUTH_TOKEN_TTL_MS;
5568
+ const every = Math.max(1000, raw || 1000);
5569
+ // The bounded-defer ceiling: a non-finite/negative value disables the bound
5570
+ // (pure defer-while-busy, the pre-#294 behaviour).
5571
+ const staleBound = Number.isFinite(Number(maxStaleMs)) && Number(maxStaleMs) >= 0 ? Number(maxStaleMs) : Infinity;
5572
+ let inFlight = false;
5573
+ const tick = async () => {
5574
+ if (inFlight) return; // a slow gh must not stack overlapping refreshes
5575
+ if (isIdle && !isIdle()) {
5576
+ // A job is active — normally defer so we don't even spawn gh mid-job. But
5577
+ // bound that defer: a PRIMED cache stale beyond `staleBound` must refresh
5578
+ // anyway (still async/non-blocking) rather than reach every subsequent job
5579
+ // for the worker's whole busy lifetime under sustained saturation (Copilot
5580
+ // review on #294). An UNSET cache is never primed mid-job (no bound applies),
5581
+ // so a logged-out worker stays anonymous until it goes idle, as before.
5582
+ const cached = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
5583
+ const staleBeyondBound = cached && (Date.now() - cached.lastCheck) >= staleBound;
5584
+ if (!staleBeyondBound) return; // defer to the next idle tick
5585
+ }
5586
+ inFlight = true;
5587
+ try {
5588
+ const { snapshotChanged } = await refreshGhAuthTokenAsync({ _spawn });
5589
+ if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
5590
+ logger.info(describeGhAuthToken(ghAuthTokenInfo()));
5591
+ }
5592
+ if (snapshotChanged && onChange) {
5593
+ try { onChange(); } catch { /* best effort — republishing is advisory */ }
5594
+ }
5595
+ } catch {
5596
+ // A failed tick (gh wedged, spawn error) must never crash the worker or
5597
+ // surface as an unhandled rejection — the next tick retries.
5598
+ } finally {
5599
+ inFlight = false;
5600
+ }
5601
+ };
5602
+ const t = setInterval(() => { tick(); }, every);
5603
+ // Unref so a dead worker runtime (serving handles gone) can exit and be
5604
+ // restarted — this maintenance refresh must not keep a non-serving PID alive
5605
+ // (see the header comment). Guarded for fake-timer test harnesses.
5606
+ if (typeof t.unref === 'function') t.unref();
5607
+ t.stop = () => clearInterval(t);
5608
+ return t;
5609
+ }
5610
+
5611
+ // Classify a git/gh failure as an AUTH (or auth-scope) failure — the cases where
5612
+ // the cached credential, not the request, is the problem: a 401/403, git's
5613
+ // credential prompts on an anonymous/rejected clone, or GitHub's OAuth-scope
5614
+ // refusal ("refusing to allow an OAuth App to create or update workflow …
5615
+ // without `workflow` scope" — the exact escalation from issue #293). Matched on
5616
+ // the ALREADY-REDACTED detail string (gitErrorDetail/describeGitFailure output),
5617
+ // so the token value is never inspected here. Deliberately conservative: a
5618
+ // non-fast-forward reject, a missing repo (404) or a network timeout are NOT
5619
+ // auth failures and must not trigger a token drop + retry.
5620
+ function isGitAuthFailure(detail) {
5621
+ const s = String(detail ?? '');
5622
+ if (!s) return false;
5623
+ return /HTTP\s*40[13]\b|\b40[13]\s*\((Unauthorized|Forbidden)\)|returned error:\s*40[13]\b|could not read (Username|Password)|Authentication failed|authentication required|Permission to .+ denied|refusing to allow an OAuth App|without .?workflow.? scope|Bad credentials|Resource not accessible by (integration|personal access token)/i.test(s);
5624
+ }
5625
+
4818
5626
  // Read the operator's own git identity from their GLOBAL config, using the real
4819
5627
  // host environment (process.env) rather than a job's sanitized gitEnv — so it
4820
5628
  // resolves even on the anonymous clone path, where gitEnv points
@@ -4942,6 +5750,82 @@ function normalizeAuthRef(authRef) {
4942
5750
  return { kind: 'custom', name: trimmed };
4943
5751
  }
4944
5752
 
5753
+ // The env-var NAME the github clone/push credential is forwarded to the harness
5754
+ // under. Single source of truth so `resolveJobSecrets` (which builds the
5755
+ // `resolved` secret map) and the clone-recovery rotation glue (which keeps that
5756
+ // map in sync when the token rotates) can NEVER drift on the key: a custom
5757
+ // authRef uses its own name; the default github credential uses GITHUB_TOKEN.
5758
+ // Returns null when no credential is forwarded (a non-github provider with no
5759
+ // custom ref, or a blank/invalid authRef).
5760
+ function ghCredentialEnvName({ provider, authRef }) {
5761
+ const prov = provider || 'github';
5762
+ const ref = normalizeAuthRef(authRef);
5763
+ if (ref.kind === 'invalid') return null;
5764
+ if (ref.kind === 'custom') return ref.name;
5765
+ return prov === 'github' ? 'GITHUB_TOKEN' : null;
5766
+ }
5767
+
5768
+ // When clone recovery ROTATES the cache-derived credential during provisioning
5769
+ // (a stale gh token re-read after an auth failure), the `resolved` secret map —
5770
+ // built BEFORE provisioning and later forwarded to the harness as `secretEnv` —
5771
+ // still holds the STALE token, so an allowPr harness's own `gh`/`git` runs would
5772
+ // use a credential the clone already proved dead (Copilot review on #293). Sync
5773
+ // the map so it hands the harness the SAME credential provisioning just adopted.
5774
+ // Guard rails: only a 'gh-cache'-sourced credential may be swapped (an explicit
5775
+ // `env`/`custom` identity is deliberate and never silently replaced), only when
5776
+ // a token actually exists, and ONLY when that key was already forwarded (never
5777
+ // inject a brand-new secret the job did not request). Pure: mutates and returns
5778
+ // `resolved`.
5779
+ function applyRotatedGhCredential({ resolved, provider, authRef, source, token }) {
5780
+ if (!resolved || source !== 'gh-cache' || !token) return resolved;
5781
+ const key = ghCredentialEnvName({ provider, authRef });
5782
+ if (key && Object.prototype.hasOwnProperty.call(resolved, key)) resolved[key] = token;
5783
+ return resolved;
5784
+ }
5785
+
5786
+ // Resolve the github clone/push credential AND record where it came from. The
5787
+ // default credential (env GITHUB_TOKEN) falls back to the gh CLI's stored token
5788
+ // so `gh auth login` setups work without exporting GITHUB_TOKEN. A custom
5789
+ // `authRef` is honored strictly (env/secret resolver only, no gh fallback) so a
5790
+ // misconfigured named secret surfaces as missing rather than silently borrowing
5791
+ // the operator's gh login. An authRef that is present but blank is a
5792
+ // misconfiguration and yields no token (never the gh fallback). `ghAuthToken`
5793
+ // is injectable for testing.
5794
+ // The returned `source` matters downstream: the #293 auth-failure recovery
5795
+ // (force-re-read `gh auth token`, retry once — NON-destructive, never drops the
5796
+ // cache first) is only legitimate when
5797
+ // the token CAME from that cache ('gh-cache'). Retrying an explicit
5798
+ // GITHUB_TOKEN ('env') or a custom authRef ('custom') failure with the
5799
+ // operator's unrelated gh login would silently swap the configured identity
5800
+ // mid-job (Copilot review on #293), so those sources never take the fallback.
5801
+ // A DEFAULT github lookup that found NO cached gh token ALSO reports
5802
+ // 'gh-cache' (with `token: null`) rather than 'none': it is still the gh-cache
5803
+ // credential path, merely a cache MISS. Tagging it 'gh-cache' keeps the
5804
+ // clone/push auth-failure recovery ELIGIBLE so a worker that cached
5805
+ // "logged out" can still self-heal (a forced re-read recovers null → fresh
5806
+ // token after the operator runs `gh auth login`) instead of failing the job
5807
+ // anonymous and waiting for the periodic TTL refresh (Copilot review on #293).
5808
+ // 'none' is reserved for a non-github provider and an invalid/blank authRef —
5809
+ // neither may borrow the gh login.
5810
+ // Returns { token, source: 'env' | 'custom' | 'gh-cache' | 'none' }.
5811
+ function githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
5812
+ const prov = provider || 'github';
5813
+ const ref = normalizeAuthRef(authRef);
5814
+ if (ref.kind === 'invalid') return { token: null, source: 'none' };
5815
+ const usesDefault = prov === 'github' && ref.kind === 'default';
5816
+ const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
5817
+ const resolved = name ? (secretResolver.resolve(name) || null) : null;
5818
+ if (resolved) return { token: resolved, source: ref.kind === 'custom' ? 'custom' : 'env' };
5819
+ if (!resolved && usesDefault) {
5820
+ const token = ghAuthToken() || null;
5821
+ // A hit AND a miss both report 'gh-cache' (see the header): the miss keeps
5822
+ // the auth-failure recovery eligible so a post-`gh auth login` forced
5823
+ // re-read can still recover this job.
5824
+ return { token, source: 'gh-cache' };
5825
+ }
5826
+ return { token: null, source: 'none' };
5827
+ }
5828
+
4945
5829
  // Resolve the github clone/push credential. The default credential (env
4946
5830
  // GITHUB_TOKEN) falls back to the gh CLI's stored token so `gh auth login`
4947
5831
  // setups work without exporting GITHUB_TOKEN. A custom `authRef` is honored
@@ -4949,16 +5833,10 @@ function normalizeAuthRef(authRef) {
4949
5833
  // secret surfaces as missing rather than silently borrowing the operator's gh
4950
5834
  // login. An authRef that is present but blank is a misconfiguration and yields no
4951
5835
  // token (never the gh fallback). `ghAuthToken` is injectable for testing.
4952
- // Returns a token or null.
5836
+ // Returns a token or null. (Callers that need the credential's SOURCE — e.g. to
5837
+ // gate the #293 gh-token auth-failure retry — use githubCloneTokenWithSource.)
4953
5838
  function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
4954
- const prov = provider || 'github';
4955
- const ref = normalizeAuthRef(authRef);
4956
- if (ref.kind === 'invalid') return null;
4957
- const usesDefault = prov === 'github' && ref.kind === 'default';
4958
- const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
4959
- let token = name ? (secretResolver.resolve(name) || null) : null;
4960
- if (!token && usesDefault) token = ghAuthToken() || null;
4961
- return token;
5839
+ return githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken }).token;
4962
5840
  }
4963
5841
 
4964
5842
  // Clone repo into <runDir>/workspace and check out / create the working branch.
@@ -4973,11 +5851,11 @@ function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghA
4973
5851
  // resume sides agree on which no-create push targets are recoverable (#270).
4974
5852
  const CONVENTIONAL_BASE_BRANCHES = new Set(['main', 'master']);
4975
5853
 
4976
- function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, _runGit = null }) {
5854
+ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, credentialSource = null, _runGit = null }) {
4977
5855
  // Test-only seam mirroring finalizeGit: route every git call through `runGitFn` so a
4978
5856
  // deterministic test can observe the per-call timeouts drawn from the shared
4979
5857
  // provisioning budget below. Defaults to the module `runGit` (production unchanged).
4980
- const runGitFn = _runGit || runGit;
5858
+ let runGitFn = _runGit || runGit;
4981
5859
  // #222: the git ops below are blocking spawnSync calls that cannot observe an
4982
5860
  // AbortSignal mid-call, so provisioning is made signal-aware by rechecking
4983
5861
  // BETWEEN operations: if a lock-loss race won, bail before the NEXT side effect
@@ -5003,7 +5881,9 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5003
5881
  // already-lost lease without starting any repository side effect.
5004
5882
  throwIfAborted('entry');
5005
5883
  const askpass = writeAskpass(runDir, token);
5006
- const gitEnv = {
5884
+ // `let` (not const): the #293 clone auth-failure retry swaps in a FRESH token
5885
+ // (rewritten askpass + GIT_TOKEN) and re-runs the clone with an updated env.
5886
+ let gitEnv = {
5007
5887
  ...process.env,
5008
5888
  GIT_TERMINAL_PROMPT: '0',
5009
5889
  GIT_CONFIG_NOSYSTEM: '1',
@@ -5091,8 +5971,80 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5091
5971
  const detailNote = detail && detail !== 'unknown error' ? ` — last git output: ${detail}` : '';
5092
5972
  throw new ProvisionError(`git clone timed out after ${effectiveTimeoutMs}ms — the repo may be too large, or the network stalled; raise the timeout (repository.cloneTimeoutMs in the envelope, or the worker's --clone-timeout flag) or scope the clone with filter/singleBranch/depth${detailNote}`);
5093
5973
  }
5094
- throw new ProvisionError(`git clone failed: ${gitErrorDetail(clone, token)}`);
5974
+ const detail = gitErrorDetail(clone, token);
5975
+ // #293: an AUTH failure here usually means the cached gh token went stale
5976
+ // (the operator re-logged in / rotated it after this worker primed). Force a
5977
+ // fresh re-read and retry the clone ONCE with it before reporting failure —
5978
+ // a rotated credential self-heals instead of shedding the job. Only when the
5979
+ // re-read yields a token DIFFERENT from the one that failed (a same-token
5980
+ // retry can't fix a genuine permission denial). The recovery is
5981
+ // NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
5982
+ // the cache first, so a transiently-failing re-read can't erase a still-valid
5983
+ // — possibly newer — cached credential (Copilot review on #294). The retry
5984
+ // re-runs the whole clone with the new token threaded through the askpass
5985
+ // env, re-anchored to the remaining provisioning budget.
5986
+ // GATED on the credential's SOURCE (Copilot review on #293): this fallback
5987
+ // replaces the failed token with the operator's `gh auth token`, so it is
5988
+ // only legitimate when the failed token CAME from that same gh cache
5989
+ // ('gh-cache'). An explicit GITHUB_TOKEN ('env') or custom authRef
5990
+ // ('custom') is a deliberately configured identity — silently swapping it
5991
+ // for the operator's gh login would violate githubCloneToken's documented
5992
+ // precedence and could act as a different identity, so those sources fail
5993
+ // without the gh retry.
5994
+ // The `token` guard is deliberately ABSENT (Copilot review on #293): a
5995
+ // default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
5996
+ // a worker that cached "logged out" recovers null → fresh token after the
5997
+ // operator runs `gh auth login` instead of failing the job anonymous. That
5998
+ // recovery must REBUILD the credential plumbing the anonymous path omitted
5999
+ // (the askpass helper, GIT_ASKPASS/GIT_TOKEN, and the username-embedded
6000
+ // remote) — a bare GIT_TOKEN swap leaves the clone unauthenticated.
6001
+ if (credentialSource === 'gh-cache' && isGitAuthFailure(detail)) {
6002
+ const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: provTimeoutMs() });
6003
+ if (fresh) {
6004
+ logger?.info?.(`git clone auth failed${cs} — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the clone once`);
6005
+ throwIfAborted('clone-auth-retry');
6006
+ // Rebuild the credential env for the fresh token. On the null→fresh
6007
+ // recovery the original env was the ANONYMOUS one (no askpass helper,
6008
+ // GIT_CONFIG_GLOBAL neutralized), so re-mint the askpass helper and
6009
+ // restore global config; on a stale-token rotation this is a no-op
6010
+ // re-point of the same GIT_TOKEN.
6011
+ const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
6012
+ const retryAskpass = writeAskpass(runDir, fresh);
6013
+ if (retryAskpass) {
6014
+ retryEnv.GIT_ASKPASS = retryAskpass;
6015
+ // The anonymous path set GIT_CONFIG_GLOBAL=/dev/null so a no-token
6016
+ // clone can't inherit operator credentials; the fresh token is ours,
6017
+ // so global config (e.g. http.proxy) may apply again.
6018
+ delete retryEnv.GIT_CONFIG_GLOBAL;
6019
+ }
6020
+ // Rebuild the clone argv so the remote carries the fresh credential's
6021
+ // username (authUrl embeds x-access-token only when a token is present);
6022
+ // the anonymous remote had no username, so git would never consult the
6023
+ // askpass helper for it. `workspaceDir` stays last; the remote is the
6024
+ // argument immediately before it (cloneArgs.push(remote, workspaceDir)).
6025
+ const retryCloneArgs = cloneArgs.slice();
6026
+ retryCloneArgs[retryCloneArgs.length - 2] = authUrl(repo.url, repo.provider || 'github', true);
6027
+ const retry = runGitFn(retryCloneArgs, { env: retryEnv, timeoutMs: provTimeoutMs() });
6028
+ if (retry.status === 0) {
6029
+ // Adopt the fresh token for the REST of provisioning + the returned
6030
+ // gitEnv (finalizeGit pushes with it) so the whole job runs on the
6031
+ // rotated credential.
6032
+ token = fresh;
6033
+ gitEnv = retryEnv;
6034
+ return provisionRepoAfterClone();
6035
+ }
6036
+ throw new ProvisionError(`git clone failed after a fresh gh token retry: ${gitErrorDetail(retry, fresh)}`);
6037
+ }
6038
+ }
6039
+ throw new ProvisionError(`git clone failed: ${detail}`);
5095
6040
  }
6041
+ return provisionRepoAfterClone();
6042
+
6043
+ // Everything after a successful clone — hoisted into a closure so the
6044
+ // auth-retry path above re-enters it without duplicating the body. The clone
6045
+ // result itself is not read below (only its success mattered); the workspace
6046
+ // on disk is the real output.
6047
+ function provisionRepoAfterClone() {
5096
6048
 
5097
6049
  // #222: the clone (the dominant blocking op and the first repo side effect) is
5098
6050
  // done. Recheck the abort signal before the remaining side effects/network ops
@@ -5626,7 +6578,16 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5626
6578
  // the authoritative result so eligibility (auto checkpointing) doesn't
5627
6579
  // under-detect an authenticated clone.
5628
6580
  const authenticated = !!token || isAuthenticatedRemote(repo.url);
5629
- return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
6581
+ // `effectiveToken` is the credential the workspace ACTUALLY runs on after any
6582
+ // #293 auth-failure retry rotated it mid-provisioning (the local `token` is
6583
+ // reassigned on adoption). The caller (workAgent) must use THIS — not its own
6584
+ // pre-provisioning resolution — for the post-provisioning credential users
6585
+ // (finalizeGit's PR reconcile/push, the checkpoint GC/ref-delete askpass), or
6586
+ // a recovered clone would keep failing with the stale credential (Copilot
6587
+ // review on #293). `credentialSource` is echoed back so finalizeGit can gate
6588
+ // its own push-retry on the same source rule.
6589
+ return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, effectiveToken: token || null, credentialSource, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
6590
+ } // end provisionRepoAfterClone
5630
6591
  }
5631
6592
 
5632
6593
  // Look up a PR for this branch (2a does NOT open it — the harness does, driven
@@ -5978,6 +6939,17 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
5978
6939
  restored,
5979
6940
  priorNotRestored,
5980
6941
  flushTimeoutMs: cfg.flushTimeoutMs,
6942
+ // Adopt a credential the job ROTATED after this checkpointer was created —
6943
+ // e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
6944
+ // (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
6945
+ // its own askpass purely from `token`, so re-binding `token` here is enough
6946
+ // to make the cleanup push authenticate with the live credential instead of
6947
+ // the one the remote already rejected — otherwise the WIP ref lingers until a
6948
+ // later GC (Copilot review on #293). Ignored when the token is unchanged or
6949
+ // falsy; the live checkpointer has already stopped by the time this is called.
6950
+ updateCredential(nextToken) {
6951
+ if (nextToken && nextToken !== token) token = nextToken;
6952
+ },
5981
6953
  // True when the workspace still holds checkpoint content NOT captured by a
5982
6954
  // commit, so the pushed branch does not represent it (an uncommitted restored
5983
6955
  // or working delta that lives only on the WIP ref). The discard-on-ack gate
@@ -6069,7 +7041,7 @@ function computeRelayCloseReason(result, runCompleted, gitResult) {
6069
7041
  // branch.push), and reconcile the agent-opened PR (when task.allowPr). A push
6070
7042
  // failure is reported (pushError) rather than thrown — the process model decides
6071
7043
  // what to do next, and re-running the agent would be non-idempotent.
6072
- function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, _runGit = null }) {
7044
+ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, credentialSource = null, _runGit = null, _pushGit = null, _reconcilePr = null, _postAttribution = null }) {
6073
7045
  const log = logger || getLogger(); // route observability through the injected logger (tests) or the host logger
6074
7046
  // Test-only seam: the late remote-reachability FILTER scans (movedStray /
6075
7047
  // unpushedBranchCommits / final stranded) AND the remote-tip VERIFICATION calls
@@ -6080,8 +7052,25 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6080
7052
  // probes cannot be made to fail deterministically. Defaults to the module `runGit`, so
6081
7053
  // production behaviour is unchanged.
6082
7054
  const lateScanGit = _runGit || runGit;
7055
+ // Test-only seam for the branch PUSH itself (the initial push AND the single
7056
+ // post-auth-failure retry). Routed separately from `lateScanGit` so a
7057
+ // hermetic test can drive the central push-recovery branches — changed-token
7058
+ // retry success, the unchanged-token guard, the credential-source gate, and
7059
+ // retry-still-fails — without a live remote, while the local enumeration and
7060
+ // late remote-reachability scans keep their own runner (Copilot review on
7061
+ // #293). Defaults to the module `runGit`, so production behaviour is unchanged.
7062
+ const pushGit = _pushGit || runGit;
7063
+ const reconcilePr = _reconcilePr || reconcileAgentPr;
7064
+ const postAttribution = _postAttribution || postAgentAttribution;
6083
7065
  const cs = corr ? ` [${corr}]` : ''; // #229 cross-channel correlation suffix
6084
7066
  const out = { branch: workingBranch, baseSha: startSha || null, headSha: null, commits: [], pushed: false, remote: null, pr: null };
7067
+ // The credential this finalize was HANDED. The push auth-failure recovery below
7068
+ // may rotate `token` to a freshly re-read gh credential mid-finalize; capturing
7069
+ // the original lets the return expose `rotatedToken` so the CALLER can propagate
7070
+ // the rotation to subsystems that captured the pre-rotation credential — notably
7071
+ // the WIP-checkpoint cleanup, whose `discardAfterAck()` would otherwise push a
7072
+ // ref deletion with the already-rejected token (Copilot review on #293).
7073
+ const originalToken = token;
6085
7074
 
6086
7075
  // Bound EVERY finalize git op against ONE shared deadline derived from the recovery
6087
7076
  // window (budgetMs) — the LOCAL graph scans below AND the sequential NETWORK ops
@@ -6465,7 +7454,100 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6465
7454
  // qualified heads ref resolves unambiguously; --set-upstream still tracks the
6466
7455
  // local branch.
6467
7456
  const pushTimeout = netTimeoutMs();
6468
- const push = runGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
7457
+ let push = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
7458
+ // #293: an AUTH rejection (401/403, or the OAuth `workflow`-scope refusal)
7459
+ // usually means the cached gh token went stale mid-run. Force a fresh
7460
+ // re-read and retry the push ONCE with it before stranding the commits — a
7461
+ // rotated credential self-heals instead of a `pushFailed`. Only when the
7462
+ // re-read yields a token DIFFERENT from the one that failed (a same-token
7463
+ // retry can't fix a genuine permission denial), and the retry runs with the
7464
+ // fresh token in GIT_TOKEN so the askpass hands it to git. The recovery is
7465
+ // NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
7466
+ // the cache first, so a transiently-failing re-read can't erase a still-valid
7467
+ // — possibly newer — cached credential (Copilot review on #294).
7468
+ // GATED on the credential's SOURCE (Copilot review on #293): only a token
7469
+ // that CAME from the gh cache ('gh-cache') may be replaced by a fresh
7470
+ // `gh auth token` read — an explicit GITHUB_TOKEN ('env') or custom authRef
7471
+ // ('custom') is a deliberately configured identity that must fail as
7472
+ // configured, never be silently swapped for the operator's gh login.
7473
+ // The `token` guard is deliberately ABSENT (Copilot review on #293): a
7474
+ // default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
7475
+ // a worker that cached "logged out" recovers null → fresh token after the
7476
+ // operator runs `gh auth login` instead of stranding the commits. That
7477
+ // recovery must REBUILD the credential plumbing the anonymous gitEnv lacked
7478
+ // (the askpass helper + GIT_ASKPASS, and global git config) — a bare
7479
+ // GIT_TOKEN swap leaves the push unauthenticated.
7480
+ // The credential that PRODUCED the ORIGINAL `push` — preserved separately so
7481
+ // the failure report below redacts the value that actually appeared in that
7482
+ // push's output, even once `token` is swapped to the fresh credential for
7483
+ // subsequent authenticated operations after a retry (Copilot review on #293).
7484
+ let pushFailureToken = token;
7485
+ // The timeout that bounded the reported `push` — promoted alongside
7486
+ // pushFailureToken when a failed retry is promoted to the reported result, so
7487
+ // describeGitFailure attributes the RETRY's timeout, not the original push's
7488
+ // (Copilot review on #293).
7489
+ let pushFailureTimeout = pushTimeout;
7490
+ if (push.status !== 0 && credentialSource === 'gh-cache' && isGitAuthFailure(describeGitFailure('git push', push, { token, timeoutMs: pushTimeout }))) {
7491
+ const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: netTimeoutMs() });
7492
+ if (fresh) {
7493
+ log.info?.(`finalizeGit${cs}: git push auth failed — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the push once`);
7494
+ const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
7495
+ // Reuse the clone-time askpass helper when gitEnv already carries one
7496
+ // (the token-backed path): the helper emits GIT_TOKEN at runtime, so
7497
+ // pointing GIT_TOKEN at the fresh credential is enough — no rewrite.
7498
+ // On the null→fresh recovery the original gitEnv was the ANONYMOUS one
7499
+ // (no askpass helper, GIT_CONFIG_GLOBAL neutralized), so mint one and
7500
+ // restore global config. Mint it in the PARENT run directory
7501
+ // (dirname(workspaceDir) === runDir), NEVER inside workspaceDir: writing
7502
+ // askpass.js/.sh into the checked-out repo after the final commits leaves
7503
+ // untracked files that make hasUncommittedChanges() retain the WIP
7504
+ // checkpoint and pollute a preserved workspace (Copilot review on #294).
7505
+ if (!retryEnv.GIT_ASKPASS) {
7506
+ const retryAskpass = writeAskpass(dirname(workspaceDir), fresh);
7507
+ if (retryAskpass) {
7508
+ retryEnv.GIT_ASKPASS = retryAskpass;
7509
+ delete retryEnv.GIT_CONFIG_GLOBAL;
7510
+ }
7511
+ }
7512
+ const retryTimeout = netTimeoutMs();
7513
+ const retry = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: retryEnv, timeoutMs: retryTimeout });
7514
+ // Rebind the git env to the FRESH credential whenever the retry ran —
7515
+ // regardless of its exit code — so the post-push remote-tip verification
7516
+ // below (`ls-remote`/`fetch`, all `env: gitEnv`) queries origin with the
7517
+ // credential the retry used, not the REJECTED old token. A retry that
7518
+ // exits nonzero because the server accepted the ref before a client-side
7519
+ // timeout would otherwise be verified with the stale token, ls-remote
7520
+ // would fail to authenticate, and a landed push would be misreported as
7521
+ // pushFailed/stranded (Copilot review on #293).
7522
+ gitEnv = retryEnv;
7523
+ if (retry.status === 0) {
7524
+ // Adopt the fresh token for the rest of finalization (the PR reconcile
7525
+ // + attribution below read `token`) and record the success.
7526
+ token = fresh;
7527
+ push = retry;
7528
+ } else {
7529
+ // The retry ALSO exited nonzero. It may still have LANDED the ref before
7530
+ // a client-side timeout (the transport-hiccup case verified below); if it
7531
+ // did, `landed` flips on and pushError is never reported. But if it did
7532
+ // NOT land, the RETRY — not the original stale-token rejection — is the
7533
+ // failure that actually stranded the commits: the original failure is
7534
+ // already RECOVERED (we re-read the token and retried), so reporting it
7535
+ // would point at a resolved cause instead of the real one. Promote `retry`
7536
+ // to the reported push result and carry ITS redaction credential (`fresh`,
7537
+ // which appeared in the retry's output) and ITS timeout into the final
7538
+ // error. The ORIGINAL rejected token must NOT be carried into the
7539
+ // subsequent authenticated operations (PR reconcile + attribution read
7540
+ // `token`): a landed retry would otherwise reconcile/attribute with the
7541
+ // rejected old token and fail despite the push. Adopt `fresh` for all
7542
+ // later ops; `gitEnv` is already rebound above (Copilot review on #293).
7543
+ push = retry;
7544
+ pushFailureToken = fresh;
7545
+ pushFailureTimeout = retryTimeout;
7546
+ token = fresh;
7547
+ log.warn?.(`finalizeGit${cs}: push still failed with the fresh gh token — ${oneLineLog(describeGitFailure('git push', retry, { token: fresh, timeoutMs: retryTimeout }))}`);
7548
+ }
7549
+ }
7550
+ }
6469
7551
  if (push.status === 0) out.pushed = true;
6470
7552
  else {
6471
7553
  // A nonzero `git push` does NOT prove the ref was not updated: a timeout or
@@ -6547,7 +7629,7 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6547
7629
  // new commit in this throwaway workspace. Surface the at-risk SHAs explicitly
6548
7630
  // (issue #231, defense #2) so the failure is a hard, actionable signal with
6549
7631
  // the commits to recover — not a soft pushError that reads as "completed".
6550
- out.pushError = describeGitFailure('git push', push, { token, timeoutMs: pushTimeout });
7632
+ out.pushError = describeGitFailure('git push', push, { token: pushFailureToken, timeoutMs: pushFailureTimeout });
6551
7633
  out.pushFailed = true;
6552
7634
  // Union the work-branch commit list into the stranded set: out.commits is now
6553
7635
  // the promoted branchCommits, but if HEAD sat off the branch and promotion
@@ -6627,13 +7709,18 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
6627
7709
  // envelope + completion note carry only the `pushFailed`/`strandedCommits`
6628
7710
  // recovery signal, not a misleading PR reference.
6629
7711
  if (workingBranch && envelope.task?.allowPr && !out.pushFailed) {
6630
- out.pr = reconcileAgentPr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
7712
+ out.pr = reconcilePr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
6631
7713
  // Record the agent's authorship on the PR (commits carry the operator's
6632
7714
  // identity now, so this preserves the machine-generated provenance).
6633
7715
  if (out.pr?.found && out.pr.number && (envelope.repository?.provider || 'github') === 'github') {
6634
- out.attribution = postAgentAttribution({ workspaceDir, token, number: out.pr.number });
7716
+ out.attribution = postAttribution({ workspaceDir, token, number: out.pr.number });
6635
7717
  }
6636
7718
  }
7719
+ // Surface a mid-finalize credential rotation (the push auth-failure recovery
7720
+ // re-read a fresh gh token) so the caller can propagate it to subsystems that
7721
+ // captured the pre-rotation credential — the WIP-checkpoint cleanup in
7722
+ // particular (Copilot review on #293). Absent when the token was never rotated.
7723
+ if (token && token !== originalToken) out.rotatedToken = token;
6637
7724
  return out;
6638
7725
  }
6639
7726
 
@@ -9578,9 +10665,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
9578
10665
  * `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
9579
10666
  * longer running here, so it rides the marker as a flagged, observational entry
9580
10667
  * without making an idle worker look busy or inflating the drain's in-flight count.
9581
- * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object }} fields
10668
+ * @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
9582
10669
  */
9583
- function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }) {
10670
+ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
9584
10671
  const jobList = Array.isArray(jobs) ? jobs : [];
9585
10672
  return {
9586
10673
  pid,
@@ -9593,6 +10680,11 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }
9593
10680
  // SDK client and begun leasing), the producer stamps this; null until then.
9594
10681
  // The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
9595
10682
  readyAt: readyAt ?? null,
10683
+ // #293: the cached gh auth token's observable identity — { present,
10684
+ // fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
10685
+ // `supervisor status` can show the credential's age/scopes per worker. Null
10686
+ // on a worker that has not primed the cache yet (or predates the field).
10687
+ ghAuth: ghAuth ?? null,
9596
10688
  };
9597
10689
  }
9598
10690
 
@@ -10193,10 +11285,27 @@ async function workAgent(req, flags, ctx) {
10193
11285
  // for why per-job setup.env is deliberately excluded), so PATH resolves a stable
10194
11286
  // representative harness binary.
10195
11287
  const agentInstanceProbeOff = String(process.env.NANO_AGENT_INSTANCE || '').trim().toLowerCase() === 'off';
11288
+ // Harness transport protocol, resolved with env-override-then-profile precedence and
11289
+ // reduced to the EFFECTIVE protocol the worker actually runs (a container sandbox is
11290
+ // pipe-only regardless of `protocol`). Computed here — ahead of the durable-transcript
11291
+ // diagnostic below and the #275 mismatch refusal later — so the startup "ACP note" is
11292
+ // keyed on the transport the worker WILL use, not a brittle reparse of the command
11293
+ // line (issue #289 review).
11294
+ const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
11295
+ const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
11296
+ const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
10196
11297
  const sdkSupportsAgentInstance =
10197
11298
  !!camunda &&
10198
11299
  typeof camunda.createAgentInstance === 'function' &&
10199
11300
  typeof camunda.updateAgentInstance === 'function';
11301
+ // #289: best-effort host c8ctl / SDK versions, so the startup warning and the
11302
+ // per-job "producer unavailable" diagnostic can name the package that must be
11303
+ // upgraded (the macbook workers ran @camunda8/cli 4.1.0-alpha.2 bundling
11304
+ // @camunda8/orchestration-cluster-api 9.1.5, which has no AgentInstance verbs).
11305
+ // Resolved from the HOST c8ctl CLI's dependency graph (rooted at the real entry /
11306
+ // CLI package), not this plugin's sibling node_modules, so the nested host SDK
11307
+ // actually resolves (issue #289 review). Null when unresolvable.
11308
+ const hostSdkInfo = probeHostSdkInfo();
10200
11309
  // The producer's `usable` gate also requires the ACP classifier (agent-instance.mjs);
10201
11310
  // without it the producer is inert regardless of SDK support, so exclude the probe too.
10202
11311
  const acpClassifierAvailable = typeof agenticSessionAcp?.classifyUpdate === 'function';
@@ -10267,13 +11376,113 @@ async function workAgent(req, flags, ctx) {
10267
11376
  const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
10268
11377
  logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
10269
11378
  logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
11379
+ // #289: fail LOUDLY at worker start — not silently per job — when this worker
11380
+ // cannot record a durable AgentInstance transcript on an agentic profile. Every
11381
+ // `nano work` profile runs an agent harness, so any worker that services external
11382
+ // agent jobs needs the host c8ctl SDK's createAgentInstance/updateAgentInstance
11383
+ // verbs. When the host SDK lacks them (e.g. @camunda8/cli 4.1.0-alpha.2 bundling
11384
+ // @camunda8/orchestration-cluster-api 9.1.5), EVERY job silently records no
11385
+ // transcript — the exact macbook-fleet defect. Name the host versions and the fix so
11386
+ // it is actionable from the startup log alone, rather than a per-job "producer
11387
+ // unavailable" line that merges two causes. NANO_AGENT_INSTANCE=off suppresses the
11388
+ // warning (transcripts were intentionally disabled).
11389
+ if (!agentInstanceProbeOff) {
11390
+ if (!sdkSupportsAgentInstance) {
11391
+ logger.warn(
11392
+ `[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the host c8ctl SDK lacks ` +
11393
+ `createAgentInstance/updateAgentInstance, so NO AgentInstance transcript will be ` +
11394
+ `recorded for any job this worker runs. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}. ` +
11395
+ `Upgrade the host c8ctl (the plugin version is irrelevant — the SDK ships with c8ctl): ` +
11396
+ `@camunda8/cli must bundle @camunda8/orchestration-cluster-api >= 10.0.0-alpha.45 ` +
11397
+ `(the first with the AgentInstance verbs). Set NANO_AGENT_INSTANCE=off to silence this ` +
11398
+ `if transcripts are intentionally disabled.`,
11399
+ );
11400
+ } else if (!acpClassifierAvailable) {
11401
+ logger.warn(
11402
+ `[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the ACP classifier ` +
11403
+ `(agenticSessionAcp.classifyUpdate) is unavailable, so no AgentInstance transcript ` +
11404
+ `will be recorded. This is unexpected (it is a static import) — the plugin install may ` +
11405
+ `be corrupt. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}.`,
11406
+ );
11407
+ } else {
11408
+ // Transcripts CAN be recorded, but only for an ACP harness: a non-ACP harness
11409
+ // emits no session/update stream for the producer to classify, so it records
11410
+ // only the opening CONFIGURATION turn (no body). Surface this once at startup so
11411
+ // an operator running e.g. `copilot --allow-all` (no --acp) knows why a run has a
11412
+ // header-only transcript. Key the note on the EFFECTIVE transport the worker will
11413
+ // actually use (`effectiveProtocol === 'acp'`), NOT a reparse of the command line:
11414
+ // a `protocol: acp` profile is driven over ACP even when the command carries no
11415
+ // literal `--acp` (`ensureAcpFlag` appends it at spawn), and a hand-reparse also
11416
+ // misses supported selectors (`acp` subcommand, `*-acp` adapter, `--experimental-acp`)
11417
+ // while a container sandbox runs pipe-only regardless of `protocol` (issue #289 review).
11418
+ const protoNote = effectiveProtocol === 'acp'
11419
+ ? ''
11420
+ : ` NOTE: the harness does not run in ACP mode (effective protocol "${effectiveProtocol}") — a non-ACP harness emits no ACP session/update stream, so only the opening CONFIGURATION turn is recorded (no body). Run the harness in ACP mode (--protocol acp on the host executor) for a full transcript.`;
11421
+ logger.info(`[${name}] Durable agent transcripts enabled (host SDK: ${formatHostSdkInfo(hostSdkInfo)}).${protoNote}`);
11422
+ }
11423
+ }
10270
11424
  // Warm the gh-token cache now, off the job-handling path: githubCloneToken()
10271
11425
  // may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
10272
11426
  // credential fallback, and doing that inside a job handler would block the
10273
11427
  // event loop — stalling the lock heartbeat and delaying the job itself.
10274
11428
  // Priming here pays that cost once at startup so every later lookup is a warm
10275
- // cache hit.
10276
- primeGhAuthToken();
11429
+ // cache hit. Pass the logger so the token's fingerprint + scopes (never the
11430
+ // value) land in the worker log at startup — issue #293's "which credential is
11431
+ // this worker actually holding?" line.
11432
+ primeGhAuthToken({ logger });
11433
+ // …and keep it fresh for the worker's lifetime (issue #293): a process-lifetime
11434
+ // memo went silently stale the moment the operator refreshed the gh login, and
11435
+ // every job kept getting the OLD token until a restart. The TTL timer re-reads
11436
+ // `gh auth token` — ASYNCHRONOUSLY (spawn, never spawnSync) and DEFERRED while
11437
+ // a job is active (`isIdle`) — so the refresh can never block the event loop /
11438
+ // lock heartbeat the way a synchronous prime on the interval could (Copilot
11439
+ // review on #293). When the cached identity actually changes, `onChange`
11440
+ // republishes the activity marker so `supervisor status` shows the adopted
11441
+ // credential even while the worker sits idle. The supervisor's
11442
+ // `reload-credentials` broadcast (SIGUSR1) forces the same re-read on demand
11443
+ // (non-destructive — an atomic swap, never a drop).
11444
+ // `activeJobs`/`writeActivity` are declared below this point but are only
11445
+ // dereferenced when a tick fires (≥1s later) or a change lands — long after
11446
+ // the whole module body has run — so the closures capture the initialized
11447
+ // bindings (TDZ-safe). The handle is tracked so the stop-signal teardown can
11448
+ // clear it (a ref'd interval would otherwise keep the process alive past
11449
+ // SIGTERM/SIGINT — the --auto liveness test relies on the poll timer being
11450
+ // the sole keepalive).
11451
+ const ghRefreshTimer = startGhAuthTokenRefresh({
11452
+ logger,
11453
+ isIdle: () => activeJobs.size === 0,
11454
+ onChange: () => writeActivity(),
11455
+ });
11456
+ // `supervisor reload-credentials` asks every worker to re-read its gh token
11457
+ // without draining in-flight jobs. The daemon signals SIGUSR1; the handler
11458
+ // re-reads the CLI ASYNCHRONOUSLY (spawn, never spawnSync) and serialized
11459
+ // against the TTL timer, so even a worker mid-job keeps its event loop / lock
11460
+ // heartbeat responsive while the new credential is adopted. Deliberately NOT
11461
+ // the SIGUSR2 drain path — reloading a credential must not park the fleet.
11462
+ // Windows cannot deliver SIGUSR1 (Node terminates the process), so the daemon
11463
+ // refuses the op there. `writeActivityRef` is late-bound to `writeActivity`
11464
+ // once it is defined below so the handler can republish the token's
11465
+ // age/scopes once the async re-read settles.
11466
+ let writeActivityRef = null;
11467
+ if (process.platform !== 'win32') {
11468
+ process.on('SIGUSR1', () => {
11469
+ // Use the ASYNC refresh (spawn, never spawnSync) so re-reading/replacing
11470
+ // the cached credential can never pause the event loop / lock heartbeat of a
11471
+ // worker servicing a job — the whole promise of `reload-credentials` is
11472
+ // that it does NOT disrupt in-flight jobs (Copilot review on #293). It is
11473
+ // serialized against the periodic timer via `ghAuthRefreshInFlight`.
11474
+ // Deliberately NOT `invalidateGhAuthToken()` first: the async `force` read
11475
+ // re-reads `gh auth token` regardless of TTL and swaps the cache ATOMICALLY
11476
+ // when the new value lands, so the worker keeps serving the still-valid old
11477
+ // token for the ~10s read window instead of handing out null meanwhile.
11478
+ refreshGhAuthTokenAsync({ force: true })
11479
+ .then(({ changed }) => {
11480
+ logger.info(`Credentials reload requested — re-read the gh auth token: ${describeGhAuthToken(ghAuthTokenInfo())}${changed ? ' (token changed)' : ''}`);
11481
+ if (writeActivityRef) writeActivityRef();
11482
+ })
11483
+ .catch(() => { /* best effort — a wedged gh must not crash the worker */ });
11484
+ });
11485
+ }
10277
11486
  logger.info('Polling for work — press Ctrl-C to stop.');
10278
11487
 
10279
11488
  // When launched under the supervisor, report per-job activity — which job(s)
@@ -10458,7 +11667,10 @@ async function workAgent(req, flags, ctx) {
10458
11667
  // Bound the last-activation guard map (TTL + settle-aware size cap). Shared
10459
11668
  // with recordJobStart so a standalone worker still prunes it.
10460
11669
  pruneLastActivation(nowMs);
10461
- const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt });
11670
+ // #293: publish the cached gh token's identity (fingerprint/scopes/age —
11671
+ // never the value) so `supervisor status` can answer "is this worker holding
11672
+ // a stale token?" in one line.
11673
+ const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
10462
11674
  const tmp = `${activityFile}.${process.pid}.tmp`;
10463
11675
  try {
10464
11676
  mkdirSync(dirname(activityFile), { recursive: true });
@@ -10469,6 +11681,9 @@ async function workAgent(req, flags, ctx) {
10469
11681
  /* best effort — activity is advisory, never fail a job over it */
10470
11682
  }
10471
11683
  };
11684
+ // Late-bind the SIGUSR1 (reload-credentials) handler's republish hook now that
11685
+ // writeActivity exists (#293).
11686
+ writeActivityRef = writeActivity;
10472
11687
  // The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
10473
11688
  // host connection carries presence/steer/ownership/transcript for EVERY
10474
11689
  // supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
@@ -10745,14 +11960,14 @@ async function workAgent(req, flags, ctx) {
10745
11960
  logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
10746
11961
  }
10747
11962
 
10748
- // #110: the role's harness protocol (pipe|acp) and ACP permission policy
10749
- // (yolo|escalate|filter), resolved with the same env-override-then-profile
10750
- // precedence as terminal. `NANO_AGENTIC_PROTOCOL`/`NANO_AGENTIC_PERMISSION`
10751
- // override a one-off worker; otherwise the hire profile decides; else the safe
10752
- // defaults (pipe/yolo). escalate/filter are carried through verbatim — the
10753
- // acp-executor task enforces yolo and interim-handles the reserved policies.
10754
- const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
10755
- const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
11963
+ // #110: the role's ACP permission policy (yolo|escalate|filter), resolved with
11964
+ // the same env-override-then-profile precedence as terminal. The harness protocol
11965
+ // (`envProtocol`/`roleProtocol`/`effectiveProtocol`) is resolved earlier, ahead of
11966
+ // the durable-transcript startup diagnostic (issue #289 review).
11967
+ // `NANO_AGENTIC_PERMISSION` overrides a one-off worker; otherwise the hire profile
11968
+ // decides; else the safe default (yolo). escalate/filter are carried through
11969
+ // verbatim — the acp-executor task enforces yolo and interim-handles the reserved
11970
+ // policies.
10756
11971
  const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
10757
11972
  const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
10758
11973
 
@@ -10767,8 +11982,7 @@ async function workAgent(req, flags, ctx) {
10767
11982
  // container profile's command STILL pipes plain JSON to an ACP harness — the
10768
11983
  // same husk. `effectiveHarnessProtocol` reduces a container's protocol to `pipe`
10769
11984
  // so the refusal covers that pipe-only executor too, not just the host (thread
10770
- // 4118327651).
10771
- const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
11985
+ // 4118327651). `effectiveProtocol` is computed earlier (issue #289 review).
10772
11986
  const startupMismatch = detectProtocolMismatch({ command: profile.command, args: effectiveArgs, protocol: effectiveProtocol });
10773
11987
  if (startupMismatch) {
10774
11988
  // Non-restartable config failure: under the supervisor this must exit with
@@ -10894,7 +12108,14 @@ async function workAgent(req, flags, ctx) {
10894
12108
  // Assemble + normalize the task envelope from headers (defaults) and
10895
12109
  // variables (overrides), then resolve any secrets it references.
10896
12110
  const envelope = normalizeTaskEnvelope(job.customHeaders ?? {}, job.variables ?? {}, { basePromptOverride });
10897
- const { resolved, missing, names } = resolveJobSecrets(secretResolver, envelope);
12111
+ const { resolved, missing, names } = await resolveJobSecrets(secretResolver, envelope);
12112
+ // #222: recheck after the awaited secret resolution (stage: secrets) —
12113
+ // resolveJobSecrets AWAITS the gh-cache-miss forced refresh, so a
12114
+ // force-stop / lease loss can win DURING that await. Gate BEFORE the
12115
+ // missing-secret settlement below so an abort that landed mid-resolve
12116
+ // returns WITHOUT settling (which would race the force-stop yield),
12117
+ // mirroring the post-prompt-fetch gate above.
12118
+ if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'secrets', logger })) return;
10898
12119
  if (missing.length > 0) {
10899
12120
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
10900
12121
  const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
@@ -10998,7 +12219,21 @@ async function workAgent(req, flags, ctx) {
10998
12219
  } else if (agentInstanceProducer.retryPending) {
10999
12220
  logger.info(`[${jobType}] AgentInstance producer not yet active for external agent job (${aiCorr}) — the initial createAgentInstance did not mint a key in time (transient rejection, no-key response, or bounded-out create), but the create stays armed to retry on the ACP hot path, so a run that recovers can still record a durable transcript (job completion unaffected).`);
11000
12221
  } else {
11001
- logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: the host SDK lacks createAgentInstance/updateAgentInstance or the ACP classifier is unavailable; continuing without a durable transcript (job completion unaffected).`);
12222
+ // #289: report WHICH precondition failed (SDK create, SDK update, ACP
12223
+ // classifier, or not-external) plus the host SDK versions, instead of
12224
+ // merging the causes into one opaque line — so the no-transcript cause is
12225
+ // root-causable from this line alone. A PERMANENT create rejection (a 4xx
12226
+ // the engine will never mint on retry) is NOT a construction-time
12227
+ // precondition failure — activation WAS attempted and permanently
12228
+ // rejected — so report it distinctly rather than as "activation not
12229
+ // attempted: unknown precondition" (issue #289 review).
12230
+ const rejectedStatus = agentInstanceProducer.permanentlyRejected;
12231
+ if (rejectedStatus != null) {
12232
+ logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation attempted but permanently rejected (HTTP ${rejectedStatus}); host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
12233
+ } else {
12234
+ const reason = agentInstanceProducer.unavailableReason || 'unknown precondition';
12235
+ logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: ${reason}; host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
12236
+ }
11002
12237
  }
11003
12238
  } catch (err) {
11004
12239
  logger.warn(`[${jobType}] AgentInstance producer activate() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; continuing without a durable transcript (job completion unaffected).`);
@@ -11137,15 +12372,38 @@ async function workAgent(req, flags, ctx) {
11137
12372
  let cwd;
11138
12373
  let extraEnv;
11139
12374
  let repoToken = null;
12375
+ let repoCredentialSource = null;
11140
12376
  if (hasRepo) {
11141
12377
  const provider = envelope.repository.provider || 'github';
11142
12378
  const authRef = envelope.repository.authRef;
11143
- repoToken = githubCloneToken({ provider, authRef, secretResolver }); // absent → anonymous clone
12379
+ // Resolve the credential AND its source once: the source gates whether
12380
+ // a later auth failure may fall back to the operator's `gh auth token`
12381
+ // (only a 'gh-cache' token may — never an explicit GITHUB_TOKEN or a
12382
+ // custom authRef; Copilot review on #293).
12383
+ const cred = githubCloneTokenWithSource({ provider, authRef, secretResolver });
12384
+ repoToken = cred.token; // absent → anonymous clone
12385
+ repoCredentialSource = cred.source;
11144
12386
  try {
11145
12387
  mkdirSync(workerNsDir, { recursive: true });
11146
12388
  runDir = mkdtempSync(join(workerNsDir, 'run-'));
11147
12389
  liveRunDirs.add(runDir);
11148
- provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal });
12390
+ provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal, credentialSource: repoCredentialSource });
12391
+ // #293: provisioning's clone auth-retry may have ROTATED the token
12392
+ // (adopting the fresh gh token into the workspace's askpass env).
12393
+ // Adopt that effective credential for the rest of the job —
12394
+ // finalizeGit's push/PR-reconcile and the checkpoint GC/ref-delete
12395
+ // askpass — so they don't keep failing with the stale pre-provision
12396
+ // token (Copilot review on #293).
12397
+ if (provisioned.effectiveToken) repoToken = provisioned.effectiveToken;
12398
+ if (provisioned.credentialSource) repoCredentialSource = provisioned.credentialSource;
12399
+ // #293: when the rotation above swapped a gh-cache credential, the
12400
+ // `resolved` secret map (built pre-provisioning, forwarded to the
12401
+ // harness as `secretEnv`) still holds the STALE token — sync it so an
12402
+ // allowPr harness's own gh/git runs use the credential provisioning
12403
+ // just adopted, not the one the clone proved dead (Copilot review on
12404
+ // #293). No-op for an env/custom identity, an unchanged token, or a
12405
+ // key the job never forwarded.
12406
+ applyRotatedGhCredential({ resolved, provider, authRef, source: repoCredentialSource, token: repoToken });
11149
12407
  if (provisioned.baseFetchError) {
11150
12408
  logger.warn(`[${jobType}] job ${job.jobKey} base fetch failed (${aiCorr}) — ${oneLineLog(provisioned.baseFetchError)}; base...head diffs may be unavailable`);
11151
12409
  }
@@ -11498,6 +12756,7 @@ async function workAgent(req, flags, ctx) {
11498
12756
  hasPrBranch: provisioned.hasPrBranch,
11499
12757
  envelope,
11500
12758
  token: repoToken,
12759
+ credentialSource: repoCredentialSource,
11501
12760
  logger,
11502
12761
  corr: aiCorr,
11503
12762
  keepRuns,
@@ -11542,6 +12801,17 @@ async function workAgent(req, flags, ctx) {
11542
12801
  } else if (provisioned) {
11543
12802
  gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
11544
12803
  }
12804
+ // Propagate a mid-finalize credential rotation (finalizeGit's push
12805
+ // auth-failure recovery re-read a fresh gh token) to every subsystem that
12806
+ // captured the pre-rotation token: the outer `repoToken` (later redaction /
12807
+ // consumers) and the WIP-checkpoint cleanup, whose post-ack ref deletion
12808
+ // would otherwise authenticate with the credential origin already rejected
12809
+ // and leave the remote WIP ref behind until a GC sweep (Copilot review on
12810
+ // #293). No-op when the token was not rotated.
12811
+ if (gitResult?.rotatedToken) {
12812
+ repoToken = gitResult.rotatedToken;
12813
+ checkpointing?.updateCredential?.(gitResult.rotatedToken);
12814
+ }
11545
12815
  // #264: the work is durable on the pushed branch, so the WIP ref goes —
11546
12816
  // but only once the engine acks job.complete (below): until then a lost
11547
12817
  // lease or failed settle redelivers the job, and the next activation must
@@ -12038,6 +13308,7 @@ async function workAgent(req, flags, ctx) {
12038
13308
  if (finished) return;
12039
13309
  finished = true;
12040
13310
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
13311
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12041
13312
  if (reaperTimer) clearInterval(reaperTimer);
12042
13313
  if (runDirTimer) clearInterval(runDirTimer);
12043
13314
  if (agenticPlane) {
@@ -12068,6 +13339,7 @@ async function workAgent(req, flags, ctx) {
12068
13339
  aborting = true;
12069
13340
  draining = true;
12070
13341
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
13342
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12071
13343
  logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
12072
13344
  // Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
12073
13345
  // clears the ownership registry, so we can yield each one afterwards.
@@ -12113,6 +13385,11 @@ async function workAgent(req, flags, ctx) {
12113
13385
  if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
12114
13386
  const n0 = inFlightCount();
12115
13387
  logger.info(`Received ${signal} — draining: polling stopped; waiting on ${n0} in-flight job(s) to finish (send SIGTERM to abort).`);
13388
+ // The refresh timer is ref'd (so it fires on cadence even under a busy
13389
+ // loop) and there may be NO in-flight job during a drain — clear it now
13390
+ // so it can't keep the process alive once the drain completes and the
13391
+ // runtime is interrupted.
13392
+ if (ghRefreshTimer) clearInterval(ghRefreshTimer);
12116
13393
  // Poll until every in-flight job has settled and released its slot, then
12117
13394
  // interrupt the now-idle runtime and exit. Wait INDEFINITELY — a force
12118
13395
  // abort (SIGTERM) is the only escape hatch.
@@ -12508,8 +13785,9 @@ function formatDuration(ms) {
12508
13785
  return `${d}d${h % 24}h`;
12509
13786
  }
12510
13787
 
12511
- /** Project a live/stored worker record to a status row (pure w.r.t. `now`). */
12512
- function summarizeSupervisorWorker(w, now = Date.now()) {
13788
+ /** Project a live/stored worker record to a status row (pure w.r.t. `now`).
13789
+ * `readActivity` is an injectable seam over readWorkerActivity (testing). */
13790
+ function summarizeSupervisorWorker(w, now = Date.now(), { readActivity = readWorkerActivity } = {}) {
12513
13791
  const alive = isPidAlive(w.pid);
12514
13792
  // Absolute base epoch (ms) the worker started, so an attached console can
12515
13793
  // re-age `uptimeMs` locally on each tick (see reageSupervisorStatus) without
@@ -12523,8 +13801,9 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12523
13801
  let activity = null; // { state: 'busy'|'idle', jobs: [{ key, type, sinceMs, sinceEpochMs }] }
12524
13802
  let engine = null; // job-polling engine base URL this worker reported (#99)
12525
13803
  let agentic = null; // { status, mode, url, discovered } agentic-channel state (#99)
13804
+ let ghAuth = null; // #293: { present, fingerprint, scopes, since, ageMs, lastCheckAgoMs } — never the token
12526
13805
  if (alive) {
12527
- const act = readWorkerActivity(w.id);
13806
+ const act = readActivity(w.id);
12528
13807
  if (act && act.pid === w.pid) {
12529
13808
  const jobs = Array.isArray(act.jobs)
12530
13809
  ? act.jobs
@@ -12556,6 +13835,24 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12556
13835
  // stale incarnation can't show a dead worker as connected to a hub.
12557
13836
  engine = typeof act.engine === 'string' && act.engine ? act.engine : null;
12558
13837
  agentic = act.agentic && typeof act.agentic === 'object' ? act.agentic : null;
13838
+ // #293: the gh token identity rides the same marker. Re-age the ticking
13839
+ // durations from the stamped `since`/marker `updatedAt` so a console that
13840
+ // re-ages locally (and a status snapshot read seconds later) shows a
13841
+ // current age, not the write-time one. Never carries the token value.
13842
+ if (act.ghAuth && typeof act.ghAuth === 'object') {
13843
+ const g = act.ghAuth;
13844
+ const ageMs = Number.isFinite(g.since) ? Math.max(0, now - g.since) : (Number.isFinite(g.ageMs) ? g.ageMs : null);
13845
+ const lastCheckAgoMs = Number.isFinite(act.updatedAt) && Number.isFinite(g.lastCheckAgoMs)
13846
+ ? Math.max(0, now - act.updatedAt + g.lastCheckAgoMs)
13847
+ : (Number.isFinite(g.lastCheckAgoMs) ? g.lastCheckAgoMs : null);
13848
+ // Coerce the identity strings at the (untrusted) marker boundary: a
13849
+ // malformed/tampered/older marker could carry a non-string `scopes` or
13850
+ // `fingerprint` (a number, array, object), and a downstream string op
13851
+ // (e.g. `scopes.split(',')` in supervisorGhAuthCell) would THROW and
13852
+ // crash `supervisor status` (Copilot review on #294). Keep only real
13853
+ // strings; anything else degrades to null (rendered as `?`/unknown).
13854
+ ghAuth = { present: g.present === true, fingerprint: typeof g.fingerprint === 'string' ? g.fingerprint : null, scopes: typeof g.scopes === 'string' ? g.scopes : null, since: Number.isFinite(g.since) ? g.since : null, ageMs, lastCheckAgoMs };
13855
+ }
12559
13856
  }
12560
13857
  // No marker (or a stale-pid one): leave activity null → rendered as unknown.
12561
13858
  }
@@ -12580,6 +13877,7 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
12580
13877
  activity,
12581
13878
  engine,
12582
13879
  agentic,
13880
+ ghAuth,
12583
13881
  };
12584
13882
  }
12585
13883
 
@@ -12614,6 +13912,16 @@ function supervisorStatusSignature(workers) {
12614
13912
  // change is a real transition that must repaint attached consoles (#99).
12615
13913
  w.engine ?? '',
12616
13914
  w.agentic ? (w.agentic.status ?? '') : null,
13915
+ // #293: the gh credential IDENTITY — presence, fingerprint, scopes, and the
13916
+ // token's `since` epoch. A token rotation / scope change / logout on an
13917
+ // otherwise-idle worker is a real transition the attached console must
13918
+ // repaint for (the GH AUTH cell changes), so fold it into the signature.
13919
+ // Deliberately EXCLUDES the ticking durations (ageMs, lastCheckAgoMs) —
13920
+ // those merely elapse and are re-aged locally (see reageSupervisorStatus),
13921
+ // so including them would make every clock tick a spurious "change".
13922
+ w.ghAuth
13923
+ ? `${w.ghAuth.present ? 1 : 0}\u0000${w.ghAuth.fingerprint ?? ''}\u0000${w.ghAuth.scopes ?? ''}\u0000${w.ghAuth.since ?? ''}`
13924
+ : null,
12617
13925
  ]),
12618
13926
  );
12619
13927
  }
@@ -12676,6 +13984,32 @@ function supervisorAgenticCell(w) {
12676
13984
  return String(w.agentic.status);
12677
13985
  }
12678
13986
 
13987
+ /**
13988
+ * GH AUTH cell (#293): the cached gh token's age + scopes, so "this worker is
13989
+ * holding a stale token" is a one-line check instead of a manual
13990
+ * /proc/<pid>/environ vs `gh auth token` comparison. `-` for a down/stopping
13991
+ * worker, `?` for a live worker not (yet) reporting or on an older build whose
13992
+ * marker predates the field, `none` when gh is missing/not logged in. The
13993
+ * fingerprint is available in the status JSON (and the worker log) when two
13994
+ * workers need to be told apart; the table keeps the cell narrow (age + scopes).
13995
+ */
13996
+ function supervisorGhAuthCell(w) {
13997
+ if (w.state !== 'running') return '-';
13998
+ if (!w.activity) return '?'; // alive but not reporting
13999
+ const g = w.ghAuth;
14000
+ if (!g) return '?'; // marker predates the ghAuth field
14001
+ if (!g.present) return 'none';
14002
+ const age = g.ageMs != null ? formatDuration(g.ageMs) : '?';
14003
+ // Defensive against a malformed activity marker (untrusted JSON): only call
14004
+ // string ops on a real string, so a non-string `scopes` (a corrupt/older/
14005
+ // tampered marker) renders `?` instead of throwing and crashing the whole
14006
+ // status table (Copilot review on #294). statusFromActivity already coerces at
14007
+ // the marker boundary; this is the belt-and-braces at the render edge, which
14008
+ // also runs on frames built elsewhere (e.g. reageSupervisorStatus passthrough).
14009
+ const scopes = typeof g.scopes === 'string' && g.scopes ? g.scopes.split(',').map((s) => s.trim()).filter(Boolean).join(',') : '?';
14010
+ return `${age} [${scopes}]`;
14011
+ }
14012
+
12679
14013
  /**
12680
14014
  * Re-age a supervisor status snapshot to `now`, recomputing the ticking
12681
14015
  * durations (`uptimeMs`, per-job `sinceMs`) from the absolute base epochs the
@@ -12718,7 +14052,17 @@ function reageSupervisorStatus(status, now = Date.now()) {
12718
14052
  const state = jobs.some((j) => j && typeof j === 'object' && !j.settlementPending) ? 'busy' : 'idle';
12719
14053
  activity = { ...activity, jobs, state };
12720
14054
  }
12721
- return { ...w, uptimeMs, activity };
14055
+ // #293: re-age the gh credential's AGE from its absolute `since` epoch, the
14056
+ // same way uptime/job-age tick locally — otherwise an attached console that
14057
+ // re-ages the cached snapshot on every tick (never rebuilding it via
14058
+ // summarizeSupervisorWorker) would freeze the GH AUTH cell's age at its
14059
+ // snapshot-time value. `lastCheckAgoMs` is RELATIVE (no absolute base rides
14060
+ // the frame), so it cannot be re-aged here and is left as-is.
14061
+ let ghAuth = w.ghAuth;
14062
+ if (ghAuth && typeof ghAuth === 'object' && Number.isFinite(ghAuth.since)) {
14063
+ ghAuth = { ...ghAuth, ageMs: Math.max(0, now - ghAuth.since) };
14064
+ }
14065
+ return { ...w, uptimeMs, activity, ghAuth };
12722
14066
  }),
12723
14067
  };
12724
14068
  }
@@ -12878,6 +14222,7 @@ function formatSupervisorStatus(status) {
12878
14222
  state: String(w.state) === 'down' && w.configFailed === true ? 'down (config)' : String(w.state),
12879
14223
  engine: supervisorEngineCell(w),
12880
14224
  agentic: supervisorAgenticCell(w),
14225
+ ghAuth: supervisorGhAuthCell(w),
12881
14226
  job: supervisorJobCell(w),
12882
14227
  pid: w.pid ? String(w.pid) : '-',
12883
14228
  restarts: String(w.restarts),
@@ -12887,8 +14232,8 @@ function formatSupervisorStatus(status) {
12887
14232
  // ENGINE + AGENTIC sit early (just after STATE) so the pinned live view's
12888
14233
  // width clamp (which trims from the right) drops the least-critical columns
12889
14234
  // (LAST EXIT, UPTIME) first and keeps the visibility diagnostics visible.
12890
- const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
12891
- const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'job', 'pid', 'restarts', 'uptime', 'last'];
14235
+ const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', ghAuth: 'GH AUTH', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
14236
+ const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'ghAuth', 'job', 'pid', 'restarts', 'uptime', 'last'];
12892
14237
  const width = {};
12893
14238
  for (const c of cols) width[c] = Math.max(head[c].length, ...rows.map((r) => r[c].length));
12894
14239
  const fmt = (r) => ' ' + cols.map((c) => r[c].padEnd(width[c])).join(' ');
@@ -13800,6 +15145,40 @@ async function runSupervisorDaemon() {
13800
15145
  attachClients.add(sock);
13801
15146
  sock.write(encodeFrame(statusFrame(false)));
13802
15147
  break;
15148
+ case 'reload-credentials': {
15149
+ // #293: tell every worker to re-read its gh auth token (SIGUSR1,
15150
+ // async — the still-valid old token is served until the new one
15151
+ // atomically lands) WITHOUT draining in-flight jobs — the operator
15152
+ // re-logged in and wants
15153
+ // the fleet to pick up the fresh credential now, not after a restart.
15154
+ // Windows cannot deliver SIGUSR1 (Node terminates the process), so
15155
+ // refuse there; the TTL refresh still adopts a rotated token.
15156
+ if (osPlatform() === 'win32') {
15157
+ sock.write(encodeFrame({ ok: false, error: 'reload-credentials is not supported on Windows (it relies on SIGUSR1, which Windows cannot deliver) — the workers\' TTL refresh still picks up a new token, or use `supervisor restart`', final: true }));
15158
+ break;
15159
+ }
15160
+ const rawCreds = Array.isArray(req.targets)
15161
+ ? req.targets.flatMap((t) => resolveTargets(t))
15162
+ : resolveTargets(req.target);
15163
+ const credIds = [...new Set(rawCreds)];
15164
+ const signaled = [];
15165
+ const failed = [];
15166
+ for (const id of credIds) {
15167
+ const w = workers.get(id);
15168
+ const alive = !!(w && isPidAlive(w.pid));
15169
+ // #293: only signal a worker whose SIGUSR1 (reload) listener is
15170
+ // PROVEN installed (reloadCredentialsReady) — otherwise the signal
15171
+ // hits Node's default action (start the inspector) and the daemon
15172
+ // would wrongly record "reloaded". Skipped workers land in `failed`
15173
+ // so `ok` stays honest; they pick up the fresh token at their own
15174
+ // startup prime.
15175
+ if (!reloadCredentialsReady(w, alive ? readWorkerActivity(id) : null, alive)) { failed.push(id); continue; }
15176
+ try { process.kill(w.pid, 'SIGUSR1'); signaled.push(id); }
15177
+ catch { failed.push(id); }
15178
+ }
15179
+ sock.write(encodeFrame({ ok: failed.length === 0, type: 'credentials-reloaded', signaled, failed, final: true }));
15180
+ break;
15181
+ }
13803
15182
  case 'stop': {
13804
15183
  // #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
13805
15184
  // jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
@@ -14305,6 +15684,45 @@ async function supervisorReloadCmd(req) {
14305
15684
  if (outcome === 'error') process.exit(1);
14306
15685
  }
14307
15686
 
15687
+ /**
15688
+ * Re-read every supervised worker's cached gh auth token WITHOUT draining
15689
+ * in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
15690
+ * worker re-reads `gh auth token` NON-DESTRUCTIVELY (a forced async re-read that
15691
+ * swaps the cache atomically when the new value lands — the still-valid old
15692
+ * token is served meanwhile, never dropped), and logs the new fingerprint/
15693
+ * scopes. Use after a `gh auth login`/`refresh` so the fleet picks up the fresh
15694
+ * credential immediately instead of waiting for the TTL refresh or a restart.
15695
+ */
15696
+ async function supervisorReloadCredentialsCmd(req) {
15697
+ const logger = getLogger();
15698
+ const target = req.positional[1] || 'all';
15699
+ const running = await liveSupervisor();
15700
+ if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
15701
+ const socketPath = running.socket || getSupervisorSocketPath();
15702
+ let res;
15703
+ try {
15704
+ res = await supervisorRequest({ op: 'reload-credentials', target }, { socketPath });
15705
+ } catch (err) {
15706
+ logger.error(`Could not reach the supervisor control socket to reload credentials: ${err?.message || err}`);
15707
+ process.exit(1);
15708
+ }
15709
+ // A structured credentials-reloaded result carries `signaled`/`failed` arrays
15710
+ // even on a PARTIAL failure (daemon returns `ok:false` with no `error` when
15711
+ // some workers were down). Report those lists BEFORE the generic failure guard
15712
+ // — exactly as the rolling-reload client does — so an operator can see which
15713
+ // workers picked up the new token and which are down, instead of a bare
15714
+ // "reload-credentials failed". The generic guard then only catches a truly
15715
+ // unstructured failure (unreachable socket, the Windows refusal with `error`).
15716
+ if (!res) { logger.error('reload-credentials failed'); process.exit(1); }
15717
+ const hasStructured = Array.isArray(res.signaled) || Array.isArray(res.failed);
15718
+ if (res.ok === false && !hasStructured) { logger.error(res.error || 'reload-credentials failed'); process.exit(1); }
15719
+ const signaled = Array.isArray(res.signaled) ? res.signaled : [];
15720
+ const failed = Array.isArray(res.failed) ? res.failed : [];
15721
+ if (signaled.length > 0) logger.info(`Told ${signaled.length} worker(s) to re-read their gh auth token: ${signaled.join(', ')}. Each re-reads \`gh auth token\` (non-destructively — the still-valid old token is served until the new one lands) and logs the new fingerprint/scopes.`);
15722
+ else logger.warn('No running workers matched — nothing to reload.');
15723
+ if (failed.length > 0) { logger.error(`Could not signal: ${failed.join(', ')} (down or gone).`); process.exit(1); }
15724
+ }
15725
+
14308
15726
  /**
14309
15727
  * Count the in-flight jobs across a fleet snapshot (array of
14310
15728
  * `summarizeSupervisorWorker` results) — the number an operator is waiting on
@@ -14515,6 +15933,10 @@ async function attachSupervisorConsole(state) {
14515
15933
  out(`• removed: ${frame.removed.join(', ') || '(none matched)'}.`);
14516
15934
  } else if (frame.type === 'restarted') {
14517
15935
  out(`• restarted: ${frame.restarted.join(', ') || '(none matched)'}.`);
15936
+ } else if (frame.type === 'credentials-reloaded') {
15937
+ const sig = Array.isArray(frame.signaled) ? frame.signaled : [];
15938
+ const fail = Array.isArray(frame.failed) ? frame.failed : [];
15939
+ out(`• credentials reload signaled: ${sig.join(', ') || '(none matched)'}${fail.length ? ` — could not signal: ${fail.join(', ')}` : ''}.`);
14518
15940
  } else if (frame.ok === false) {
14519
15941
  out(`! ${frame.error}`);
14520
15942
  }
@@ -14560,10 +15982,15 @@ async function attachSupervisorConsole(state) {
14560
15982
  case '': break;
14561
15983
  case 'help':
14562
15984
  out('Commands: status | add <profile> [--name <worker>] [--instances <n>] [work flags] |');
14563
- out(' remove <id|profile|all> | restart <id|profile|all> |');
15985
+ out(' remove <id|profile|all> | restart <id|profile|all> | reload-credentials [id|profile|all] |');
14564
15986
  out(' logs [id] | detach | stop | help');
14565
15987
  break;
14566
15988
  case 'status': sock.write(encodeFrame({ op: 'status' })); break;
15989
+ case 'reload-credentials': {
15990
+ const target = parts.shift() || 'all';
15991
+ sock.write(encodeFrame({ op: 'reload-credentials', target }));
15992
+ break;
15993
+ }
14567
15994
  case 'add': {
14568
15995
  const profile = parts.shift();
14569
15996
  if (!profile) { out('usage: add <profile> [--name <worker>] [--instances <n>] [work flags]'); break; }
@@ -15109,6 +16536,9 @@ async function supervisorCommand(req, flags, ctx) {
15109
16536
  case 'reload':
15110
16537
  await supervisorReloadCmd(req);
15111
16538
  return;
16539
+ case 'reload-credentials':
16540
+ await supervisorReloadCredentialsCmd(req);
16541
+ return;
15112
16542
  case 'stop':
15113
16543
  await supervisorStopCmd(coerceBool(flags?.force, false));
15114
16544
  return;
@@ -15117,7 +16547,7 @@ async function supervisorCommand(req, flags, ctx) {
15117
16547
  supervisorLogsCmd(req);
15118
16548
  return;
15119
16549
  default:
15120
- getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach`);
16550
+ getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
15121
16551
  process.exit(1);
15122
16552
  }
15123
16553
  }
@@ -17922,8 +19352,30 @@ export {
17922
19352
  namespaceHasLiveHarness,
17923
19353
  authUrl,
17924
19354
  githubCloneToken,
19355
+ githubCloneTokenWithSource,
17925
19356
  ghAuthTokenFromCli,
17926
19357
  primeGhAuthToken,
19358
+ refreshGhAuthToken,
19359
+ refreshGhAuthTokenAsync,
19360
+ invalidateGhAuthToken,
19361
+ recoverGhTokenAfterAuthFailure,
19362
+ setGhAuthTokenReader,
19363
+ resetGhAuthTokenReader,
19364
+ classifyGhAuthRead,
19365
+ GH_AUTH_READ_FAILED,
19366
+ adoptGhAuthToken,
19367
+ adoptGhAuthTokenAsync,
19368
+ readGhAuthTokenAsync,
19369
+ ghCredentialEnvName,
19370
+ applyRotatedGhCredential,
19371
+ ghAuthTokenInfo,
19372
+ describeGhAuthToken,
19373
+ ghTokenFingerprint,
19374
+ ghTokenScopes,
19375
+ ghTokenScopesAsync,
19376
+ isGitAuthFailure,
19377
+ startGhAuthTokenRefresh,
19378
+ GH_AUTH_TOKEN_TTL_MS,
17927
19379
  ghAuthEnv,
17928
19380
  redactToken,
17929
19381
  agentRunsRoot,
@@ -18004,10 +19456,12 @@ export {
18004
19456
  countSupervisorInFlight,
18005
19457
  supervisorEngineCell,
18006
19458
  supervisorAgenticCell,
19459
+ supervisorGhAuthCell,
18007
19460
  agenticStateForTarget,
18008
19461
  normalizeAgenticMessage,
18009
19462
  buildActivityPayload,
18010
19463
  activityMarkerReadyFor,
19464
+ reloadCredentialsReady,
18011
19465
  waitForChildExit,
18012
19466
  supervisorWorkerActivityFile,
18013
19467
  SETTLEMENT_PENDING_GHOST_TTL_MS,
@@ -18019,6 +19473,7 @@ export {
18019
19473
  supervisorStartCmd,
18020
19474
  supervisorAddCmd,
18021
19475
  supervisorReloadCmd,
19476
+ supervisorReloadCredentialsCmd,
18022
19477
  workforceReloadCmd,
18023
19478
  runningSupervisor,
18024
19479
  readSupervisorState,
@@ -18127,6 +19582,7 @@ export const metadata = {
18127
19582
  { command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
18128
19583
  { command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
18129
19584
  { command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
19585
+ { command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
18130
19586
  { command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
18131
19587
  { command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
18132
19588
  { command: 'c8ctl nano workforce add qwen --instances 2 --roles pr-review,feature', description: "Add an entry mapped to explicit job types (<rank>:pr-review, <rank>:feature, where <rank> is the qwen hire's rank at start) — does not mutate the hired profile" },
@@ -18367,7 +19823,7 @@ function printUsage() {
18367
19823
  console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--terminal pty|pipe] [--protocol pipe|acp] [--permission yolo|escalate|filter] [--env NAME=VALUE ...] [--list]');
18368
19824
  console.log(' c8ctl nano assign <profileName> <cap[,cap...]> [--name <n>] [--capabilities <a,b>]');
18369
19825
  console.log(' c8ctl nano work <profileName> [--auto [--auto-scope <p>]] [--arg <switch> ...] [--recovery-window <ms>] [--idle-timeout <ms>] [--job-timeout <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
18370
- console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach] ... (manage many workers from one terminal)');
19826
+ console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach] ... (manage many workers from one terminal)');
18371
19827
  console.log(' c8ctl nano workforce [add|remove|list|start|status|stop|reload] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
18372
19828
  console.log('');
18373
19829
  console.log('Subcommands:');